示例#1
0
 def test_two_nodes_fail(self):
     # Create container1
     # Kill container1 servers excepting one of the primaries
     # Delete container1 directly to the one primary still up
     # Restart other container1 servers
     # Get to a final state
     # Assert all container1 servers indicate container1 is gone (happens
     #   because the one node that knew about the delete replicated to the
     #   others.)
     # Assert account level also indicates container1 is gone
     container1 = 'container-%s' % uuid4()
     cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
     client.put_container(self.url, self.token, container1)
     cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
     kill_server(cnodes[0]['port'], self.port2server, self.pids)
     kill_server(cnodes[1]['port'], self.port2server, self.pids)
     direct_client.direct_delete_container(cnodes[2], cpart, self.account,
                                           container1)
     start_server(cnodes[0]['port'], self.port2server, self.pids)
     start_server(cnodes[1]['port'], self.port2server, self.pids)
     start_server(cnp_port, self.port2server, self.pids)
     get_to_final_state()
     for cnode in cnodes:
         exc = None
         try:
             direct_client.direct_get_container(cnode, cpart, self.account,
                                                container1)
         except client.ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
     headers, containers = client.get_account(self.url, self.token)
     self.assertEquals(headers['x-account-container-count'], '0')
     self.assertEquals(headers['x-account-object-count'], '0')
     self.assertEquals(headers['x-account-bytes-used'], '0')
 def test_main(self):
     # Create container
     # Kill container servers excepting two of the primaries
     # Create container/obj
     # Restart other primary server
     # Assert it does not know about container/obj
     # Run the object-updaters
     # Assert the other primary server now knows about container/obj
     container = 'container-%s' % uuid4()
     client.put_container(self.url, self.token, container)
     cpart, cnodes = self.container_ring.get_nodes(self.account, container)
     cnode = cnodes[0]
     kill_nonprimary_server(cnodes, self.port2server, self.pids)
     kill_server(cnode['port'], self.port2server, self.pids)
     obj = 'object-%s' % uuid4()
     client.put_object(self.url, self.token, container, obj, '')
     start_server(cnode['port'], self.port2server, self.pids)
     self.assert_(not direct_client.direct_get_container(
         cnode, cpart, self.account, container)[1])
     processes = []
     for node in xrange(1, 5):
         processes.append(Popen(['swift-object-updater',
                                 '/etc/swift/object-server/%d.conf' % node,
                                 'once']))
     for process in processes:
         process.wait()
     objs = [o['name'] for o in direct_client.direct_get_container(
         cnode, cpart, self.account, container)[1]]
     self.assert_(obj in objs)
示例#3
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        # Kill container servers excepting two of the primaries
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        kill_nonprimary_server(cnodes, self.ipport2server)
        kill_server((cnode['ip'], cnode['port']), self.ipport2server)

        # Create container/obj
        obj = 'object-%s' % uuid4()
        client.put_object(self.url, self.token, container, obj, '')

        # Restart other primary server
        start_server((cnode['ip'], cnode['port']), self.ipport2server)

        # Assert it does not know about container/obj
        self.assertFalse(
            direct_client.direct_get_container(cnode, cpart, self.account,
                                               container)[1])

        # Run the object-updaters
        Manager(['object-updater']).once()

        # Assert the other primary server now knows about container/obj
        objs = [
            o['name'] for o in direct_client.direct_get_container(
                cnode, cpart, self.account, container)[1]
        ]
        self.assertTrue(obj in objs)
    def test_async_update_after_PUT(self):
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url, self.token, 'c1',
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(direct_client.direct_get_container(
            cnodes[0], cpart, self.account, 'c1')[1])

        # Run the object-updaters to be sure updates are done
        Manager(['object-updater']).once()

        # check the re-started container server has update with override values
        obj = direct_client.direct_get_container(
            cnodes[0], cpart, self.account, 'c1')[1][0]
        self.assertEqual(meta['etag'], obj['hash'])
        self.assertEqual(len(content), obj['bytes'])
示例#5
0
    def test_all_nodes_fail(self):
        # Create container1
        container1 = 'container-%s' % uuid4()
        cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
        client.put_container(self.url, self.token, container1)
        client.put_object(self.url, self.token, container1, 'obj1', 'data1')

        # All primaries go down
        for cnode in cnodes:
            kill_server((cnode['ip'], cnode['port']), self.ipport2server)

        # Can't GET the container
        with self.assertRaises(client.ClientException) as caught:
            client.get_container(self.url, self.token, container1)
        self.assertEqual(caught.exception.http_status, 503)

        # But we can still write objects! The old info is still in memcache
        client.put_object(self.url, self.token, container1, 'obj2', 'data2')

        # Can't POST the container, either
        with self.assertRaises(client.ClientException) as caught:
            client.post_container(self.url, self.token, container1, {})
        self.assertEqual(caught.exception.http_status, 503)

        # Though it *does* evict the cache
        with self.assertRaises(client.ClientException) as caught:
            client.put_object(self.url, self.token, container1, 'obj3', 'x')
        self.assertEqual(caught.exception.http_status, 503)
示例#6
0
 def test_one_node_fails(self):
     # Create container1
     # Kill container1 servers excepting two of the primaries
     # Delete container1
     # Restart other container1 primary server
     # Create container1/object1 (allowed because at least server thinks the
     #   container exists)
     # Get to a final state
     # Assert all container1 servers indicate container1 is alive and
     #   well with object1
     # Assert account level also indicates container1 is alive and
     #   well with object1
     container1 = 'container-%s' % uuid4()
     cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
     client.put_container(self.url, self.token, container1)
     kill_nonprimary_server(cnodes, self.port2server, self.pids)
     kill_server(cnodes[0]['port'], self.port2server, self.pids)
     client.delete_container(self.url, self.token, container1)
     start_server(cnodes[0]['port'], self.port2server, self.pids)
     client.put_object(self.url, self.token, container1, 'object1', '123')
     get_to_final_state()
     for cnode in cnodes:
         self.assertEquals(
             [o['name'] for o in direct_client.direct_get_container(
                 cnode, cpart, self.account, container1)[1]],
             ['object1'])
     headers, containers = client.get_account(self.url, self.token)
     self.assertEquals(headers['x-account-container-count'], '1')
     self.assertEquals(headers['x-account-object-count'], '1')
     self.assertEquals(headers['x-account-bytes-used'], '3')
 def test_one_node_fails(self):
     # Create container1
     # Kill container1 servers excepting two of the primaries
     # Delete container1
     # Restart other container1 primary server
     # Create container1/object1 (allowed because at least server thinks the
     #   container exists)
     # Get to a final state
     # Assert all container1 servers indicate container1 is alive and
     #   well with object1
     # Assert account level also indicates container1 is alive and
     #   well with object1
     container1 = 'container-%s' % uuid4()
     cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
     client.put_container(self.url, self.token, container1)
     kill_nonprimary_server(cnodes, self.port2server, self.pids)
     kill_server(cnodes[0]['port'], self.port2server, self.pids)
     client.delete_container(self.url, self.token, container1)
     start_server(cnodes[0]['port'], self.port2server, self.pids)
     client.put_object(self.url, self.token, container1, 'object1', '123')
     get_to_final_state()
     for cnode in cnodes:
         self.assertEquals([
             o['name'] for o in direct_client.direct_get_container(
                 cnode, cpart, self.account, container1)[1]
         ], ['object1'])
     headers, containers = client.get_account(self.url, self.token)
     self.assertEquals(headers['x-account-container-count'], '1')
     self.assertEquals(headers['x-account-object-count'], '1')
     self.assertEquals(headers['x-account-bytes-used'], '3')
示例#8
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        # Kill container servers excepting two of the primaries
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        kill_nonprimary_server(cnodes, self.ipport2server, self.pids)
        kill_server((cnode['ip'], cnode['port']),
                    self.ipport2server, self.pids)

        # Create container/obj
        obj = 'object-%s' % uuid4()
        client.put_object(self.url, self.token, container, obj, '')

        # Restart other primary server
        start_server((cnode['ip'], cnode['port']),
                     self.ipport2server, self.pids)

        # Assert it does not know about container/obj
        self.assert_(not direct_client.direct_get_container(
            cnode, cpart, self.account, container)[1])

        # Run the object-updaters
        Manager(['object-updater']).once()

        # Assert the other primary server now knows about container/obj
        objs = [o['name'] for o in direct_client.direct_get_container(
            cnode, cpart, self.account, container)[1]]
        self.assert_(obj in objs)
 def test_two_nodes_fail(self):
     # Create container1
     # Kill container1 servers excepting one of the primaries
     # Delete container1 directly to the one primary still up
     # Restart other container1 servers
     # Get to a final state
     # Assert all container1 servers indicate container1 is gone (happens
     #   because the one node that knew about the delete replicated to the
     #   others.)
     # Assert account level also indicates container1 is gone
     container1 = 'container-%s' % uuid4()
     cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
     client.put_container(self.url, self.token, container1)
     cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
     kill_server(cnodes[0]['port'], self.port2server, self.pids)
     kill_server(cnodes[1]['port'], self.port2server, self.pids)
     direct_client.direct_delete_container(cnodes[2], cpart, self.account,
                                           container1)
     start_server(cnodes[0]['port'], self.port2server, self.pids)
     start_server(cnodes[1]['port'], self.port2server, self.pids)
     start_server(cnp_port, self.port2server, self.pids)
     get_to_final_state()
     for cnode in cnodes:
         exc = None
         try:
             direct_client.direct_get_container(cnode, cpart, self.account,
                                                container1)
         except client.ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
     headers, containers = client.get_account(self.url, self.token)
     self.assertEquals(headers['x-account-container-count'], '0')
     self.assertEquals(headers['x-account-object-count'], '0')
     self.assertEquals(headers['x-account-bytes-used'], '0')
示例#10
0
 def test_main(self):
     # Create container
     # Kill container servers excepting two of the primaries
     # Create container/obj
     # Restart other primary server
     # Assert it does not know about container/obj
     # Run the object-updaters
     # Assert the other primary server now knows about container/obj
     container = 'container-%s' % uuid4()
     client.put_container(self.url, self.token, container)
     cpart, cnodes = self.container_ring.get_nodes(self.account, container)
     cnode = cnodes[0]
     kill_nonprimary_server(cnodes, self.port2server, self.pids)
     kill_server(cnode['port'], self.port2server, self.pids)
     obj = 'object-%s' % uuid4()
     client.put_object(self.url, self.token, container, obj, '')
     start_server(cnode['port'], self.port2server, self.pids)
     self.assert_(not direct_client.direct_get_container(
         cnode, cpart, self.account, container)[1])
     processes = []
     for node in xrange(1, 5):
         processes.append(
             Popen([
                 'swift-object-updater', self.configs['object'] % node,
                 'once'
             ]))
     for process in processes:
         process.wait()
     objs = [
         o['name'] for o in direct_client.direct_get_container(
             cnode, cpart, self.account, container)[1]
     ]
     self.assert_(obj in objs)
    def test_async_updates_after_PUT_and_POST(self):
        # verify correct update values when PUT update and POST updates are
        # missed but then async updates are sent
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url,
                             self.token,
                             'c1',
                             headers={'X-Storage-Policy': self.policy.name})

        # PUT and POST to object while one container server is stopped so that
        # we force async updates to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url,
                          self.token,
                          'c1',
                          'o1',
                          contents=content,
                          content_type='test/ctype')
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # use internal client for POST so we can force fast-post mode
        int_client = self.make_internal_client(object_post_as_copy=False)
        int_client.set_object_metadata(self.account, 'c1', 'o1',
                                       {'X-Object-Meta-Fruit': 'Tomato'})
        self.assertEqual('Tomato',
                         int_client.get_object_metadata(
                             self.account, 'c1',
                             'o1')['x-object-meta-fruit'])  # sanity

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(
            direct_client.direct_get_container(cnodes[0], cpart, self.account,
                                               'c1')[1])

        # Run the object-updaters to send the async pendings
        Manager(['object-updater']).once()

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(
                cnode, cpart, self.account, 'c1')[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]['bytes'])
            self.assertEqual('test/ctype', listing[0]['content_type'])
            listing_etags.add(listing[0]['hash'])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, 'c1')
        self.assertEqual(1, len(listing))
        self.assertEqual('o1', listing[0]['name'])
        self.assertEqual(len(content), listing[0]['bytes'])
        self.assertEqual(meta['etag'], listing[0]['hash'])
        self.assertEqual('test/ctype', listing[0]['content_type'])
示例#12
0
    def test_main(self):
        # Create container
        container = "container-%s" % uuid4()
        client.put_container(self.url, self.token, container)

        # Kill container servers excepting two of the primaries
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        kill_nonprimary_server(cnodes, self.ipport2server)
        kill_server((cnode["ip"], cnode["port"]), self.ipport2server)

        # Create container/obj
        obj = "object-%s" % uuid4()
        client.put_object(self.url, self.token, container, obj, "")

        # Restart other primary server
        start_server((cnode["ip"], cnode["port"]), self.ipport2server)

        # Assert it does not know about container/obj
        self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1])

        # Run the object-updaters
        Manager(["object-updater"]).once()

        # Assert the other primary server now knows about container/obj
        objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
        self.assertIn(obj, objs)
    def test_missing_container(self):
        # In this test, we need to put container at handoff devices, so we
        # need container devices more than replica count
        if len(self.container_ring.devs) <= self.container_ring.replica_count:
            raise SkipTest("Need devices more that replica count")

        container = 'container-%s' % uuid4()
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)

        # Kill all primary container servers
        for cnode in cnodes:
            kill_server((cnode['ip'], cnode['port']), self.ipport2server)

        # Create container, and all of its replicas are placed at handoff
        # device
        try:
            client.put_container(self.url, self.token, container)
        except ClientException as err:
            # if the cluster doesn't have enough devices, swift may return
            # error (ex. When we only have 4 devices in 3-replica cluster).
            self.assertEqual(err.http_status, 503)

        # Assert handoff device has a container replica
        another_cnode = next(self.container_ring.get_more_nodes(cpart))
        direct_client.direct_get_container(
            another_cnode, cpart, self.account, container)

        # Restart all primary container servers
        for cnode in cnodes:
            start_server((cnode['ip'], cnode['port']), self.ipport2server)

        # Create container/obj
        obj = 'object-%s' % uuid4()
        client.put_object(self.url, self.token, container, obj, '')

        # Run the object-updater
        Manager(['object-updater']).once()

        # Run the container-replicator, and now, container replicas
        # at handoff device get moved to primary servers
        Manager(['container-replicator']).once()

        # Assert container replicas in primary servers, just moved by
        # replicator don't know about the object
        for cnode in cnodes:
            self.assertFalse(direct_client.direct_get_container(
                cnode, cpart, self.account, container)[1])

        # since the container is empty - we can delete it!
        client.delete_container(self.url, self.token, container)

        # Re-run the object-updaters and now container replicas in primary
        # container servers should get updated
        Manager(['object-updater']).once()

        # Assert all primary container servers know about container/obj
        for cnode in cnodes:
            objs = [o['name'] for o in direct_client.direct_get_container(
                    cnode, cpart, self.account, container)[1]]
            self.assertIn(obj, objs)
示例#14
0
    def test_async_update_after_PUT(self):
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url,
                             self.token,
                             'c1',
                             headers={'X-Storage-Policy': self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(
            direct_client.direct_get_container(cnodes[0], cpart, self.account,
                                               'c1')[1])

        # Run the object-updaters to be sure updates are done
        Manager(['object-updater']).once()

        # check the re-started container server has update with override values
        obj = direct_client.direct_get_container(cnodes[0], cpart,
                                                 self.account, 'c1')[1][0]
        self.assertEqual(meta['etag'], obj['hash'])
        self.assertEqual(len(content), obj['bytes'])
示例#15
0
    def test_missing_container(self):
        # In this test, we need to put container at handoff devices, so we
        # need container devices more than replica count
        if len(self.container_ring.devs) <= self.container_ring.replica_count:
            raise SkipTest("Need devices more that replica count")

        container = "container-%s" % uuid4()
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)

        # Kill all primary container servers
        for cnode in cnodes:
            kill_server((cnode["ip"], cnode["port"]), self.ipport2server)

        # Create container, and all of its replicas are placed at handoff
        # device
        try:
            client.put_container(self.url, self.token, container)
        except ClientException as err:
            # if the cluster doesn't have enough devices, swift may return
            # error (ex. When we only have 4 devices in 3-replica cluster).
            self.assertEqual(err.http_status, 503)

        # Assert handoff device has a container replica
        another_cnode = self.container_ring.get_more_nodes(cpart).next()
        direct_client.direct_get_container(another_cnode, cpart, self.account, container)

        # Restart all primary container servers
        for cnode in cnodes:
            start_server((cnode["ip"], cnode["port"]), self.ipport2server)

        # Create container/obj
        obj = "object-%s" % uuid4()
        client.put_object(self.url, self.token, container, obj, "")

        # Run the object-updater
        Manager(["object-updater"]).once()

        # Run the container-replicator, and now, container replicas
        # at handoff device get moved to primary servers
        Manager(["container-replicator"]).once()

        # Assert container replicas in primary servers, just moved by
        # replicator don't know about the object
        for cnode in cnodes:
            self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1])

        # since the container is empty - we can delete it!
        client.delete_container(self.url, self.token, container)

        # Re-run the object-updaters and now container replicas in primary
        # container servers should get updated
        Manager(["object-updater"]).once()

        # Assert all primary container servers know about container/obj
        for cnode in cnodes:
            objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
            self.assertIn(obj, objs)
示例#16
0
    def test_update_during_POST_only(self):
        # verify correct update values when PUT update is missed but then a
        # POST update succeeds *before* the PUT async pending update is sent
        cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
        client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
        content = u"stuff"
        client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
        meta = client.head_object(self.url, self.token, "c1", "o1")

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
        self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])

        # use internal client for POST so we can force fast-post mode
        int_client = self.make_internal_client(object_post_as_copy=False)
        int_client.set_object_metadata(self.account, "c1", "o1", {"X-Object-Meta-Fruit": "Tomato"})
        self.assertEqual(
            "Tomato", int_client.get_object_metadata(self.account, "c1", "o1")["x-object-meta-fruit"]
        )  # sanity

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]["bytes"])
            self.assertEqual("test/ctype", listing[0]["content_type"])
            listing_etags.add(listing[0]["hash"])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, "c1")
        self.assertEqual(1, len(listing))
        self.assertEqual("o1", listing[0]["name"])
        self.assertEqual(len(content), listing[0]["bytes"])
        self.assertEqual(meta["etag"], listing[0]["hash"])
        self.assertEqual("test/ctype", listing[0]["content_type"])

        # Run the object-updaters to send the async pending from the PUT
        Manager(["object-updater"]).once()

        # check container listing metadata is still correct
        for cnode in cnodes:
            listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]["bytes"])
            self.assertEqual("test/ctype", listing[0]["content_type"])
            listing_etags.add(listing[0]["hash"])
        self.assertEqual(1, len(listing_etags))
示例#17
0
    def test_missing_primaries(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url,
                             self.token,
                             container,
                             headers={'X-Storage-Policy': self.policy.name})

        # Create container/obj (goes to all three primaries)
        obj = 'object-%s' % uuid4()
        client.put_object(self.url, self.token, container, obj, b'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != b'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Kill all primaries obj server
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(self.account, container,
                                                   obj)
        for onode in onodes:
            kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Indirectly (i.e., through proxy) try to GET object, it should return
        # a 503, since all primaries will Timeout and handoffs return a 404.
        try:
            client.get_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 503)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Restart the first container/obj primary server again
        onode = onodes[0]
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # Send a delete that will reach first primary and handoff.
        # Sure, the DELETE will return a 404 since the handoff doesn't
        # have a .data file, but object server will still write a
        # Tombstone in the handoff node!
        try:
            client.delete_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 404)

        # kill the first container/obj primary server again
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # a new GET should return a 404, since all primaries will Timeout
        # and the handoff will return a 404 but this time with a tombstone
        try:
            client.get_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
    def test_async_updates_after_PUT_and_POST(self):
        # verify correct update values when PUT update and POST updates are
        # missed but then async updates are sent
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url, self.token, 'c1',
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # PUT and POST to object while one container server is stopped so that
        # we force async updates to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url, self.token, 'c1', 'o1', contents=content,
                          content_type='test/ctype')
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # use internal client for POST so we can force fast-post mode
        int_client = self.make_internal_client(object_post_as_copy=False)
        int_client.set_object_metadata(
            self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
        self.assertEqual(
            'Tomato',
            int_client.get_object_metadata(self.account, 'c1', 'o1')
            ['x-object-meta-fruit'])  # sanity

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(direct_client.direct_get_container(
            cnodes[0], cpart, self.account, 'c1')[1])

        # Run the object-updaters to send the async pendings
        Manager(['object-updater']).once()

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(
                cnode, cpart, self.account, 'c1')[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]['bytes'])
            self.assertEqual('test/ctype', listing[0]['content_type'])
            listing_etags.add(listing[0]['hash'])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, 'c1')
        self.assertEqual(1, len(listing))
        self.assertEqual('o1', listing[0]['name'])
        self.assertEqual(len(content), listing[0]['bytes'])
        self.assertEqual(meta['etag'], listing[0]['hash'])
        self.assertEqual('test/ctype', listing[0]['content_type'])
    def test_async_update_after_PUT(self):
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url,
                             self.token,
                             'c1',
                             headers={'X-Storage-Policy': self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url,
                          self.token,
                          'c1',
                          'o1',
                          contents=content,
                          content_type='test/ctype')
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(
            direct_client.direct_get_container(cnodes[0], cpart, self.account,
                                               'c1')[1])

        # Run the object-updaters to be sure updates are done
        Manager(['object-updater']).once()

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(
                cnode, cpart, self.account, 'c1')[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]['bytes'])
            self.assertEqual('test/ctype', listing[0]['content_type'])
            listing_etags.add(listing[0]['hash'])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, 'c1')
        self.assertEqual(1, len(listing))
        self.assertEqual('o1', listing[0]['name'])
        self.assertEqual(len(content), listing[0]['bytes'])
        self.assertEqual(meta['etag'], listing[0]['hash'])
        self.assertEqual('test/ctype', listing[0]['content_type'])
示例#20
0
    def test_update_during_POST_only(self):
        # verify correct update values when PUT update is missed but then a
        # POST update succeeds *before* the PUT async pending update is sent
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url,
                             self.token,
                             'c1',
                             headers={'X-Storage-Policy': self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(
            direct_client.direct_get_container(cnodes[0], cpart, self.account,
                                               'c1')[1])

        # use internal client for POST so we can force fast-post mode
        int_client = self.make_internal_client(object_post_as_copy=False)
        int_client.set_object_metadata(self.account, 'c1', 'o1',
                                       {'X-Object-Meta-Fruit': 'Tomato'})
        self.assertEqual('Tomato',
                         int_client.get_object_metadata(
                             self.account, 'c1',
                             'o1')['x-object-meta-fruit'])  # sanity

        # check the re-started container server has update with override values
        obj = direct_client.direct_get_container(cnodes[0], cpart,
                                                 self.account, 'c1')[1][0]
        self.assertEqual(meta['etag'], obj['hash'])
        self.assertEqual(len(content), obj['bytes'])

        # Run the object-updaters to send the async pending from the PUT
        Manager(['object-updater']).once()

        # check container listing metadata is still correct
        obj = direct_client.direct_get_container(cnodes[0], cpart,
                                                 self.account, 'c1')[1][0]
        self.assertEqual(meta['etag'], obj['hash'])
        self.assertEqual(len(content), obj['bytes'])
示例#21
0
    def test_two_nodes_fail(self):
        # Create container1
        container1 = 'container-%s' % uuid4()
        cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
        client.put_container(self.url, self.token, container1)

        # Kill container1 servers excepting one of the primaries
        cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
                                            self.pids)
        kill_server((cnodes[0]['ip'], cnodes[0]['port']),
                    self.ipport2server, self.pids)
        kill_server((cnodes[1]['ip'], cnodes[1]['port']),
                    self.ipport2server, self.pids)

        # Delete container1 directly to the one primary still up
        direct_client.direct_delete_container(cnodes[2], cpart, self.account,
                                              container1)

        # Restart other container1 servers
        start_server((cnodes[0]['ip'], cnodes[0]['port']),
                     self.ipport2server, self.pids)
        start_server((cnodes[1]['ip'], cnodes[1]['port']),
                     self.ipport2server, self.pids)
        start_server(cnp_ipport, self.ipport2server, self.pids)

        # Get to a final state
        self.get_to_final_state()

        # Assert all container1 servers indicate container1 is gone (happens
        #   because the one node that knew about the delete replicated to the
        #   others.)
        for cnode in cnodes:
            try:
                direct_client.direct_get_container(cnode, cpart, self.account,
                                                   container1)
            except ClientException as err:
                self.assertEqual(err.http_status, 404)
            else:
                self.fail("Expected ClientException but didn't get it")

        # Assert account level also indicates container1 is gone
        headers, containers = client.get_account(self.url, self.token)
        self.assertEqual(headers['x-account-container-count'], '0')
        self.assertEqual(headers['x-account-object-count'], '0')
        self.assertEqual(headers['x-account-bytes-used'], '0')
    def test_two_nodes_fail(self):
        # Create container1
        container1 = 'container-%s' % uuid4()
        cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
        client.put_container(self.url, self.token, container1)

        # Kill container1 servers excepting one of the primaries
        cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
                                            self.pids)
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
                    self.pids)
        kill_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
                    self.pids)

        # Delete container1 directly to the one primary still up
        direct_client.direct_delete_container(cnodes[2], cpart, self.account,
                                              container1)

        # Restart other container1 servers
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
                     self.pids)
        start_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
                     self.pids)
        start_server(cnp_ipport, self.ipport2server, self.pids)

        # Get to a final state
        self.get_to_final_state()

        # Assert all container1 servers indicate container1 is gone (happens
        #   because the one node that knew about the delete replicated to the
        #   others.)
        for cnode in cnodes:
            try:
                direct_client.direct_get_container(cnode, cpart, self.account,
                                                   container1)
            except ClientException as err:
                self.assertEqual(err.http_status, 404)
            else:
                self.fail("Expected ClientException but didn't get it")

        # Assert account level also indicates container1 is gone
        headers, containers = client.get_account(self.url, self.token)
        self.assertEqual(headers['x-account-container-count'], '0')
        self.assertEqual(headers['x-account-object-count'], '0')
        self.assertEqual(headers['x-account-bytes-used'], '0')
    def test_update_during_POST_only(self):
        # verify correct update values when PUT update is missed but then a
        # POST update succeeds *before* the PUT async pending update is sent
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url, self.token, 'c1',
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(direct_client.direct_get_container(
            cnodes[0], cpart, self.account, 'c1')[1])

        # use internal client for POST so we can force fast-post mode
        int_client = self.make_internal_client(object_post_as_copy=False)
        int_client.set_object_metadata(
            self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
        self.assertEqual(
            'Tomato',
            int_client.get_object_metadata(self.account, 'c1', 'o1')
            ['x-object-meta-fruit'])  # sanity

        # check the re-started container server has update with override values
        obj = direct_client.direct_get_container(
            cnodes[0], cpart, self.account, 'c1')[1][0]
        self.assertEqual(meta['etag'], obj['hash'])
        self.assertEqual(len(content), obj['bytes'])

        # Run the object-updaters to send the async pending from the PUT
        Manager(['object-updater']).once()

        # check container listing metadata is still correct
        obj = direct_client.direct_get_container(
            cnodes[0], cpart, self.account, 'c1')[1][0]
        self.assertEqual(meta['etag'], obj['hash'])
        self.assertEqual(len(content), obj['bytes'])
    def test_async_update_after_PUT(self):
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url, self.token, 'c1',
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url, self.token, 'c1', 'o1', contents=content,
                          content_type='test/ctype')
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(direct_client.direct_get_container(
            cnodes[0], cpart, self.account, 'c1')[1])

        # Run the object-updaters to be sure updates are done
        Manager(['object-updater']).once()

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(
                cnode, cpart, self.account, 'c1')[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]['bytes'])
            self.assertEqual('test/ctype', listing[0]['content_type'])
            listing_etags.add(listing[0]['hash'])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, 'c1')
        self.assertEqual(1, len(listing))
        self.assertEqual('o1', listing[0]['name'])
        self.assertEqual(len(content), listing[0]['bytes'])
        self.assertEqual(meta['etag'], listing[0]['hash'])
        self.assertEqual('test/ctype', listing[0]['content_type'])
示例#25
0
    def test_async_update_after_PUT(self):
        cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
        client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
        content = u"stuff"
        client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
        meta = client.head_object(self.url, self.token, "c1", "o1")

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
        self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])

        # Run the object-updaters to be sure updates are done
        Manager(["object-updater"]).once()

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]["bytes"])
            self.assertEqual("test/ctype", listing[0]["content_type"])
            listing_etags.add(listing[0]["hash"])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, "c1")
        self.assertEqual(1, len(listing))
        self.assertEqual("o1", listing[0]["name"])
        self.assertEqual(len(content), listing[0]["bytes"])
        self.assertEqual(meta["etag"], listing[0]["hash"])
        self.assertEqual("test/ctype", listing[0]["content_type"])
    def test_one_node_fails(self):
        # Create container1
        container1 = "container-%s" % uuid4()
        cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
        client.put_container(self.url, self.token, container1)

        # Kill container1 servers excepting two of the primaries
        kill_nonprimary_server(cnodes, self.port2server, self.pids)
        kill_server(cnodes[0]["port"], self.port2server, self.pids)

        # Delete container1
        client.delete_container(self.url, self.token, container1)

        # Restart other container1 primary server
        start_server(cnodes[0]["port"], self.port2server, self.pids)

        # Create container1/object1 (allowed because at least server thinks the
        #   container exists)
        client.put_object(self.url, self.token, container1, "object1", "123")

        # Get to a final state
        self.get_to_final_state()

        # Assert all container1 servers indicate container1 is alive and
        #   well with object1
        for cnode in cnodes:
            self.assertEquals(
                [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container1)[1]],
                ["object1"],
            )

        # Assert account level also indicates container1 is alive and
        #   well with object1
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers["x-account-container-count"], "1")
        self.assertEquals(headers["x-account-object-count"], "1")
        self.assertEquals(headers["x-account-bytes-used"], "3")
示例#27
0
    def test_main(self):
        # Create container1 and container2
        # Assert account level sees them
        # Create container2/object1
        # Assert account level doesn't see it yet
        # Get to final state
        # Assert account level now sees the container2/object1
        # Kill account servers excepting two of the primaries
        # Delete container1
        # Assert account level knows container1 is gone but doesn't know about
        #   container2/object2 yet
        # Put container2/object2
        # Run container updaters
        # Assert account level now knows about container2/object2
        # Restart other primary account server
        # Assert that server doesn't know about container1's deletion or the
        #   new container2/object2 yet
        # Get to final state
        # Assert that server is now up to date

        container1 = 'container1'
        client.put_container(self.url, self.token, container1)
        container2 = 'container2'
        client.put_container(self.url, self.token, container2)
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '0')
        self.assertEquals(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
        self.assert_(found1)
        self.assert_(found2)

        client.put_object(self.url, self.token, container2, 'object1', '1234')
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '0')
        self.assertEquals(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
        self.assert_(found1)
        self.assert_(found2)

        get_to_final_state()
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assert_(found1)
        self.assert_(found2)

        apart, anodes = self.account_ring.get_nodes(self.account)
        kill_nonprimary_server(anodes, self.port2server, self.pids)
        kill_server(anodes[0]['port'], self.port2server, self.pids)

        client.delete_container(self.url, self.token, container1)
        client.put_object(self.url, self.token, container2, 'object2', '12345')
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assert_(not found1)
        self.assert_(found2)

        processes = []
        for node in xrange(1, 5):
            processes.append(Popen([
                'swift-container-updater',
                self.configs['container'] % node,
                'once']))
        for process in processes:
            process.wait()
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '2')
        self.assertEquals(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 2)
                self.assertEquals(container['bytes'], 9)
        self.assert_(not found1)
        self.assert_(found2)

        start_server(anodes[0]['port'], self.port2server, self.pids)

        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assert_(found1)
        self.assert_(found2)

        get_to_final_state()
        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '2')
        self.assertEquals(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 2)
                self.assertEquals(container['bytes'], 9)
        self.assert_(not found1)
        self.assert_(found2)
示例#28
0
    def test_stale_reads(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container,
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # Kill one primary obj server
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Create container/obj (goes to two primaries and one handoff)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Stash the on disk data from a primary for future comparison with the
        # handoff - this may not equal 'VERIFY' if for example the proxy has
        # crypto enabled
        direct_get_data = direct_client.direct_get_object(
            onodes[1], opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]

        # Restart the first container/obj primary server again
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # send a delete request to primaries
        client.delete_object(self.url, self.token, container, obj)

        # there should be .ts files in all primaries now
        for node in onodes:
            try:
                direct_client.direct_get_object(
                    node, opart, self.account, container, obj, headers={
                        'X-Backend-Storage-Policy-Index': self.policy.idx})
            except ClientException as err:
                self.assertEqual(err.http_status, 404)
            else:
                self.fail("Expected ClientException but didn't get it")

        # verify that handoff still has the data, DELETEs should have gone
        # only to primaries
        another_onode = next(self.object_ring.get_more_nodes(opart))
        handoff_data = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        self.assertEqual(handoff_data, direct_get_data)

        # Indirectly (i.e., through proxy) try to GET object, it should return
        # a 404, before bug #1560574, the proxy would return the stale object
        # from the handoff
        try:
            client.get_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#29
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container,
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # Kill one container/obj primary server
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Create container/obj (goes to two primary servers and one handoff)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Kill other two container/obj primary servers
        #   to ensure GET handoff works
        for node in onodes[1:]:
            kill_server((node['ip'], node['port']), self.ipport2server)

        # Indirectly through proxy assert we can get container/obj
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Restart those other two container/obj primary servers
        for node in onodes[1:]:
            start_server((node['ip'], node['port']), self.ipport2server)

        # We've indirectly verified the handoff node has the container/object,
        #   but let's directly verify it.
        another_onode = next(self.object_ring.get_more_nodes(opart))
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # drop a tempfile in the handoff's datadir, like it might have
        # had if there was an rsync failure while it was previously a
        # primary
        handoff_device_path = self.device_dir('object', another_onode)
        data_filename = None
        for root, dirs, files in os.walk(handoff_device_path):
            for filename in files:
                if filename.endswith('.data'):
                    data_filename = filename
                    temp_filename = '.%s.6MbL6r' % data_filename
                    temp_filepath = os.path.join(root, temp_filename)
        if not data_filename:
            self.fail('Did not find any data files on %r' %
                      handoff_device_path)
        open(temp_filepath, 'w')

        # Assert container listing (via proxy and directly) has container/obj
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj not in objs:
                raise Exception(
                    'Container server %s:%s did not know about object' %
                    (cnode['ip'], cnode['port']))

        # Bring the first container/obj primary server back up
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # Assert that it doesn't have container/obj yet
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) / 10
            Manager(['object-replicator']).once(number=node_id)
        try:
            another_port_num = another_onode['replication_port']
        except KeyError:
            another_port_num = another_onode['port']
        another_num = (another_port_num - 6000) / 10
        Manager(['object-replicator']).once(number=another_num)

        # Assert the first container/obj primary server now has container/obj
        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # and that it does *not* have a temporary rsync dropping!
        found_data_filename = False
        primary_device_path = self.device_dir('object', onode)
        for root, dirs, files in os.walk(primary_device_path):
            for filename in files:
                if filename.endswith('.6MbL6r'):
                    self.fail('Found unexpected file %s' %
                              os.path.join(root, filename))
                if filename == data_filename:
                    found_data_filename = True
        self.assertTrue(found_data_filename,
                        'Did not find data file %r on %r' % (
                            data_filename, primary_device_path))

        # Assert the handoff server no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Kill the first container/obj primary server again (we have two
        #   primaries and the handoff up now)
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Delete container/obj
        try:
            client.delete_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            if self.object_ring.replica_count > 2:
                raise
            # Object DELETE returning 503 for (404, 204)
            # remove this with fix for
            # https://bugs.launchpad.net/swift/+bug/1318375
            self.assertEqual(503, err.http_status)

        # Assert we can't head container/obj
        try:
            client.head_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Assert container/obj is not in the container listing, both indirectly
        #   and directly
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj in objs:
            raise Exception('Container listing still knew about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj in objs:
                raise Exception(
                    'Container server %s:%s still knew about object' %
                    (cnode['ip'], cnode['port']))

        # Restart the first container/obj primary server again
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # Assert it still has container/obj
        direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) / 10
            Manager(['object-replicator']).once(number=node_id)
        another_node_id = (another_port_num - 6000) / 10
        Manager(['object-replicator']).once(number=another_node_id)

        # Assert primary node no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#30
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url,
                             self.token,
                             container,
                             headers={'X-Storage-Policy': self.policy.name})

        # Kill one container/obj primary server
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(self.account, container,
                                                   obj)
        onode = onodes[0]
        kill_server((onode['ip'], onode['port']), self.ipport2server,
                    self.pids)

        # Create container/obj (goes to two primary servers and one handoff)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Kill other two container/obj primary servers
        #   to ensure GET handoff works
        for node in onodes[1:]:
            kill_server((node['ip'], node['port']), self.ipport2server,
                        self.pids)

        # Indirectly through proxy assert we can get container/obj
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Restart those other two container/obj primary servers
        for node in onodes[1:]:
            start_server((node['ip'], node['port']), self.ipport2server,
                         self.pids)

        # We've indirectly verified the handoff node has the container/object,
        #   but let's directly verify it.
        another_onode = next(self.object_ring.get_more_nodes(opart))
        odata = direct_client.direct_get_object(
            another_onode,
            opart,
            self.account,
            container,
            obj,
            headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # Assert container listing (via proxy and directly) has container/obj
        objs = [
            o['name']
            for o in client.get_container(self.url, self.token, container)[1]
        ]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        for cnode in cnodes:
            objs = [
                o['name'] for o in direct_client.direct_get_container(
                    cnode, cpart, self.account, container)[1]
            ]
            if obj not in objs:
                raise Exception(
                    'Container server %s:%s did not know about object' %
                    (cnode['ip'], cnode['port']))

        # Bring the first container/obj primary server back up
        start_server((onode['ip'], onode['port']), self.ipport2server,
                     self.pids)

        # Assert that it doesn't have container/obj yet
        try:
            direct_client.direct_get_object(
                onode,
                opart,
                self.account,
                container,
                obj,
                headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) / 10
            Manager(['object-replicator']).once(number=node_id)
        try:
            another_port_num = another_onode['replication_port']
        except KeyError:
            another_port_num = another_onode['port']
        another_num = (another_port_num - 6000) / 10
        Manager(['object-replicator']).once(number=another_num)

        # Assert the first container/obj primary server now has container/obj
        odata = direct_client.direct_get_object(
            onode,
            opart,
            self.account,
            container,
            obj,
            headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # Assert the handoff server no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode,
                opart,
                self.account,
                container,
                obj,
                headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Kill the first container/obj primary server again (we have two
        #   primaries and the handoff up now)
        kill_server((onode['ip'], onode['port']), self.ipport2server,
                    self.pids)

        # Delete container/obj
        try:
            client.delete_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            if self.object_ring.replica_count > 2:
                raise
            # Object DELETE returning 503 for (404, 204)
            # remove this with fix for
            # https://bugs.launchpad.net/swift/+bug/1318375
            self.assertEqual(503, err.http_status)

        # Assert we can't head container/obj
        try:
            client.head_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Assert container/obj is not in the container listing, both indirectly
        #   and directly
        objs = [
            o['name']
            for o in client.get_container(self.url, self.token, container)[1]
        ]
        if obj in objs:
            raise Exception('Container listing still knew about object')
        for cnode in cnodes:
            objs = [
                o['name'] for o in direct_client.direct_get_container(
                    cnode, cpart, self.account, container)[1]
            ]
            if obj in objs:
                raise Exception(
                    'Container server %s:%s still knew about object' %
                    (cnode['ip'], cnode['port']))

        # Restart the first container/obj primary server again
        start_server((onode['ip'], onode['port']), self.ipport2server,
                     self.pids)

        # Assert it still has container/obj
        direct_client.direct_get_object(
            onode,
            opart,
            self.account,
            container,
            obj,
            headers={'X-Backend-Storage-Policy-Index': self.policy.idx})

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) / 10
            Manager(['object-replicator']).once(number=node_id)
        another_node_id = (another_port_num - 6000) / 10
        Manager(['object-replicator']).once(number=another_node_id)

        # Assert primary node no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode,
                opart,
                self.account,
                container,
                obj,
                headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#31
0
    def test_main(self):
        # Create container
        # Kill one container/obj primary server
        # Create container/obj (goes to two primary servers and one handoff)
        # Kill other two container/obj primary servers
        # Indirectly through proxy assert we can get container/obj
        # Restart those other two container/obj primary servers
        # Directly to handoff server assert we can get container/obj
        # Assert container listing (via proxy and directly) has container/obj
        # Bring the first container/obj primary server back up
        # Assert that it doesn't have container/obj yet
        # Run object replication, ensuring we run the handoff node last so it
        #   should remove its extra handoff partition
        # Assert the first container/obj primary server now has container/obj
        # Assert the handoff server no longer has container/obj
        # Kill the first container/obj primary server again (we have two
        #   primaries and the handoff up now)
        # Delete container/obj
        # Assert we can't head container/obj
        # Assert container/obj is not in the container listing, both indirectly
        #   and directly
        # Restart the first container/obj primary server again
        # Assert it still has container/obj
        # Run object replication, ensuring we run the handoff node last so it
        #   should remove its extra handoff partition
        # Assert primary node no longer has container/obj
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(self.account, container,
                                                   obj)
        onode = onodes[0]
        kill_server(onode['port'], self.port2server, self.pids)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        # Kill all primaries to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node['port'], self.port2server, self.pids)
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        for node in onodes[1:]:
            start_server(node['port'], self.port2server, self.pids)
        # We've indirectly verified the handoff node has the object, but let's
        # directly verify it.
        another_onode = self.object_ring.get_more_nodes(opart).next()
        odata = direct_client.direct_get_object(another_onode, opart,
                                                self.account, container,
                                                obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        objs = [
            o['name']
            for o in client.get_container(self.url, self.token, container)[1]
        ]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        for cnode in cnodes:
            objs = [
                o['name'] for o in direct_client.direct_get_container(
                    cnode, cpart, self.account, container)[1]
            ]
            if obj not in objs:
                raise Exception(
                    'Container server %s:%s did not know about object' %
                    (cnode['ip'], cnode['port']))
        start_server(onode['port'], self.port2server, self.pids)
        exc = None
        try:
            direct_client.direct_get_object(onode, opart, self.account,
                                            container, obj)
        except direct_client.ClientException, err:
            exc = err
示例#32
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container,
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # Kill one container/obj primary server
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Create container/obj (goes to two primary servers and one handoff)
        client.put_object(self.url, self.token, container, obj, b'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != b'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Stash the on disk data from a primary for future comparison with the
        # handoff - this may not equal 'VERIFY' if for example the proxy has
        # crypto enabled
        direct_get_data = direct_client.direct_get_object(
            onodes[1], opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]

        # Kill other two container/obj primary servers
        #   to ensure GET handoff works
        for node in onodes[1:]:
            kill_server((node['ip'], node['port']), self.ipport2server)

        # Indirectly through proxy assert we can get container/obj
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != b'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Restart those other two container/obj primary servers
        for node in onodes[1:]:
            start_server((node['ip'], node['port']), self.ipport2server)

        # We've indirectly verified the handoff node has the container/object,
        #   but let's directly verify it.
        another_onode = next(self.object_ring.get_more_nodes(opart))
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        self.assertEqual(direct_get_data, odata)

        # drop a tempfile in the handoff's datadir, like it might have
        # had if there was an rsync failure while it was previously a
        # primary
        handoff_device_path = self.device_dir(another_onode)
        data_filename = None
        for root, dirs, files in os.walk(handoff_device_path):
            for filename in files:
                if filename.endswith('.data'):
                    data_filename = filename
                    temp_filename = '.%s.6MbL6r' % data_filename
                    temp_filepath = os.path.join(root, temp_filename)
        if not data_filename:
            self.fail('Did not find any data files on %r' %
                      handoff_device_path)
        open(temp_filepath, 'w')

        # Assert container listing (via proxy and directly) has container/obj
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj not in objs:
                raise Exception(
                    'Container server %s:%s did not know about object' %
                    (cnode['ip'], cnode['port']))

        # Bring the first container/obj primary server back up
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # Assert that it doesn't have container/obj yet
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) // 10
            Manager(['object-replicator']).once(number=node_id)
        try:
            another_port_num = another_onode['replication_port']
        except KeyError:
            another_port_num = another_onode['port']
        another_num = (another_port_num - 6000) // 10
        Manager(['object-replicator']).once(number=another_num)

        # Assert the first container/obj primary server now has container/obj
        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        self.assertEqual(direct_get_data, odata)

        # and that it does *not* have a temporary rsync dropping!
        found_data_filename = False
        primary_device_path = self.device_dir(onode)
        for root, dirs, files in os.walk(primary_device_path):
            for filename in files:
                if filename.endswith('.6MbL6r'):
                    self.fail('Found unexpected file %s' %
                              os.path.join(root, filename))
                if filename == data_filename:
                    found_data_filename = True
        self.assertTrue(found_data_filename,
                        'Did not find data file %r on %r' % (
                            data_filename, primary_device_path))

        # Assert the handoff server no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Kill the first container/obj primary server again (we have two
        #   primaries and the handoff up now)
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Delete container/obj
        try:
            client.delete_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            if self.object_ring.replica_count > 2:
                raise
            # Object DELETE returning 503 for (404, 204)
            # remove this with fix for
            # https://bugs.launchpad.net/swift/+bug/1318375
            self.assertEqual(503, err.http_status)

        # Assert we can't head container/obj
        try:
            client.head_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Assert container/obj is not in the container listing, both indirectly
        #   and directly
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj in objs:
            raise Exception('Container listing still knew about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj in objs:
                raise Exception(
                    'Container server %s:%s still knew about object' %
                    (cnode['ip'], cnode['port']))

        # Restart the first container/obj primary server again
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # Assert it still has container/obj
        direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) // 10
            Manager(['object-replicator']).once(number=node_id)
        another_node_id = (another_port_num - 6000) // 10
        Manager(['object-replicator']).once(number=another_node_id)

        # Assert primary node no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#33
0
    def test_metadata_sync(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container,
                             headers={'X-Storage-Policy': self.policy.name,
                                      'X-Container-Meta-A': '1',
                                      'X-Container-Meta-B': '1',
                                      'X-Container-Meta-C': '1'})

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes.pop()
        # 2 of 3 container servers are temporarily down
        for node in cnodes:
            kill_server((node['ip'], node['port']),
                        self.ipport2server)

        # Put some meta on the lone server, to make sure it's merged properly
        # This will 503 (since we don't have a quorum), but we don't care (!)
        try:
            client.post_container(self.url, self.token, container,
                                  headers={'X-Container-Meta-A': '2',
                                           'X-Container-Meta-B': '2',
                                           'X-Container-Meta-D': '2'})
        except ClientException:
            pass

        # object updates come to only one container server
        for _ in range(self.object_puts):
            obj = 'object-%s' % uuid4()
            client.put_object(self.url, self.token, container, obj, 'VERIFY')

        # 2 container servers make comeback
        for node in cnodes:
            start_server((node['ip'], node['port']),
                         self.ipport2server)
        # But, container-server which got object updates is down
        kill_server((cnode['ip'], cnode['port']),
                    self.ipport2server)

        # Metadata update will be applied to 2 container servers
        # (equal to quorum)
        client.post_container(self.url, self.token, container,
                              headers={'X-Container-Meta-B': '3',
                                       'X-Container-Meta-E': '3'})
        # container-server which got object updates makes comeback
        start_server((cnode['ip'], cnode['port']),
                     self.ipport2server)

        # other nodes have no objects
        for node in cnodes:
            resp_headers = direct_client.direct_head_container(
                node, cpart, self.account, container)
            self.assertIn(resp_headers.get('x-container-object-count'),
                          (None, '0', 0))

        # If container-replicator on the node which got the object updates
        # runs in first, db file may be replicated by rsync to other
        # containers. In that case, the db file does not information about
        # metadata, so metadata should be synced before replication
        Manager(['container-replicator']).once(
            number=self.config_number(cnode))

        expected_meta = {
            'x-container-meta-a': '2',
            'x-container-meta-b': '3',
            'x-container-meta-c': '1',
            'x-container-meta-d': '2',
            'x-container-meta-e': '3',
        }

        # node that got the object updates now has the meta
        resp_headers = direct_client.direct_head_container(
            cnode, cpart, self.account, container)
        for header, value in expected_meta.items():
            self.assertIn(header, resp_headers)
            self.assertEqual(value, resp_headers[header])
        self.assertNotIn(resp_headers.get('x-container-object-count'),
                         (None, '0', 0))

        # other nodes still have the meta, as well as objects
        for node in cnodes:
            resp_headers = direct_client.direct_head_container(
                node, cpart, self.account, container)
            for header, value in expected_meta.items():
                self.assertIn(header, resp_headers)
                self.assertEqual(value, resp_headers[header])
            self.assertNotIn(resp_headers.get('x-container-object-count'),
                             (None, '0', 0))

        # and after full pass on remaining nodes
        for node in cnodes:
            Manager(['container-replicator']).once(
                number=self.config_number(node))

        # ... all is right
        for node in cnodes + [cnode]:
            resp_headers = direct_client.direct_head_container(
                node, cpart, self.account, container)
            for header, value in expected_meta.items():
                self.assertIn(header, resp_headers)
                self.assertEqual(value, resp_headers[header])
            self.assertNotIn(resp_headers.get('x-container-object-count'),
                             (None, '0', 0))
    def test_main(self):
        # Create container
        # Kill one container/obj primary server
        # Delete the "objects" directory on the primary server
        # Create container/obj (goes to two primary servers and one handoff)
        # Kill other two container/obj primary servers
        # Indirectly through proxy assert we can get container/obj
        # Restart those other two container/obj primary servers
        # Directly to handoff server assert we can get container/obj
        # Assert container listing (via proxy and directly) has container/obj
        # Bring the first container/obj primary server back up
        # Assert that it doesn't have container/obj yet
        # Run object replication for first container/obj primary server
        # Run object replication for handoff node
        # Assert the first container/obj primary server now has container/obj
        # Assert the handoff server no longer has container/obj

        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server(onode['port'], self.port2server, self.pids)
        obj_dir = '%s/objects' % self._get_objects_dir(onode)
        shutil.rmtree(obj_dir, True)
        self.assertFalse(os.path.exists(obj_dir))
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        # Kill all primaries to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node['port'], self.port2server, self.pids)
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        for node in onodes[1:]:
            start_server(node['port'], self.port2server, self.pids)
            self.assertFalse(os.path.exists(obj_dir))
            # We've indirectly verified the handoff node has the object, but
            # let's directly verify it.
        another_onode = self.object_ring.get_more_nodes(opart).next()
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        timeout = time.time() + 5
        found_objs_on_cnode = []
        while time.time() < timeout:
            for cnode in [c for c in cnodes if cnodes not in
                          found_objs_on_cnode]:
                objs = [o['name'] for o in
                        direct_client.direct_get_container(
                            cnode, cpart, self.account, container)[1]]
                if obj in objs:
                    found_objs_on_cnode.append(cnode)
            if len(found_objs_on_cnode) >= len(cnodes):
                break
            time.sleep(0.3)
        if len(found_objs_on_cnode) < len(cnodes):
            missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
                       cnodes if cnode not in found_objs_on_cnode]
            raise Exception('Container servers %r did not know about object' %
                            missing)
        start_server(onode['port'], self.port2server, self.pids)
        self.assertFalse(os.path.exists(obj_dir))
        exc = None
        try:
            direct_client.direct_get_object(onode, opart, self.account,
                                            container, obj)
        except direct_client.ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)
        self.assertFalse(os.path.exists(obj_dir))

        try:
            port_num = onode['replication_port']
        except KeyError:
            port_num = onode['port']
        try:
            another_port_num = another_onode['replication_port']
        except KeyError:
            another_port_num = another_onode['port']
        call(['swift-object-replicator',
              self.configs['object-replicator'] %
              ((port_num - 6000) / 10), 'once'])
        call(['swift-object-replicator',
              self.configs['object-replicator'] %
              ((another_port_num - 6000) / 10), 'once'])

        odata = direct_client.direct_get_object(onode, opart, self.account,
                                                container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        exc = None
        try:
            direct_client.direct_get_object(another_onode, opart, self.account,
                                            container, obj)
        except direct_client.ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)
示例#35
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container,
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]

        # Kill one container/obj primary server
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Delete the default data directory for objects on the primary server
        obj_dir = '%s/%s' % (self._get_objects_dir(onode),
                             get_data_dir(self.policy))
        shutil.rmtree(obj_dir, True)
        self.assertFalse(os.path.exists(obj_dir))

        # Create container/obj (goes to two primary servers and one handoff)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Stash the on disk data from a primary for future comparison with the
        # handoff - this may not equal 'VERIFY' if for example the proxy has
        # crypto enabled
        direct_get_data = direct_client.direct_get_object(
            onodes[1], opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]

        # Kill other two container/obj primary servers
        #  to ensure GET handoff works
        for node in onodes[1:]:
            kill_server((node['ip'], node['port']), self.ipport2server)

        # Indirectly through proxy assert we can get container/obj
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        # Restart those other two container/obj primary servers
        for node in onodes[1:]:
            start_server((node['ip'], node['port']), self.ipport2server)
            self.assertFalse(os.path.exists(obj_dir))
            # We've indirectly verified the handoff node has the object, but
            # let's directly verify it.

        # Directly to handoff server assert we can get container/obj
        another_onode = next(self.object_ring.get_more_nodes(opart))
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj,
            headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        self.assertEqual(direct_get_data, odata)

        # Assert container listing (via proxy and directly) has container/obj
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        timeout = time.time() + 5
        found_objs_on_cnode = []
        while time.time() < timeout:
            for cnode in [c for c in cnodes if cnodes not in
                          found_objs_on_cnode]:
                objs = [o['name'] for o in
                        direct_client.direct_get_container(
                            cnode, cpart, self.account, container)[1]]
                if obj in objs:
                    found_objs_on_cnode.append(cnode)
            if len(found_objs_on_cnode) >= len(cnodes):
                break
            time.sleep(0.3)
        if len(found_objs_on_cnode) < len(cnodes):
            missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
                       cnodes if cnode not in found_objs_on_cnode]
            raise Exception('Container servers %r did not know about object' %
                            missing)

        # Bring the first container/obj primary server back up
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # Assert that it doesn't have container/obj yet
        self.assertFalse(os.path.exists(obj_dir))
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
            self.assertFalse(os.path.exists(obj_dir))
        else:
            self.fail("Expected ClientException but didn't get it")

        # Run object replication for first container/obj primary server
        _, num = get_server_number(
            (onode['ip'], onode.get('replication_port', onode['port'])),
            self.ipport2server)
        Manager(['object-replicator']).once(number=num)

        # Run object replication for handoff node
        _, another_num = get_server_number(
            (another_onode['ip'],
             another_onode.get('replication_port', another_onode['port'])),
            self.ipport2server)
        Manager(['object-replicator']).once(number=another_num)

        # Assert the first container/obj primary server now has container/obj
        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        self.assertEqual(direct_get_data, odata)

        # Assert the handoff server no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#36
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container,
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]

        # Kill one container/obj primary server
        kill_server(onode['port'], self.port2server, self.pids)

        # Delete the default data directory for objects on the primary server
        obj_dir = '%s/%s' % (self._get_objects_dir(onode),
                             get_data_dir(self.policy))
        shutil.rmtree(obj_dir, True)
        self.assertFalse(os.path.exists(obj_dir))

        # Create container/obj (goes to two primary servers and one handoff)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Kill other two container/obj primary servers
        #  to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node['port'], self.port2server, self.pids)

        # Indirectly through proxy assert we can get container/obj
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        # Restart those other two container/obj primary servers
        for node in onodes[1:]:
            start_server(node['port'], self.port2server, self.pids)
            self.assertFalse(os.path.exists(obj_dir))
            # We've indirectly verified the handoff node has the object, but
            # let's directly verify it.

        # Directly to handoff server assert we can get container/obj
        another_onode = next(self.object_ring.get_more_nodes(opart))
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj,
            headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # Assert container listing (via proxy and directly) has container/obj
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        timeout = time.time() + 5
        found_objs_on_cnode = []
        while time.time() < timeout:
            for cnode in [c for c in cnodes if cnodes not in
                          found_objs_on_cnode]:
                objs = [o['name'] for o in
                        direct_client.direct_get_container(
                            cnode, cpart, self.account, container)[1]]
                if obj in objs:
                    found_objs_on_cnode.append(cnode)
            if len(found_objs_on_cnode) >= len(cnodes):
                break
            time.sleep(0.3)
        if len(found_objs_on_cnode) < len(cnodes):
            missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
                       cnodes if cnode not in found_objs_on_cnode]
            raise Exception('Container servers %r did not know about object' %
                            missing)

        # Bring the first container/obj primary server back up
        start_server(onode['port'], self.port2server, self.pids)

        # Assert that it doesn't have container/obj yet
        self.assertFalse(os.path.exists(obj_dir))
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEquals(err.http_status, 404)
            self.assertFalse(os.path.exists(obj_dir))
        else:
            self.fail("Expected ClientException but didn't get it")

        try:
            port_num = onode['replication_port']
        except KeyError:
            port_num = onode['port']
        try:
            another_port_num = another_onode['replication_port']
        except KeyError:
            another_port_num = another_onode['port']

        # Run object replication for first container/obj primary server
        num = (port_num - 6000) / 10
        Manager(['object-replicator']).once(number=num)

        # Run object replication for handoff node
        another_num = (another_port_num - 6000) / 10
        Manager(['object-replicator']).once(number=another_num)

        # Assert the first container/obj primary server now has container/obj
        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # Assert the handoff server no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEquals(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#37
0
    def test_main(self):
        # Create container
        # Kill one container/obj primary server
        # Delete the "objects" directory on the primary server
        # Create container/obj (goes to two primary servers and one handoff)
        # Kill other two container/obj primary servers
        # Indirectly through proxy assert we can get container/obj
        # Restart those other two container/obj primary servers
        # Directly to handoff server assert we can get container/obj
        # Assert container listing (via proxy and directly) has container/obj
        # Bring the first container/obj primary server back up
        # Assert that it doesn't have container/obj yet
        # Run object replication for first container/obj primary server
        # Run object replication for handoff node
        # Assert the first container/obj primary server now has container/obj
        # Assert the handoff server no longer has container/obj

        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(self.account, container,
                                                   obj)
        onode = onodes[0]
        kill_server(onode['port'], self.port2server, self.pids)
        obj_dir = '%s/objects' % self._get_objects_dir(onode)
        shutil.rmtree(obj_dir, True)
        self.assertFalse(os.path.exists(obj_dir))
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        # Kill all primaries to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node['port'], self.port2server, self.pids)
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        for node in onodes[1:]:
            start_server(node['port'], self.port2server, self.pids)
            self.assertFalse(os.path.exists(obj_dir))
            # We've indirectly verified the handoff node has the object, but
            # let's directly verify it.
        another_onode = self.object_ring.get_more_nodes(opart).next()
        odata = direct_client.direct_get_object(another_onode, opart,
                                                self.account, container,
                                                obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        objs = [
            o['name']
            for o in client.get_container(self.url, self.token, container)[1]
        ]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        timeout = time.time() + 5
        found_objs_on_cnode = []
        while time.time() < timeout:
            for cnode in [
                    c for c in cnodes if cnodes not in found_objs_on_cnode
            ]:
                objs = [
                    o['name'] for o in direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]
                ]
                if obj in objs:
                    found_objs_on_cnode.append(cnode)
            if len(found_objs_on_cnode) >= len(cnodes):
                break
            time.sleep(0.3)
        if len(found_objs_on_cnode) < len(cnodes):
            missing = [
                '%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes
                if cnode not in found_objs_on_cnode
            ]
            raise Exception('Container servers %r did not know about object' %
                            missing)
        start_server(onode['port'], self.port2server, self.pids)
        self.assertFalse(os.path.exists(obj_dir))
        exc = None
        try:
            direct_client.direct_get_object(onode, opart, self.account,
                                            container, obj)
        except ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)
        self.assertFalse(os.path.exists(obj_dir))

        try:
            port_num = onode['replication_port']
        except KeyError:
            port_num = onode['port']
        try:
            another_port_num = another_onode['replication_port']
        except KeyError:
            another_port_num = another_onode['port']
        call([
            'swift-object-replicator',
            self.configs['object-replicator'] % ((port_num - 6000) / 10),
            'once'
        ])
        call([
            'swift-object-replicator', self.configs['object-replicator'] %
            ((another_port_num - 6000) / 10), 'once'
        ])

        odata = direct_client.direct_get_object(onode, opart, self.account,
                                                container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        exc = None
        try:
            direct_client.direct_get_object(another_onode, opart, self.account,
                                            container, obj)
        except ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)
示例#38
0
    def test_main(self):
        # Create container1 and container2
        # Assert account level sees them
        # Create container2/object1
        # Assert account level doesn't see it yet
        # Get to final state
        # Assert account level now sees the container2/object1
        # Kill account servers excepting two of the primaries
        # Delete container1
        # Assert account level knows container1 is gone but doesn't know about
        #   container2/object2 yet
        # Put container2/object2
        # Run container updaters
        # Assert account level now knows about container2/object2
        # Restart other primary account server
        # Assert that server doesn't know about container1's deletion or the
        #   new container2/object2 yet
        # Get to final state
        # Assert that server is now up to date

        container1 = 'container1'
        client.put_container(self.url, self.token, container1)
        container2 = 'container2'
        client.put_container(self.url, self.token, container2)
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '0')
        self.assertEquals(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
        self.assert_(found1)
        self.assert_(found2)

        client.put_object(self.url, self.token, container2, 'object1', '1234')
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '0')
        self.assertEquals(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
        self.assert_(found1)
        self.assert_(found2)

        get_to_final_state()
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assert_(found1)
        self.assert_(found2)

        apart, anodes = self.account_ring.get_nodes(self.account)
        kill_nonprimary_server(anodes, self.port2server, self.pids)
        kill_server(anodes[0]['port'], self.port2server, self.pids)

        client.delete_container(self.url, self.token, container1)
        client.put_object(self.url, self.token, container2, 'object2', '12345')
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assert_(not found1)
        self.assert_(found2)

        processes = []
        for node in xrange(1, 5):
            processes.append(
                Popen([
                    'swift-container-updater',
                    '/etc/swift/container-server/%d.conf' % node, 'once'
                ]))
        for process in processes:
            process.wait()
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '2')
        self.assertEquals(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 2)
                self.assertEquals(container['bytes'], 9)
        self.assert_(not found1)
        self.assert_(found2)

        start_server(anodes[0]['port'], self.port2server, self.pids)

        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assert_(found1)
        self.assert_(found2)

        get_to_final_state()
        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '2')
        self.assertEquals(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 2)
                self.assertEquals(container['bytes'], 9)
        self.assert_(not found1)
        self.assert_(found2)
示例#39
0
    def test_main(self):
        # Create container
        # Kill one container/obj primary server
        # Create container/obj (goes to two primary servers and one handoff)
        # Kill other two container/obj primary servers
        # Indirectly through proxy assert we can get container/obj
        # Restart those other two container/obj primary servers
        # Directly to handoff server assert we can get container/obj
        # Assert container listing (via proxy and directly) has container/obj
        # Bring the first container/obj primary server back up
        # Assert that it doesn't have container/obj yet
        # Run object replication, ensuring we run the handoff node last so it
        #   should remove its extra handoff partition
        # Assert the first container/obj primary server now has container/obj
        # Assert the handoff server no longer has container/obj
        # Kill the first container/obj primary server again (we have two
        #   primaries and the handoff up now)
        # Delete container/obj
        # Assert we can't head container/obj
        # Assert container/obj is not in the container listing, both indirectly
        #   and directly
        # Restart the first container/obj primary server again
        # Assert it still has container/obj
        # Run object replication, ensuring we run the handoff node last so it
        #   should remove its extra handoff partition
        # Assert primary node no longer has container/obj
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server(onode['port'], self.port2server, self.pids)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        # Kill all primaries to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node['port'], self.port2server, self.pids)
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        for node in onodes[1:]:
            start_server(node['port'], self.port2server, self.pids)
        # We've indirectly verified the handoff node has the object, but let's
        # directly verify it.
        another_onode = self.object_ring.get_more_nodes(opart).next()
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj not in objs:
                raise Exception(
                    'Container server %s:%s did not know about object' %
                    (cnode['ip'], cnode['port']))
        start_server(onode['port'], self.port2server, self.pids)
        exc = None
        try:
            direct_client.direct_get_object(onode, opart, self.account,
                                            container, obj)
        except direct_client.ClientException, err:
            exc = err
    def test_main(self):
        # Create container1 and container2
        container1 = 'container1'
        client.put_container(self.url, self.token, container1)
        container2 = 'container2'
        client.put_container(self.url, self.token, container2)

        # Assert account level sees them
        headers, containers = client.get_account(self.url, self.token)
        self.assertEqual(headers['x-account-container-count'], '2')
        self.assertEqual(headers['x-account-object-count'], '0')
        self.assertEqual(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEqual(container['count'], 0)
                self.assertEqual(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEqual(container['count'], 0)
                self.assertEqual(container['bytes'], 0)
        self.assertTrue(found1)
        self.assertTrue(found2)

        # Create container2/object1
        client.put_object(self.url, self.token, container2, 'object1', '1234')

        # Assert account level doesn't see it yet
        headers, containers = client.get_account(self.url, self.token)
        self.assertEqual(headers['x-account-container-count'], '2')
        self.assertEqual(headers['x-account-object-count'], '0')
        self.assertEqual(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEqual(container['count'], 0)
                self.assertEqual(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEqual(container['count'], 0)
                self.assertEqual(container['bytes'], 0)
        self.assertTrue(found1)
        self.assertTrue(found2)

        # Get to final state
        self.get_to_final_state()

        # Assert account level now sees the container2/object1
        headers, containers = client.get_account(self.url, self.token)
        self.assertEqual(headers['x-account-container-count'], '2')
        self.assertEqual(headers['x-account-object-count'], '1')
        self.assertEqual(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEqual(container['count'], 0)
                self.assertEqual(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEqual(container['count'], 1)
                self.assertEqual(container['bytes'], 4)
        self.assertTrue(found1)
        self.assertTrue(found2)

        apart, anodes = self.account_ring.get_nodes(self.account)
        kill_nonprimary_server(anodes, self.ipport2server)
        kill_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server)
        # Kill account servers excepting two of the primaries

        # Delete container1
        client.delete_container(self.url, self.token, container1)

        # Put container2/object2
        client.put_object(self.url, self.token, container2, 'object2', '12345')

        # Assert account level knows container1 is gone but doesn't know about
        #   container2/object2 yet
        headers, containers = client.get_account(self.url, self.token)
        self.assertEqual(headers['x-account-container-count'], '1')
        self.assertEqual(headers['x-account-object-count'], '1')
        self.assertEqual(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEqual(container['count'], 1)
                self.assertEqual(container['bytes'], 4)
        self.assertFalse(found1)
        self.assertTrue(found2)

        # Run container updaters
        Manager(['container-updater']).once()

        # Assert account level now knows about container2/object2
        headers, containers = client.get_account(self.url, self.token)
        self.assertEqual(headers['x-account-container-count'], '1')
        self.assertEqual(headers['x-account-object-count'], '2')
        self.assertEqual(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEqual(container['count'], 2)
                self.assertEqual(container['bytes'], 9)
        self.assertFalse(found1)
        self.assertTrue(found2)

        # Restart other primary account server
        start_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server)

        # Assert that server doesn't know about container1's deletion or the
        #   new container2/object2 yet
        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEqual(headers['x-account-container-count'], '2')
        self.assertEqual(headers['x-account-object-count'], '1')
        self.assertEqual(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEqual(container['count'], 1)
                self.assertEqual(container['bytes'], 4)
        self.assertTrue(found1)
        self.assertTrue(found2)

        # Get to final state
        self.get_to_final_state()

        # Assert that server is now up to date
        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEqual(headers['x-account-container-count'], '1')
        self.assertEqual(headers['x-account-object-count'], '2')
        self.assertEqual(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEqual(container['count'], 2)
                self.assertEqual(container['bytes'], 9)
                self.assertEqual(container['bytes'], 9)
        self.assertFalse(found1)
        self.assertTrue(found2)
示例#41
0
              self.configs['object-replicator'] %
              ((another_port_num - 6000) / 10), 'once'])
        odata = direct_client.direct_get_object(onode, opart, self.account,
                                                container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        exc = None
        try:
            direct_client.direct_get_object(another_onode, opart, self.account,
                                            container, obj)
        except direct_client.ClientException, err:
            exc = err
        self.assertEquals(exc.http_status, 404)

        kill_server(onode['port'], self.port2server, self.pids)
        client.delete_object(self.url, self.token, container, obj)
        exc = None
        try:
            client.head_object(self.url, self.token, container, obj)
        except direct_client.ClientException, err:
            exc = err
        self.assertEquals(exc.http_status, 404)
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj in objs:
            raise Exception('Container listing still knew about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
示例#42
0
    def test_stale_reads(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container,
                             headers={'X-Storage-Policy':
                                      self.policy.name})

        # Kill one primary obj server
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server((onode['ip'], onode['port']), self.ipport2server)

        # Create container/obj (goes to two primaries and one handoff)
        client.put_object(self.url, self.token, container, obj, b'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != b'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Stash the on disk data from a primary for future comparison with the
        # handoff - this may not equal 'VERIFY' if for example the proxy has
        # crypto enabled
        direct_get_data = direct_client.direct_get_object(
            onodes[1], opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]

        # Restart the first container/obj primary server again
        start_server((onode['ip'], onode['port']), self.ipport2server)

        # send a delete request to primaries
        client.delete_object(self.url, self.token, container, obj)

        # there should be .ts files in all primaries now
        for node in onodes:
            try:
                direct_client.direct_get_object(
                    node, opart, self.account, container, obj, headers={
                        'X-Backend-Storage-Policy-Index': self.policy.idx})
            except ClientException as err:
                self.assertEqual(err.http_status, 404)
            else:
                self.fail("Expected ClientException but didn't get it")

        # verify that handoff still has the data, DELETEs should have gone
        # only to primaries
        another_onode = next(self.object_ring.get_more_nodes(opart))
        handoff_data = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        self.assertEqual(handoff_data, direct_get_data)

        # Indirectly (i.e., through proxy) try to GET object, it should return
        # a 404, before bug #1560574, the proxy would return the stale object
        # from the handoff
        try:
            client.get_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEqual(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#43
0
    def test_main(self):
        # Create container
        # Kill one container/obj primary server
        # Delete the "objects" directory on the primary server
        # Create container/obj (goes to two primary servers and one handoff)
        # Kill other two container/obj primary servers
        # Indirectly through proxy assert we can get container/obj
        # Restart those other two container/obj primary servers
        # Directly to handoff server assert we can get container/obj
        # Assert container listing (via proxy and directly) has container/obj
        # Bring the first container/obj primary server back up
        # Assert that it doesn't have container/obj yet
        # Run object replication for first container/obj primary server
        # Run object replication for handoff node
        # Assert the first container/obj primary server now has container/obj
        # Assert the handoff server no longer has container/obj

        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server(onode['port'], self.port2server, self.pids)
        obj_dir = '%s/objects' % self._get_objects_dir(onode)
        shutil.rmtree(obj_dir, True)
        self.assertFalse(os.path.exists(obj_dir))
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        # Kill all primaries to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node['port'], self.port2server, self.pids)
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))
        for node in onodes[1:]:
            start_server(node['port'], self.port2server, self.pids)
            self.assertFalse(os.path.exists(obj_dir))
            # We've indirectly verified the handoff node has the object, but
            # let's directly verify it.
        another_onode = self.object_ring.get_more_nodes(opart).next()
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj not in objs:
                raise Exception(
                    'Container server %s:%s did not know about object' %
                    (cnode['ip'], cnode['port']))
        start_server(onode['port'], self.port2server, self.pids)
        self.assertFalse(os.path.exists(obj_dir))
        exc = None
        try:
            direct_client.direct_get_object(onode, opart, self.account,
                                            container, obj)
        except direct_client.ClientException, err:
            exc = err
示例#44
0
            ((another_onode['port'] - 6000) / 10), 'once'
        ])
        odata = direct_client.direct_get_object(onode, opart, self.account,
                                                container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))
        exc = None
        try:
            direct_client.direct_get_object(another_onode, opart, self.account,
                                            container, obj)
        except direct_client.ClientException, err:
            exc = err
        self.assertEquals(exc.http_status, 404)

        kill_server(onode['port'], self.port2server, self.pids)
        client.delete_object(self.url, self.token, container, obj)
        exc = None
        try:
            client.head_object(self.url, self.token, container, obj)
        except direct_client.ClientException, err:
            exc = err
        self.assertEquals(exc.http_status, 404)
        objs = [
            o['name']
            for o in client.get_container(self.url, self.token, container)[1]
        ]
        if obj in objs:
            raise Exception('Container listing still knew about object')
        for cnode in cnodes:
            objs = [
示例#45
0
    def test_metadata_sync(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url,
                             self.token,
                             container,
                             headers={
                                 'X-Storage-Policy': self.policy.name,
                                 'X-Container-Meta-A': '1',
                                 'X-Container-Meta-B': '1',
                                 'X-Container-Meta-C': '1'
                             })

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes.pop()
        # 2 of 3 container servers are temporarily down
        for node in cnodes:
            kill_server((node['ip'], node['port']), self.ipport2server)

        # Put some meta on the lone server, to make sure it's merged properly
        # This will 503 (since we don't have a quorum), but we don't care (!)
        try:
            client.post_container(self.url,
                                  self.token,
                                  container,
                                  headers={
                                      'X-Container-Meta-A': '2',
                                      'X-Container-Meta-B': '2',
                                      'X-Container-Meta-D': '2'
                                  })
        except ClientException:
            pass

        # object updates come to only one container server
        for _ in range(self.object_puts):
            obj = 'object-%s' % uuid4()
            client.put_object(self.url, self.token, container, obj, 'VERIFY')

        # 2 container servers make comeback
        for node in cnodes:
            start_server((node['ip'], node['port']), self.ipport2server)
        # But, container-server which got object updates is down
        kill_server((cnode['ip'], cnode['port']), self.ipport2server)

        # Metadata update will be applied to 2 container servers
        # (equal to quorum)
        client.post_container(self.url,
                              self.token,
                              container,
                              headers={
                                  'X-Container-Meta-B': '3',
                                  'X-Container-Meta-E': '3'
                              })
        # container-server which got object updates makes comeback
        start_server((cnode['ip'], cnode['port']), self.ipport2server)

        # other nodes have no objects
        for node in cnodes:
            resp_headers = direct_client.direct_head_container(
                node, cpart, self.account, container)
            self.assertIn(resp_headers.get('x-container-object-count'),
                          (None, '0', 0))

        # If container-replicator on the node which got the object updates
        # runs in first, db file may be replicated by rsync to other
        # containers. In that case, the db file does not information about
        # metadata, so metadata should be synced before replication
        Manager(['container-replicator'
                 ]).once(number=self.config_number(cnode))

        expected_meta = {
            'x-container-meta-a': '2',
            'x-container-meta-b': '2',
            'x-container-meta-c': '1',
            'x-container-meta-d': '2',
        }

        # node that got the object updates still doesn't have the meta
        resp_headers = direct_client.direct_head_container(
            cnode, cpart, self.account, container)
        for header, value in expected_meta.items():
            self.assertIn(header, resp_headers)
            self.assertEqual(value, resp_headers[header])
        self.assertNotIn(resp_headers.get('x-container-object-count'),
                         (None, '0', 0))

        expected_meta = {
            'x-container-meta-a': '2',
            'x-container-meta-b': '3',
            'x-container-meta-c': '1',
            'x-container-meta-d': '2',
            'x-container-meta-e': '3',
        }

        # other nodes still have the meta, as well as objects
        for node in cnodes:
            resp_headers = direct_client.direct_head_container(
                node, cpart, self.account, container)
            for header, value in expected_meta.items():
                self.assertIn(header, resp_headers)
                self.assertEqual(value, resp_headers[header])
            self.assertNotIn(resp_headers.get('x-container-object-count'),
                             (None, '0', 0))

        # and after full pass on remaining nodes
        for node in cnodes:
            Manager(['container-replicator'
                     ]).once(number=self.config_number(node))

        # ... all is right
        for node in cnodes + [cnode]:
            resp_headers = direct_client.direct_head_container(
                node, cpart, self.account, container)
            for header, value in expected_meta.items():
                self.assertIn(header, resp_headers)
                self.assertEqual(value, resp_headers[header])
            self.assertNotIn(resp_headers.get('x-container-object-count'),
                             (None, '0', 0))
示例#46
0
    def test_main(self):
        # Create container
        # Kill one container/obj primary server
        # Create container/obj (goes to two primary servers and one handoff)
        # Kill other two container/obj primary servers
        # Indirectly through proxy assert we can get container/obj
        # Restart those other two container/obj primary servers
        # Directly to handoff server assert we can get container/obj
        # Assert container listing (via proxy and directly) has container/obj
        # Bring the first container/obj primary server back up
        # Assert that it doesn't have container/obj yet
        # Run object replication, ensuring we run the handoff node last so it
        #   should remove its extra handoff partition
        # Assert the first container/obj primary server now has container/obj
        # Assert the handoff server no longer has container/obj
        # Kill the first container/obj primary server again (we have two
        #   primaries and the handoff up now)
        # Delete container/obj
        # Assert we can't head container/obj
        # Assert container/obj is not in the container listing, both indirectly
        #   and directly
        # Restart the first container/obj primary server again
        # Assert it still has container/obj
        # Run object replication, ensuring we run the handoff node last so it
        #   should remove its extra handoff partition
        # Assert primary node no longer has container/obj
        container = "container-%s" % uuid4()
        client.put_container(self.url, self.token, container)

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = "object-%s" % uuid4()
        opart, onodes = self.object_ring.get_nodes(self.account, container, obj)
        onode = onodes[0]
        kill_server(onode["port"], self.port2server, self.pids)
        client.put_object(self.url, self.token, container, obj, "VERIFY")
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != "VERIFY":
            raise Exception("Object GET did not return VERIFY, instead it " "returned: %s" % repr(odata))
        # Kill all primaries to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node["port"], self.port2server, self.pids)
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != "VERIFY":
            raise Exception("Object GET did not return VERIFY, instead it " "returned: %s" % repr(odata))
        for node in onodes[1:]:
            start_server(node["port"], self.port2server, self.pids)
        # We've indirectly verified the handoff node has the object, but let's
        # directly verify it.
        another_onode = self.object_ring.get_more_nodes(opart).next()
        odata = direct_client.direct_get_object(another_onode, opart, self.account, container, obj)[-1]
        if odata != "VERIFY":
            raise Exception("Direct object GET did not return VERIFY, instead " "it returned: %s" % repr(odata))
        objs = [o["name"] for o in client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception("Container listing did not know about object")
        for cnode in cnodes:
            objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
            if obj not in objs:
                raise Exception("Container server %s:%s did not know about object" % (cnode["ip"], cnode["port"]))
        start_server(onode["port"], self.port2server, self.pids)
        exc = None
        try:
            direct_client.direct_get_object(onode, opart, self.account, container, obj)
        except ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)
        # Run the extra server last so it'll remove its extra partition
        processes = []
        for node in onodes:
            try:
                port_num = node["replication_port"]
            except KeyError:
                port_num = node["port"]
            processes.append(
                Popen(["swift-object-replicator", self.configs["object-replicator"] % ((port_num - 6000) / 10), "once"])
            )
        for process in processes:
            process.wait()
        try:
            another_port_num = another_onode["replication_port"]
        except KeyError:
            another_port_num = another_onode["port"]
        call(["swift-object-replicator", self.configs["object-replicator"] % ((another_port_num - 6000) / 10), "once"])
        odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1]
        if odata != "VERIFY":
            raise Exception("Direct object GET did not return VERIFY, instead " "it returned: %s" % repr(odata))
        exc = None
        try:
            direct_client.direct_get_object(another_onode, opart, self.account, container, obj)
        except ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)

        kill_server(onode["port"], self.port2server, self.pids)
        client.delete_object(self.url, self.token, container, obj)
        exc = None
        try:
            client.head_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)
        objs = [o["name"] for o in client.get_container(self.url, self.token, container)[1]]
        if obj in objs:
            raise Exception("Container listing still knew about object")
        for cnode in cnodes:
            objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
            if obj in objs:
                raise Exception("Container server %s:%s still knew about object" % (cnode["ip"], cnode["port"]))
        start_server(onode["port"], self.port2server, self.pids)
        direct_client.direct_get_object(onode, opart, self.account, container, obj)
        # Run the extra server last so it'll remove its extra partition
        processes = []
        for node in onodes:
            try:
                port_num = node["replication_port"]
            except KeyError:
                port_num = node["port"]
            processes.append(
                Popen(["swift-object-replicator", self.configs["object-replicator"] % ((port_num - 6000) / 10), "once"])
            )
        for process in processes:
            process.wait()
        call(["swift-object-replicator", self.configs["object-replicator"] % ((another_port_num - 6000) / 10), "once"])
        exc = None
        try:
            direct_client.direct_get_object(another_onode, opart, self.account, container, obj)
        except ClientException as err:
            exc = err
        self.assertEquals(exc.http_status, 404)
示例#47
0
    def test_main(self):
        # Create container
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)

        # Kill one container/obj primary server
        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        cnode = cnodes[0]
        obj = 'object-%s' % uuid4()
        opart, onodes = self.object_ring.get_nodes(
            self.account, container, obj)
        onode = onodes[0]
        kill_server(onode['port'], self.port2server, self.pids)

        # Create container/obj (goes to two primary servers and one handoff)
        client.put_object(self.url, self.token, container, obj, 'VERIFY')
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Kill other two container/obj primary servers
        #   to ensure GET handoff works
        for node in onodes[1:]:
            kill_server(node['port'], self.port2server, self.pids)

        # Indirectly through proxy assert we can get container/obj
        odata = client.get_object(self.url, self.token, container, obj)[-1]
        if odata != 'VERIFY':
            raise Exception('Object GET did not return VERIFY, instead it '
                            'returned: %s' % repr(odata))

        # Restart those other two container/obj primary servers
        for node in onodes[1:]:
            start_server(node['port'], self.port2server, self.pids)

        # We've indirectly verified the handoff node has the container/object,
        #   but let's directly verify it.
        another_onode = self.object_ring.get_more_nodes(opart).next()
        odata = direct_client.direct_get_object(
            another_onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # Assert container listing (via proxy and directly) has container/obj
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj not in objs:
            raise Exception('Container listing did not know about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj not in objs:
                raise Exception(
                    'Container server %s:%s did not know about object' %
                    (cnode['ip'], cnode['port']))

        # Bring the first container/obj primary server back up
        start_server(onode['port'], self.port2server, self.pids)

        # Assert that it doesn't have container/obj yet
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEquals(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) / 10
            Manager(['object-replicator']).once(number=node_id)
        try:
            another_port_num = another_onode['replication_port']
        except KeyError:
            another_port_num = another_onode['port']
        another_num = (another_port_num - 6000) / 10
        Manager(['object-replicator']).once(number=another_num)

        # Assert the first container/obj primary server now has container/obj
        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        if odata != 'VERIFY':
            raise Exception('Direct object GET did not return VERIFY, instead '
                            'it returned: %s' % repr(odata))

        # Assert the handoff server no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEquals(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Kill the first container/obj primary server again (we have two
        #   primaries and the handoff up now)
        kill_server(onode['port'], self.port2server, self.pids)

        # Delete container/obj
        try:
            client.delete_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            if self.object_ring.replica_count > 2:
                raise
            # Object DELETE returning 503 for (404, 204)
            # remove this with fix for
            # https://bugs.launchpad.net/swift/+bug/1318375
            self.assertEqual(503, err.http_status)

        # Assert we can't head container/obj
        try:
            client.head_object(self.url, self.token, container, obj)
        except client.ClientException as err:
            self.assertEquals(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")

        # Assert container/obj is not in the container listing, both indirectly
        #   and directly
        objs = [o['name'] for o in
                client.get_container(self.url, self.token, container)[1]]
        if obj in objs:
            raise Exception('Container listing still knew about object')
        for cnode in cnodes:
            objs = [o['name'] for o in
                    direct_client.direct_get_container(
                        cnode, cpart, self.account, container)[1]]
            if obj in objs:
                raise Exception(
                    'Container server %s:%s still knew about object' %
                    (cnode['ip'], cnode['port']))

        # Restart the first container/obj primary server again
        start_server(onode['port'], self.port2server, self.pids)

        # Assert it still has container/obj
        direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})

        # Run object replication, ensuring we run the handoff node last so it
        #   will remove its extra handoff partition
        for node in onodes:
            try:
                port_num = node['replication_port']
            except KeyError:
                port_num = node['port']
            node_id = (port_num - 6000) / 10
            Manager(['object-replicator']).once(number=node_id)
        another_node_id = (another_port_num - 6000) / 10
        Manager(['object-replicator']).once(number=another_node_id)

        # Assert primary node no longer has container/obj
        try:
            direct_client.direct_get_object(
                another_onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
        except ClientException as err:
            self.assertEquals(err.http_status, 404)
        else:
            self.fail("Expected ClientException but didn't get it")
示例#48
0
    def test_main(self):
        # Create container1 and container2
        container1 = 'container1'
        client.put_container(self.url, self.token, container1)
        container2 = 'container2'
        client.put_container(self.url, self.token, container2)

        # Assert account level sees them
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '0')
        self.assertEquals(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
        self.assertTrue(found1)
        self.assertTrue(found2)

        # Create container2/object1
        client.put_object(self.url, self.token, container2, 'object1', '1234')

        # Assert account level doesn't see it yet
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '0')
        self.assertEquals(headers['x-account-bytes-used'], '0')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
        self.assertTrue(found1)
        self.assertTrue(found2)

        # Get to final state
        self.get_to_final_state()

        # Assert account level now sees the container2/object1
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
                self.assertEquals(container['count'], 0)
                self.assertEquals(container['bytes'], 0)
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assertTrue(found1)
        self.assertTrue(found2)

        apart, anodes = self.account_ring.get_nodes(self.account)
        kill_nonprimary_server(anodes, self.ipport2server, self.pids)
        kill_server((anodes[0]['ip'], anodes[0]['port']),
                    self.ipport2server, self.pids)
        # Kill account servers excepting two of the primaries

        # Delete container1
        client.delete_container(self.url, self.token, container1)

        # Put container2/object2
        client.put_object(self.url, self.token, container2, 'object2', '12345')

        # Assert account level knows container1 is gone but doesn't know about
        #   container2/object2 yet
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assertTrue(not found1)
        self.assertTrue(found2)

        # Run container updaters
        Manager(['container-updater']).once()

        # Assert account level now knows about container2/object2
        headers, containers = client.get_account(self.url, self.token)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '2')
        self.assertEquals(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 2)
                self.assertEquals(container['bytes'], 9)
        self.assertTrue(not found1)
        self.assertTrue(found2)

        # Restart other primary account server
        start_server((anodes[0]['ip'], anodes[0]['port']),
                     self.ipport2server, self.pids)

        # Assert that server doesn't know about container1's deletion or the
        #   new container2/object2 yet
        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEquals(headers['x-account-container-count'], '2')
        self.assertEquals(headers['x-account-object-count'], '1')
        self.assertEquals(headers['x-account-bytes-used'], '4')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 1)
                self.assertEquals(container['bytes'], 4)
        self.assertTrue(found1)
        self.assertTrue(found2)

        # Get to final state
        self.get_to_final_state()

        # Assert that server is now up to date
        headers, containers = \
            direct_client.direct_get_account(anodes[0], apart, self.account)
        self.assertEquals(headers['x-account-container-count'], '1')
        self.assertEquals(headers['x-account-object-count'], '2')
        self.assertEquals(headers['x-account-bytes-used'], '9')
        found1 = False
        found2 = False
        for container in containers:
            if container['name'] == container1:
                found1 = True
            elif container['name'] == container2:
                found2 = True
                self.assertEquals(container['count'], 2)
                self.assertEquals(container['bytes'], 9)
        self.assertTrue(not found1)
        self.assertTrue(found2)
示例#49
0
    def test_update_pending(self):

        # Create container
        container = 'contx'
        client.put_container(self.url, self.token, container)

        part, nodes = self.account_ring.get_nodes(self.account)
        anode = nodes[0]

        # Stop a quorum of account servers
        # This allows the put to continue later.
        kill_nonprimary_server(nodes, self.ipport2server)
        kill_server((anode['ip'], anode['port']), self.ipport2server)

        # Put object
        # This creates an outstanding update.
        client.put_object(self.url, self.token, container, 'object1', b'123')

        cont_db_files = self.get_container_db_files(container)
        self.assertEqual(len(cont_db_files), 3)

        # Collect the observable state from containers
        outstanding_files = []
        for cfile in cont_db_files:
            broker = ContainerBroker(cfile)
            try:
                info = broker.get_info()
            except LockTimeout:
                self.fail('LockTimeout at %s' % (cfile, ))
            if Timestamp(info['put_timestamp']) <= 0:
                self.fail('No put_timestamp at %s' % (cfile, ))
            # Correct even if reported_put_timestamp is zero.
            if info['put_timestamp'] > info['reported_put_timestamp']:
                outstanding_files.append(cfile)
        self.assertGreater(len(outstanding_files), 0)

        # At this point the users shut everything down and screw up the
        # hash in swift.conf. But we destroy the account DB instead.
        files = self.get_account_db_files(self.account)
        for afile in files:
            os.unlink(afile)

        # Restart the stopped primary server
        start_server((anode['ip'], anode['port']), self.ipport2server)

        # Make sure updaters run
        Manager(['container-updater']).once()

        # Collect the observable state from containers again and examine it
        outstanding_files_new = []
        for cfile in cont_db_files:

            # We aren't catching DatabaseConnectionError, because
            # we only want to approve of DBs that were quarantined,
            # and not otherwise damaged. So if the code below throws
            # an exception for other reason, we want the test to fail.
            if not os.path.exists(cfile):
                continue

            broker = ContainerBroker(cfile)
            try:
                info = broker.get_info()
            except LockTimeout:
                self.fail('LockTimeout at %s' % (cfile, ))
            if Timestamp(info['put_timestamp']) <= 0:
                self.fail('No put_timestamp at %s' % (cfile, ))
            # Correct even if reported_put_timestamp is zero.
            if info['put_timestamp'] > info['reported_put_timestamp']:
                outstanding_files_new.append(cfile)
        self.assertLengthEqual(outstanding_files_new, 0)

        self.get_to_final_state()