Beispiel #1
0
    def test_PUT_encryption_override(self):
        # set crypto override to disable encryption.
        # simulate another middleware wanting to set footers
        other_footers = {
            'Etag': 'other etag',
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
                'other override'}
        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'PUT',
               'swift.crypto.override': True,
               'swift.callback.update_footers':
                   lambda footers: footers.update(other_footers)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('201 Created', resp.status)

        # verify that other middleware's footers made it to app
        req_hdrs = self.app.headers[0]
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify object is NOT encrypted by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        self.assertEqual(body, get_req.get_response(self.app).body)
Beispiel #2
0
    def test_bucket_GET_max_keys(self):
        bucket_name = "junk"

        req = Request.blank(
            "/%s?max-keys=5" % bucket_name,
            environ={"REQUEST_METHOD": "GET"},
            headers={"Authorization": "AWS test:tester:hmac"},
        )
        status, headers, body = self.call_swift3(req)
        elem = fromstring(body, "ListBucketResult")
        self.assertEquals(elem.find("./MaxKeys").text, "5")
        _, path = self.swift.calls[-1]
        _, query_string = path.split("?")
        args = dict(cgi.parse_qsl(query_string))
        self.assert_(args["limit"] == "6")

        req = Request.blank(
            "/%s?max-keys=5000" % bucket_name,
            environ={"REQUEST_METHOD": "GET"},
            headers={"Authorization": "AWS test:tester:hmac"},
        )
        status, headers, body = self.call_swift3(req)
        elem = fromstring(body, "ListBucketResult")
        self.assertEquals(elem.find("./MaxKeys").text, "5000")
        _, path = self.swift.calls[-1]
        _, query_string = path.split("?")
        args = dict(cgi.parse_qsl(query_string))
        self.assertEquals(args["limit"], "1001")
Beispiel #3
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env["swift.copy_hook"] = self.copy_hook(
            env.get("swift.copy_hook", lambda src_req, src_resp, sink_req: src_resp)
        )

        try:
            if req.method == "PUT" and req.params.get("multipart-manifest") == "put":
                return self.handle_multipart_put(req, start_response)
            if req.method == "DELETE" and req.params.get("multipart-manifest") == "delete":
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == "GET" or req.method == "HEAD":
                return self.handle_multipart_get_or_head(req, start_response)
            if "X-Static-Large-Object" in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body="X-Static-Large-Object is a reserved header. "
                    "To create a static large object add query param "
                    "multipart-manifest=put.",
                )
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Beispiel #4
0
    def test_GETorHEAD_base(self):
        base = Controller(self.app)
        req = Request.blank("/v1/a/c/o/with/slashes")
        ring = FakeRing()
        nodes = list(ring.get_part_nodes(0)) + list(ring.get_more_nodes(0))
        with patch("swift.proxy.controllers.base." "http_connect", fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, "object", iter(nodes), "part", "/a/c/o/with/slashes")
        self.assertTrue("swift.object/a/c/o/with/slashes" in resp.environ)
        self.assertEqual(resp.environ["swift.object/a/c/o/with/slashes"]["status"], 200)
        req = Request.blank("/v1/a/c/o")
        with patch("swift.proxy.controllers.base." "http_connect", fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, "object", iter(nodes), "part", "/a/c/o")
        self.assertTrue("swift.object/a/c/o" in resp.environ)
        self.assertEqual(resp.environ["swift.object/a/c/o"]["status"], 200)
        req = Request.blank("/v1/a/c")
        with patch("swift.proxy.controllers.base." "http_connect", fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, "container", iter(nodes), "part", "/a/c")
        self.assertTrue("swift.container/a/c" in resp.environ)
        self.assertEqual(resp.environ["swift.container/a/c"]["status"], 200)

        req = Request.blank("/v1/a")
        with patch("swift.proxy.controllers.base." "http_connect", fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, "account", iter(nodes), "part", "/a")
        self.assertTrue("swift.account/a" in resp.environ)
        self.assertEqual(resp.environ["swift.account/a"]["status"], 200)
Beispiel #5
0
    def test_proxy_client_logging(self):
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank(
            "/",
            environ={"REQUEST_METHOD": "GET", "REMOTE_ADDR": "1.2.3.4", "HTTP_X_FORWARDED_FOR": "4.5.6.7,8.9.10.11"},
        )
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[0], "4.5.6.7")  # client ip
        self.assertEquals(log_parts[1], "1.2.3.4")  # remote addr

        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank(
            "/", environ={"REQUEST_METHOD": "GET", "REMOTE_ADDR": "1.2.3.4", "HTTP_X_CLUSTER_CLIENT_IP": "4.5.6.7"}
        )
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[0], "4.5.6.7")  # client ip
        self.assertEquals(log_parts[1], "1.2.3.4")  # remote addr
 def test_container4unknown_memcache(self):
     fake_memcache = FakeMemcache()
     self.assertEquals(fake_memcache.store, {})
     resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache}).get_response(self.test_staticweb)
     self.assertEquals(resp.status_int, 301)
     self.assertEquals(fake_memcache.store, {"/staticweb/v1/a/c4": ("index.html", "error.html", "t", "listing.css")})
     self.assert_(self.test_staticweb.app.get_c4_called)
     self.test_staticweb.app.get_c4_called = False
     resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache}).get_response(self.test_staticweb)
     self.assertEquals(resp.status_int, 301)
     self.assert_(not self.test_staticweb.app.get_c4_called)
     self.assertEquals(fake_memcache.store, {"/staticweb/v1/a/c4": ("index.html", "error.html", "t", "listing.css")})
     resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache, "REQUEST_METHOD": "PUT"}).get_response(
         self.test_staticweb
     )
     self.assertEquals(resp.status_int, 200)
     self.assertEquals(fake_memcache.store, {})
     resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache}).get_response(self.test_staticweb)
     self.assertEquals(resp.status_int, 301)
     self.assertEquals(fake_memcache.store, {"/staticweb/v1/a/c4": ("index.html", "error.html", "t", "listing.css")})
     resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache, "REQUEST_METHOD": "POST"}).get_response(
         self.test_staticweb
     )
     self.assertEquals(resp.status_int, 200)
     self.assertEquals(fake_memcache.store, {})
    def test_upload_size(self):
        # Using default policy
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {"log_headers": "yes"})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c/o/foo", environ={"REQUEST_METHOD": "PUT", "wsgi.input": BytesIO(b"some stuff")})
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[11], str(len("FAKE APP")))
        self.assertEquals(log_parts[10], str(len("some stuff")))
        self.assertUpdateStats(
            [
                ("object.PUT.200.xfer", len("some stuff") + len("FAKE APP")),
                ("object.policy.0.PUT.200.xfer", len("some stuff") + len("FAKE APP")),
            ],
            app,
        )

        # Using a non-existent policy
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx="-1"), {"log_headers": "yes"})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c/o/foo", environ={"REQUEST_METHOD": "PUT", "wsgi.input": BytesIO(b"some stuff")})
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[11], str(len("FAKE APP")))
        self.assertEquals(log_parts[10], str(len("some stuff")))
        self.assertUpdateStats([("object.PUT.200.xfer", len("some stuff") + len("FAKE APP"))], app)
Beispiel #8
0
    def test_swift_owner(self):
        owner_headers = {
            "x-container-read": "value",
            "x-container-write": "value",
            "x-container-sync-key": "value",
            "x-container-sync-to": "value",
        }
        controller = proxy_server.ContainerController(self.app, "a", "c")

        req = Request.blank("/a/c")
        with mock.patch(
            "swift.proxy.controllers.base.http_connect", fake_http_connect(200, 200, headers=owner_headers)
        ):
            resp = controller.HEAD(req)
        self.assertEquals(2, resp.status_int // 100)
        for key in owner_headers:
            self.assertTrue(key not in resp.headers)

        req = Request.blank("/a/c", environ={"swift_owner": True})
        with mock.patch(
            "swift.proxy.controllers.base.http_connect", fake_http_connect(200, 200, headers=owner_headers)
        ):
            resp = controller.HEAD(req)
        self.assertEquals(2, resp.status_int // 100)
        for key in owner_headers:
            self.assertTrue(key in resp.headers)
Beispiel #9
0
    def test_cdn_get_no_content(self):
        prev_data = json.dumps(
            {"account": "acc", "container": "cont", "ttl": 1234, "logs_enabled": True, "cdn_enabled": True}
        )
        self.test_origin.app = FakeApp(
            iter([("204 No Content", {}, prev_data), ("304 No Content", {}, "")])  # call to _get_cdn_data
        )  # call to get obj
        req = Request.blank(
            "http://1234.r34.origin_cdn.com:8080/obj1.jpg",
            environ={"REQUEST_METHOD": "HEAD", "swift.cdn_hash": "abcd", "swift.cdn_object_name": "obj1.jpg"},
        )
        resp = req.get_response(self.test_origin)
        self.assertEquals(resp.status_int, 304)

        self.test_origin.app = FakeApp(
            iter([("204 No Content", {}, prev_data), ("404 No Content", {}, "")])  # call to _get_cdn_data
        )  # call to get obj
        req = Request.blank(
            "http://1234.r34.origin_cdn.com:8080/obj1.jpg",
            environ={"REQUEST_METHOD": "HEAD", "swift.cdn_hash": "abcd", "swift.cdn_object_name": "obj1.jpg"},
        )
        resp = req.get_response(self.test_origin)
        self.assertEquals(resp.status_int, 404)

        self.test_origin.app = FakeApp(
            iter([("204 No Content", {}, prev_data), ("416 No Content", {}, "")])  # call to _get_cdn_data
        )  # call to get obj
        req = Request.blank(
            "http://1234.r34.origin_cdn.com:8080/obj1.jpg",
            environ={"REQUEST_METHOD": "HEAD", "swift.cdn_hash": "abcd", "swift.cdn_object_name": "obj1.jpg"},
        )
        resp = req.get_response(self.test_origin)
        self.assertEquals(resp.status_int, 416)
Beispiel #10
0
    def test_canonical_uri_sigv2(self):
        environ = {"HTTP_HOST": "bucket1.s3.test.com", "REQUEST_METHOD": "GET"}

        headers = {"Authorization": "AWS test:tester:hmac", "X-Amz-Date": self.get_date_header()}

        # Virtual hosted-style
        with patch("swift3.cfg.CONF.storage_domain", "s3.test.com"):
            req = Request.blank("/", environ=environ, headers=headers)
            sigv2_req = S3_Request(req.environ)
            uri = sigv2_req._canonical_uri()
            self.assertEqual(uri, "/bucket1/")
            self.assertEqual(req.environ["PATH_INFO"], "/")

            req = Request.blank("/obj1", environ=environ, headers=headers)
            sigv2_req = S3_Request(req.environ)
            uri = sigv2_req._canonical_uri()
            self.assertEqual(uri, "/bucket1/obj1")
            self.assertEqual(req.environ["PATH_INFO"], "/obj1")

        environ = {"HTTP_HOST": "s3.test.com", "REQUEST_METHOD": "GET"}

        # Path-style
        with patch("swift3.cfg.CONF.storage_domain", ""):
            req = Request.blank("/", environ=environ, headers=headers)
            sigv2_req = S3_Request(req.environ)
            uri = sigv2_req._canonical_uri()

            self.assertEqual(uri, "/")
            self.assertEqual(req.environ["PATH_INFO"], "/")

            req = Request.blank("/bucket1/obj1", environ=environ, headers=headers)
            sigv2_req = S3_Request(req.environ)
            uri = sigv2_req._canonical_uri()
            self.assertEqual(uri, "/bucket1/obj1")
            self.assertEqual(req.environ["PATH_INFO"], "/bucket1/obj1")
Beispiel #11
0
 def test_v2_non_obj_response(self):
     # account
     req = Request.blank("/endpoints/v2/a")
     resp = req.get_response(self.list_endpoints)
     expected = {
         "endpoints": [
             "http://10.1.2.1:6000/sdc1/0/a",
             "http://10.1.1.1:6000/sda1/0/a",
             "http://10.1.1.1:6000/sdb1/0/a",
         ],
         "headers": {},
     }
     # container
     self.assertEqual(resp.body, json.dumps(expected))
     req = Request.blank("/endpoints/v2/a/c")
     resp = req.get_response(self.list_endpoints)
     expected = {
         "endpoints": [
             "http://10.1.2.2:6000/sdd1/0/a/c",
             "http://10.1.1.1:6000/sda1/0/a/c",
             "http://10.1.2.1:6000/sdc1/0/a/c",
         ],
         "headers": {},
     }
     self.assertEqual(resp.body, json.dumps(expected))
Beispiel #12
0
def make_pre_authed_request(env, method=None, path=None, body=None, headers=None, agent="Swift"):
    """
    Makes a new swob.Request based on the current env but with the
    parameters specified. Note that this request will be preauthorized.

    :param env: The WSGI environment to base the new request on.
    :param method: HTTP method of new request; default is from
                   the original env.
    :param path: HTTP path of new request; default is from the
                 original env. path should be compatible with what you
                 would send to Request.blank. path should be quoted and it
                 can include a query string. for example:
                 '/a%20space?unicode_str%E8%AA%9E=y%20es'
    :param body: HTTP body of new request; empty by default.
    :param headers: Extra HTTP headers of new request; None by
                    default.
    :param agent: The HTTP user agent to use; default 'Swift'. You
                  can put %(orig)s in the agent to have it replaced
                  with the original env's HTTP_USER_AGENT, such as
                  '%(orig)s StaticWeb'. You also set agent to None to
                  use the original env's HTTP_USER_AGENT or '' to
                  have no HTTP_USER_AGENT.
    :returns: Fresh swob.Request object.
    """
    query_string = None
    if path and "?" in path:
        path, query_string = path.split("?", 1)
    newenv = make_pre_authed_env(env, method, path=unquote(path), agent=agent, query_string=query_string)
    if not headers:
        headers = {}
    if body:
        return Request.blank(path, environ=newenv, body=body, headers=headers)
    else:
        return Request.blank(path, environ=newenv, headers=headers)
Beispiel #13
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        try:
            if obj:
                if req.method == "PUT" and req.params.get("multipart-manifest") == "put":
                    return self.handle_multipart_put(req, start_response)
                if req.method == "DELETE" and req.params.get("multipart-manifest") == "delete":
                    return self.handle_multipart_delete(req)(env, start_response)
                if "X-Static-Large-Object" in req.headers:
                    raise HTTPBadRequest(
                        request=req,
                        body="X-Static-Large-Object is a reserved header. "
                        "To create a static large object add query param "
                        "multipart-manifest=put.",
                    )
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Beispiel #14
0
    def __call__(self, env, start_response):
        # If override is set in env, then just pass along
        if config_true_value(env.get("swift.crypto.override")):
            return self.app(env, start_response)

        req = Request(env)

        if self.disable_encryption and req.method in ("PUT", "POST"):
            return self.app(env, start_response)
        try:
            req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ("GET", "HEAD"):
            handler = EncrypterObjContext(self, self.logger).handle_get_or_head
        elif req.method == "PUT":
            handler = EncrypterObjContext(self, self.logger).handle_put
        elif req.method == "POST":
            handler = EncrypterObjContext(self, self.logger).handle_post
        else:
            # anything else
            return self.app(env, start_response)

        try:
            return handler(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
Beispiel #15
0
    def test_with_memcache(self):
        def my_lookup(d):
            return 0, "%s.example.com" % d

        cname_lookup.lookup_cname = my_lookup

        class memcache_stub(object):
            def __init__(self):
                self.cache = {}

            def get(self, key):
                return self.cache.get(key, None)

            def set(self, key, value, *a, **kw):
                self.cache[key] = value

        memcache = memcache_stub()
        req = Request.blank(
            "/", environ={"REQUEST_METHOD": "GET", "swift.cache": memcache}, headers={"Host": "mysite.com"}
        )
        resp = self.app(req.environ, start_response)
        self.assertEquals(resp, "FAKE APP")
        req = Request.blank(
            "/", environ={"REQUEST_METHOD": "GET", "swift.cache": memcache}, headers={"Host": "mysite.com"}
        )
        resp = self.app(req.environ, start_response)
        self.assertEquals(resp, "FAKE APP")
Beispiel #16
0
    def test_admin_setup(self):
        # PUTs for account and 16 .hash's
        self.test_origin.app = FakeApp(iter([("204 No Content", {}, "") for i in xrange(201)]))
        resp = Request.blank(
            "/origin/.prep",
            environ={"REQUEST_METHOD": "PUT"},
            headers={"X-Origin-Admin-User": ".origin_admin", "X-Origin-Admin-Key": "unittest"},
        ).get_response(self.test_origin)
        self.assertEquals(resp.status_int, 204)
        self.assertEquals(self.test_origin.app.calls, 101)

        self.test_origin.app = FakeApp(iter([("404 Not Found", {}, "")]))
        req = Request.blank(
            "/origin/.prep",
            environ={"REQUEST_METHOD": "PUT"},
            headers={"X-Origin-Admin-User": ".origin_admin", "X-Origin-Admin-Key": "unittest"},
        )
        self.assertRaises(Exception, req.get_response, self.test_origin)

        self.test_origin.app = FakeApp(iter([("204 No Content", {}, ""), ("404 Not Found", {}, "")]))
        req = Request.blank(
            "/origin/.prep",
            environ={"REQUEST_METHOD": "PUT"},
            headers={"X-Origin-Admin-User": ".origin_admin", "X-Origin-Admin-Key": "unittest"},
        )
        self.assertRaises(Exception, req.get_response, self.test_origin)
Beispiel #17
0
    def test_bucket_acl_PUT(self):
        elem = Element("AccessControlPolicy")
        owner = SubElement(elem, "Owner")
        SubElement(owner, "ID").text = "id"
        acl = SubElement(elem, "AccessControlList")
        grant = SubElement(acl, "Grant")
        grantee = SubElement(grant, "Grantee", nsmap={"xsi": XMLNS_XSI})
        grantee.set("{%s}type" % XMLNS_XSI, "Group")
        SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AllUsers"
        SubElement(grant, "Permission").text = "READ"

        xml = tostring(elem)
        req = Request.blank(
            "/bucket?acl",
            environ={"REQUEST_METHOD": "PUT"},
            headers={"Authorization": "AWS test:tester:hmac", "Date": self.get_date_header()},
            body=xml,
        )
        status, headers, body = self.call_swift3(req)
        self.assertEquals(status.split()[0], "200")

        req = Request.blank(
            "/bucket?acl",
            environ={"REQUEST_METHOD": "PUT", "wsgi.input": StringIO(xml)},
            headers={
                "Authorization": "AWS test:tester:hmac",
                "Date": self.get_date_header(),
                "Transfer-Encoding": "chunked",
            },
        )
        self.assertIsNone(req.content_length)
        self.assertIsNone(req.message_length())
        status, headers, body = self.call_swift3(req)
        self.assertEquals(status.split()[0], "200")
Beispiel #18
0
    def test_origin_db_post_fail(self):
        self.test_origin.app = FakeApp(
            iter([("204 No Content", {}, ""), ("404 Not Found", {}, "")])  # call to _get_cdn_data  # put to .hash
        )
        req = Request.blank("http://origin_db.com:8080/v1/acc/cont", environ={"REQUEST_METHOD": "PUT"})
        resp = req.get_response(self.test_origin)
        self.assertEquals(resp.status_int, 500)

        self.test_origin.app = FakeApp(
            iter(
                [
                    ("204 No Content", {}, ""),  # call to _get_cdn_data
                    ("204 No Content", {}, ""),  # put to .hash
                    ("404 Not Found", {}, ""),  # HEAD check to list container
                    ("404 Not Found", {}, ""),  # PUT to list container
                ]
            )
        )
        req = Request.blank("http://origin_db.com:8080/v1/acc/cont", environ={"REQUEST_METHOD": "PUT"})
        resp = req.get_response(self.test_origin)
        self.assertEquals(resp.status_int, 500)

        self.test_origin.app = FakeApp(
            iter(
                [
                    ("204 No Content", {}, ""),  # call to _get_cdn_data
                    ("204 No Content", {}, ""),  # put to .hash
                    ("204 No Content", {}, ""),  # HEAD check to list container
                    ("404 Not Found", {}, ""),  # PUT to list container
                ]
            )
        )
        req = Request.blank("http://origin_db.com:8080/v1/acc/cont", environ={"REQUEST_METHOD": "PUT"})
        resp = req.get_response(self.test_origin)
        self.assertEquals(resp.status_int, 500)
Beispiel #19
0
    def test_handle_multipart_put_bad_data(self):
        bad_data = json.dumps([{"path": "/cont/object", "etag": "etagoftheobj", "size_bytes": "lala"}])
        req = Request.blank(
            "/test_good/AUTH_test/c/man?multipart-manifest=put", environ={"REQUEST_METHOD": "PUT"}, body=bad_data
        )
        self.assertRaises(HTTPException, self.slo.handle_multipart_put, req)

        for bad_data in [
            json.dumps([{"path": "/cont", "etag": "etagoftheobj", "size_bytes": 100}]),
            json.dumps("asdf"),
            json.dumps(None),
            json.dumps(5),
            "not json",
            "1234",
            None,
            "",
            json.dumps({"path": None}),
            json.dumps([{"path": "/c/o", "etag": None, "size_bytes": 12}]),
            json.dumps([{"path": "/c/o", "etag": "asdf", "size_bytes": "sd"}]),
            json.dumps([{"path": 12, "etag": "etagoftheobj", "size_bytes": 100}]),
            json.dumps([{"path": u"/cont/object\u2661", "etag": "etagoftheobj", "size_bytes": 100}]),
            json.dumps([{"path": 12, "size_bytes": 100}]),
            json.dumps([{"path": 12, "size_bytes": 100}]),
            json.dumps([{"path": None, "etag": "etagoftheobj", "size_bytes": 100}]),
        ]:
            req = Request.blank(
                "/test_good/AUTH_test/c/man?multipart-manifest=put", environ={"REQUEST_METHOD": "PUT"}, body=bad_data
            )
            self.assertRaises(HTTPException, self.slo.handle_multipart_put, req)
    def test_PUT_object_with_long_names(self):
        for i in (220, 221):
            longname = "o" * i
            path = "/V1.0/a/c2/" + longname

            with nested(
                patch(
                    "swiftonfile.swift.common.middleware." "check_constraints.get_container_info",
                    self.container2_info_mock,
                ),
                patch("swiftonfile.swift.common.middleware." "check_constraints.POLICIES", self.policies_mock),
            ):
                resp = Request.blank(path, environ={"REQUEST_METHOD": "PUT"}).get_response(self.test_check)
                self.assertEquals(resp.status_int, 200)

        longname = "o" * 222
        path = "/V1.0/a/c2/" + longname

        with nested(
            patch(
                "swiftonfile.swift.common.middleware.check_constraints." "get_container_info", self.container2_info_mock
            ),
            patch("swiftonfile.swift.common.middleware.check_constraints." "POLICIES", self.policies_mock),
        ):
            resp = Request.blank(path, environ={"REQUEST_METHOD": "PUT"}).get_response(self.test_check)
            self.assertEquals(resp.status_int, 400)
            self.assertTrue("too long" in resp.body)
    def test_policy_index(self):
        # Policy index can be specified by X-Backend-Storage-Policy-Index
        # in the request header for object API
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx="1"), {})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "PUT"})
        resp = app(req.environ, start_response)
        "".join(resp)
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[20], "1")

        # Policy index can be specified by X-Backend-Storage-Policy-Index
        # in the response header for container API
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c", environ={"REQUEST_METHOD": "GET"})

        def fake_call(app, env, start_response):
            start_response(
                app.response_str,
                [
                    ("Content-Type", "text/plain"),
                    ("Content-Length", str(sum(map(len, app.body)))),
                    ("X-Backend-Storage-Policy-Index", "1"),
                ],
            )
            while env["wsgi.input"].read(5):
                pass
            return app.body

        with mock.patch.object(FakeApp, "__call__", fake_call):
            resp = app(req.environ, start_response)
            "".join(resp)
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[20], "1")
Beispiel #22
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        self.account_name = account
        self.container_name = container
        self.object_name = obj

        # Save off original request method (COPY/POST) in case it gets mutated
        # into PUT during handling. This way logging can display the method
        # the client actually sent.
        req.environ["swift.orig_req_method"] = req.method

        if req.method == "PUT" and req.headers.get("X-Copy-From"):
            return self.handle_PUT(req, start_response)
        elif req.method == "COPY":
            return self.handle_COPY(req, start_response)
        elif req.method == "POST" and self.object_post_as_copy:
            return self.handle_object_post_as_copy(req, start_response)
        elif req.method == "OPTIONS":
            # Does not interfere with OPTIONS response from (account,container)
            # servers and /info response.
            return self.handle_OPTIONS(req, start_response)

        return self.app(env, start_response)
Beispiel #23
0
    def test_PUT_encryption_override(self):
        # set crypto override to disable encryption.
        # simulate another middleware wanting to set footers
        other_footers = {
            "Etag": "other etag",
            "X-Object-Sysmeta-Other": "other sysmeta",
            "X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
        }
        body = "FAKE APP"
        env = {
            "REQUEST_METHOD": "PUT",
            "swift.crypto.override": True,
            "swift.callback.update_footers": lambda footers: footers.update(other_footers),
        }
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("201 Created", resp.status)

        # verify that other middleware's footers made it to app
        req_hdrs = self.app.headers[0]
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify object is NOT encrypted by getting direct from the app
        get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
        self.assertEqual(body, get_req.get_response(self.app).body)
    def __call__(self, env, start_response):
        # making a duplicate, because if this is a COPY request, we will
        # modify the PATH_INFO to find out if the 'Destination' is in a
        # versioned container
        req = Request(env.copy())
        try:
            (version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # In case allow_versioned_writes is set in the filter configuration,
        # the middleware becomes the authority on whether object
        # versioning is enabled or not. In case it is not set, then
        # the option in the container configuration is still checked
        # for backwards compatibility

        # For a container request, first just check if option is set,
        # can be either true or false.
        # If set, check if enabled when actually trying to set container
        # header. If not set, let request be handled by container server
        # for backwards compatibility.
        # For an object request, also check if option is set (either T or F).
        # If set, check if enabled when checking versions container in
        # sysmeta property. If it is not set check 'versions' property in
        # container_info
        allow_versioned_writes = self.conf.get("allow_versioned_writes")
        if allow_versioned_writes and container and not obj:
            return self.container_request(req, start_response, allow_versioned_writes)
        elif obj and req.method in ("PUT", "COPY", "DELETE"):
            return self.object_request(req, version, account, container, obj, allow_versioned_writes)(
                env, start_response
            )
        else:
            return self.app(env, start_response)
Beispiel #25
0
 def create_container(self, req, container_path):
     """
     Checks if the container exists and if not try to create it.
     :params container_path: an unquoted path to a container to be created
     :returns: True if created container, False if container exists
     :raises: CreateContainerError when unable to create container
     """
     new_env = req.environ.copy()
     new_env["PATH_INFO"] = container_path
     new_env["swift.source"] = "EA"
     new_env["REQUEST_METHOD"] = "HEAD"
     head_cont_req = Request.blank(container_path, environ=new_env)
     resp = head_cont_req.get_response(self.app)
     if resp.is_success:
         return False
     if resp.status_int == 404:
         new_env = req.environ.copy()
         new_env["PATH_INFO"] = container_path
         new_env["swift.source"] = "EA"
         new_env["REQUEST_METHOD"] = "PUT"
         create_cont_req = Request.blank(container_path, environ=new_env)
         resp = create_cont_req.get_response(self.app)
         if resp.is_success:
             return True
     raise CreateContainerError("Create Container Failed: " + container_path, resp.status_int, resp.status)
Beispiel #26
0
    def test_container7listing(self):
        # container7 has web-listings = f, web-error=error.html
        resp = Request.blank("/v1/a/c7/").get_response(self.test_staticweb)
        self.assertEqual(resp.status_int, 404)
        self.assertIn("Web Listing Disabled", resp.body)

        # expect 301 if auth'd but no trailing '/'
        resp = Request.blank("/v1/a/c7").get_response(self.test_staticweb)
        self.assertEqual(resp.status_int, 301)

        # expect default 401 if request is not auth'd and no trailing '/'
        test_staticweb = FakeAuthFilter(staticweb.filter_factory({})(self.app), deny_listing=True, deny_objects=True)
        resp = Request.blank("/v1/a/c7").get_response(test_staticweb)
        self.assertEqual(resp.status_int, 401)
        self.assertNotIn("Hey, you're not authorized to see this!", resp.body)

        # expect custom 401 if request is not auth'd for listing
        test_staticweb = FakeAuthFilter(staticweb.filter_factory({})(self.app), deny_listing=True)
        resp = Request.blank("/v1/a/c7/").get_response(test_staticweb)
        self.assertEqual(resp.status_int, 401)
        self.assertIn("Hey, you're not authorized to see this!", resp.body)

        # expect default 401 if request is not auth'd for listing or object GET
        test_staticweb = FakeAuthFilter(staticweb.filter_factory({})(self.app), deny_listing=True, deny_objects=True)
        resp = Request.blank("/v1/a/c7/").get_response(test_staticweb)
        self.assertEqual(resp.status_int, 401)
        self.assertNotIn("Hey, you're not authorized to see this!", resp.body)
Beispiel #27
0
    def test_bucket_GET_max_keys(self):
        class FakeApp(object):
            def __call__(self, env, start_response):
                self.query_string = env["QUERY_STRING"]
                start_response("200 OK", [])
                return "[]"

        fake_app = FakeApp()
        local_app = swift3.filter_factory({})(fake_app)
        bucket_name = "junk"

        req = Request.blank(
            "/%s" % bucket_name,
            environ={"REQUEST_METHOD": "GET", "QUERY_STRING": "max-keys=5"},
            headers={"Authorization": "AWS test:tester:hmac"},
        )
        resp = local_app(req.environ, lambda *args: None)
        dom = xml.dom.minidom.parseString("".join(resp))
        self.assertEquals(dom.getElementsByTagName("MaxKeys")[0].childNodes[0].nodeValue, "5")
        args = dict(cgi.parse_qsl(fake_app.query_string))
        self.assert_(args["limit"] == "6")

        req = Request.blank(
            "/%s" % bucket_name,
            environ={"REQUEST_METHOD": "GET", "QUERY_STRING": "max-keys=5000"},
            headers={"Authorization": "AWS test:tester:hmac"},
        )
        resp = local_app(req.environ, lambda *args: None)
        dom = xml.dom.minidom.parseString("".join(resp))
        self.assertEquals(dom.getElementsByTagName("MaxKeys")[0].childNodes[0].nodeValue, "1000")
        args = dict(cgi.parse_qsl(fake_app.query_string))
        self.assertEquals(args["limit"], "1001")
    def test_get_name_and_placement_object_replication(self):
        # yup, suffixes are sent '-'.joined in the path
        path = "/device/part/012-345-678-9ab-cde"
        req = Request.blank(path, headers={"X-Backend-Storage-Policy-Index": "0"})
        device, partition, suffix_parts, policy = get_name_and_placement(req, 2, 3, True)
        self.assertEqual(device, "device")
        self.assertEqual(partition, "part")
        self.assertEqual(suffix_parts, "012-345-678-9ab-cde")
        self.assertEqual(policy, POLICIES[0])
        self.assertEqual(policy.policy_type, EC_POLICY)

        path = "/device/part"
        req = Request.blank(path, headers={"X-Backend-Storage-Policy-Index": "1"})
        device, partition, suffix_parts, policy = get_name_and_placement(req, 2, 3, True)
        self.assertEqual(device, "device")
        self.assertEqual(partition, "part")
        self.assertEqual(suffix_parts, None)  # false-y
        self.assertEqual(policy, POLICIES[1])
        self.assertEqual(policy.policy_type, REPL_POLICY)

        path = "/device/part/"  # with a trailing slash
        req = Request.blank(path, headers={"X-Backend-Storage-Policy-Index": "1"})
        device, partition, suffix_parts, policy = get_name_and_placement(req, 2, 3, True)
        self.assertEqual(device, "device")
        self.assertEqual(partition, "part")
        self.assertEqual(suffix_parts, "")  # still false-y
        self.assertEqual(policy, POLICIES[1])
        self.assertEqual(policy.policy_type, REPL_POLICY)
Beispiel #29
0
    def test_sysmeta_replaced_by_PUT(self):
        path = "/v1/a/c/o"

        env = {"REQUEST_METHOD": "PUT"}
        hdrs = dict(self.original_sysmeta_headers_1)
        hdrs.update(self.original_sysmeta_headers_2)
        hdrs.update(self.original_meta_headers_1)
        hdrs.update(self.original_meta_headers_2)
        req = Request.blank(path, environ=env, headers=hdrs, body="x")
        resp = req.get_response(self.app)
        self._assertStatus(resp, 201)

        env = {"REQUEST_METHOD": "PUT"}
        hdrs = dict(self.changed_sysmeta_headers)
        hdrs.update(self.new_sysmeta_headers)
        hdrs.update(self.changed_meta_headers)
        hdrs.update(self.new_meta_headers)
        hdrs.update(self.bad_headers)
        req = Request.blank(path, environ=env, headers=hdrs, body="x")
        resp = req.get_response(self.app)
        self._assertStatus(resp, 201)

        req = Request.blank(path, environ={})
        resp = req.get_response(self.app)
        self._assertStatus(resp, 200)
        self._assertInHeaders(resp, self.changed_sysmeta_headers)
        self._assertInHeaders(resp, self.new_sysmeta_headers)
        self._assertNotInHeaders(resp, self.original_sysmeta_headers_2)
        self._assertInHeaders(resp, self.changed_meta_headers)
        self._assertInHeaders(resp, self.new_meta_headers)
        self._assertNotInHeaders(resp, self.original_meta_headers_2)
Beispiel #30
0
 def test_domain_remap_account(self):
     req = Request.blank("/", environ={"REQUEST_METHOD": "GET"}, headers={"Host": "AUTH_a.example.com"})
     resp = self.app(req.environ, start_response)
     self.assertEquals(resp, "/v1/AUTH_a")
     req = Request.blank("/", environ={"REQUEST_METHOD": "GET"}, headers={"Host": "AUTH-uuid.example.com"})
     resp = self.app(req.environ, start_response)
     self.assertEquals(resp, "/v1/AUTH_uuid")