コード例 #1
0
ファイル: test_bucket.py プロジェクト: jgmerritt/swift
    def test_bucket_GET_max_keys(self):
        bucket_name = 'junk'

        req = Request.blank('/%s?max-keys=5' % bucket_name,
                            environ={'REQUEST_METHOD': 'GET'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_s3api(req)
        elem = fromstring(body, 'ListBucketResult')
        self.assertEqual(elem.find('./MaxKeys').text, '5')
        _, path = self.swift.calls[-1]
        _, query_string = path.split('?')
        args = dict(cgi.parse_qsl(query_string))
        self.assertEqual(args['limit'], '6')

        req = Request.blank('/%s?max-keys=5000' % bucket_name,
                            environ={'REQUEST_METHOD': 'GET'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_s3api(req)
        elem = fromstring(body, 'ListBucketResult')
        self.assertEqual(elem.find('./MaxKeys').text, '5000')
        _, path = self.swift.calls[-1]
        _, query_string = path.split('?')
        args = dict(cgi.parse_qsl(query_string))
        self.assertEqual(args['limit'], '1001')
コード例 #2
0
ファイル: test_acl.py プロジェクト: tumf/swift3
    def test_bucket_acl_PUT(self):
        elem = Element('AccessControlPolicy')
        owner = SubElement(elem, 'Owner')
        SubElement(owner, 'ID').text = 'id'
        acl = SubElement(elem, 'AccessControlList')
        grant = SubElement(acl, 'Grant')
        grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
        grantee.set('{%s}type' % XMLNS_XSI, 'Group')
        SubElement(grantee, 'URI').text = \
            'http://acs.amazonaws.com/groups/global/AllUsers'
        SubElement(grant, 'Permission').text = 'READ'

        xml = tostring(elem)
        req = Request.blank('/bucket?acl',
                            environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Authorization': 'AWS test:tester:hmac'},
                            body=xml)
        status, headers, body = self.call_swift3(req)
        self.assertEquals(status.split()[0], '200')

        req = Request.blank('/bucket?acl',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'wsgi.input': StringIO(xml)},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Transfer-Encoding': 'chunked'})
        self.assertIsNone(req.content_length)
        self.assertIsNone(req.message_length())
        status, headers, body = self.call_swift3(req)
        self.assertEquals(status.split()[0], '200')
コード例 #3
0
ファイル: test_proxy_logging.py プロジェクト: hbhdytf/mac
    def test_proxy_client_logging(self):
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank('/', environ={
            'REQUEST_METHOD': 'GET',
            'REMOTE_ADDR': '1.2.3.4',
            'HTTP_X_FORWARDED_FOR': '4.5.6.7,8.9.10.11'})
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[0], '4.5.6.7')  # client ip
        self.assertEquals(log_parts[1], '1.2.3.4')  # remote addr

        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank('/', environ={
            'REQUEST_METHOD': 'GET',
            'REMOTE_ADDR': '1.2.3.4',
            'HTTP_X_CLUSTER_CLIENT_IP': '4.5.6.7'})
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[0], '4.5.6.7')  # client ip
        self.assertEquals(log_parts[1], '1.2.3.4')  # remote addr
コード例 #4
0
ファイル: test_bucket.py プロジェクト: jgmerritt/swift
    def test_bucket_GET_v2_with_delimiter_max_keys(self):
        bucket_name = 'junk'
        req = Request.blank(
            '/%s?list-type=2&delimiter=a&max-keys=2' % bucket_name,
            environ={'REQUEST_METHOD': 'GET'},
            headers={'Authorization': 'AWS test:tester:hmac',
                     'Date': self.get_date_header()})
        status, headers, body = self.call_s3api(req)
        self.assertEqual(status.split()[0], '200')
        elem = fromstring(body, 'ListBucketResult')
        next_token = elem.find('./NextContinuationToken')
        self.assertIsNotNone(next_token)
        self.assertEqual(elem.find('./MaxKeys').text, '2')
        self.assertEqual(elem.find('./IsTruncated').text, 'true')

        req = Request.blank(
            '/%s?list-type=2&delimiter=a&max-keys=2&continuation-token=%s' %
            (bucket_name, next_token.text),
            environ={'REQUEST_METHOD': 'GET'},
            headers={'Authorization': 'AWS test:tester:hmac',
                     'Date': self.get_date_header()})
        status, headers, body = self.call_s3api(req)
        self.assertEqual(status.split()[0], '200')
        elem = fromstring(body, 'ListBucketResult')
        names = [o.find('./Key').text for o in elem.iterchildren('Contents')]
        self.assertEqual(names[0], 'lily')
コード例 #5
0
ファイル: test_bulk.py プロジェクト: 2015-ucsc-hp/swift
    def test_extract_tar_works(self):
        # On systems where $TMPDIR is long (like OS X), we need to do this
        # or else every upload will fail due to the path being too long.
        self.app.max_pathlen += len(self.testdir)
        for compress_format in ['', 'gz', 'bz2']:
            base_name = 'base_works_%s' % compress_format
            dir_tree = [
                {base_name: [{'sub_dir1': ['sub1_file1', 'sub1_file2']},
                             {'sub_dir2': ['sub2_file1', u'test obj \u2661']},
                             'sub_file1',
                             {'sub_dir3': [{'sub4_dir1': '../sub4 file1'}]},
                             {'sub_dir4': None},
                             ]}]

            build_dir_tree(self.testdir, dir_tree)
            mode = 'w'
            extension = ''
            if compress_format:
                mode += ':' + compress_format
                extension += '.' + compress_format
            tar = tarfile.open(name=os.path.join(self.testdir,
                                                 'tar_works.tar' + extension),
                               mode=mode)
            tar.add(os.path.join(self.testdir, base_name))
            tar.close()
            req = Request.blank('/tar_works/acc/cont/')
            req.environ['wsgi.input'] = open(
                os.path.join(self.testdir, 'tar_works.tar' + extension))
            req.headers['transfer-encoding'] = 'chunked'
            resp_body = self.handle_extract_and_iter(req, compress_format)
            resp_data = utils.json.loads(resp_body)
            self.assertEquals(resp_data['Number Files Created'], 6)

            # test out xml
            req = Request.blank('/tar_works/acc/cont/')
            req.environ['wsgi.input'] = open(
                os.path.join(self.testdir, 'tar_works.tar' + extension))
            req.headers['transfer-encoding'] = 'chunked'
            resp_body = self.handle_extract_and_iter(
                req, compress_format, 'application/xml')
            self.assert_('<response_status>201 Created</response_status>' in
                         resp_body)
            self.assert_('<number_files_created>6</number_files_created>' in
                         resp_body)

            # test out nonexistent format
            req = Request.blank('/tar_works/acc/cont/?extract-archive=tar',
                                headers={'Accept': 'good_xml'})
            req.environ['REQUEST_METHOD'] = 'PUT'
            req.environ['wsgi.input'] = open(
                os.path.join(self.testdir, 'tar_works.tar' + extension))
            req.headers['transfer-encoding'] = 'chunked'

            def fake_start_response(*args, **kwargs):
                pass

            app_iter = self.bulk(req.environ, fake_start_response)
            resp_body = ''.join([i for i in app_iter])

            self.assert_('Response Status: 406' in resp_body)
コード例 #6
0
ファイル: test_base.py プロジェクト: mahak/swift
    def test_get_account_info_cache(self):
        # Works with fake apps that return ints in the headers
        cached = {'status': 404,
                  'bytes': 3333,
                  'total_object_count': 10}
        req = Request.blank("/v1/account/cont",
                            environ={'swift.cache': FakeCache(cached)})
        resp = get_account_info(req.environ, FakeApp())
        self.assertEqual(resp['bytes'], 3333)
        self.assertEqual(resp['total_object_count'], 10)
        self.assertEqual(resp['status'], 404)

        # Works with strings too, like you get when parsing HTTP headers
        # that came in through a socket from the account server
        cached = {'status': 404,
                  'bytes': '3333',
                  'container_count': '234',
                  'total_object_count': '10',
                  'meta': {}}
        req = Request.blank("/v1/account/cont",
                            environ={'swift.cache': FakeCache(cached)})
        resp = get_account_info(req.environ, FakeApp())
        self.assertEqual(resp['status'], 404)
        self.assertEqual(resp['bytes'], 3333)
        self.assertEqual(resp['container_count'], 234)
        self.assertEqual(resp['meta'], {})
        self.assertEqual(resp['total_object_count'], 10)
コード例 #7
0
ファイル: test_proxy_logging.py プロジェクト: hbhdytf/mac
    def test_policy_index(self):
        # Policy index can be specified by X-Backend-Storage-Policy-Index
        # in the request header for object API
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx='1'), {})
        app.access_logger = FakeLogger()
        req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
        resp = app(req.environ, start_response)
        ''.join(resp)
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[20], '1')

        # Policy index can be specified by X-Backend-Storage-Policy-Index
        # in the response header for container API
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})

        def fake_call(app, env, start_response):
            start_response(app.response_str,
                           [('Content-Type', 'text/plain'),
                            ('Content-Length', str(sum(map(len, app.body)))),
                            ('X-Backend-Storage-Policy-Index', '1')])
            while env['wsgi.input'].read(5):
                pass
            return app.body

        with mock.patch.object(FakeApp, '__call__', fake_call):
            resp = app(req.environ, start_response)
            ''.join(resp)
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[20], '1')
コード例 #8
0
ファイル: copy.py プロジェクト: ISCAS-VDI/swift-base
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        self.account_name = account
        self.container_name = container
        self.object_name = obj

        # Save off original request method (COPY/POST) in case it gets mutated
        # into PUT during handling. This way logging can display the method
        # the client actually sent.
        req.environ['swift.orig_req_method'] = req.method

        if req.method == 'PUT' and req.headers.get('X-Copy-From'):
            return self.handle_PUT(req, start_response)
        elif req.method == 'COPY':
            return self.handle_COPY(req, start_response)
        elif req.method == 'POST' and self.object_post_as_copy:
            return self.handle_object_post_as_copy(req, start_response)
        elif req.method == 'OPTIONS':
            # Does not interfere with OPTIONS response from (account,container)
            # servers and /info response.
            return self.handle_OPTIONS(req, start_response)

        return self.app(env, start_response)
コード例 #9
0
ファイル: encrypter.py プロジェクト: jgmerritt/swift
    def __call__(self, env, start_response):
        # If override is set in env, then just pass along
        if config_true_value(env.get('swift.crypto.override')):
            return self.app(env, start_response)

        req = Request(env)

        if self.disable_encryption and req.method in ('PUT', 'POST'):
            return self.app(env, start_response)
        try:
            req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('GET', 'HEAD'):
            handler = EncrypterObjContext(self, self.logger).handle_get_or_head
        elif req.method == 'PUT':
            handler = EncrypterObjContext(self, self.logger).handle_put
        elif req.method == 'POST':
            handler = EncrypterObjContext(self, self.logger).handle_post
        else:
            # anything else
            return self.app(env, start_response)

        try:
            return handler(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
コード例 #10
0
ファイル: test_s3request.py プロジェクト: mahak/swift
    def test_check_signature_multi_bytes_secret_failure(self):
        # Test v2 check_signature with multi bytes invalid secret
        req = Request.blank('/photos/puppy.jpg', headers={
            'Host': 'johnsmith.s3.amazonaws.com',
            'Date': 'Tue, 27 Mar 2007 19:36:42 +0000',
            'Authorization': ('AWS AKIAIOSFODNN7EXAMPLE:'
                              'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
        })
        sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
        # This is a failure case with utf-8 non-ascii multi-bytes charactor
        # but we expect to return just False instead of exceptions
        self.assertFalse(sigv2_req.check_signature(
            u'\u30c9\u30e9\u30b4\u30f3'))

        # Test v4 check_signature with multi bytes invalid secret
        amz_date_header = self.get_v4_amz_date_header()
        req = Request.blank('/photos/puppy.jpg', headers={
            'Authorization':
                'AWS4-HMAC-SHA256 '
                'Credential=test/%s/us-east-1/s3/aws4_request, '
                'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
                'Signature=X' % amz_date_header.split('T', 1)[0],
            'X-Amz-Content-SHA256': '0123456789',
            'X-Amz-Date': amz_date_header
        })
        sigv4_req = SigV4Request(
            req.environ, storage_domain='s3.amazonaws.com')
        self.assertFalse(sigv4_req.check_signature(
            u'\u30c9\u30e9\u30b4\u30f3'))
コード例 #11
0
ファイル: test_proxy_logging.py プロジェクト: JioCloud/swift
    def test_proxy_client_logging(self):
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank(
            "/",
            environ={"REQUEST_METHOD": "GET", "REMOTE_ADDR": "1.2.3.4", "HTTP_X_FORWARDED_FOR": "4.5.6.7,8.9.10.11"},
        )
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[0], "4.5.6.7")  # client ip
        self.assertEquals(log_parts[1], "1.2.3.4")  # remote addr

        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank(
            "/", environ={"REQUEST_METHOD": "GET", "REMOTE_ADDR": "1.2.3.4", "HTTP_X_CLUSTER_CLIENT_IP": "4.5.6.7"}
        )
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEquals(log_parts[0], "4.5.6.7")  # client ip
        self.assertEquals(log_parts[1], "1.2.3.4")  # remote addr
コード例 #12
0
ファイル: test_encrypter.py プロジェクト: jgmerritt/swift
    def test_PUT_encryption_override(self):
        # set crypto override to disable encryption.
        # simulate another middleware wanting to set footers
        other_footers = {
            'Etag': 'other etag',
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
                'other override'}
        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'PUT',
               'swift.crypto.override': True,
               'swift.callback.update_footers':
                   lambda footers: footers.update(other_footers)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('201 Created', resp.status)

        # verify that other middleware's footers made it to app
        req_hdrs = self.app.headers[0]
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify object is NOT encrypted by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        self.assertEqual(body, get_req.get_response(self.app).body)
コード例 #13
0
ファイル: test_s3request.py プロジェクト: mahak/swift
    def test_to_swift_req_subrequest_proxy_access_log(self):
        container = 'bucket'
        obj = 'obj'
        method = 'GET'

        # force_swift_request_proxy_log is True
        req = Request.blank('/%s/%s' % (container, obj),
                            environ={'REQUEST_METHOD': method,
                                     'swift.proxy_access_log_made': True},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        with patch.object(Request, 'get_response') as m_swift_resp, \
                patch.object(Request, 'remote_user', 'authorized'):
            m_swift_resp.return_value = FakeSwiftResponse()
            s3_req = S3AclRequest(
                req.environ, MagicMock(), force_request_log=True)
            sw_req = s3_req.to_swift_req(method, container, obj)
            self.assertFalse(sw_req.environ['swift.proxy_access_log_made'])

        # force_swift_request_proxy_log is False
        req = Request.blank('/%s/%s' % (container, obj),
                            environ={'REQUEST_METHOD': method,
                                     'swift.proxy_access_log_made': True},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        with patch.object(Request, 'get_response') as m_swift_resp, \
                patch.object(Request, 'remote_user', 'authorized'):
            m_swift_resp.return_value = FakeSwiftResponse()
            s3_req = S3AclRequest(
                req.environ, MagicMock(), force_request_log=False)
            sw_req = s3_req.to_swift_req(method, container, obj)
            self.assertTrue(sw_req.environ['swift.proxy_access_log_made'])
コード例 #14
0
ファイル: slo.py プロジェクト: iloveyou416068/swift-1
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env['swift.copy_hook'] = self.copy_hook(
            env.get('swift.copy_hook',
                    lambda src_req, src_resp, sink_req: src_resp))

        try:
            if req.method == 'PUT' and \
                    req.params.get('multipart-manifest') == 'put':
                return self.handle_multipart_put(req, start_response)
            if req.method == 'DELETE' and \
                    req.params.get('multipart-manifest') == 'delete':
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == 'GET' or req.method == 'HEAD':
                return self.handle_multipart_get_or_head(req, start_response)
            if 'X-Static-Large-Object' in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body='X-Static-Large-Object is a reserved header. '
                    'To create a static large object add query param '
                    'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
コード例 #15
0
    def test_check_object_creation_copy(self):
        headers = {'Content-Length': '0',
                   'X-Copy-From': 'c/o2',
                   'Content-Type': 'text/plain'}
        self.assertEquals(constraints.check_object_creation(Request.blank(
            '/', headers=headers), 'object_name'), None)

        headers = {'Content-Length': '1',
                   'X-Copy-From': 'c/o2',
                   'Content-Type': 'text/plain'}
        self.assertEquals(constraints.check_object_creation(Request.blank(
            '/', headers=headers), 'object_name').status_int,
            HTTP_BAD_REQUEST)

        headers = {'Transfer-Encoding': 'chunked',
                   'X-Copy-From': 'c/o2',
                   'Content-Type': 'text/plain'}
        self.assertEquals(constraints.check_object_creation(Request.blank(
            '/', headers=headers), 'object_name'), None)

        # a content-length header is always required
        headers = {'X-Copy-From': 'c/o2',
                   'Content-Type': 'text/plain'}
        self.assertEquals(constraints.check_object_creation(Request.blank(
            '/', headers=headers), 'object_name').status_int,
            HTTP_LENGTH_REQUIRED)
コード例 #16
0
ファイル: test_base.py プロジェクト: 10389030/swift
    def test_GETorHEAD_base(self):
        base = Controller(self.app)
        req = Request.blank('/v1/a/c/o/with/slashes')
        with patch('swift.proxy.controllers.base.'
                   'http_connect', fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
                                       '/a/c/o/with/slashes')
        self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ)
        self.assertEqual(
            resp.environ['swift.object/a/c/o/with/slashes']['status'], 200)
        req = Request.blank('/v1/a/c/o')
        with patch('swift.proxy.controllers.base.'
                   'http_connect', fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
                                       '/a/c/o')
        self.assertTrue('swift.object/a/c/o' in resp.environ)
        self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200)
        req = Request.blank('/v1/a/c')
        with patch('swift.proxy.controllers.base.'
                   'http_connect', fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part',
                                       '/a/c')
        self.assertTrue('swift.container/a/c' in resp.environ)
        self.assertEqual(resp.environ['swift.container/a/c']['status'], 200)

        req = Request.blank('/v1/a')
        with patch('swift.proxy.controllers.base.'
                   'http_connect', fake_http_connect(200)):
            resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part',
                                       '/a')
        self.assertTrue('swift.account/a' in resp.environ)
        self.assertEqual(resp.environ['swift.account/a']['status'], 200)
コード例 #17
0
ファイル: test_slo.py プロジェクト: mygoda/openstack
    def test_handle_multipart_put_bad_data(self):
        bad_data = json.dumps([{"path": "/cont/object", "etag": "etagoftheobj", "size_bytes": "lala"}])
        req = Request.blank(
            "/test_good/AUTH_test/c/man?multipart-manifest=put", environ={"REQUEST_METHOD": "PUT"}, body=bad_data
        )
        self.assertRaises(HTTPException, self.slo.handle_multipart_put, req)

        for bad_data in [
            json.dumps([{"path": "/cont", "etag": "etagoftheobj", "size_bytes": 100}]),
            json.dumps("asdf"),
            json.dumps(None),
            json.dumps(5),
            "not json",
            "1234",
            None,
            "",
            json.dumps({"path": None}),
            json.dumps([{"path": "/c/o", "etag": None, "size_bytes": 12}]),
            json.dumps([{"path": "/c/o", "etag": "asdf", "size_bytes": "sd"}]),
            json.dumps([{"path": 12, "etag": "etagoftheobj", "size_bytes": 100}]),
            json.dumps([{"path": u"/cont/object\u2661", "etag": "etagoftheobj", "size_bytes": 100}]),
            json.dumps([{"path": 12, "size_bytes": 100}]),
            json.dumps([{"path": 12, "size_bytes": 100}]),
            json.dumps([{"path": None, "etag": "etagoftheobj", "size_bytes": 100}]),
        ]:
            req = Request.blank(
                "/test_good/AUTH_test/c/man?multipart-manifest=put", environ={"REQUEST_METHOD": "PUT"}, body=bad_data
            )
            self.assertRaises(HTTPException, self.slo.handle_multipart_put, req)
コード例 #18
0
ファイル: encrypter.py プロジェクト: fvennetier/oio-swift
    def __call__(self, env, start_response):
        if config_true_value(env.get('swift.crypto.override')):
            return self.app(env, start_response)

        req = Request(env)

        if self.disable_encryption and req.method in ('PUT', 'POST'):
            return self.app(env, start_response)
        try:
            req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        fetch_crypto_keys = env.get(CRYPTO_KEY_CALLBACK)
        if fetch_crypto_keys is not None:
            try:
                fetch_crypto_keys()
            except HTTPException as exc:
                if MISSING_KEY_MSG in exc.body:
                    if req.method in ('PUT', 'POST'):
                        # No key, just upload without encryption
                        env['swift.crypto.override'] = True
                    # else:
                    #   let the thing fail later,
                    #   if a key is required for decoding
                else:
                    raise
            except Exception:
                # Let the parent class handle other exceptions
                pass
        res = super(Encrypter, self).__call__(env, start_response)
        return res
コード例 #19
0
ファイル: test_base.py プロジェクト: 10389030/swift
    def test_get_account_info_cache(self):
        # The original test that we prefer to preserve
        cached = {'status': 404,
                  'bytes': 3333,
                  'total_object_count': 10}
        req = Request.blank("/v1/account/cont",
                            environ={'swift.cache': FakeCache(cached)})
        with patch('swift.proxy.controllers.base.'
                   '_prepare_pre_auth_info_request', FakeRequest):
            resp = get_account_info(req.environ, 'xxx')
        self.assertEquals(resp['bytes'], 3333)
        self.assertEquals(resp['total_object_count'], 10)
        self.assertEquals(resp['status'], 404)

        # Here is a more realistic test
        cached = {'status': 404,
                  'bytes': '3333',
                  'container_count': '234',
                  'total_object_count': '10',
                  'meta': {}}
        req = Request.blank("/v1/account/cont",
                            environ={'swift.cache': FakeCache(cached)})
        with patch('swift.proxy.controllers.base.'
                   '_prepare_pre_auth_info_request', FakeRequest):
            resp = get_account_info(req.environ, 'xxx')
        self.assertEquals(resp['status'], 404)
        self.assertEquals(resp['bytes'], '3333')
        self.assertEquals(resp['container_count'], 234)
        self.assertEquals(resp['meta'], {})
        self.assertEquals(resp['total_object_count'], '10')
コード例 #20
0
 def test_parse_account_that_looks_like_version(self):
     """
     Demonstrate the failure mode for accounts that looks like versions,
     if you can make _parse_path better and this is the *only* test that
     fails you can delete it ;)
     """
     bad_accounts = (
         'v3.0', 'verybaddaccountwithnoprefix',
     )
     for bad_account in bad_accounts:
         req = Request.blank('/endpoints/%s/c/o' % bad_account)
         self.assertRaises(ValueError,
                           self.list_endpoints._parse_path, req)
     even_worse_accounts = {
         'v1': 1.0,
         'v2.0': 2.0,
     }
     for bad_account, guessed_version in even_worse_accounts.items():
         req = Request.blank('/endpoints/%s/c/o' % bad_account)
         version, account, container, obj = \
             self.list_endpoints._parse_path(req)
         self.assertEqual(version, guessed_version)
         self.assertEqual(account, 'c')
         self.assertEqual(container, 'o')
         self.assertEqual(obj, None)
コード例 #21
0
ファイル: test_base.py プロジェクト: bouncestorage/swift
    def test_get_account_info_cache(self):
        # The original test that we prefer to preserve
        cached = {'status': 404,
                  'bytes': 3333,
                  'total_object_count': 10}
        req = Request.blank("/v1/account/cont",
                            environ={'swift.cache': FakeCache(cached)})
        resp = get_account_info(req.environ, FakeApp())
        self.assertEqual(resp['bytes'], 3333)
        self.assertEqual(resp['total_object_count'], 10)
        self.assertEqual(resp['status'], 404)

        # Here is a more realistic test
        cached = {'status': 404,
                  'bytes': '3333',
                  'container_count': '234',
                  'total_object_count': '10',
                  'meta': {}}
        req = Request.blank("/v1/account/cont",
                            environ={'swift.cache': FakeCache(cached)})
        resp = get_account_info(req.environ, FakeApp())
        self.assertEqual(resp['status'], 404)
        self.assertEqual(resp['bytes'], '3333')
        self.assertEqual(resp['container_count'], 234)
        self.assertEqual(resp['meta'], {})
        self.assertEqual(resp['total_object_count'], '10')
コード例 #22
0
    def test_policy_index(self):
        # Policy index can be specified by X-Backend-Storage-Policy-Index
        # in the request header for object API
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx="1"), {})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "PUT"})
        resp = app(req.environ, start_response)
        "".join(resp)
        log_parts = self._log_parts(app)
        self.assertEqual(log_parts[20], "1")

        # Policy index can be specified by X-Backend-Storage-Policy-Index
        # in the response header for container API
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c", environ={"REQUEST_METHOD": "GET"})

        def fake_call(app, env, start_response):
            start_response(
                app.response_str,
                [
                    ("Content-Type", "text/plain"),
                    ("Content-Length", str(sum(map(len, app.body)))),
                    ("X-Backend-Storage-Policy-Index", "1"),
                ],
            )
            while env["wsgi.input"].read(5):
                pass
            return app.body

        with mock.patch.object(FakeApp, "__call__", fake_call):
            resp = app(req.environ, start_response)
            "".join(resp)
        log_parts = self._log_parts(app)
        self.assertEqual(log_parts[20], "1")
コード例 #23
0
ファイル: bulk.py プロジェクト: hbhdytf/mac
 def create_container(self, req, container_path):
     """
     Checks if the container exists and if not try to create it.
     :params container_path: an unquoted path to a container to be created
     :returns: True if created container, False if container exists
     :raises: CreateContainerError when unable to create container
     """
     new_env = req.environ.copy()
     new_env['PATH_INFO'] = container_path
     new_env['swift.source'] = 'EA'
     new_env['REQUEST_METHOD'] = 'HEAD'
     head_cont_req = Request.blank(container_path, environ=new_env)
     resp = head_cont_req.get_response(self.app)
     if resp.is_success:
         return False
     if resp.status_int == 404:
         new_env = req.environ.copy()
         new_env['PATH_INFO'] = container_path
         new_env['swift.source'] = 'EA'
         new_env['REQUEST_METHOD'] = 'PUT'
         create_cont_req = Request.blank(container_path, environ=new_env)
         resp = create_cont_req.get_response(self.app)
         if resp.is_success:
             return True
     raise CreateContainerError(
         "Create Container Failed: " + container_path,
         resp.status_int, resp.status)
コード例 #24
0
ファイル: test_encrypter.py プロジェクト: notmyname/swift
    def test_PUT_encryption_override(self):
        # set crypto override to disable encryption.
        # simulate another middleware wanting to set footers
        other_footers = {
            "Etag": "other etag",
            "X-Object-Sysmeta-Other": "other sysmeta",
            "X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
        }
        body = "FAKE APP"
        env = {
            "REQUEST_METHOD": "PUT",
            "swift.crypto.override": True,
            "swift.callback.update_footers": lambda footers: footers.update(other_footers),
        }
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("201 Created", resp.status)

        # verify that other middleware's footers made it to app
        req_hdrs = self.app.headers[0]
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify object is NOT encrypted by getting direct from the app
        get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
        self.assertEqual(body, get_req.get_response(self.app).body)
コード例 #25
0
ファイル: test_request.py プロジェクト: notmyname/swift3
    def test_headers_to_sign_sigv4(self):
        environ = {
            'REQUEST_METHOD': 'GET'}

        # host and x-amz-date
        x_amz_date = self.get_v4_amz_date_header()
        headers = {
            'Authorization':
                'AWS4-HMAC-SHA256 '
                'Credential=test/20130524/US/s3/aws4_request, '
                'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
                'Signature=X',
            'X-Amz-Content-SHA256': '0123456789',
            'Date': self.get_date_header(),
            'X-Amz-Date': x_amz_date}

        req = Request.blank('/', environ=environ, headers=headers)
        sigv4_req = SigV4Request(req.environ)

        headers_to_sign = sigv4_req._headers_to_sign()
        self.assertEqual(['host', 'x-amz-content-sha256', 'x-amz-date'],
                         sorted(headers_to_sign.keys()))
        self.assertEqual(headers_to_sign['host'], 'localhost:80')
        self.assertEqual(headers_to_sign['x-amz-date'], x_amz_date)
        self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789')

        # no x-amz-date
        headers = {
            'Authorization':
                'AWS4-HMAC-SHA256 '
                'Credential=test/20130524/US/s3/aws4_request, '
                'SignedHeaders=host;x-amz-content-sha256,'
                'Signature=X',
            'X-Amz-Content-SHA256': '0123456789',
            'Date': self.get_date_header()}

        req = Request.blank('/', environ=environ, headers=headers)
        sigv4_req = SigV4Request(req.environ)

        headers_to_sign = sigv4_req._headers_to_sign()
        self.assertEqual(['host', 'x-amz-content-sha256'],
                         sorted(headers_to_sign.keys()))
        self.assertEqual(headers_to_sign['host'], 'localhost:80')
        self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789')

        # SignedHeaders says, host and x-amz-date included but there is not
        # X-Amz-Date header
        headers = {
            'Authorization':
                'AWS4-HMAC-SHA256 '
                'Credential=test/20130524/US/s3/aws4_request, '
                'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
                'Signature=X',
            'X-Amz-Content-SHA256': '0123456789',
            'Date': self.get_date_header()}

        req = Request.blank('/', environ=environ, headers=headers)
        with self.assertRaises(SignatureDoesNotMatch):
            sigv4_req = SigV4Request(req.environ)
            sigv4_req._headers_to_sign()
コード例 #26
0
ファイル: test_bucket.py プロジェクト: jgmerritt/swift
    def test_bucket_PUT(self):
        req = Request.blank('/bucket',
                            environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_s3api(req)
        self.assertEqual(body, '')
        self.assertEqual(status.split()[0], '200')
        self.assertEqual(headers['Location'], '/bucket')

        # Apparently some clients will include a chunked transfer-encoding
        # even with no body
        req = Request.blank('/bucket',
                            environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header(),
                                     'Transfer-Encoding': 'chunked'})
        status, headers, body = self.call_s3api(req)
        self.assertEqual(body, '')
        self.assertEqual(status.split()[0], '200')
        self.assertEqual(headers['Location'], '/bucket')

        with UnreadableInput(self) as fake_input:
            req = Request.blank(
                '/bucket',
                environ={'REQUEST_METHOD': 'PUT',
                         'wsgi.input': fake_input},
                headers={'Authorization': 'AWS test:tester:hmac',
                         'Date': self.get_date_header()})
            status, headers, body = self.call_s3api(req)
        self.assertEqual(body, '')
        self.assertEqual(status.split()[0], '200')
        self.assertEqual(headers['Location'], '/bucket')
コード例 #27
0
ファイル: test_sysmeta.py プロジェクト: igusher/swift
    def test_sysmeta_replaced_by_PUT(self):
        path = '/v1/a/c/o'

        env = {'REQUEST_METHOD': 'PUT'}
        hdrs = dict(self.original_sysmeta_headers_1)
        hdrs.update(self.original_sysmeta_headers_2)
        hdrs.update(self.original_meta_headers_1)
        hdrs.update(self.original_meta_headers_2)
        req = Request.blank(path, environ=env, headers=hdrs, body='x')
        resp = req.get_response(self.app)
        self._assertStatus(resp, 201)

        env = {'REQUEST_METHOD': 'PUT'}
        hdrs = dict(self.changed_sysmeta_headers)
        hdrs.update(self.new_sysmeta_headers)
        hdrs.update(self.changed_meta_headers)
        hdrs.update(self.new_meta_headers)
        hdrs.update(self.bad_headers)
        req = Request.blank(path, environ=env, headers=hdrs, body='x')
        resp = req.get_response(self.app)
        self._assertStatus(resp, 201)

        req = Request.blank(path, environ={})
        resp = req.get_response(self.app)
        self._assertStatus(resp, 200)
        self._assertInHeaders(resp, self.changed_sysmeta_headers)
        self._assertInHeaders(resp, self.new_sysmeta_headers)
        self._assertNotInHeaders(resp, self.original_sysmeta_headers_2)
        self._assertInHeaders(resp, self.changed_meta_headers)
        self._assertInHeaders(resp, self.new_meta_headers)
        self._assertNotInHeaders(resp, self.original_meta_headers_2)
コード例 #28
0
    def test_upload_size(self):
        # Using default policy
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {"log_headers": "yes"})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c/o/foo", environ={"REQUEST_METHOD": "PUT", "wsgi.input": BytesIO(b"some stuff")})
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEqual(log_parts[11], str(len("FAKE APP")))
        self.assertEqual(log_parts[10], str(len("some stuff")))
        self.assertUpdateStats(
            [
                ("object.PUT.200.xfer", len("some stuff") + len("FAKE APP")),
                ("object.policy.0.PUT.200.xfer", len("some stuff") + len("FAKE APP")),
            ],
            app,
        )

        # Using a non-existent policy
        app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx="-1"), {"log_headers": "yes"})
        app.access_logger = FakeLogger()
        req = Request.blank("/v1/a/c/o/foo", environ={"REQUEST_METHOD": "PUT", "wsgi.input": BytesIO(b"some stuff")})
        resp = app(req.environ, start_response)
        # exhaust generator
        [x for x in resp]
        log_parts = self._log_parts(app)
        self.assertEqual(log_parts[11], str(len("FAKE APP")))
        self.assertEqual(log_parts[10], str(len("some stuff")))
        self.assertUpdateStats([("object.PUT.200.xfer", len("some stuff") + len("FAKE APP"))], app)
コード例 #29
0
ファイル: test_bucket.py プロジェクト: jgmerritt/swift
    def test_bucket_GET_v2_fetch_owner(self):
        bucket_name = 'junk'
        req = Request.blank('/%s?list-type=2' % bucket_name,
                            environ={'REQUEST_METHOD': 'GET'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_s3api(req)
        self.assertEqual(status.split()[0], '200')

        elem = fromstring(body, 'ListBucketResult')
        name = elem.find('./Name').text
        self.assertEqual(name, bucket_name)

        objects = elem.iterchildren('Contents')
        for o in objects:
            self.assertIsNone(o.find('./Owner'))

        req = Request.blank('/%s?list-type=2&fetch-owner=true' % bucket_name,
                            environ={'REQUEST_METHOD': 'GET'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_s3api(req)
        self.assertEqual(status.split()[0], '200')

        elem = fromstring(body, 'ListBucketResult')
        name = elem.find('./Name').text
        self.assertEqual(name, bucket_name)

        objects = elem.iterchildren('Contents')
        for o in objects:
            self.assertIsNotNone(o.find('./Owner'))
コード例 #30
0
ファイル: test_swift3.py プロジェクト: blaxter/swift3
    def test_bucket_GET_max_keys(self):
        class FakeApp(object):
            def __call__(self, env, start_response):
                self.query_string = env['QUERY_STRING']
                start_response('200 OK', [])
                return '[]'
        fake_app = FakeApp()
        local_app = swift3.filter_factory({})(fake_app)
        bucket_name = 'junk'

        req = Request.blank('/%s' % bucket_name,
                environ={'REQUEST_METHOD': 'GET',
                         'QUERY_STRING': 'max-keys=5'},
                headers={'Authorization': 'AWS test:tester:hmac'})
        resp = local_app(req.environ, lambda *args: None)
        dom = xml.dom.minidom.parseString("".join(resp))
        self.assertEquals(dom.getElementsByTagName('MaxKeys')[0].
                childNodes[0].nodeValue, '5')
        args = dict(cgi.parse_qsl(fake_app.query_string))
        self.assert_(args['limit'] == '6')

        req = Request.blank('/%s' % bucket_name,
                environ={'REQUEST_METHOD': 'GET',
                         'QUERY_STRING': 'max-keys=5000'},
                headers={'Authorization': 'AWS test:tester:hmac'})
        resp = local_app(req.environ, lambda *args: None)
        dom = xml.dom.minidom.parseString("".join(resp))
        self.assertEquals(dom.getElementsByTagName('MaxKeys')[0].
                childNodes[0].nodeValue, '1000')
        args = dict(cgi.parse_qsl(fake_app.query_string))
        self.assertEquals(args['limit'], '1001')
コード例 #31
0
ファイル: obj.py プロジェクト: kvite/swift
    def PUT(self, req):
        """HTTP PUT request handler."""
        if req.if_none_match is not None and '*' not in req.if_none_match:
            # Sending an etag with if-none-match isn't currently supported
            return HTTPBadRequest(request=req, content_type='text/plain',
                                  body='If-None-Match only supports *')
        container_info = self.container_info(
            self.account_name, self.container_name, req)
        policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                       container_info['storage_policy'])
        obj_ring = self.app.get_object_ring(policy_index)

        # pass the policy index to storage nodes via req header
        req.headers['X-Backend-Storage-Policy-Index'] = policy_index
        container_partition = container_info['partition']
        containers = container_info['nodes']
        req.acl = container_info['write_acl']
        req.environ['swift_sync_key'] = container_info['sync_key']
        object_versions = container_info['versions']
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp

        if not containers:
            return HTTPNotFound(request=req)

        # Sometimes the 'content-type' header exists, but is set to None.
        content_type_manually_set = True
        detect_content_type = \
            config_true_value(req.headers.get('x-detect-content-type'))
        if detect_content_type or not req.headers.get('content-type'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            req.headers['Content-Type'] = guessed_type or \
                'application/octet-stream'
            if detect_content_type:
                req.headers.pop('x-detect-content-type')
            else:
                content_type_manually_set = False

        error_response = check_object_creation(req, self.object_name) or \
            check_content_type(req)
        if error_response:
            return error_response

        partition, nodes = obj_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)

        # do a HEAD request for container sync and checking object versions
        if 'x-timestamp' in req.headers or \
                (object_versions and not
                 req.environ.get('swift_versioned_copy')):
            # make sure proxy-server uses the right policy index
            _headers = {'X-Backend-Storage-Policy-Index': policy_index,
                        'X-Newest': 'True'}
            hreq = Request.blank(req.path_info, headers=_headers,
                                 environ={'REQUEST_METHOD': 'HEAD'})
            hresp = self.GETorHEAD_base(
                hreq, _('Object'), obj_ring, partition,
                hreq.swift_entity_path)

        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req_timestamp = Timestamp(req.headers['X-Timestamp'])
                if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
                        hresp.environ['swift_x_timestamp'] >= req_timestamp:
                    return HTTPAccepted(request=req)
            except ValueError:
                return HTTPBadRequest(
                    request=req, content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                         'was %r' % req.headers['x-timestamp'])
            req.headers['X-Timestamp'] = req_timestamp.internal
        else:
            req.headers['X-Timestamp'] = Timestamp(time.time()).internal

        if object_versions and not req.environ.get('swift_versioned_copy'):
            if hresp.status_int != HTTP_NOT_FOUND:
                # This is a version manifest and needs to be handled
                # differently. First copy the existing data to a new object,
                # then write the data from this request to the version manifest
                # object.
                lcontainer = object_versions.split('/')[0]
                prefix_len = '%03x' % len(self.object_name)
                lprefix = prefix_len + self.object_name + '/'
                ts_source = hresp.environ.get('swift_x_timestamp')
                if ts_source is None:
                    ts_source = time.mktime(time.strptime(
                                            hresp.headers['last-modified'],
                                            '%a, %d %b %Y %H:%M:%S GMT'))
                new_ts = Timestamp(ts_source).internal
                vers_obj_name = lprefix + new_ts
                copy_headers = {
                    'Destination': '%s/%s' % (lcontainer, vers_obj_name)}
                copy_environ = {'REQUEST_METHOD': 'COPY',
                                'swift_versioned_copy': True
                                }
                copy_req = Request.blank(req.path_info, headers=copy_headers,
                                         environ=copy_environ)
                copy_resp = self.COPY(copy_req)
                if is_client_error(copy_resp.status_int):
                    # missing container or bad permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)

        reader = req.environ['wsgi.input'].read
        data_source = iter(lambda: reader(self.app.client_chunk_size), '')
        source_header = req.headers.get('X-Copy-From')
        source_resp = None
        if source_header:
            if req.environ.get('swift.orig_req_method', req.method) != 'POST':
                req.environ.setdefault('swift.log_info', []).append(
                    'x-copy-from:%s' % source_header)
            ver, acct, _rest = req.split_path(2, 3, True)
            src_account_name = req.headers.get('X-Copy-From-Account', None)
            if src_account_name:
                src_account_name = check_account_format(req, src_account_name)
            else:
                src_account_name = acct
            src_container_name, src_obj_name = check_copy_from_header(req)
            source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
                            src_container_name, src_obj_name)
            source_req = req.copy_get()

            # make sure the source request uses it's container_info
            source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
            source_req.path_info = source_header
            source_req.headers['X-Newest'] = 'true'
            orig_obj_name = self.object_name
            orig_container_name = self.container_name
            orig_account_name = self.account_name
            self.object_name = src_obj_name
            self.container_name = src_container_name
            self.account_name = src_account_name
            sink_req = Request.blank(req.path_info,
                                     environ=req.environ, headers=req.headers)
            source_resp = self.GET(source_req)

            # This gives middlewares a way to change the source; for example,
            # this lets you COPY a SLO manifest and have the new object be the
            # concatenation of the segments (like what a GET request gives
            # the client), not a copy of the manifest file.
            hook = req.environ.get(
                'swift.copy_hook',
                (lambda source_req, source_resp, sink_req: source_resp))
            source_resp = hook(source_req, source_resp, sink_req)

            if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
                return source_resp
            self.object_name = orig_obj_name
            self.container_name = orig_container_name
            self.account_name = orig_account_name
            data_source = iter(source_resp.app_iter)
            sink_req.content_length = source_resp.content_length
            if sink_req.content_length is None:
                # This indicates a transfer-encoding: chunked source object,
                # which currently only happens because there are more than
                # CONTAINER_LISTING_LIMIT segments in a segmented object. In
                # this case, we're going to refuse to do the server-side copy.
                return HTTPRequestEntityTooLarge(request=req)
            if sink_req.content_length > constraints.MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=req)
            sink_req.etag = source_resp.etag

            # we no longer need the X-Copy-From header
            del sink_req.headers['X-Copy-From']
            if 'X-Copy-From-Account' in sink_req.headers:
                del sink_req.headers['X-Copy-From-Account']
            if not content_type_manually_set:
                sink_req.headers['Content-Type'] = \
                    source_resp.headers['Content-Type']
            if config_true_value(
                    sink_req.headers.get('x-fresh-metadata', 'false')):
                # post-as-copy: ignore new sysmeta, copy existing sysmeta
                condition = lambda k: is_sys_meta('object', k)
                remove_items(sink_req.headers, condition)
                copy_header_subset(source_resp, sink_req, condition)
            else:
                # copy/update existing sysmeta and user meta
                copy_headers_into(source_resp, sink_req)
                copy_headers_into(req, sink_req)

            # copy over x-static-large-object for POSTs and manifest copies
            if 'X-Static-Large-Object' in source_resp.headers and \
                    req.params.get('multipart-manifest') == 'get':
                sink_req.headers['X-Static-Large-Object'] = \
                    source_resp.headers['X-Static-Large-Object']

            req = sink_req

        req, delete_at_container, delete_at_part, \
            delete_at_nodes = self._config_obj_expiration(req)

        node_iter = GreenthreadSafeIterator(
            self.iter_nodes_local_first(obj_ring, partition))
        pile = GreenPile(len(nodes))
        te = req.headers.get('transfer-encoding', '')
        chunked = ('chunked' in te)

        outgoing_headers = self._backend_requests(
            req, len(nodes), container_partition, containers,
            delete_at_container, delete_at_part, delete_at_nodes)

        for nheaders in outgoing_headers:
            # RFC2616:8.2.3 disallows 100-continue without a body
            if (req.content_length > 0) or chunked:
                nheaders['Expect'] = '100-continue'
            pile.spawn(self._connect_put_node, node_iter, partition,
                       req.swift_entity_path, nheaders,
                       self.app.logger.thread_locals)

        conns = [conn for conn in pile if conn]
        min_conns = quorum_size(len(nodes))

        if req.if_none_match is not None and '*' in req.if_none_match:
            statuses = [conn.resp.status for conn in conns if conn.resp]
            if HTTP_PRECONDITION_FAILED in statuses:
                # If we find any copy of the file, it shouldn't be uploaded
                self.app.logger.debug(
                    _('Object PUT returning 412, %(statuses)r'),
                    {'statuses': statuses})
                return HTTPPreconditionFailed(request=req)

        if len(conns) < min_conns:
            self.app.logger.error(
                _('Object PUT returning 503, %(conns)s/%(nodes)s '
                  'required connections'),
                {'conns': len(conns), 'nodes': min_conns})
            return HTTPServiceUnavailable(request=req)
        bytes_transferred = 0
        try:
            with ContextPool(len(nodes)) as pool:
                for conn in conns:
                    conn.failed = False
                    conn.queue = Queue(self.app.put_queue_depth)
                    pool.spawn(self._send_file, conn, req.path)
                while True:
                    with ChunkReadTimeout(self.app.client_timeout):
                        try:
                            chunk = next(data_source)
                        except StopIteration:
                            if chunked:
                                for conn in conns:
                                    conn.queue.put('0\r\n\r\n')
                            break
                    bytes_transferred += len(chunk)
                    if bytes_transferred > constraints.MAX_FILE_SIZE:
                        return HTTPRequestEntityTooLarge(request=req)
                    for conn in list(conns):
                        if not conn.failed:
                            conn.queue.put(
                                '%x\r\n%s\r\n' % (len(chunk), chunk)
                                if chunked else chunk)
                        else:
                            conns.remove(conn)
                    if len(conns) < min_conns:
                        self.app.logger.error(_(
                            'Object PUT exceptions during'
                            ' send, %(conns)s/%(nodes)s required connections'),
                            {'conns': len(conns), 'nodes': min_conns})
                        return HTTPServiceUnavailable(request=req)
                for conn in conns:
                    if conn.queue.unfinished_tasks:
                        conn.queue.join()
            conns = [conn for conn in conns if not conn.failed]
        except ChunkReadTimeout as err:
            self.app.logger.warn(
                _('ERROR Client read timeout (%ss)'), err.seconds)
            self.app.logger.increment('client_timeouts')
            return HTTPRequestTimeout(request=req)
        except (Exception, Timeout):
            self.app.logger.exception(
                _('ERROR Exception causing client disconnect'))
            return HTTPClientDisconnect(request=req)
        if req.content_length and bytes_transferred < req.content_length:
            req.client_disconnect = True
            self.app.logger.warn(
                _('Client disconnected without sending enough data'))
            self.app.logger.increment('client_disconnects')
            return HTTPClientDisconnect(request=req)

        statuses, reasons, bodies, etags = self._get_put_responses(req, conns,
                                                                   nodes)

        if len(etags) > 1:
            self.app.logger.error(
                _('Object servers returned %s mismatched etags'), len(etags))
            return HTTPServerError(request=req)
        etag = etags.pop() if len(etags) else None
        resp = self.best_response(req, statuses, reasons, bodies,
                                  _('Object PUT'), etag=etag)
        if source_header:
            acct, path = source_header.split('/', 3)[2:4]
            resp.headers['X-Copied-From-Account'] = quote(acct)
            resp.headers['X-Copied-From'] = quote(path)
            if 'last-modified' in source_resp.headers:
                resp.headers['X-Copied-From-Last-Modified'] = \
                    source_resp.headers['last-modified']
            copy_headers_into(req, resp)
        resp.last_modified = math.ceil(
            float(Timestamp(req.headers['X-Timestamp'])))
        return resp
コード例 #32
0
ファイル: copy.py プロジェクト: zhoujian1210/swift
    def handle_PUT(self, req, start_response):
        if req.content_length:
            return HTTPBadRequest(body='Copy requests require a zero byte '
                                  'body',
                                  request=req,
                                  content_type='text/plain')(req.environ,
                                                             start_response)

        # Form the path of source object to be fetched
        ver, acct, _rest = req.split_path(2, 3, True)
        src_account_name = req.headers.get('X-Copy-From-Account')
        if src_account_name:
            src_account_name = check_account_format(req,
                                                    unquote(src_account_name))
        else:
            src_account_name = acct
        src_container_name, src_obj_name = _check_copy_from_header(req)
        source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
                                        src_container_name, src_obj_name)

        # GET the source object, bail out on error
        ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
        source_resp = self._get_source_object(ssc_ctx, source_path, req)
        if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
            return source_resp(source_resp.environ, start_response)

        # Create a new Request object based on the original request instance.
        # This will preserve original request environ including headers.
        sink_req = Request.blank(req.path_info, environ=req.environ)

        def is_object_sysmeta(k):
            return is_sys_meta('object', k)

        if config_true_value(req.headers.get('x-fresh-metadata', 'false')):
            # x-fresh-metadata only applies to copy, not post-as-copy: ignore
            # existing user metadata, update existing sysmeta with new
            copy_header_subset(source_resp, sink_req, is_object_sysmeta)
            copy_header_subset(req, sink_req, is_object_sysmeta)
        else:
            # First copy existing sysmeta, user meta and other headers from the
            # source to the sink, apart from headers that are conditionally
            # copied below and timestamps.
            exclude_headers = ('x-static-large-object', 'x-object-manifest',
                               'etag', 'content-type', 'x-timestamp',
                               'x-backend-timestamp')
            copy_header_subset(source_resp, sink_req,
                               lambda k: k.lower() not in exclude_headers)
            # now update with original req headers
            sink_req.headers.update(req.headers)

        params = sink_req.params
        if params.get('multipart-manifest') == 'get':
            if 'X-Static-Large-Object' in source_resp.headers:
                params['multipart-manifest'] = 'put'
            if 'X-Object-Manifest' in source_resp.headers:
                del params['multipart-manifest']
                sink_req.headers['X-Object-Manifest'] = \
                    source_resp.headers['X-Object-Manifest']
            sink_req.params = params

        # Set swift.source, data source, content length and etag
        # for the PUT request
        sink_req.environ['swift.source'] = 'SSC'
        sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
        sink_req.content_length = source_resp.content_length
        if (source_resp.status_int == HTTP_OK
                and 'X-Static-Large-Object' not in source_resp.headers
                and ('X-Object-Manifest' not in source_resp.headers
                     or req.params.get('multipart-manifest') == 'get')):
            # copy source etag so that copied content is verified, unless:
            #  - not a 200 OK response: source etag may not match the actual
            #    content, for example with a 206 Partial Content response to a
            #    ranged request
            #  - SLO manifest: etag cannot be specified in manifest PUT; SLO
            #    generates its own etag value which may differ from source
            #  - SLO: etag in SLO response is not hash of actual content
            #  - DLO: etag in DLO response is not hash of actual content
            sink_req.headers['Etag'] = source_resp.etag
        else:
            # since we're not copying the source etag, make sure that any
            # container update override values are not copied.
            remove_items(
                sink_req.headers, lambda k: k.startswith(
                    'X-Object-Sysmeta-Container-Update-Override-'))

        # We no longer need these headers
        sink_req.headers.pop('X-Copy-From', None)
        sink_req.headers.pop('X-Copy-From-Account', None)

        # If the copy request does not explicitly override content-type,
        # use the one present in the source object.
        if not req.headers.get('content-type'):
            sink_req.headers['Content-Type'] = \
                source_resp.headers['Content-Type']

        # Create response headers for PUT response
        resp_headers = self._create_response_headers(source_path, source_resp,
                                                     sink_req)

        put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
        close_if_possible(source_resp.app_iter)
        return put_resp
コード例 #33
0
    def __call__(self, env, start_response):
        """WSGI Application entry point for the Swift Object Server."""
        start_time = time.time()
        req = Request(env)
        self.logger.txn_id = req.headers.get('x-trans-id', None)

        if not check_utf8(req.path_info):
            res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
        else:
            try:
                # disallow methods which have not been marked 'public'
                try:
                    if req.method not in self.allowed_methods:
                        raise AttributeError('Not allowed method.')
                except AttributeError:
                    res = HTTPMethodNotAllowed()
                else:
                    method = getattr(self, req.method)
                    res = method(req)
            except DiskFileCollision:
                res = HTTPForbidden(request=req)
            except HTTPException as error_response:
                res = error_response
            except (Exception, Timeout):
                self.logger.exception(
                    _('ERROR __call__ error with %(method)s'
                      ' %(path)s '), {
                          'method': req.method,
                          'path': req.path
                      })
                res = HTTPInternalServerError(body=traceback.format_exc())
        trans_time = time.time() - start_time
        if self.log_requests:
            log_line = get_log_line(req, res, trans_time, '')
            if req.method in ('REPLICATE', 'SSYNC') or \
                    'X-Backend-Replication' in req.headers:
                self.logger.debug(log_line)
            else:
                self.logger.info(log_line)
        if req.method in ('PUT', 'DELETE'):
            slow = self.slow - trans_time
            if slow > 0:
                sleep(slow)

        # To be able to zero-copy send the object, we need a few things.
        # First, we have to be responding successfully to a GET, or else we're
        # not sending the object. Second, we have to be able to extract the
        # socket file descriptor from the WSGI input object. Third, the
        # diskfile has to support zero-copy send.
        #
        # There's a good chance that this could work for 206 responses too,
        # but the common case is sending the whole object, so we'll start
        # there.
        if req.method == 'GET' and res.status_int == 200 and \
           isinstance(env['wsgi.input'], wsgi.Input):
            app_iter = getattr(res, 'app_iter', None)
            checker = getattr(app_iter, 'can_zero_copy_send', None)
            if checker and checker():
                # For any kind of zero-copy thing like sendfile or splice, we
                # need the file descriptor. Eventlet doesn't provide a clean
                # way of getting that, so we resort to this.
                wsock = env['wsgi.input'].get_socket()
                wsockfd = wsock.fileno()

                # Don't call zero_copy_send() until after we force the HTTP
                # headers out of Eventlet and into the socket.
                def zero_copy_iter():
                    # If possible, set TCP_CORK so that headers don't
                    # immediately go on the wire, but instead, wait for some
                    # response body to make the TCP frames as large as
                    # possible (and hence as few packets as possible).
                    #
                    # On non-Linux systems, we might consider TCP_NODELAY, but
                    # since the only known zero-copy-capable diskfile uses
                    # Linux-specific syscalls, we'll defer that work until
                    # someone needs it.
                    if hasattr(socket, 'TCP_CORK'):
                        wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK,
                                         1)
                    yield EventletPlungerString()
                    try:
                        app_iter.zero_copy_send(wsockfd)
                    except Exception:
                        self.logger.exception("zero_copy_send() blew up")
                        raise
                    yield ''

                # Get headers ready to go out
                res(env, start_response)
                return zero_copy_iter()
            else:
                return res(env, start_response)
        else:
            return res(env, start_response)
コード例 #34
0
ファイル: proxy_logging.py プロジェクト: chenbaihu/icehouse
    def __call__(self, env, start_response):
        start_response_args = [None]
        input_proxy = InputProxy(env['wsgi.input'])
        env['wsgi.input'] = input_proxy
        start_time = time.time()

        def my_start_response(status, headers, exc_info=None):
            start_response_args[0] = (status, list(headers), exc_info)

        def status_int_for_logging(client_disconnect=False, start_status=None):
            # log disconnected clients as '499' status code
            if client_disconnect or input_proxy.client_disconnect:
                ret_status_int = 499
            elif start_status is None:
                ret_status_int = int(
                    start_response_args[0][0].split(' ', 1)[0])
            else:
                ret_status_int = start_status
            return ret_status_int

        def iter_response(iterable):
            iterator = iter(iterable)
            try:
                chunk = iterator.next()
                while not chunk:
                    chunk = iterator.next()
            except StopIteration:
                chunk = ''
            for h, v in start_response_args[0][1]:
                if h.lower() in ('content-length', 'transfer-encoding'):
                    break
            else:
                if not chunk:
                    start_response_args[0][1].append(('content-length', '0'))
                elif isinstance(iterable, list):
                    start_response_args[0][1].append(
                        ('content-length', str(sum(len(i) for i in iterable))))
            start_response(*start_response_args[0])
            req = Request(env)

            # Log timing information for time-to-first-byte (GET requests only)
            method = self.method_from_req(req)
            if method == 'GET' and not self.req_already_logged(req):
                status_int = status_int_for_logging()
                metric_name = self.statsd_metric_name(req, status_int, method)
                if metric_name:
                    self.access_logger.timing_since(
                        metric_name + '.first-byte.timing', start_time)

            bytes_sent = 0
            client_disconnect = False
            try:
                while chunk:
                    bytes_sent += len(chunk)
                    yield chunk
                    chunk = iterator.next()
            except GeneratorExit:  # generator was closed before we finished
                client_disconnect = True
                raise
            finally:
                status_int = status_int_for_logging(client_disconnect)
                self.log_request(
                    req, status_int, input_proxy.bytes_received, bytes_sent,
                    start_time, time.time())

        try:
            iterable = self.app(env, my_start_response)
        except Exception:
            req = Request(env)
            status_int = status_int_for_logging(start_status=500)
            self.log_request(
                req, status_int, input_proxy.bytes_received, 0, start_time,
                time.time())
            raise
        else:
            return iter_response(iterable)
コード例 #35
0
 def test_cache_middleware(self):
     req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'})
     resp = self.app(req.environ, start_response)
     self.assertTrue('swift.cache' in resp)
     self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing))
コード例 #36
0
ファイル: recon.py プロジェクト: TheUtils/swift
 def __call__(self, env, start_response):
     req = Request(env)
     if req.path.startswith('/recon/'):
         return self.GET(req)(env, start_response)
     else:
         return self.app(env, start_response)
コード例 #37
0
ファイル: obj.py プロジェクト: kvite/swift
    def DELETE(self, req):
        """HTTP DELETE request handler."""
        container_info = self.container_info(
            self.account_name, self.container_name, req)
        # pass the policy index to storage nodes via req header
        policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                       container_info['storage_policy'])
        obj_ring = self.app.get_object_ring(policy_index)
        # pass the policy index to storage nodes via req header
        req.headers['X-Backend-Storage-Policy-Index'] = policy_index
        container_partition = container_info['partition']
        containers = container_info['nodes']
        req.acl = container_info['write_acl']
        req.environ['swift_sync_key'] = container_info['sync_key']
        object_versions = container_info['versions']
        if object_versions:
            # this is a version manifest and needs to be handled differently
            object_versions = unquote(object_versions)
            lcontainer = object_versions.split('/')[0]
            prefix_len = '%03x' % len(self.object_name)
            lprefix = prefix_len + self.object_name + '/'
            item_list = []
            try:
                for _item in self._listing_iter(lcontainer, lprefix,
                                                req.environ):
                    item_list.append(_item)
            except ListingIterNotFound:
                # no worries, last_item is None
                pass
            except ListingIterNotAuthorized as err:
                return err.aresp
            except ListingIterError:
                return HTTPServerError(request=req)

            while len(item_list) > 0:
                previous_version = item_list.pop()
                # there are older versions so copy the previous version to the
                # current object and delete the previous version
                orig_container = self.container_name
                orig_obj = self.object_name
                self.container_name = lcontainer
                self.object_name = previous_version['name'].encode('utf-8')

                copy_path = '/v1/' + self.account_name + '/' + \
                            self.container_name + '/' + self.object_name

                copy_headers = {'X-Newest': 'True',
                                'Destination': orig_container + '/' + orig_obj
                                }
                copy_environ = {'REQUEST_METHOD': 'COPY',
                                'swift_versioned_copy': True
                                }
                creq = Request.blank(copy_path, headers=copy_headers,
                                     environ=copy_environ)
                copy_resp = self.COPY(creq)
                if copy_resp.status_int == HTTP_NOT_FOUND:
                    # the version isn't there so we'll try with previous
                    self.container_name = orig_container
                    self.object_name = orig_obj
                    continue
                if is_client_error(copy_resp.status_int):
                    # some user error, maybe permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)
                # reset these because the COPY changed them
                self.container_name = lcontainer
                self.object_name = previous_version['name'].encode('utf-8')
                new_del_req = Request.blank(copy_path, environ=req.environ)
                container_info = self.container_info(
                    self.account_name, self.container_name, req)
                policy_idx = container_info['storage_policy']
                obj_ring = self.app.get_object_ring(policy_idx)
                # pass the policy index to storage nodes via req header
                new_del_req.headers['X-Backend-Storage-Policy-Index'] = \
                    policy_idx
                container_partition = container_info['partition']
                containers = container_info['nodes']
                new_del_req.acl = container_info['write_acl']
                new_del_req.path_info = copy_path
                req = new_del_req
                # remove 'X-If-Delete-At', since it is not for the older copy
                if 'X-If-Delete-At' in req.headers:
                    del req.headers['X-If-Delete-At']
                break
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not containers:
            return HTTPNotFound(request=req)
        partition, nodes = obj_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)
        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req_timestamp = Timestamp(req.headers['X-Timestamp'])
            except ValueError:
                return HTTPBadRequest(
                    request=req, content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                         'was %r' % req.headers['x-timestamp'])
            req.headers['X-Timestamp'] = req_timestamp.internal
        else:
            req.headers['X-Timestamp'] = Timestamp(time.time()).internal

        headers = self._backend_requests(
            req, len(nodes), container_partition, containers)
        # When deleting objects treat a 404 status as 204.
        status_overrides = {404: 204}
        resp = self.make_requests(req, obj_ring,
                                  partition, 'DELETE', req.swift_entity_path,
                                  headers, overrides=status_overrides)
        return resp
コード例 #38
0
    def test_object_multi_DELETE_with_error(self):
        self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
                            swob.HTTPNoContent, {}, None)
        self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
                            swob.HTTPNotFound, {}, None)
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
                            swob.HTTPForbidden, {}, None)
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key4', swob.HTTPOk,
                            {'x-static-large-object': 'True'}, None)
        slo_delete_resp = {
            'Number Not Found':
            0,
            'Response Status':
            '400 Bad Request',
            'Errors': [["/bucket+segments/obj1", "403 Forbidden"],
                       ["/bucket+segments/obj2", "403 Forbidden"]],
            'Response Body':
            '',
            'Number Deleted':
            8
        }
        self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key4', swob.HTTPOk,
                            {}, json.dumps(slo_delete_resp))

        elem = Element('Delete')
        for key in ['Key1', 'Key2', 'Key3', 'Key4']:
            obj = SubElement(elem, 'Object')
            SubElement(obj, 'Key').text = key
        body = tostring(elem, use_s3ns=False)
        content_md5 = md5(body).digest().encode('base64').strip()

        req = Request.blank('/bucket?delete',
                            environ={'REQUEST_METHOD': 'POST'},
                            headers={
                                'Authorization': 'AWS test:tester:hmac',
                                'Content-Type': 'multipart/form-data',
                                'Date': self.get_date_header(),
                                'Content-MD5': content_md5
                            },
                            body=body)
        status, headers, body = self.call_s3api(req)
        self.assertEqual(status.split()[0], '200')

        elem = fromstring(body)
        self.assertEqual(len(elem.findall('Deleted')), 2)
        self.assertEqual(len(elem.findall('Error')), 2)
        self.assertEqual(
            [(el.find('Code').text, el.find('Message').text)
             for el in elem.findall('Error')],
            [('AccessDenied', 'Access Denied.'),
             ('SLODeleteError', '\n'.join([
                 '400 Bad Request', '/bucket+segments/obj1: 403 Forbidden',
                 '/bucket+segments/obj2: 403 Forbidden'
             ]))])
        self.assertEqual(self.swift.calls, [
            ('HEAD', '/v1/AUTH_test/bucket'),
            ('HEAD', '/v1/AUTH_test/bucket/Key1'),
            ('DELETE', '/v1/AUTH_test/bucket/Key1'),
            ('HEAD', '/v1/AUTH_test/bucket/Key2'),
            ('HEAD', '/v1/AUTH_test/bucket/Key3'),
            ('HEAD', '/v1/AUTH_test/bucket/Key4'),
            ('DELETE', '/v1/AUTH_test/bucket/Key4?multipart-manifest=delete'),
        ])
コード例 #39
0
 def test_check_metadata_good(self):
     headers = {'X-Object-Meta-Name': 'Value'}
     self.assertIsNone(
         constraints.check_metadata(Request.blank('/', headers=headers),
                                    'object'))
コード例 #40
0
    def test_get_endpoint(self):
        # Expected results for objects taken from test_ring
        # Expected results for others computed by manually invoking
        # ring.get_nodes().
        resp = Request.blank('/endpoints/a/c/o1').get_response(
            self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(resp.content_type, 'application/json')
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sdb1/1/a/c/o1",
            "http://10.1.2.2:6000/sdd1/1/a/c/o1"
        ])

        # Here, 'o1/' is the object name.
        resp = Request.blank('/endpoints/a/c/o1/').get_response(
            self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sdb1/3/a/c/o1/",
            "http://10.1.2.2:6000/sdd1/3/a/c/o1/"
        ])

        resp = Request.blank('/endpoints/a/c2').get_response(
            self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sda1/2/a/c2",
            "http://10.1.2.1:6000/sdc1/2/a/c2"
        ])

        resp = Request.blank('/endpoints/a1').get_response(self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.2.1:6000/sdc1/0/a1", "http://10.1.1.1:6000/sda1/0/a1",
            "http://10.1.1.1:6000/sdb1/0/a1"
        ])

        resp = Request.blank('/endpoints/').get_response(self.list_endpoints)
        self.assertEquals(resp.status_int, 400)

        resp = Request.blank('/endpoints/a/c 2').get_response(
            self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sdb1/3/a/c%202",
            "http://10.1.2.2:6000/sdd1/3/a/c%202"
        ])

        resp = Request.blank('/endpoints/a/c%202').get_response(
            self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sdb1/3/a/c%202",
            "http://10.1.2.2:6000/sdd1/3/a/c%202"
        ])

        resp = Request.blank('/endpoints/ac%20count/con%20tainer/ob%20ject') \
            .get_response(self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sdb1/3/ac%20count/con%20tainer/ob%20ject",
            "http://10.1.2.2:6000/sdd1/3/ac%20count/con%20tainer/ob%20ject"
        ])

        resp = Request.blank('/endpoints/a/c/o1', {'REQUEST_METHOD': 'POST'}) \
            .get_response(self.list_endpoints)
        self.assertEquals(resp.status_int, 405)
        self.assertEquals(resp.status, '405 Method Not Allowed')
        self.assertEquals(resp.headers['allow'], 'GET')

        resp = Request.blank('/not-endpoints').get_response(
            self.list_endpoints)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(resp.status, '200 OK')
        self.assertEquals(resp.body, 'FakeApp')

        # test custom path with trailing slash
        custom_path_le = list_endpoints.filter_factory({
            'swift_dir':
            self.testdir,
            'list_endpoints_path':
            '/some/another/path/'
        })(self.app)
        resp = Request.blank('/some/another/path/a/c/o1') \
            .get_response(custom_path_le)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(resp.content_type, 'application/json')
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sdb1/1/a/c/o1",
            "http://10.1.2.2:6000/sdd1/1/a/c/o1"
        ])

        # test ustom path without trailing slash
        custom_path_le = list_endpoints.filter_factory({
            'swift_dir':
            self.testdir,
            'list_endpoints_path':
            '/some/another/path'
        })(self.app)
        resp = Request.blank('/some/another/path/a/c/o1') \
            .get_response(custom_path_le)
        self.assertEquals(resp.status_int, 200)
        self.assertEquals(resp.content_type, 'application/json')
        self.assertEquals(json.loads(resp.body), [
            "http://10.1.1.1:6000/sdb1/1/a/c/o1",
            "http://10.1.2.2:6000/sdd1/1/a/c/o1"
        ])
コード例 #41
0
    def test_check_delete_headers(self):

        # X-Delete-After
        headers = {'X-Delete-After': '60'}
        resp = constraints.check_delete_headers(
            Request.blank('/', headers=headers))
        self.assertTrue(isinstance(resp, Request))
        self.assertTrue('x-delete-at' in resp.headers)

        headers = {'X-Delete-After': 'abc'}
        try:
            resp = constraints.check_delete_headers(
                Request.blank('/', headers=headers))
        except HTTPException as e:
            self.assertEqual(e.status_int, HTTP_BAD_REQUEST)
            self.assertTrue('Non-integer X-Delete-After' in e.body)
        else:
            self.fail("Should have failed with HTTPBadRequest")

        headers = {'X-Delete-After': '60.1'}
        try:
            resp = constraints.check_delete_headers(
                Request.blank('/', headers=headers))
        except HTTPException as e:
            self.assertEqual(e.status_int, HTTP_BAD_REQUEST)
            self.assertTrue('Non-integer X-Delete-After' in e.body)
        else:
            self.fail("Should have failed with HTTPBadRequest")

        headers = {'X-Delete-After': '-1'}
        try:
            resp = constraints.check_delete_headers(
                Request.blank('/', headers=headers))
        except HTTPException as e:
            self.assertEqual(e.status_int, HTTP_BAD_REQUEST)
            self.assertTrue('X-Delete-After in past' in e.body)
        else:
            self.fail("Should have failed with HTTPBadRequest")

        # X-Delete-At
        t = str(int(time.time() + 100))
        headers = {'X-Delete-At': t}
        resp = constraints.check_delete_headers(
            Request.blank('/', headers=headers))
        self.assertTrue(isinstance(resp, Request))
        self.assertTrue('x-delete-at' in resp.headers)
        self.assertEqual(resp.headers.get('X-Delete-At'), t)

        headers = {'X-Delete-At': 'abc'}
        try:
            resp = constraints.check_delete_headers(
                Request.blank('/', headers=headers))
        except HTTPException as e:
            self.assertEqual(e.status_int, HTTP_BAD_REQUEST)
            self.assertTrue('Non-integer X-Delete-At' in e.body)
        else:
            self.fail("Should have failed with HTTPBadRequest")

        t = str(int(time.time() + 100)) + '.1'
        headers = {'X-Delete-At': t}
        try:
            resp = constraints.check_delete_headers(
                Request.blank('/', headers=headers))
        except HTTPException as e:
            self.assertEqual(e.status_int, HTTP_BAD_REQUEST)
            self.assertTrue('Non-integer X-Delete-At' in e.body)
        else:
            self.fail("Should have failed with HTTPBadRequest")

        t = str(int(time.time()))
        headers = {'X-Delete-At': t}
        try:
            resp = constraints.check_delete_headers(
                Request.blank('/', headers=headers))
        except HTTPException as e:
            self.assertEqual(e.status_int, HTTP_BAD_REQUEST)
            self.assertTrue('X-Delete-At in past' in e.body)
        else:
            self.fail("Should have failed with HTTPBadRequest")

        t = str(int(time.time() - 1))
        headers = {'X-Delete-At': t}
        try:
            resp = constraints.check_delete_headers(
                Request.blank('/', headers=headers))
        except HTTPException as e:
            self.assertEqual(e.status_int, HTTP_BAD_REQUEST)
            self.assertTrue('X-Delete-At in past' in e.body)
        else:
            self.fail("Should have failed with HTTPBadRequest")
コード例 #42
0
 def test_check_metadata_empty_name(self):
     headers = {'X-Object-Meta-': 'Value'}
     resp = constraints.check_metadata(Request.blank('/', headers=headers),
                                       'object')
     self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
     self.assertIn('Metadata name cannot be empty', resp.body)
コード例 #43
0
    def handle_multipart_put(self, req, start_response):
        """
        Will handle the PUT of a SLO manifest.
        Heads every object in manifest to check if is valid and if so will
        save a manifest generated from the user input. Uses WSGIContext to
        call self and start_response and returns a WSGI iterator.

        :params req: a swob.Request with an obj in path
        :raises: HttpException on errors
        """
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(req.environ, start_response)
        if req.content_length > self.max_manifest_size:
            raise HTTPRequestEntityTooLarge("Manifest File > %d bytes" %
                                            self.max_manifest_size)
        if req.headers.get('X-Copy-From'):
            raise HTTPMethodNotAllowed(
                'Multipart Manifest PUTs cannot be COPY requests')
        if req.content_length is None and \
                req.headers.get('transfer-encoding', '').lower() != 'chunked':
            raise HTTPLengthRequired(request=req)
        parsed_data = parse_input(req.body_file.read(self.max_manifest_size))
        problem_segments = []

        if len(parsed_data) > self.max_manifest_segments:
            raise HTTPRequestEntityTooLarge(
                'Number of segments must be <= %d' %
                self.max_manifest_segments)
        total_size = 0
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            out_content_type = 'text/plain'
        data_for_storage = []
        slo_etag = md5()
        for index, seg_dict in enumerate(parsed_data):
            obj_name = seg_dict['path']
            if isinstance(obj_name, unicode):
                obj_name = obj_name.encode('utf-8')
            obj_path = '/'.join(['', vrs, account, obj_name.lstrip('/')])
            if req.path == quote(obj_path):
                raise HTTPConflict('Manifest object name "%s" '
                                   'cannot be included in the manifest' %
                                   obj_name)
            try:
                seg_size = int(seg_dict['size_bytes'])
            except (ValueError, TypeError):
                raise HTTPBadRequest('Invalid Manifest File')
            if seg_size < self.min_segment_size and \
                    index < len(parsed_data) - 1:
                raise HTTPBadRequest(
                    'Each segment, except the last, must be at least '
                    '%d bytes.' % self.min_segment_size)

            new_env = req.environ.copy()
            new_env['PATH_INFO'] = obj_path
            new_env['REQUEST_METHOD'] = 'HEAD'
            new_env['swift.source'] = 'SLO'
            del (new_env['wsgi.input'])
            del (new_env['QUERY_STRING'])
            new_env['CONTENT_LENGTH'] = 0
            new_env['HTTP_USER_AGENT'] = \
                '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT')
            head_seg_resp = \
                Request.blank(obj_path, new_env).get_response(self)
            if head_seg_resp.is_success:
                total_size += seg_size
                if seg_size != head_seg_resp.content_length:
                    problem_segments.append([quote(obj_name), 'Size Mismatch'])
                if seg_dict['etag'] == head_seg_resp.etag:
                    slo_etag.update(seg_dict['etag'])
                else:
                    problem_segments.append([quote(obj_name), 'Etag Mismatch'])
                if head_seg_resp.last_modified:
                    last_modified = head_seg_resp.last_modified
                else:
                    # shouldn't happen
                    last_modified = datetime.now()

                last_modified_formatted = \
                    last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f')
                seg_data = {
                    'name': '/' + seg_dict['path'].lstrip('/'),
                    'bytes': seg_size,
                    'hash': seg_dict['etag'],
                    'content_type': head_seg_resp.content_type,
                    'last_modified': last_modified_formatted
                }
                if config_true_value(
                        head_seg_resp.headers.get('X-Static-Large-Object')):
                    seg_data['sub_slo'] = True
                data_for_storage.append(seg_data)

            else:
                problem_segments.append(
                    [quote(obj_name), head_seg_resp.status])
        if problem_segments:
            resp_body = get_response_body(out_content_type, {},
                                          problem_segments)
            raise HTTPBadRequest(resp_body, content_type=out_content_type)
        env = req.environ

        if not env.get('CONTENT_TYPE'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream'
        env['swift.content_type_overridden'] = True
        env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size
        env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True'
        json_data = json.dumps(data_for_storage)
        env['CONTENT_LENGTH'] = str(len(json_data))
        env['wsgi.input'] = StringIO(json_data)

        slo_put_context = SloPutContext(self, slo_etag)
        return slo_put_context.handle_slo_put(req, start_response)
コード例 #44
0
 def test_check_metadata_empty(self):
     headers = {}
     self.assertIsNone(
         constraints.check_metadata(Request.blank('/', headers=headers),
                                    'object'))
コード例 #45
0
ファイル: test_obj.py プロジェクト: logorn/oio-swift
 def test_GET_not_found(self):
     req = Request.blank('/v1/a/c/o')
     self.storage.object_fetch = Mock(side_effect=exc.NoSuchObject)
     resp = req.get_response(self.app)
     self.assertEqual(resp.status_int, 404)
コード例 #46
0
 def test_check_object_creation_bad_content_type(self):
     headers = {'Transfer-Encoding': 'chunked', 'Content-Type': '\xff\xff'}
     resp = constraints.check_object_creation(
         Request.blank('/', headers=headers), 'object_name')
     self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
     self.assertTrue('Content-Type' in resp.body)
コード例 #47
0
ファイル: test_obj.py プロジェクト: logorn/oio-swift
 def test_PUT_requires_length(self):
     req = Request.blank('/v1/a/c/o', method='PUT')
     resp = req.get_response(self.app)
     self.assertEqual(resp.status_int, 411)
コード例 #48
0
    def make_request(self,
                     method,
                     path,
                     headers,
                     acceptable_statuses,
                     body_file=None,
                     params=None):
        """Makes a request to Swift with retries.

        :param method: HTTP method of request.
        :param path: Path of request.
        :param headers: Headers to be sent with request.
        :param acceptable_statuses: List of acceptable statuses for request.
        :param body_file: Body file to be passed along with request,
                          defaults to None.
        :param params: A dict of params to be set in request query string,
                       defaults to None.

        :returns: Response object on success.

        :raises UnexpectedResponse: Exception raised when make_request() fails
                                    to get a response with an acceptable status
        :raises Exception: Exception is raised when code fails in an
                           unexpected way.
        """

        headers = dict(headers)
        headers['user-agent'] = self.user_agent
        for attempt in range(self.request_tries):
            resp = exc_type = exc_value = exc_traceback = None
            req = Request.blank(path,
                                environ={'REQUEST_METHOD': method},
                                headers=headers)
            if body_file is not None:
                if hasattr(body_file, 'seek'):
                    body_file.seek(0)
                req.body_file = body_file
            if params:
                req.params = params
            try:
                resp = req.get_response(self.app)
            except (Exception, Timeout):
                exc_type, exc_value, exc_traceback = exc_info()
            else:
                if resp.status_int in acceptable_statuses or \
                        resp.status_int // 100 in acceptable_statuses:
                    return resp
                elif not is_server_error(resp.status_int):
                    # No sense retrying when we expect the same result
                    break
            # sleep only between tries, not after each one
            if attempt < self.request_tries - 1:
                if resp:
                    # always close any resp.app_iter before we discard it
                    with closing_if_possible(resp.app_iter):
                        # for non 2XX requests it's safe and useful to drain
                        # the response body so we log the correct status code
                        if resp.status_int // 100 != 2:
                            for iter_body in resp.app_iter:
                                pass
                sleep(2**(attempt + 1))
        if resp:
            msg = 'Unexpected response: %s' % resp.status
            if resp.status_int // 100 != 2 and resp.body:
                # provide additional context (and drain the response body) for
                # non 2XX responses
                msg += ' (%s)' % resp.body
            raise UnexpectedResponse(msg, resp)
        if exc_type:
            # To make pep8 tool happy, in place of raise t, v, tb:
            six.reraise(exc_type, exc_value, exc_traceback)
コード例 #49
0
    def test_PUT_req(self):
        body_key = os.urandom(32)
        object_key = fetch_crypto_keys()['object']
        plaintext = b'FAKE APP'
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)
        ciphertext_etag = md5hex(ciphertext)

        env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {
            'etag': plaintext_etag,
            'content-type': 'text/plain',
            'content-length': str(len(plaintext)),
            'x-object-meta-etag': 'not to be confused with the Etag!',
            'x-object-meta-test': 'encrypt me',
            'x-object-sysmeta-test': 'do not encrypt me'
        }
        req = Request.blank('/v1/a/c/o',
                            environ=env,
                            body=plaintext,
                            headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        with mock.patch(
                'swift.common.middleware.crypto.crypto_utils.'
                'Crypto.create_random_key',
                return_value=body_key):
            resp = req.get_response(self.encrypter)
        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual('PUT', self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify body crypto meta
        actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
        actual = json.loads(urlparse.unquote_plus(actual))
        self.assertEqual(Crypto().cipher, actual['cipher'])
        self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))

        # verify wrapped body key
        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
        self.assertEqual(expected_wrapped_key,
                         base64.b64decode(actual['body_key']['key']))
        self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv']))
        self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])

        # verify etag
        self.assertEqual(ciphertext_etag, req_hdrs['Etag'])

        encrypted_etag, _junk, etag_meta = \
            req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
        # verify crypto_meta was appended to this etag
        self.assertTrue(etag_meta)
        actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])

        # verify encrypted version of plaintext etag
        actual = base64.b64decode(encrypted_etag)
        etag_iv = base64.b64decode(actual_meta['iv'])
        enc_etag = encrypt(plaintext_etag.encode('ascii'), object_key, etag_iv)
        self.assertEqual(enc_etag, actual)

        # verify etag MAC for conditional requests
        actual_hmac = base64.b64decode(
            req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac'])
        exp_hmac = hmac.new(object_key, plaintext_etag.encode('ascii'),
                            hashlib.sha256).digest()
        self.assertEqual(actual_hmac, exp_hmac)

        # verify encrypted etag for container update
        self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
                      req_hdrs)
        parts = req_hdrs[
            'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
        self.assertEqual(2, len(parts))

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = 'swift_meta='
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len(crypto_meta_tag):]))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])
        self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])

        cont_key = fetch_crypto_keys()['container']
        cont_etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        exp_etag = encrypt(plaintext_etag.encode('ascii'), cont_key,
                           cont_etag_iv)
        self.assertEqual(exp_etag, base64.b64decode(parts[0]))

        # content-type is not encrypted
        self.assertEqual('text/plain', req_hdrs['Content-Type'])

        # user meta is encrypted
        self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
        self._verify_user_metadata(req_hdrs, 'Etag',
                                   'not to be confused with the Etag!',
                                   object_key)

        # sysmeta is not encrypted
        self.assertEqual('do not encrypt me',
                         req_hdrs['X-Object-Sysmeta-Test'])

        # verify object is encrypted by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        resp = get_req.get_response(self.app)
        self.assertEqual(ciphertext, resp.body)
        self.assertEqual(ciphertext_etag, resp.headers['Etag'])
コード例 #50
0
ファイル: test_obj.py プロジェクト: logorn/oio-swift
 def test_PUT_if_none_match_not_star(self):
     req = Request.blank('/v1/a/c/o', method='PUT')
     req.headers['if-none-match'] = 'foo'
     req.headers['content-length'] = '0'
     resp = req.get_response(self.app)
     self.assertEqual(resp.status_int, 400)
コード例 #51
0
    def _test_PUT_with_other_footers(self, override_etag):
        # verify handling of another middleware's footer callback
        body_key = os.urandom(32)
        object_key = fetch_crypto_keys()['object']
        plaintext = b'FAKE APP'
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)
        ciphertext_etag = md5hex(ciphertext)
        other_footers = {
            'Etag': plaintext_etag,
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Size':
            'other override',
            'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag
        }

        env = {
            'REQUEST_METHOD':
            'PUT',
            CRYPTO_KEY_CALLBACK:
            fetch_crypto_keys,
            'swift.callback.update_footers':
            lambda footers: footers.update(other_footers)
        }
        hdrs = {
            'content-type': 'text/plain',
            'content-length': str(len(plaintext)),
            'Etag': 'correct etag is in footers'
        }
        req = Request.blank('/v1/a/c/o',
                            environ=env,
                            body=plaintext,
                            headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})

        with mock.patch(
                'swift.common.middleware.crypto.crypto_utils.'
                'Crypto.create_random_key', lambda *args: body_key):
            resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual('PUT', self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify that other middleware's footers made it to app, including any
        # container update overrides but nothing Etag-related
        other_footers.pop('Etag')
        other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify encryption footers are ok
        encrypted_etag, _junk, etag_meta = \
            req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
        self.assertTrue(etag_meta)
        actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])

        self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
        actual = base64.b64decode(encrypted_etag)
        etag_iv = base64.b64decode(actual_meta['iv'])
        exp_etag = encrypt(plaintext_etag.encode('ascii'), object_key, etag_iv)
        self.assertEqual(exp_etag, actual)

        # verify encrypted etag for container update
        self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
                      req_hdrs)
        parts = req_hdrs[
            'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
        self.assertEqual(2, len(parts))

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = 'swift_meta='
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len(crypto_meta_tag):]))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])

        cont_key = fetch_crypto_keys()['container']
        cont_etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        exp_etag = encrypt(override_etag.encode('ascii'), cont_key,
                           cont_etag_iv)
        self.assertEqual(exp_etag, base64.b64decode(parts[0]))

        # verify body crypto meta
        actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
        actual = json.loads(urlparse.unquote_plus(actual))
        self.assertEqual(Crypto().cipher, actual['cipher'])
        self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))

        # verify wrapped body key
        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
        self.assertEqual(expected_wrapped_key,
                         base64.b64decode(actual['body_key']['key']))
        self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv']))
        self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
コード例 #52
0
 def _make_request(self, path, **kwargs):
     req = Request.blank(path, **kwargs)
     req.environ['swift.cache'] = FakeMemcache()
     return req
コード例 #53
0
 def do_test(host):
     req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
                         headers={'Host': host})
     return app(req.environ, start_response)
コード例 #54
0
    def test_PUT_nothing_read(self):
        # simulate an artificial scenario of a downstream filter/app not
        # actually reading the input stream from encrypter.
        class NonReadingApp(object):
            def __call__(self, env, start_response):
                # note: no read from wsgi.input
                req = Request(env)
                env['swift.callback.update_footers'](req.headers)
                call_headers.append(req.headers)
                resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
                return resp(env, start_response)

        env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {
            'content-type': 'text/plain',
            'content-length': 0,
            'etag': 'etag from client'
        }
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))
        self.assertEqual('etag from client', call_headers[0]['etag'])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))

        # check that an upstream footer callback gets called
        other_footers = {
            'Etag': EMPTY_ETAG,
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'
        }
        env.update({
            'swift.callback.update_footers':
            lambda footers: footers.update(other_footers)
        })
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))

        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))

        # verify encrypted override etag for container update.
        self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
                      call_headers[0])
        parts = call_headers[0][
            'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
        self.assertEqual(2, len(parts))
        cont_key = fetch_crypto_keys()['container']

        param = parts[1].strip()
        crypto_meta_tag = 'swift_meta='
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len(crypto_meta_tag):]))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])
        self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])

        cont_etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt(b'other override', cont_key, cont_etag_iv),
                         base64.b64decode(parts[0]))

        # verify that other middleware's footers made it to app
        other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
        for k, v in other_footers.items():
            self.assertEqual(v, call_headers[0][k])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))

        # if upstream footer override etag is for an empty body then check that
        # it is not encrypted
        other_footers = {
            'Etag': EMPTY_ETAG,
            'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG
        }
        env.update({
            'swift.callback.update_footers':
            lambda footers: footers.update(other_footers)
        })
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))

        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))

        # verify that other middleware's footers made it to app
        for k, v in other_footers.items():
            self.assertEqual(v, call_headers[0][k])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))

        # if upstream footer override etag is an empty string then check that
        # it is not encrypted
        other_footers = {
            'Etag': EMPTY_ETAG,
            'X-Object-Sysmeta-Container-Update-Override-Etag': ''
        }
        env.update({
            'swift.callback.update_footers':
            lambda footers: footers.update(other_footers)
        })
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))

        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))

        # verify that other middleware's footers made it to app
        for k, v in other_footers.items():
            self.assertEqual(v, call_headers[0][k])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
コード例 #55
0
 def test_cname_matching_ending_not_domain(self):
     req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
                         headers={'Host': 'foo.com'})
     resp = self.app(req.environ, start_response)
     self.assertEqual(resp,
                      ['CNAME lookup failed to resolve to a valid domain'])
コード例 #56
0
 def test_PUT_app_exception(self):
     app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {})
     req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
     with self.assertRaises(HTTPException) as catcher:
         req.get_response(app)
     self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)
コード例 #57
0
 def test_something_weird(self):
     req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
                         headers={'Host': 'mysite.com'})
     resp = self.app(req.environ, start_response)
     self.assertEqual(resp,
                      ['CNAME lookup failed to resolve to a valid domain'])
コード例 #58
0
 def do_test(lookup_back):
     req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
                         headers={'Host': 'c.a.example.com'})
     module = 'swift.common.middleware.cname_lookup.lookup_cname'
     with mock.patch(module, lambda d, r: (0, lookup_back)):
         return app(req.environ, start_response)
コード例 #59
0
ファイル: test_obj.py プロジェクト: sohonet/swift3
    def test_object_HEAD_error(self):
        # HEAD does not return the body even an error response in the
        # specifications of the REST API.
        # So, check the response code for error test of HEAD.
        req = Request.blank('/bucket/object',
                            environ={'REQUEST_METHOD': 'HEAD'},
                            headers={
                                'Authorization': 'AWS test:tester:hmac',
                                'Date': self.get_date_header()
                            })
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                            swob.HTTPUnauthorized, {}, None)
        status, headers, body = self.call_swift3(req)
        self.assertEqual(status.split()[0], '403')
        self.assertEqual(body, '')  # sanity

        req = Request.blank('/bucket/object',
                            environ={'REQUEST_METHOD': 'HEAD'},
                            headers={
                                'Authorization': 'AWS test:tester:hmac',
                                'Date': self.get_date_header()
                            })
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                            swob.HTTPForbidden, {}, None)
        status, headers, body = self.call_swift3(req)
        self.assertEqual(status.split()[0], '403')
        self.assertEqual(body, '')  # sanity

        req = Request.blank('/bucket/object',
                            environ={'REQUEST_METHOD': 'HEAD'},
                            headers={
                                'Authorization': 'AWS test:tester:hmac',
                                'Date': self.get_date_header()
                            })
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                            swob.HTTPNotFound, {}, None)
        status, headers, body = self.call_swift3(req)
        self.assertEqual(status.split()[0], '404')
        self.assertEqual(body, '')  # sanity

        req = Request.blank('/bucket/object',
                            environ={'REQUEST_METHOD': 'HEAD'},
                            headers={
                                'Authorization': 'AWS test:tester:hmac',
                                'Date': self.get_date_header()
                            })
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                            swob.HTTPPreconditionFailed, {}, None)
        status, headers, body = self.call_swift3(req)
        self.assertEqual(status.split()[0], '412')
        self.assertEqual(body, '')  # sanity

        req = Request.blank('/bucket/object',
                            environ={'REQUEST_METHOD': 'HEAD'},
                            headers={
                                'Authorization': 'AWS test:tester:hmac',
                                'Date': self.get_date_header()
                            })
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                            swob.HTTPServerError, {}, None)
        status, headers, body = self.call_swift3(req)
        self.assertEqual(status.split()[0], '500')
        self.assertEqual(body, '')  # sanity

        req = Request.blank('/bucket/object',
                            environ={'REQUEST_METHOD': 'HEAD'},
                            headers={
                                'Authorization': 'AWS test:tester:hmac',
                                'Date': self.get_date_header()
                            })
        self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                            swob.HTTPServiceUnavailable, {}, None)
        status, headers, body = self.call_swift3(req)
        self.assertEqual(status.split()[0], '500')
        self.assertEqual(body, '')  # sanity
コード例 #60
0
    def test_caching(self):
        fail_to_resolve = ['CNAME lookup failed to resolve to a valid domain']

        class memcache_stub(object):
            def __init__(self):
                self.cache = {}

            def get(self, key):
                return self.cache.get(key, None)

            def set(self, key, value, *a, **kw):
                self.cache[key] = value

        module = 'swift.common.middleware.cname_lookup.lookup_cname'
        dns_module = 'dns.resolver.Resolver.query'
        memcache = memcache_stub()

        with mock.patch(module) as m:
            m.return_value = (3600, 'c.example.com')
            req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
                                              'swift.cache': memcache},
                                headers={'Host': 'mysite2.com'})
            resp = self.app(req.environ, start_response)
            self.assertEqual(resp, ['FAKE APP'])
            self.assertEqual(m.call_count, 1)
            self.assertEqual(memcache.cache.get('cname-mysite2.com'),
                             'c.example.com')
            req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
                                              'swift.cache': memcache},
                                headers={'Host': 'mysite2.com'})
            resp = self.app(req.environ, start_response)
            self.assertEqual(resp, ['FAKE APP'])
            self.assertEqual(m.call_count, 1)
            self.assertEqual(memcache.cache.get('cname-mysite2.com'),
                             'c.example.com')

        for exc, num in ((dns.resolver.NXDOMAIN(), 3),
                         (dns.resolver.NoAnswer(), 4)):
            with mock.patch(dns_module) as m:
                m.side_effect = exc
                req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
                                                  'swift.cache': memcache},
                                    headers={'Host': 'mysite%d.com' % num})
                resp = self.app(req.environ, start_response)
                self.assertEqual(resp, fail_to_resolve)
                self.assertEqual(m.call_count, 1)
                self.assertEqual(memcache.cache.get('cname-mysite3.com'),
                                 False)
                req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
                                                  'swift.cache': memcache},
                                    headers={'Host': 'mysite%d.com' % num})
                resp = self.app(req.environ, start_response)
                self.assertEqual(resp, fail_to_resolve)
                self.assertEqual(m.call_count, 1)
                self.assertEqual(
                    memcache.cache.get('cname-mysite%d.com' % num), False)

        with mock.patch(dns_module) as m:
            m.side_effect = dns.exception.DNSException()
            req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
                                              'swift.cache': memcache},
                                headers={'Host': 'mysite5.com'})
            resp = self.app(req.environ, start_response)
            self.assertEqual(resp, fail_to_resolve)
            self.assertEqual(m.call_count, 1)
            self.assertFalse('cname-mysite5.com' in memcache.cache)
            req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
                                              'swift.cache': memcache},
                                headers={'Host': 'mysite5.com'})
            resp = self.app(req.environ, start_response)
            self.assertEqual(resp, fail_to_resolve)
            self.assertEqual(m.call_count, 2)
            self.assertFalse('cname-mysite5.com' in memcache.cache)