예제 #1
0
        def do_test(method, plain_etags, expected_plain_etags=None):
            env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
            match_header_value = ", ".join(plain_etags)
            req = Request.blank(
                "/v1/a/c/o", environ=env, method=method, headers={match_header_name: match_header_value}
            )
            app = FakeSwift()
            app.register(method, "/v1/a/c/o", HTTPOk, {})
            resp = req.get_response(encrypter.Encrypter(app, {}))
            self.assertEqual("200 OK", resp.status)

            self.assertEqual(1, len(app.calls), app.calls)
            self.assertEqual(method, app.calls[0][0])
            actual_headers = app.headers[0]

            # verify the alternate etag location has been specified
            if match_header_value and match_header_value != "*":
                self.assertIn("X-Backend-Etag-Is-At", actual_headers)
                self.assertEqual("X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"])

            # verify etags have been supplemented with masked values
            self.assertIn(match_header_name, actual_headers)
            actual_etags = set(actual_headers[match_header_name].split(", "))
            key = fetch_crypto_keys()["object"]
            masked_etags = [
                '"%s"' % base64.b64encode(hmac.new(key, etag.strip('"'), hashlib.sha256).digest())
                for etag in plain_etags
                if etag not in ("*", "")
            ]
            expected_etags = set((expected_plain_etags or plain_etags) + masked_etags)
            self.assertEqual(expected_etags, actual_etags)
            # check that the request environ was returned to original state
            self.assertEqual(set(plain_etags), set(req.headers[match_header_name].split(", ")))
예제 #2
0
 def test_config_true_value_on_disable_encryption(self):
     app = FakeSwift()
     self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption)
     for val in ('true', '1', 'yes', 'on', 't', 'y'):
         app = encrypter.Encrypter(app,
                                   {'disable_encryption': val})
         self.assertTrue(app.disable_encryption)
예제 #3
0
 def setUp(self):
     self.app = FakeSwift()
     conf = {'symloop_max': '2'}
     self.sym = symlink.filter_factory(conf)(self.app)
     self.sym.logger = self.app.logger
     vw_conf = {'allow_versioned_writes': 'true'}
     self.vw = versioned_writes.filter_factory(vw_conf)(self.sym)
예제 #4
0
        def do_test(method, plain_etags, expected_plain_etags=None):
            env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
            match_header_value = ', '.join(plain_etags)
            req = Request.blank(
                '/v1/a/c/o', environ=env, method=method,
                headers={match_header_name: match_header_value})
            app = FakeSwift()
            app.register(method, '/v1/a/c/o', HTTPOk, {})
            resp = req.get_response(encrypter.Encrypter(app, {}))
            self.assertEqual('200 OK', resp.status)

            self.assertEqual(1, len(app.calls), app.calls)
            self.assertEqual(method, app.calls[0][0])
            actual_headers = app.headers[0]

            # verify the alternate etag location has been specified
            if match_header_value and match_header_value != '*':
                self.assertIn('X-Backend-Etag-Is-At', actual_headers)
                self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
                                 actual_headers['X-Backend-Etag-Is-At'])

            # verify etags have been supplemented with masked values
            self.assertIn(match_header_name, actual_headers)
            actual_etags = set(actual_headers[match_header_name].split(', '))
            # masked values for secret_id None
            key = fetch_crypto_keys()['object']
            masked_etags = [
                '"%s"' % bytes_to_wsgi(base64.b64encode(hmac.new(
                    key, wsgi_to_bytes(etag.strip('"')),
                    hashlib.sha256).digest()))
                for etag in plain_etags if etag not in ('*', '')]
            # masked values for secret_id myid
            key = fetch_crypto_keys(key_id={'secret_id': 'myid'})['object']
            masked_etags_myid = [
                '"%s"' % bytes_to_wsgi(base64.b64encode(hmac.new(
                    key, wsgi_to_bytes(etag.strip('"')),
                    hashlib.sha256).digest()))
                for etag in plain_etags if etag not in ('*', '')]
            expected_etags = set((expected_plain_etags or plain_etags) +
                                 masked_etags + masked_etags_myid)
            self.assertEqual(expected_etags, actual_etags)
            # check that the request environ was returned to original state
            self.assertEqual(set(plain_etags),
                             set(req.headers[match_header_name].split(', ')))
 def setUp(self):
     conf = {'sds_default_account': 'OPENIO'}
     self.filter_conf = {
         'strip_v1': 'true',
         'swift3_compat': 'true',
         'account_first': 'true'
     }
     self.app = FakeSwift()
     self.ch = container_hierarchy.filter_factory(
         conf,
         **self.filter_conf)(self.app)
예제 #6
0
class TestEncrypter(OrigTestEncrypter):
    def setUp(self):
        self.app = FakeSwift()
        self.encrypter = encrypter.Encrypter(self.app, {})
        self.encrypter.logger = FakeLogger()

    def test_PUT_missing_key_in_header(self):
        def raise_exc():
            raise HTTPBadRequest(
                'Missing X-Amz-Server-Side-Encryption-Customer-Key header')

        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: raise_exc}
        hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        # If it does not find a key, no problem,
        # oioswift's encrypter let the request pass through,
        # and does not encrypt the object.
        self.assertEqual('201 Created', resp.status)
예제 #7
0
 def setUp(self):
     self.fake_swift = FakeSwift()
     self.app = listing_formats.ListingFilter(self.fake_swift)
     self.fake_account_listing = json.dumps([
         {'name': 'bar', 'bytes': 0, 'count': 0,
          'last_modified': '1970-01-01T00:00:00.000000'},
         {'subdir': 'foo_'},
     ])
     self.fake_container_listing = json.dumps([
         {'name': 'bar', 'hash': 'etag', 'bytes': 0,
          'content_type': 'text/plain',
          'last_modified': '1970-01-01T00:00:00.000000'},
         {'subdir': 'foo/'},
     ])
예제 #8
0
    def setUp(self):
        conf = {'sds_default_account': 'OPENIO'}
        self.filter_conf = {
            'strip_v1': 'true',
            'swift3_compat': 'true',
            'account_first': 'true',
            'stop_at_first_match': 'true',
            'pattern1': r'(\d{3})/(\d{3})/(\d)\d\d/\d\d(\d)/',
            'pattern2': r'(\d{3})/(\d)\d\d/\d\d(\d)/',
            'pattern3': r'^(cloud)/([0-9a-f][0-9a-f])',
            'pattern4': r'^(cloud)/([0-9a-f])',
            'pattern9': r'^/?([^/]+)',
        }

        if hasattr(ContainerBuilder, 'alternatives'):
            self.filter_conf['stop_at_first_match'] = 'false'

        self.app = FakeSwift()
        self.ch = regexcontainer.filter_factory(
            conf,
            **self.filter_conf)(self.app)
예제 #9
0
 def setUp(self):
     self.app = FakeSwift()
     conf = {'allow_versioned_writes': 'true'}
     self.vw = versioned_writes.filter_factory(conf)(self.app)
예제 #10
0
class VersionedWritesTestCase(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        conf = {'allow_versioned_writes': 'true'}
        self.vw = versioned_writes.filter_factory(conf)(self.app)

    def call_app(self, req, app=None, expect_exception=False):
        if app is None:
            app = self.app

        self.authorized = []

        def authorize(req):
            self.authorized.append(req)

        if 'swift.authorize' not in req.environ:
            req.environ['swift.authorize'] = authorize

        req.headers.setdefault("User-Agent", "Marula Kruger")

        status = [None]
        headers = [None]

        def start_response(s, h, ei=None):
            status[0] = s
            headers[0] = h

        body_iter = app(req.environ, start_response)
        body = ''
        caught_exc = None
        try:
            for chunk in body_iter:
                body += chunk
        except Exception as exc:
            if expect_exception:
                caught_exc = exc
            else:
                raise

        if expect_exception:
            return status[0], headers[0], body, caught_exc
        else:
            return status[0], headers[0], body

    def call_vw(self, req, **kwargs):
        return self.call_app(req, app=self.vw, **kwargs)

    def assertRequestEqual(self, req, other):
        self.assertEqual(req.method, other.method)
        self.assertEqual(req.path, other.path)

    def test_put_container(self):
        self.app.register('PUT', '/v1/a/c', swob.HTTPOk, {}, 'passed')
        req = Request.blank('/v1/a/c',
                            headers={'X-Versions-Location': 'ver_cont'},
                            environ={'REQUEST_METHOD': 'PUT'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')

        # check for sysmeta header
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('PUT', method)
        self.assertEqual('/v1/a/c', path)
        self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_container_allow_versioned_writes_false(self):
        self.vw.conf = {'allow_versioned_writes': 'false'}

        # PUT/POST container must fail as 412 when allow_versioned_writes
        # set to false
        for method in ('PUT', 'POST'):
            req = Request.blank('/v1/a/c',
                                headers={'X-Versions-Location': 'ver_cont'},
                                environ={'REQUEST_METHOD': method})
            status, headers, body = self.call_vw(req)
            self.assertEqual(status, "412 Precondition Failed")

        # GET/HEAD performs as normal
        self.app.register('GET', '/v1/a/c', swob.HTTPOk, {}, 'passed')
        self.app.register('HEAD', '/v1/a/c', swob.HTTPOk, {}, 'passed')

        for method in ('GET', 'HEAD'):
            req = Request.blank('/v1/a/c',
                                headers={'X-Versions-Location': 'ver_cont'},
                                environ={'REQUEST_METHOD': method})
            status, headers, body = self.call_vw(req)
            self.assertEqual(status, '200 OK')

    def test_remove_versions_location(self):
        self.app.register('POST', '/v1/a/c', swob.HTTPOk, {}, 'passed')
        req = Request.blank('/v1/a/c',
                            headers={'X-Remove-Versions-Location': 'x'},
                            environ={'REQUEST_METHOD': 'POST'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')

        # check for sysmeta header
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('POST', method)
        self.assertEqual('/v1/a/c', path)
        self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
        self.assertTrue('x-versions-location' in req_headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_remove_add_versions_precedence(self):
        self.app.register(
            'POST', '/v1/a/c', swob.HTTPOk,
            {'x-container-sysmeta-versions-location': 'ver_cont'},
            'passed')
        req = Request.blank('/v1/a/c',
                            headers={'X-Remove-Versions-Location': 'x',
                                     'X-Versions-Location': 'ver_cont'},
                            environ={'REQUEST_METHOD': 'POST'})

        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)

        # check for sysmeta header
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('POST', method)
        self.assertEqual('/v1/a/c', path)
        self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
        self.assertTrue('x-remove-versions-location' not in req_headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_get_container(self):
        self.app.register(
            'GET', '/v1/a/c', swob.HTTPOk,
            {'x-container-sysmeta-versions-location': 'ver_cont'}, None)
        req = Request.blank(
            '/v1/a/c',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_get_head(self):
        self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None)
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

        self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {}, None)
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'HEAD'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_put_object_no_versioning(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')

        cache = FakeCache({})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_put_first_object_success(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_PUT_versioning_with_nonzero_default_policy(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None)

        cache = FakeCache({'versions': 'ver_cont', 'storage_policy': '2'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')

        # check for 'X-Backend-Storage-Policy-Index' in HEAD request
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('HEAD', method)
        self.assertEqual('/v1/a/c/o', path)
        self.assertTrue('X-Backend-Storage-Policy-Index' in req_headers)
        self.assertEqual('2',
                         req_headers.get('X-Backend-Storage-Policy-Index'))
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_put_object_no_versioning_with_container_config_true(self):
        # set False to versions_write obsously and expect no COPY occurred
        self.vw.conf = {'allow_versioned_writes': 'false'}
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/a/c/o', swob.HTTPOk,
            {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        called_method = [method for (method, path, hdrs) in self.app._calls]
        self.assertTrue('COPY' not in called_method)

    def test_delete_object_no_versioning_with_container_config_true(self):
        # set False to versions_write obviously and expect no GET versioning
        # container and COPY called (just delete object as normal)
        self.vw.conf = {'allow_versioned_writes': 'false'}
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPNoContent, {}, 'passed')
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '204 No Content')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        called_method = \
            [method for (method, path, rheaders) in self.app._calls]
        self.assertTrue('COPY' not in called_method)
        self.assertTrue('GET' not in called_method)

    def test_copy_object_no_versioning_with_container_config_true(self):
        # set False to versions_write obviously and expect no extra
        # COPY called (just copy object as normal)
        self.vw.conf = {'allow_versioned_writes': 'false'}
        self.app.register(
            'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        called_method = \
            [method for (method, path, rheaders) in self.app._calls]
        self.assertTrue('COPY' in called_method)
        self.assertEqual(called_method.count('COPY'), 1)

    def test_new_version_success(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/a/c/o', swob.HTTPOk,
            {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
        self.app.register(
            'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    @local_tz
    def test_new_version_sysmeta_precedence(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/a/c/o', swob.HTTPOk,
            {'last-modified': 'Thu, 1 Jan 1970 00:00:00 GMT'}, 'passed')
        self.app.register(
            'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)

        # fill cache with two different values for versions location
        # new middleware should use sysmeta first
        cache = FakeCache({'versions': 'old_ver_cont',
                          'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

        # check that sysmeta header was used
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[1]
        self.assertEqual('COPY', method)
        self.assertEqual('/v1/a/c/o', path)
        self.assertEqual('ver_cont/001o/0000000000.00000',
                         req_headers['Destination'])

    def test_copy_first_version(self):
        self.app.register(
            'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None)
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'},
            headers={'Destination': 'tgt_cont/tgt_obj'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_copy_new_version(self):
        self.app.register(
            'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
            {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
        self.app.register(
            'COPY', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, None)
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'},
            headers={'Destination': 'tgt_cont/tgt_obj'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_copy_new_version_different_account(self):
        self.app.register(
            'COPY', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'HEAD', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
            {'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
        self.app.register(
            'COPY', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, None)
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/src_a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'},
            headers={'Destination': 'tgt_cont/tgt_obj',
                     'Destination-Account': 'tgt_a'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_copy_new_version_bogus_account(self):
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/src_a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'},
            headers={'Destination': 'tgt_cont/tgt_obj',
                     'Destination-Account': '/im/on/a/boat'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '412 Precondition Failed')

    def test_delete_first_object_success(self):
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
            swob.HTTPNotFound, {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_delete_latest_version_success(self):
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}, '
            '{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'COPY', '/v1/a/ver_cont/001o/2', swob.HTTPCreated,
            {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/2', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            headers={'X-If-Delete-At': 1},
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

        # check that X-If-Delete-At was removed from DELETE request
        calls = self.app.calls_with_headers
        method, path, req_headers = calls.pop()
        self.assertEqual('DELETE', method)
        self.assertTrue(path.startswith('/v1/a/ver_cont/001o/2'))
        self.assertFalse('x-if-delete-at' in req_headers or
                         'X-If-Delete-At' in req_headers)

    def test_DELETE_on_expired_versioned_object(self):
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}, '
            '{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}]')

        # expired object
        self.app.register(
            'COPY', '/v1/a/ver_cont/001o/2', swob.HTTPNotFound,
            {}, None)
        self.app.register(
            'COPY', '/v1/a/ver_cont/001o/1', swob.HTTPCreated,
            {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_denied_DELETE_of_versioned_object(self):
        authorize_call = []
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}, '
            '{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPForbidden,
            {}, None)

        def fake_authorize(req):
            authorize_call.append(req)
            return swob.HTTPForbidden()

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'swift.authorize': fake_authorize,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '403 Forbidden')
        self.assertEqual(len(authorize_call), 1)
        self.assertRequestEqual(req, authorize_call[0])
예제 #11
0
class OioRegexContainer(unittest.TestCase):
    def setUp(self):
        conf = {'sds_default_account': 'OPENIO'}
        self.filter_conf = {
            'strip_v1': 'true',
            'swift3_compat': 'true',
            'account_first': 'true',
            'stop_at_first_match': 'true',
            'pattern1': r'(\d{3})/(\d{3})/(\d)\d\d/\d\d(\d)/',
            'pattern2': r'(\d{3})/(\d)\d\d/\d\d(\d)/',
            'pattern3': r'^(cloud)/([0-9a-f][0-9a-f])',
            'pattern4': r'^(cloud)/([0-9a-f])',
            'pattern9': r'^/?([^/]+)',
        }

        if hasattr(ContainerBuilder, 'alternatives'):
            self.filter_conf['stop_at_first_match'] = 'false'

        self.app = FakeSwift()
        self.ch = regexcontainer.filter_factory(
            conf,
            **self.filter_conf)(self.app)

    def tearDown(self):
        pass

    def call_app(self, req, app=None):
        if app is None:
            app = self.app

        self.authorized = []

        def authorize(req):
            self.authorized.append(req)

        if 'swift.authorize' not in req.environ:
            req.environ['swift.authorize'] = authorize

        req.headers.setdefault("User-Agent", "Melted Cheddar")

        status = [None]
        headers = [None]

        def start_response(s, h, ei=None):
            status[0] = s
            headers[0] = h

        body_iter = app(req.environ, start_response)
        with utils.closing_if_possible(body_iter):
            body = b''.join(body_iter)

        return status[0], headers[0], body

    def call_rc(self, req):
        return self.call_app(req, app=self.ch)

    def _check_pattern(self, path_in, path_out):
        self.app.register('PUT', path_out, swob.HTTPCreated, {})
        req = Request.blank(path_in, method='PUT')
        resp = self.call_rc(req)
        self.assertEqual(resp[0], "201 Created")
        self.assertEqual(self.app.calls, [('PUT', path_out)])

    def test_pattern1(self):
        self._check_pattern(
            '/v1/a/c/111/222/456/789/o',
            '/v1/a/11122249/111/222/456/789/o')

    def test_pattern2(self):
        if self.filter_conf['stop_at_first_match'] == 'true':
            self.skipTest("require openio-sds >= 4.2")
        self.app.register('GET', '/v1/a/11122249/111/222/456/789/o',
                          swob.HTTPNotFound, {})
        self.app.register('GET', '/v1/a/11126/111/222/456/789/o',
                          swob.HTTPOk, {})
        req = Request.blank('/v1/a/c/111/222/456/789/o', method='GET')
        resp = self.call_rc(req)
        self.assertEqual(resp[0], "200 OK")
        self.assertEqual(
            self.app.calls,
            [('GET', '/v1/a/11122249/111/222/456/789/o'),
             ('GET', '/v1/a/11126/111/222/456/789/o')])

    def test_pattern3(self):
        self._check_pattern('/v1/a/c/cloud/ff_object',
                            '/v1/a/cloudff/cloud/ff_object')

    def test_pattern4(self):
        if self.filter_conf['stop_at_first_match'] == 'true':
            self.skipTest("require openio-sds >= 4.2")
        self.app.register('GET', '/v1/a/cloudff/cloud/ff_object',
                          swob.HTTPNotFound, {})
        self.app.register('GET', '/v1/a/cloudf/cloud/ff_object',
                          swob.HTTPOk, {})
        req = Request.blank('/v1/a/c/cloud/ff_object', method='GET')
        resp = self.call_rc(req)
        self.assertEqual(resp[0], "200 OK")
        self.assertEqual(
            self.app.calls,
            [('GET', '/v1/a/cloudff/cloud/ff_object'),
             ('GET', '/v1/a/cloudf/cloud/ff_object')])

    def test_pattern9(self):
        self._check_pattern('/v1/a/c/gc_regex/path/ob',
                            '/v1/a/gc_regex/gc_regex/path/ob')

    def test_get_without_matching_pattern(self):
        if self.filter_conf['stop_at_first_match'] == 'true':
            self.skipTest("require openio-sds >= 4.2")
        self.app.register('GET', '/v1/a/11122249/111/222/456/789/o',
                          swob.HTTPNotFound, {})
        self.app.register('GET', '/v1/a/11126/111/222/456/789/o',
                          swob.HTTPNotFound, {})
        self.app.register('GET', '/v1/a/111/111/222/456/789/o',
                          swob.HTTPNotFound, {})
        req = Request.blank('/v1/a/c/111/222/456/789/o', method='GET')
        resp = self.call_rc(req)
        self.assertEqual(resp[0], "404 Not Found")
        self.assertEqual(
            self.app.calls,
            [('GET', '/v1/a/11122249/111/222/456/789/o'),
             ('GET', '/v1/a/11126/111/222/456/789/o'),
             ('GET', '/v1/a/111/111/222/456/789/o')])

    def test_simple_listing(self):
        self._check_pattern('/v1/a/c/111/222/456/789/o',
                            '/v1/a/11122249/111/222/456/789/o')

        self.app.register('GET',
                          '/v1/a/11122249?prefix=/111/222/456/789/o',
                          swob.HTTPOk, {})

        req = Request.blank('/v1/a/c?prefix=/111/222/456/789/o', method='GET')
        resp = self.call_rc(req)
        self.assertEqual(resp[0], '200 OK')

    def test_fallback_listing(self):
        if self.filter_conf['stop_at_first_match'] == 'true':
            self.skipTest("require openio-sds >= 4.2")

        self.app.register('GET',
                          '/v1/a/11122249?prefix=/111/222/456/789/o',
                          swob.HTTPNotFound, {})
        self.app.register('GET',
                          '/v1/a/11126?prefix=/111/222/456/789/o',
                          swob.HTTPOk, {})

        req = Request.blank('/v1/a/c?prefix=/111/222/456/789/o', method='GET')
        resp = self.call_rc(req)
        self.assertEqual(resp[0], '200 OK')
        self.assertEqual(
            self.app.calls,
            [('GET', '/v1/a/11122249?prefix=/111/222/456/789/o'),
             ('GET', '/v1/a/11126?prefix=/111/222/456/789/o')])

    def test_swift3_mpu(self):
        self.app.register('PUT',
                          '/v1/a/cloudff+segments/cloud/ff_object',
                          swob.HTTPOk, {})
        req = Request.blank(
            '/v1/a/c+segments/cloud/ff_object', method='PUT')
        resp = self.call_rc(req)
        self.assertEqual(resp[0], '200 OK')

    def test_copy(self):
        self.app.register('PUT',
                          '/v1/a/cloudff/cloud/ff_object',
                          swob.HTTPOk, {})
        req = Request.blank(
            '/v1/a/c/cloud/ff_object', method='PUT',
            headers={'X-Copy-From': 'container/path/dir1/object'})
        resp = self.call_rc(req)
        self.assertEqual(resp[0], '200 OK')
        self.assertEqual(self.app.headers[0]['X-Copy-From'],
                         "/dir1/dir1/object")

    def test_fastcopy(self):
        self.app.register('PUT',
                          '/v1/a/cloudff/cloud/ff_object',
                          swob.HTTPOk, {})
        req = Request.blank('/v1/a/c/cloud/ff_object', method='PUT',
                            headers={'Oio-Copy-From': 'container/path/object'})
        resp = self.call_rc(req)
        self.assertEqual(resp[0], '200 OK')
        self.assertEqual(self.app.headers[0]['Oio-Copy-From'],
                         "/object/object")
예제 #12
0
class TestEncrypter(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        self.encrypter = encrypter.Encrypter(self.app, {})
        self.encrypter.logger = FakeLogger()

    def _verify_user_metadata(self, req_hdrs, name, value, key):
        # verify encrypted version of user metadata
        self.assertNotIn('X-Object-Meta-' + name, req_hdrs)
        expected_hdr = 'X-Object-Transient-Sysmeta-Crypto-Meta-' + name
        self.assertIn(expected_hdr, req_hdrs)
        enc_val, param = req_hdrs[expected_hdr].split(';')
        param = param.strip()
        self.assertTrue(param.startswith('swift_meta='))
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len('swift_meta='):]))
        self.assertEqual(Crypto.cipher, actual_meta['cipher'])
        meta_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, meta_iv)
        self.assertEqual(
            base64.b64encode(encrypt(value, key, meta_iv)),
            enc_val)
        # if there is any encrypted user metadata then this header should exist
        self.assertIn('X-Object-Transient-Sysmeta-Crypto-Meta', req_hdrs)
        common_meta = json.loads(urlparse.unquote_plus(
            req_hdrs['X-Object-Transient-Sysmeta-Crypto-Meta']))
        self.assertDictEqual({'cipher': Crypto.cipher,
                              'key_id': {'v': 'fake', 'path': '/a/c/fake'}},
                             common_meta)

    def test_PUT_req(self):
        body_key = os.urandom(32)
        object_key = fetch_crypto_keys()['object']
        plaintext = 'FAKE APP'
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)
        ciphertext_etag = md5hex(ciphertext)

        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'etag': plaintext_etag,
                'content-type': 'text/plain',
                'content-length': str(len(plaintext)),
                'x-object-meta-etag': 'not to be confused with the Etag!',
                'x-object-meta-test': 'encrypt me',
                'x-object-sysmeta-test': 'do not encrypt me'}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        with mock.patch(
            'swift.common.middleware.crypto.crypto_utils.'
            'Crypto.create_random_key',
                return_value=body_key):
            resp = req.get_response(self.encrypter)
        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual('PUT', self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify body crypto meta
        actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
        actual = json.loads(urlparse.unquote_plus(actual))
        self.assertEqual(Crypto().cipher, actual['cipher'])
        self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))

        # verify wrapped body key
        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
        self.assertEqual(expected_wrapped_key,
                         base64.b64decode(actual['body_key']['key']))
        self.assertEqual(FAKE_IV,
                         base64.b64decode(actual['body_key']['iv']))
        self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])

        # verify etag
        self.assertEqual(ciphertext_etag, req_hdrs['Etag'])

        encrypted_etag, _junk, etag_meta = \
            req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
        # verify crypto_meta was appended to this etag
        self.assertTrue(etag_meta)
        actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])

        # verify encrypted version of plaintext etag
        actual = base64.b64decode(encrypted_etag)
        etag_iv = base64.b64decode(actual_meta['iv'])
        enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
        self.assertEqual(enc_etag, actual)

        # verify etag MAC for conditional requests
        actual_hmac = base64.b64decode(
            req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac'])
        self.assertEqual(actual_hmac, hmac.new(
            object_key, plaintext_etag, hashlib.sha256).digest())

        # verify encrypted etag for container update
        self.assertIn(
            'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
        parts = req_hdrs[
            'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
        self.assertEqual(2, len(parts))

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = 'swift_meta='
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len(crypto_meta_tag):]))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])
        self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])

        cont_key = fetch_crypto_keys()['container']
        cont_etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv),
                         base64.b64decode(parts[0]))

        # content-type is not encrypted
        self.assertEqual('text/plain', req_hdrs['Content-Type'])

        # user meta is encrypted
        self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
        self._verify_user_metadata(
            req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)

        # sysmeta is not encrypted
        self.assertEqual('do not encrypt me',
                         req_hdrs['X-Object-Sysmeta-Test'])

        # verify object is encrypted by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        resp = get_req.get_response(self.app)
        self.assertEqual(ciphertext, resp.body)
        self.assertEqual(ciphertext_etag, resp.headers['Etag'])

    def test_PUT_zero_size_object(self):
        # object body encryption should be skipped for zero sized object body
        object_key = fetch_crypto_keys()['object']
        plaintext_etag = EMPTY_ETAG

        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'etag': EMPTY_ETAG,
                'content-type': 'text/plain',
                'content-length': '0',
                'x-object-meta-etag': 'not to be confused with the Etag!',
                'x-object-meta-test': 'encrypt me',
                'x-object-sysmeta-test': 'do not encrypt me'}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body='', headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})

        resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual('PUT', self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify that there is no body crypto meta
        self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs)
        # verify etag is md5 of plaintext
        self.assertEqual(EMPTY_ETAG, req_hdrs['Etag'])
        # verify there is no etag crypto meta
        self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs)
        # verify there is no container update override for etag
        self.assertNotIn(
            'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)

        # user meta is still encrypted
        self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
        self._verify_user_metadata(
            req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)

        # sysmeta is not encrypted
        self.assertEqual('do not encrypt me',
                         req_hdrs['X-Object-Sysmeta-Test'])

        # verify object is empty by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        resp = get_req.get_response(self.app)
        self.assertEqual('', resp.body)
        self.assertEqual(EMPTY_ETAG, resp.headers['Etag'])

    def _test_PUT_with_other_footers(self, override_etag):
        # verify handling of another middleware's footer callback
        body_key = os.urandom(32)
        object_key = fetch_crypto_keys()['object']
        plaintext = 'FAKE APP'
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)
        ciphertext_etag = md5hex(ciphertext)
        other_footers = {
            'Etag': plaintext_etag,
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Size':
                'other override',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
                override_etag}

        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'swift.callback.update_footers':
                   lambda footers: footers.update(other_footers)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(plaintext)),
                'Etag': 'correct etag is in footers'}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})

        with mock.patch(
            'swift.common.middleware.crypto.crypto_utils.'
            'Crypto.create_random_key',
                lambda *args: body_key):
            resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual('PUT', self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify that other middleware's footers made it to app, including any
        # container update overrides but nothing Etag-related
        other_footers.pop('Etag')
        other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify encryption footers are ok
        encrypted_etag, _junk, etag_meta = \
            req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
        self.assertTrue(etag_meta)
        actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])

        self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
        actual = base64.b64decode(encrypted_etag)
        etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)

        # verify encrypted etag for container update
        self.assertIn(
            'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
        parts = req_hdrs[
            'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
        self.assertEqual(2, len(parts))

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = 'swift_meta='
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len(crypto_meta_tag):]))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])

        cont_key = fetch_crypto_keys()['container']
        cont_etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
                         base64.b64decode(parts[0]))

        # verify body crypto meta
        actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
        actual = json.loads(urlparse.unquote_plus(actual))
        self.assertEqual(Crypto().cipher, actual['cipher'])
        self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))

        # verify wrapped body key
        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
        self.assertEqual(expected_wrapped_key,
                         base64.b64decode(actual['body_key']['key']))
        self.assertEqual(FAKE_IV,
                         base64.b64decode(actual['body_key']['iv']))
        self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])

    def test_PUT_with_other_footers(self):
        self._test_PUT_with_other_footers('override etag')

    def test_PUT_with_other_footers_and_etag_of_empty_body(self):
        # verify that an override etag value of EMPTY_ETAG will be encrypted
        # when there was a non-zero body length
        self._test_PUT_with_other_footers(EMPTY_ETAG)

    def _test_PUT_with_etag_override_in_headers(self, override_etag):
        # verify handling of another middleware's
        # container-update-override-etag in headers
        plaintext = 'FAKE APP'
        plaintext_etag = md5hex(plaintext)

        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(plaintext)),
                'Etag': plaintext_etag,
                'X-Object-Sysmeta-Container-Update-Override-Etag':
                    override_etag}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
        req_hdrs = self.app.headers[0]

        # verify encrypted etag for container update
        self.assertIn(
            'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
        parts = req_hdrs[
            'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
        self.assertEqual(2, len(parts))
        cont_key = fetch_crypto_keys()['container']

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = 'swift_meta='
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len(crypto_meta_tag):]))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])
        self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])

        cont_etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
                         base64.b64decode(parts[0]))

    def test_PUT_with_etag_override_in_headers(self):
        self._test_PUT_with_etag_override_in_headers('override_etag')

    def test_PUT_with_etag_of_empty_body_override_in_headers(self):
        # verify that an override etag value of EMPTY_ETAG will be encrypted
        # when there was a non-zero body length
        self._test_PUT_with_etag_override_in_headers(EMPTY_ETAG)

    def _test_PUT_with_empty_etag_override_in_headers(self, plaintext):
        # verify that an override etag value of '' from other middleware is
        # passed through unencrypted
        plaintext_etag = md5hex(plaintext)
        override_etag = ''
        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(plaintext)),
                'Etag': plaintext_etag,
                'X-Object-Sysmeta-Container-Update-Override-Etag':
                    override_etag}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
        req_hdrs = self.app.headers[0]
        self.assertIn(
            'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
        self.assertEqual(
            override_etag,
            req_hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'])

    def test_PUT_with_empty_etag_override_in_headers(self):
        self._test_PUT_with_empty_etag_override_in_headers('body')

    def test_PUT_with_empty_etag_override_in_headers_no_body(self):
        self._test_PUT_with_empty_etag_override_in_headers('')

    def _test_PUT_with_empty_etag_override_in_footers(self, plaintext):
        # verify that an override etag value of '' from other middleware is
        # passed through unencrypted
        plaintext_etag = md5hex(plaintext)
        override_etag = ''
        other_footers = {
            'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag}
        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'swift.callback.update_footers':
                   lambda footers: footers.update(other_footers)}

        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(plaintext)),
                'Etag': plaintext_etag}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
        req_hdrs = self.app.headers[0]
        self.assertIn(
            'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
        self.assertEqual(
            override_etag,
            req_hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'])

    def test_PUT_with_empty_etag_override_in_footers(self):
        self._test_PUT_with_empty_etag_override_in_footers('body')

    def test_PUT_with_empty_etag_override_in_footers_no_body(self):
        self._test_PUT_with_empty_etag_override_in_footers('')

    def test_PUT_with_bad_etag_in_other_footers(self):
        # verify that etag supplied in footers from other middleware overrides
        # header etag when validating inbound plaintext etags
        plaintext = 'FAKE APP'
        plaintext_etag = md5hex(plaintext)
        other_footers = {
            'Etag': 'bad etag',
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
                'other override'}

        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'swift.callback.update_footers':
                   lambda footers: footers.update(other_footers)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(plaintext)),
                'Etag': plaintext_etag}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('422 Unprocessable Entity', resp.status)
        self.assertNotIn('Etag', resp.headers)

    def test_PUT_with_bad_etag_in_headers_and_other_footers(self):
        # verify that etag supplied in headers from other middleware is used if
        # none is supplied in footers when validating inbound plaintext etags
        plaintext = 'FAKE APP'
        other_footers = {
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
                'other override'}

        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'swift.callback.update_footers':
                   lambda footers: footers.update(other_footers)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(plaintext)),
                'Etag': 'bad etag'}
        req = Request.blank(
            '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('422 Unprocessable Entity', resp.status)
        self.assertNotIn('Etag', resp.headers)

    def test_PUT_nothing_read(self):
        # simulate an artificial scenario of a downstream filter/app not
        # actually reading the input stream from encrypter.
        class NonReadingApp(object):
            def __call__(self, env, start_response):
                # note: no read from wsgi.input
                req = Request(env)
                env['swift.callback.update_footers'](req.headers)
                call_headers.append(req.headers)
                resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
                return resp(env, start_response)

        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'content-type': 'text/plain',
                'content-length': 0,
                'etag': 'etag from client'}
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))
        self.assertEqual('etag from client', call_headers[0]['etag'])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))

        # check that an upstream footer callback gets called
        other_footers = {
            'Etag': EMPTY_ETAG,
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
                'other override'}
        env.update({'swift.callback.update_footers':
                    lambda footers: footers.update(other_footers)})
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))

        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))

        # verify encrypted override etag for container update.
        self.assertIn(
            'X-Object-Sysmeta-Container-Update-Override-Etag', call_headers[0])
        parts = call_headers[0][
            'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
        self.assertEqual(2, len(parts))
        cont_key = fetch_crypto_keys()['container']

        param = parts[1].strip()
        crypto_meta_tag = 'swift_meta='
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(
            urlparse.unquote_plus(param[len(crypto_meta_tag):]))
        self.assertEqual(Crypto().cipher, actual_meta['cipher'])
        self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])

        cont_etag_iv = base64.b64decode(actual_meta['iv'])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt('other override', cont_key, cont_etag_iv),
                         base64.b64decode(parts[0]))

        # verify that other middleware's footers made it to app
        other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
        for k, v in other_footers.items():
            self.assertEqual(v, call_headers[0][k])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))

        # if upstream footer override etag is for an empty body then check that
        # it is not encrypted
        other_footers = {
            'Etag': EMPTY_ETAG,
            'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG}
        env.update({'swift.callback.update_footers':
                    lambda footers: footers.update(other_footers)})
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))

        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))

        # verify that other middleware's footers made it to app
        for k, v in other_footers.items():
            self.assertEqual(v, call_headers[0][k])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))

        # if upstream footer override etag is an empty string then check that
        # it is not encrypted
        other_footers = {
            'Etag': EMPTY_ETAG,
            'X-Object-Sysmeta-Container-Update-Override-Etag': ''}
        env.update({'swift.callback.update_footers':
                    lambda footers: footers.update(other_footers)})
        req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))

        self.assertEqual('201 Created', resp.status)
        self.assertEqual('response etag', resp.headers['Etag'])
        self.assertEqual(1, len(call_headers))

        # verify that other middleware's footers made it to app
        for k, v in other_footers.items():
            self.assertEqual(v, call_headers[0][k])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))

    def test_POST_req(self):
        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'POST',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'x-object-meta-test': 'encrypt me',
                'x-object-meta-test2': '',
                'x-object-sysmeta-test': 'do not encrypt me'}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        key = fetch_crypto_keys()['object']
        self.app.register('POST', '/v1/a/c/o', HTTPAccepted, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('202 Accepted', resp.status)
        self.assertNotIn('Etag', resp.headers)

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual('POST', self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # user meta is encrypted
        self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', key)
        # unless it had no value
        self.assertEqual('', req_hdrs['X-Object-Meta-Test2'])

        # sysmeta is not encrypted
        self.assertEqual('do not encrypt me',
                         req_hdrs['X-Object-Sysmeta-Test'])

    def _test_no_user_metadata(self, method):
        # verify that x-object-transient-sysmeta-crypto-meta is not set when
        # there is no user metadata
        env = {'REQUEST_METHOD': method,
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env, body='body')
        self.app.register(method, '/v1/a/c/o', HTTPAccepted, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('202 Accepted', resp.status)
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(method, self.app.calls[0][0])
        self.assertNotIn('x-object-transient-sysmeta-crypto-meta',
                         self.app.headers[0])

    def test_PUT_no_user_metadata(self):
        self._test_no_user_metadata('PUT')

    def test_POST_no_user_metadata(self):
        self._test_no_user_metadata('POST')

    def _test_if_match(self, method, match_header_name):
        def do_test(method, plain_etags, expected_plain_etags=None):
            env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
            match_header_value = ', '.join(plain_etags)
            req = Request.blank(
                '/v1/a/c/o', environ=env, method=method,
                headers={match_header_name: match_header_value})
            app = FakeSwift()
            app.register(method, '/v1/a/c/o', HTTPOk, {})
            resp = req.get_response(encrypter.Encrypter(app, {}))
            self.assertEqual('200 OK', resp.status)

            self.assertEqual(1, len(app.calls), app.calls)
            self.assertEqual(method, app.calls[0][0])
            actual_headers = app.headers[0]

            # verify the alternate etag location has been specified
            if match_header_value and match_header_value != '*':
                self.assertIn('X-Backend-Etag-Is-At', actual_headers)
                self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
                                 actual_headers['X-Backend-Etag-Is-At'])

            # verify etags have been supplemented with masked values
            self.assertIn(match_header_name, actual_headers)
            actual_etags = set(actual_headers[match_header_name].split(', '))
            # masked values for secret_id None
            key = fetch_crypto_keys()['object']
            masked_etags = [
                '"%s"' % base64.b64encode(hmac.new(
                    key, etag.strip('"'), hashlib.sha256).digest())
                for etag in plain_etags if etag not in ('*', '')]
            # masked values for secret_id myid
            key = fetch_crypto_keys(key_id={'secret_id': 'myid'})['object']
            masked_etags_myid = [
                '"%s"' % base64.b64encode(hmac.new(
                    key, etag.strip('"'), hashlib.sha256).digest())
                for etag in plain_etags if etag not in ('*', '')]
            expected_etags = set((expected_plain_etags or plain_etags) +
                                 masked_etags + masked_etags_myid)
            self.assertEqual(expected_etags, actual_etags)
            # check that the request environ was returned to original state
            self.assertEqual(set(plain_etags),
                             set(req.headers[match_header_name].split(', ')))

        do_test(method, [''])
        do_test(method, ['"an etag"'])
        do_test(method, ['"an etag"', '"another_etag"'])
        do_test(method, ['*'])
        # rfc2616 does not allow wildcard *and* etag but test it anyway
        do_test(method, ['*', '"an etag"'])
        # etags should be quoted but check we can cope if they are not
        do_test(
            method, ['*', 'an etag', 'another_etag'],
            expected_plain_etags=['*', '"an etag"', '"another_etag"'])

    def test_GET_if_match(self):
        self._test_if_match('GET', 'If-Match')

    def test_HEAD_if_match(self):
        self._test_if_match('HEAD', 'If-Match')

    def test_GET_if_none_match(self):
        self._test_if_match('GET', 'If-None-Match')

    def test_HEAD_if_none_match(self):
        self._test_if_match('HEAD', 'If-None-Match')

    def _test_existing_etag_is_at_header(self, method, match_header_name):
        # if another middleware has already set X-Backend-Etag-Is-At then
        # encrypter should not override that value
        env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank(
            '/v1/a/c/o', environ=env, method=method,
            headers={match_header_name: "an etag",
                     'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Other-Etag'})
        self.app.register(method, '/v1/a/c/o', HTTPOk, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('200 OK', resp.status)

        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(method, self.app.calls[0][0])
        actual_headers = self.app.headers[0]
        self.assertIn('X-Backend-Etag-Is-At', actual_headers)
        self.assertEqual(
            'X-Object-Sysmeta-Other-Etag,X-Object-Sysmeta-Crypto-Etag-Mac',
            actual_headers['X-Backend-Etag-Is-At'])
        actual_etags = set(actual_headers[match_header_name].split(', '))
        self.assertIn('"an etag"', actual_etags)

    def test_GET_if_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header('GET', 'If-Match')

    def test_HEAD_if_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header('HEAD', 'If-Match')

    def test_GET_if_none_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header('GET', 'If-None-Match')

    def test_HEAD_if_none_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header('HEAD', 'If-None-Match')

    def _test_etag_is_at_not_duplicated(self, method):
        # verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
        # X-Backend-Etag-Is-At
        key = fetch_crypto_keys()['object']
        env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank(
            '/v1/a/c/o', environ=env, method=method,
            headers={'If-Match': '"an etag"',
                     'If-None-Match': '"another etag"'})
        self.app.register(method, '/v1/a/c/o', HTTPOk, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('200 OK', resp.status)

        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(method, self.app.calls[0][0])
        actual_headers = self.app.headers[0]
        self.assertIn('X-Backend-Etag-Is-At', actual_headers)
        self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
                         actual_headers['X-Backend-Etag-Is-At'])

        self.assertIn('"%s"' % base64.b64encode(
            hmac.new(key, 'an etag', hashlib.sha256).digest()),
            actual_headers['If-Match'])
        self.assertIn('"another etag"', actual_headers['If-None-Match'])
        self.assertIn('"%s"' % base64.b64encode(
            hmac.new(key, 'another etag', hashlib.sha256).digest()),
            actual_headers['If-None-Match'])

    def test_GET_etag_is_at_not_duplicated(self):
        self._test_etag_is_at_not_duplicated('GET')

    def test_HEAD_etag_is_at_not_duplicated(self):
        self._test_etag_is_at_not_duplicated('HEAD')

    def test_PUT_response_inconsistent_etag_is_not_replaced(self):
        # if response is success but etag does not match the ciphertext md5
        # then verify that we do *not* replace it with the plaintext etag
        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated,
                          {'Etag': 'not the ciphertext etag'})
        resp = req.get_response(self.encrypter)
        self.assertEqual('201 Created', resp.status)
        self.assertEqual('not the ciphertext etag', resp.headers['Etag'])

    def test_PUT_multiseg_no_client_etag(self):
        body_key = os.urandom(32)
        chunks = ['some', 'chunks', 'of data']
        body = ''.join(chunks)
        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'wsgi.input': FileLikeIter(chunks)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})

        with mock.patch(
            'swift.common.middleware.crypto.crypto_utils.'
            'Crypto.create_random_key',
                lambda *args: body_key):
            resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        # verify object is encrypted by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        self.assertEqual(encrypt(body, body_key, FAKE_IV),
                         get_req.get_response(self.app).body)

    def test_PUT_multiseg_good_client_etag(self):
        body_key = os.urandom(32)
        chunks = ['some', 'chunks', 'of data']
        body = ''.join(chunks)
        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'wsgi.input': FileLikeIter(chunks)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body)),
                'Etag': md5hex(body)}
        req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})

        with mock.patch(
            'swift.common.middleware.crypto.crypto_utils.'
            'Crypto.create_random_key',
                lambda *args: body_key):
            resp = req.get_response(self.encrypter)

        self.assertEqual('201 Created', resp.status)
        # verify object is encrypted by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        self.assertEqual(encrypt(body, body_key, FAKE_IV),
                         get_req.get_response(self.app).body)

    def test_PUT_multiseg_bad_client_etag(self):
        chunks = ['some', 'chunks', 'of data']
        body = ''.join(chunks)
        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'wsgi.input': FileLikeIter(chunks)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body)),
                'Etag': 'badclientetag'}
        req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('422 Unprocessable Entity', resp.status)

    def test_PUT_missing_key_callback(self):
        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'PUT'}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        resp = req.get_response(self.encrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertIn('missing callback',
                      self.encrypter.logger.get_lines_for_level('error')[0])
        self.assertEqual('Unable to retrieve encryption keys.', resp.body)

    def test_PUT_error_in_key_callback(self):
        def raise_exc(*args, **kwargs):
            raise Exception('Testing')

        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'PUT',
               CRYPTO_KEY_CALLBACK: raise_exc}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        resp = req.get_response(self.encrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertIn('from callback: Testing',
                      self.encrypter.logger.get_lines_for_level('error')[0])
        self.assertEqual('Unable to retrieve encryption keys.', resp.body)

    def test_PUT_encryption_override(self):
        # set crypto override to disable encryption.
        # simulate another middleware wanting to set footers
        other_footers = {
            'Etag': 'other etag',
            'X-Object-Sysmeta-Other': 'other sysmeta',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
                'other override'}
        body = 'FAKE APP'
        env = {'REQUEST_METHOD': 'PUT',
               'swift.crypto.override': True,
               'swift.callback.update_footers':
                   lambda footers: footers.update(other_footers)}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual('201 Created', resp.status)

        # verify that other middleware's footers made it to app
        req_hdrs = self.app.headers[0]
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify object is NOT encrypted by getting direct from the app
        get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        self.assertEqual(body, get_req.get_response(self.app).body)

    def _test_constraints_checking(self, method):
        # verify that the check_metadata function is called on PUT and POST
        body = 'FAKE APP'
        env = {'REQUEST_METHOD': method,
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {'content-type': 'text/plain',
                'content-length': str(len(body))}
        req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
        mocked_func = 'swift.common.middleware.crypto.encrypter.check_metadata'
        with mock.patch(mocked_func) as mocked:
            mocked.side_effect = [HTTPBadRequest('testing')]
            resp = req.get_response(self.encrypter)
        self.assertEqual('400 Bad Request', resp.status)
        self.assertEqual(1, mocked.call_count)
        mocked.assert_called_once_with(mock.ANY, 'object')
        self.assertEqual(req.headers,
                         mocked.call_args_list[0][0][0].headers)

    def test_PUT_constraints_checking(self):
        self._test_constraints_checking('PUT')

    def test_POST_constraints_checking(self):
        self._test_constraints_checking('POST')

    def test_config_true_value_on_disable_encryption(self):
        app = FakeSwift()
        self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption)
        for val in ('true', '1', 'yes', 'on', 't', 'y'):
            app = encrypter.Encrypter(app,
                                      {'disable_encryption': val})
            self.assertTrue(app.disable_encryption)

    def test_PUT_app_exception(self):
        app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {})
        req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
        with self.assertRaises(HTTPException) as catcher:
            req.get_response(app)
        self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)

    def test_encrypt_header_val(self):
        # Prepare key and Crypto instance
        object_key = fetch_crypto_keys()['object']

        # - Normal string can be crypted
        encrypted = encrypter.encrypt_header_val(Crypto(), 'aaa', object_key)
        # sanity: return value is 2 item tuple
        self.assertEqual(2, len(encrypted))
        crypted_val, crypt_info = encrypted
        expected_crypt_val = base64.b64encode(
            encrypt('aaa', object_key, FAKE_IV))
        expected_crypt_info = {
            'cipher': 'AES_CTR_256', 'iv': 'This is an IV123'}
        self.assertEqual(expected_crypt_val, crypted_val)
        self.assertEqual(expected_crypt_info, crypt_info)

        # - Empty string raises a ValueError for safety
        with self.assertRaises(ValueError) as cm:
            encrypter.encrypt_header_val(Crypto(), '', object_key)

        self.assertEqual('empty value is not acceptable',
                         cm.exception.message)

        # - None also raises a ValueError for safety
        with self.assertRaises(ValueError) as cm:
            encrypter.encrypt_header_val(Crypto(), None, object_key)

        self.assertEqual('empty value is not acceptable',
                         cm.exception.message)
예제 #13
0
class TestDecrypterObjectRequests(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        self.decrypter = decrypter.Decrypter(self.app, {})
        self.decrypter.logger = FakeLogger()

    def _make_response_headers(self, content_length, plaintext_etag, keys,
                               body_key):
        # helper method to make a typical set of response headers for a GET or
        # HEAD request
        cont_key = keys['container']
        object_key = keys['object']
        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
                         'iv': FAKE_IV}
        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
        return HeaderKeyDict({
            'Etag': 'hashOfCiphertext',
            'content-type': 'text/plain',
            'content-length': content_length,
            'X-Object-Sysmeta-Crypto-Etag': '%s; swift_meta=%s' % (
                base64.b64encode(encrypt(plaintext_etag, object_key, FAKE_IV)),
                get_crypto_meta_header()),
            'X-Object-Sysmeta-Crypto-Body-Meta':
                get_crypto_meta_header(body_crypto_meta),
            'x-object-transient-sysmeta-crypto-meta-test':
                base64.b64encode(encrypt('encrypt me', object_key, FAKE_IV)) +
                ';swift_meta=' + get_crypto_meta_header(),
            'x-object-sysmeta-container-update-override-etag':
                encrypt_and_append_meta('encrypt me, too', cont_key),
            'x-object-sysmeta-test': 'do not encrypt me',
        })

    def _test_request_success(self, method, body):
        env = {'REQUEST_METHOD': method,
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        plaintext_etag = md5hex(body)
        body_key = os.urandom(32)
        enc_body = encrypt(body, body_key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)

        # there shouldn't be any x-object-meta- headers, but if there are
        # then the decrypted header will win where there is a name clash...
        hdrs.update({
            'x-object-meta-test': 'unexpected, overwritten by decrypted value',
            'x-object-meta-distinct': 'unexpected but distinct from encrypted'
        })
        self.app.register(
            method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('200 OK', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual('text/plain', resp.headers['Content-Type'])
        self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
        self.assertEqual('unexpected but distinct from encrypted',
                         resp.headers['x-object-meta-distinct'])
        self.assertEqual('do not encrypt me',
                         resp.headers['x-object-sysmeta-test'])
        self.assertEqual(
            'encrypt me, too',
            resp.headers['X-Object-Sysmeta-Container-Update-Override-Etag'])
        self.assertNotIn('X-Object-Sysmeta-Crypto-Body-Meta', resp.headers)
        self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', resp.headers)
        return resp

    def test_GET_success(self):
        body = 'FAKE APP'
        resp = self._test_request_success('GET', body)
        self.assertEqual(body, resp.body)

    def test_HEAD_success(self):
        body = 'FAKE APP'
        resp = self._test_request_success('HEAD', body)
        self.assertEqual('', resp.body)

    def test_headers_case(self):
        body = 'fAkE ApP'
        req = Request.blank('/v1/a/c/o', body='FaKe')
        req.environ[CRYPTO_KEY_CALLBACK] = fetch_crypto_keys
        plaintext_etag = md5hex(body)
        body_key = os.urandom(32)
        enc_body = encrypt(body, body_key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)

        hdrs.update({
            'x-Object-mEta-ignoRes-caSe': 'thIs pArt WilL bE cOol',
        })
        self.app.register(
            'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)

        status, headers, app_iter = req.call_application(self.decrypter)
        self.assertEqual(status, '200 OK')
        expected = {
            'Etag': '7f7837924188f7b511a9e3881a9f77a8',
            'X-Object-Sysmeta-Container-Update-Override-Etag':
            'encrypt me, too',
            'X-Object-Meta-Test': 'encrypt me',
            'Content-Length': '8',
            'X-Object-Meta-Ignores-Case': 'thIs pArt WilL bE cOol',
            'X-Object-Sysmeta-Test': 'do not encrypt me',
            'Content-Type': 'text/plain',
        }
        self.assertEqual(dict(headers), expected)
        self.assertEqual('fAkE ApP', ''.join(app_iter))

    def _test_412_response(self, method):
        # simulate a 412 response to a conditional GET which has an Etag header
        data = 'the object content'
        env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env, method=method)
        resp_body = 'I am sorry, you have failed to meet a precondition'
        hdrs = self._make_response_headers(
            len(resp_body), md5hex(data), fetch_crypto_keys(), 'not used')
        self.app.register(method, '/v1/a/c/o', HTTPPreconditionFailed,
                          body=resp_body, headers=hdrs)
        resp = req.get_response(self.decrypter)

        self.assertEqual('412 Precondition Failed', resp.status)
        # the response body should not be decrypted, it is already plaintext
        self.assertEqual(resp_body if method == 'GET' else '', resp.body)
        # whereas the Etag and other headers should be decrypted
        self.assertEqual(md5hex(data), resp.headers['Etag'])
        self.assertEqual('text/plain', resp.headers['Content-Type'])
        self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
        self.assertEqual('do not encrypt me',
                         resp.headers['x-object-sysmeta-test'])

    def test_GET_412_response(self):
        self._test_412_response('GET')

    def test_HEAD_412_response(self):
        self._test_412_response('HEAD')

    def _test_404_response(self, method):
        # simulate a 404 response, sanity check response headers
        env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env, method=method)
        resp_body = 'You still have not found what you are looking for'
        hdrs = {'content-type': 'text/plain',
                'content-length': len(resp_body)}
        self.app.register(method, '/v1/a/c/o', HTTPNotFound,
                          body=resp_body, headers=hdrs)
        resp = req.get_response(self.decrypter)

        self.assertEqual('404 Not Found', resp.status)
        # the response body should not be decrypted, it is already plaintext
        self.assertEqual(resp_body if method == 'GET' else '', resp.body)
        # there should be no etag header inserted by decrypter
        self.assertNotIn('Etag', resp.headers)
        self.assertEqual('text/plain', resp.headers['Content-Type'])

    def test_GET_404_response(self):
        self._test_404_response('GET')

    def test_HEAD_404_response(self):
        self._test_404_response('HEAD')

    def test_GET_missing_etag_crypto_meta(self):
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        key = fetch_crypto_keys()['object']
        enc_body = encrypt(body, key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), md5hex(body), fetch_crypto_keys(), 'not used')
        # simulate missing crypto meta from encrypted etag
        hdrs['X-Object-Sysmeta-Crypto-Etag'] = \
            base64.b64encode(encrypt(md5hex(body), key, FAKE_IV))
        self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
                          headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertIn('Error decrypting header', resp.body)
        self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def _test_override_etag_bad_meta(self, method, bad_crypto_meta):
        env = {'REQUEST_METHOD': method,
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        key = fetch_crypto_keys()['object']
        enc_body = encrypt(body, key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), md5hex(body), fetch_crypto_keys(), 'not used')
        # simulate missing crypto meta from encrypted override etag
        hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
            encrypt_and_append_meta(
                md5hex(body), key, crypto_meta=bad_crypto_meta)
        self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
                          headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertIn('Error decrypting header '
                      'X-Object-Sysmeta-Container-Update-Override-Etag',
                      self.decrypter.logger.get_lines_for_level('error')[0])
        return resp

    def test_GET_override_etag_bad_iv(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['iv'] = 'bad_iv'
        resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta)
        self.assertIn('Error decrypting header', resp.body)

    def test_HEAD_override_etag_bad_iv(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['iv'] = 'bad_iv'
        resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta)
        self.assertEqual('', resp.body)

    def test_GET_override_etag_bad_cipher(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['cipher'] = 'unknown cipher'
        resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta)
        self.assertIn('Error decrypting header', resp.body)

    def test_HEAD_override_etag_bad_cipher(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['cipher'] = 'unknown cipher'
        resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta)
        self.assertEqual('', resp.body)

    def _test_bad_key(self, method):
        # use bad key
        def bad_fetch_crypto_keys():
            keys = fetch_crypto_keys()
            keys['object'] = 'bad key'
            return keys

        env = {'REQUEST_METHOD': method,
               CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        key = fetch_crypto_keys()['object']
        enc_body = encrypt(body, key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), md5hex(body), fetch_crypto_keys(), 'not used')
        self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
                          headers=hdrs)
        return req.get_response(self.decrypter)

    def test_HEAD_with_bad_key(self):
        resp = self._test_bad_key('HEAD')
        self.assertEqual('500 Internal Error', resp.status)
        self.assertIn("Bad key for 'object'",
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_bad_key(self):
        resp = self._test_bad_key('GET')
        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Unable to retrieve encryption keys.',
                         resp.body)
        self.assertIn("Bad key for 'object'",
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def _test_bad_crypto_meta_for_user_metadata(self, method, bad_crypto_meta):
        # use bad iv for metadata headers
        env = {'REQUEST_METHOD': method,
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        key = fetch_crypto_keys()['object']
        enc_body = encrypt(body, key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), md5hex(body), fetch_crypto_keys(), 'not used')
        enc_val = base64.b64encode(encrypt('encrypt me', key, FAKE_IV))
        if bad_crypto_meta:
            enc_val += ';swift_meta=' + get_crypto_meta_header(
                crypto_meta=bad_crypto_meta)
        hdrs['x-object-transient-sysmeta-crypto-meta-test'] = enc_val
        self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
                          headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertIn(
            'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
            'Test', self.decrypter.logger.get_lines_for_level('error')[0])
        return resp

    def test_HEAD_with_missing_crypto_meta_for_user_metadata(self):
        self._test_bad_crypto_meta_for_user_metadata('HEAD', None)
        self.assertIn('Missing crypto meta in value',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_missing_crypto_meta_for_user_metadata(self):
        self._test_bad_crypto_meta_for_user_metadata('GET', None)
        self.assertIn('Missing crypto meta in value',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_HEAD_with_bad_iv_for_user_metadata(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['iv'] = 'bad_iv'
        self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta)
        self.assertIn('IV must be length 16',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_HEAD_with_missing_iv_for_user_metadata(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta.pop('iv')
        self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta)
        self.assertIn(
            'iv', self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_bad_iv_for_user_metadata(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['iv'] = 'bad_iv'
        resp = self._test_bad_crypto_meta_for_user_metadata(
            'GET', bad_crypto_meta)
        self.assertEqual('Error decrypting header', resp.body)
        self.assertIn('IV must be length 16',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_missing_iv_for_user_metadata(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta.pop('iv')
        resp = self._test_bad_crypto_meta_for_user_metadata(
            'GET', bad_crypto_meta)
        self.assertEqual('Error decrypting header', resp.body)
        self.assertIn(
            'iv', self.decrypter.logger.get_lines_for_level('error')[0])

    def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta):
        # use bad iv for object body
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        key = fetch_crypto_keys()['object']
        enc_body = encrypt(body, key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), md5hex(body), fetch_crypto_keys(), 'not used')
        hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
            get_crypto_meta_header(crypto_meta=bad_crypto_meta)
        self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
                          headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Error decrypting object', resp.body)
        self.assertIn('Error decrypting object',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_bad_iv_for_object_body(self):
        bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32))
        bad_crypto_meta['iv'] = 'bad_iv'
        self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
        self.assertIn('IV must be length 16',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_missing_iv_for_object_body(self):
        bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32))
        bad_crypto_meta.pop('iv')
        self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
        self.assertIn("Missing 'iv'",
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_bad_body_key_for_object_body(self):
        body_key_meta = {'key': 'wrapped too short key', 'iv': FAKE_IV}
        bad_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
        self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
        self.assertIn('Key must be length 32',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_with_missing_body_key_for_object_body(self):
        bad_crypto_meta = fake_get_crypto_meta()  # no key by default
        self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
        self.assertIn("Missing 'body_key'",
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def _test_req_metadata_not_encrypted(self, method):
        # check that metadata is not decrypted if it does not have crypto meta;
        # testing for case of an unencrypted POST to an object.
        env = {'REQUEST_METHOD': method,
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        plaintext_etag = md5hex(body)
        body_key = os.urandom(32)
        enc_body = encrypt(body, body_key, FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), plaintext_etag, fetch_crypto_keys(), body_key)
        hdrs.pop('x-object-transient-sysmeta-crypto-meta-test')
        hdrs['x-object-meta-test'] = 'plaintext not encrypted'
        self.app.register(
            method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('200 OK', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual('text/plain', resp.headers['Content-Type'])
        self.assertEqual('plaintext not encrypted',
                         resp.headers['x-object-meta-test'])

    def test_HEAD_metadata_not_encrypted(self):
        self._test_req_metadata_not_encrypted('HEAD')

    def test_GET_metadata_not_encrypted(self):
        self._test_req_metadata_not_encrypted('GET')

    def test_GET_unencrypted_data(self):
        # testing case of an unencrypted object with encrypted metadata from
        # a later POST
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        obj_key = fetch_crypto_keys()['object']
        hdrs = {'Etag': md5hex(body),
                'content-type': 'text/plain',
                'content-length': len(body),
                'x-object-transient-sysmeta-crypto-meta-test':
                    base64.b64encode(encrypt('encrypt me', obj_key, FAKE_IV)) +
                    ';swift_meta=' + get_crypto_meta_header(),
                'x-object-sysmeta-test': 'do not encrypt me'}
        self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual(body, resp.body)
        self.assertEqual('200 OK', resp.status)
        self.assertEqual(md5hex(body), resp.headers['Etag'])
        self.assertEqual('text/plain', resp.headers['Content-Type'])
        # POSTed user meta was encrypted
        self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
        # PUT sysmeta was not encrypted
        self.assertEqual('do not encrypt me',
                         resp.headers['x-object-sysmeta-test'])

    def test_GET_multiseg(self):
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        chunks = ['some', 'chunks', 'of data']
        body = ''.join(chunks)
        plaintext_etag = md5hex(body)
        body_key = os.urandom(32)
        ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
        enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
        hdrs = self._make_response_headers(
            sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(),
            body_key)
        self.app.register(
            'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual(body, resp.body)
        self.assertEqual('200 OK', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual('text/plain', resp.headers['Content-Type'])

    def test_GET_multiseg_with_range(self):
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        req.headers['Content-Range'] = 'bytes 3-10/17'
        chunks = ['0123', '45678', '9abcdef']
        body = ''.join(chunks)
        plaintext_etag = md5hex(body)
        body_key = os.urandom(32)
        ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
        enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
        enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]]
        hdrs = self._make_response_headers(
            sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(),
            body_key)
        hdrs['content-range'] = req.headers['Content-Range']
        self.app.register(
            'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('3456789a', resp.body)
        self.assertEqual('200 OK', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual('text/plain', resp.headers['Content-Type'])

    # Force the decrypter context updates to be less than one of our range
    # sizes to check that the decrypt context offset is setup correctly with
    # offset to first byte of range for first update and then re-used.
    # Do mocking here to have the mocked value have effect in the generator
    # function.
    @mock.patch.object(decrypter, 'DECRYPT_CHUNK_SIZE', 4)
    def test_GET_multipart_ciphertext(self):
        # build fake multipart response body
        body_key = os.urandom(32)
        plaintext = 'Cwm fjord veg balks nth pyx quiz'
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)
        parts = ((0, 3, 'text/plain'),
                 (4, 9, 'text/plain; charset=us-ascii'),
                 (24, 32, 'text/plain'))
        length = len(ciphertext)
        body = ''
        for start, end, ctype in parts:
            body += '--multipartboundary\r\n'
            body += 'Content-Type: %s\r\n' % ctype
            body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
            body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
        body += '--multipartboundary--'

        # register request with fake swift
        hdrs = self._make_response_headers(
            len(body), plaintext_etag, fetch_crypto_keys(), body_key)
        hdrs['content-type'] = \
            'multipart/byteranges;boundary=multipartboundary'
        self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body,
                          headers=hdrs)

        # issue request
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        resp = req.get_response(self.decrypter)

        self.assertEqual('206 Partial Content', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual(len(body), int(resp.headers['Content-Length']))
        self.assertEqual('multipart/byteranges;boundary=multipartboundary',
                         resp.headers['Content-Type'])

        # the multipart headers could be re-ordered, so parse response body to
        # verify expected content
        resp_lines = resp.body.split('\r\n')
        resp_lines.reverse()
        for start, end, ctype in parts:
            self.assertEqual('--multipartboundary', resp_lines.pop())
            expected_header_lines = {
                'Content-Type: %s' % ctype,
                'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)}
            resp_header_lines = {resp_lines.pop(), resp_lines.pop()}
            self.assertEqual(expected_header_lines, resp_header_lines)
            self.assertEqual('', resp_lines.pop())
            self.assertEqual(plaintext[start:end], resp_lines.pop())
        self.assertEqual('--multipartboundary--', resp_lines.pop())

        # we should have consumed the whole response body
        self.assertFalse(resp_lines)

    def test_GET_multipart_content_type(self):
        # *just* having multipart content type shouldn't trigger the mime doc
        # code path
        body_key = os.urandom(32)
        plaintext = 'Cwm fjord veg balks nth pyx quiz'
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)

        # register request with fake swift
        hdrs = self._make_response_headers(
            len(ciphertext), plaintext_etag, fetch_crypto_keys(), body_key)
        hdrs['content-type'] = \
            'multipart/byteranges;boundary=multipartboundary'
        self.app.register('GET', '/v1/a/c/o', HTTPOk, body=ciphertext,
                          headers=hdrs)

        # issue request
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        resp = req.get_response(self.decrypter)

        self.assertEqual('200 OK', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual(len(plaintext), int(resp.headers['Content-Length']))
        self.assertEqual('multipart/byteranges;boundary=multipartboundary',
                         resp.headers['Content-Type'])
        self.assertEqual(plaintext, resp.body)

    def test_GET_multipart_no_body_crypto_meta(self):
        # build fake multipart response body
        plaintext = 'Cwm fjord veg balks nth pyx quiz'
        plaintext_etag = md5hex(plaintext)
        parts = ((0, 3, 'text/plain'),
                 (4, 9, 'text/plain; charset=us-ascii'),
                 (24, 32, 'text/plain'))
        length = len(plaintext)
        body = ''
        for start, end, ctype in parts:
            body += '--multipartboundary\r\n'
            body += 'Content-Type: %s\r\n' % ctype
            body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
            body += '\r\n\r\n' + plaintext[start:end] + '\r\n'
        body += '--multipartboundary--'

        # register request with fake swift
        hdrs = {
            'Etag': plaintext_etag,
            'content-type': 'multipart/byteranges;boundary=multipartboundary',
            'content-length': len(body)}
        self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body,
                          headers=hdrs)

        # issue request
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        resp = req.get_response(self.decrypter)

        self.assertEqual('206 Partial Content', resp.status)
        self.assertEqual(plaintext_etag, resp.headers['Etag'])
        self.assertEqual(len(body), int(resp.headers['Content-Length']))
        self.assertEqual('multipart/byteranges;boundary=multipartboundary',
                         resp.headers['Content-Type'])

        # the multipart response body should be unchanged
        self.assertEqual(body, resp.body)

    def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta):
        # build fake multipart response body
        key = fetch_crypto_keys()['object']
        ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV)
        plaintext = 'Cwm fjord veg balks nth pyx quiz'
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, ctxt=ctxt)
        parts = ((0, 3, 'text/plain'),
                 (4, 9, 'text/plain; charset=us-ascii'),
                 (24, 32, 'text/plain'))
        length = len(ciphertext)
        body = ''
        for start, end, ctype in parts:
            body += '--multipartboundary\r\n'
            body += 'Content-Type: %s\r\n' % ctype
            body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
            body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
        body += '--multipartboundary--'

        # register request with fake swift
        hdrs = self._make_response_headers(
            len(body), plaintext_etag, fetch_crypto_keys(), 'not used')
        hdrs['content-type'] = \
            'multipart/byteranges;boundary=multipartboundary'
        hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
            get_crypto_meta_header(bad_crypto_meta)
        self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)

        # issue request
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        resp = req.get_response(self.decrypter)

        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Error decrypting object', resp.body)
        self.assertIn('Error decrypting object',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_multipart_bad_body_cipher(self):
        self._test_GET_multipart_bad_body_crypto_meta(
            {'cipher': 'Mystery cipher', 'iv': '1234567887654321'})
        self.assertIn('Cipher must be AES_CTR_256',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_multipart_missing_body_cipher(self):
        self._test_GET_multipart_bad_body_crypto_meta(
            {'iv': '1234567887654321'})
        self.assertIn('cipher',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_multipart_too_short_body_iv(self):
        self._test_GET_multipart_bad_body_crypto_meta(
            {'cipher': 'AES_CTR_256', 'iv': 'too short'})
        self.assertIn('IV must be length 16',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_multipart_too_long_body_iv(self):
        self._test_GET_multipart_bad_body_crypto_meta(
            {'cipher': 'AES_CTR_256', 'iv': 'a little too long'})
        self.assertIn('IV must be length 16',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_multipart_missing_body_iv(self):
        self._test_GET_multipart_bad_body_crypto_meta(
            {'cipher': 'AES_CTR_256'})
        self.assertIn('iv',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_missing_key_callback(self):
        # Do not provide keys, and do not set override flag
        env = {'REQUEST_METHOD': 'GET'}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), md5hex('not the body'), fetch_crypto_keys(), 'not used')
        self.app.register(
            'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Unable to retrieve encryption keys.',
                         resp.body)
        self.assertIn('missing callback',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_error_in_key_callback(self):
        def raise_exc():
            raise Exception('Testing')

        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: raise_exc}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
        hdrs = self._make_response_headers(
            len(body), md5hex(body), fetch_crypto_keys(), 'not used')
        self.app.register(
            'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Unable to retrieve encryption keys.',
                         resp.body)
        self.assertIn('from callback: Testing',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_cipher_mismatch_for_body(self):
        # Cipher does not match
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['cipher'] = 'unknown_cipher'
        hdrs = self._make_response_headers(
            len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used')
        hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
            get_crypto_meta_header(crypto_meta=bad_crypto_meta)
        self.app.register(
            'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Error decrypting object', resp.body)
        self.assertIn('Error decrypting object',
                      self.decrypter.logger.get_lines_for_level('error')[0])
        self.assertIn('Bad crypto meta: Cipher',
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_cipher_mismatch_for_metadata(self):
        # Cipher does not match
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        key = fetch_crypto_keys()['object']
        enc_body = encrypt(body, key, FAKE_IV)
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['cipher'] = 'unknown_cipher'
        hdrs = self._make_response_headers(
            len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used')
        hdrs.update({'x-object-transient-sysmeta-crypto-meta-test':
                     base64.b64encode(encrypt('encrypt me', key, FAKE_IV)) +
                     ';swift_meta=' +
                     get_crypto_meta_header(crypto_meta=bad_crypto_meta)})
        self.app.register(
            'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Error decrypting header', resp.body)
        self.assertIn(
            'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
            'Test', self.decrypter.logger.get_lines_for_level('error')[0])

    def test_GET_decryption_override(self):
        # This covers the case of an old un-encrypted object
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
               'swift.crypto.override': True}
        req = Request.blank('/v1/a/c/o', environ=env)
        body = 'FAKE APP'
        hdrs = {'Etag': md5hex(body),
                'content-type': 'text/plain',
                'content-length': len(body),
                'x-object-meta-test': 'do not encrypt me',
                'x-object-sysmeta-test': 'do not encrypt me'}
        self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
        resp = req.get_response(self.decrypter)
        self.assertEqual(body, resp.body)
        self.assertEqual('200 OK', resp.status)
        self.assertEqual(md5hex(body), resp.headers['Etag'])
        self.assertEqual('text/plain', resp.headers['Content-Type'])
        self.assertEqual('do not encrypt me',
                         resp.headers['x-object-meta-test'])
        self.assertEqual('do not encrypt me',
                         resp.headers['x-object-sysmeta-test'])
예제 #14
0
class TestEncrypter(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        self.encrypter = encrypter.Encrypter(self.app, {})
        self.encrypter.logger = FakeLogger()

    def _verify_user_metadata(self, req_hdrs, name, value, key):
        # verify encrypted version of user metadata
        self.assertNotIn("X-Object-Meta-" + name, req_hdrs)
        expected_hdr = "X-Object-Transient-Sysmeta-Crypto-Meta-" + name
        self.assertIn(expected_hdr, req_hdrs)
        enc_val, param = req_hdrs[expected_hdr].split(";")
        param = param.strip()
        self.assertTrue(param.startswith("swift_meta="))
        actual_meta = json.loads(urllib.unquote_plus(param[len("swift_meta=") :]))
        self.assertEqual(Crypto.cipher, actual_meta["cipher"])
        meta_iv = base64.b64decode(actual_meta["iv"])
        self.assertEqual(FAKE_IV, meta_iv)
        self.assertEqual(base64.b64encode(encrypt(value, key, meta_iv)), enc_val)
        # if there is any encrypted user metadata then this header should exist
        self.assertIn("X-Object-Transient-Sysmeta-Crypto-Meta", req_hdrs)
        common_meta = json.loads(urllib.unquote_plus(req_hdrs["X-Object-Transient-Sysmeta-Crypto-Meta"]))
        self.assertDictEqual({"cipher": Crypto.cipher, "key_id": {"v": "fake", "path": "/a/c/fake"}}, common_meta)

    def test_PUT_req(self):
        body_key = os.urandom(32)
        object_key = fetch_crypto_keys()["object"]
        plaintext = "FAKE APP"
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)
        ciphertext_etag = md5hex(ciphertext)

        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {
            "etag": plaintext_etag,
            "content-type": "text/plain",
            "content-length": str(len(plaintext)),
            "x-object-meta-etag": "not to be confused with the Etag!",
            "x-object-meta-test": "encrypt me",
            "x-object-sysmeta-test": "do not encrypt me",
        }
        req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        with mock.patch(
            "swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", return_value=body_key
        ):
            resp = req.get_response(self.encrypter)
        self.assertEqual("201 Created", resp.status)
        self.assertEqual(plaintext_etag, resp.headers["Etag"])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual("PUT", self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify body crypto meta
        actual = req_hdrs["X-Object-Sysmeta-Crypto-Body-Meta"]
        actual = json.loads(urllib.unquote_plus(actual))
        self.assertEqual(Crypto().cipher, actual["cipher"])
        self.assertEqual(FAKE_IV, base64.b64decode(actual["iv"]))

        # verify wrapped body key
        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
        self.assertEqual(expected_wrapped_key, base64.b64decode(actual["body_key"]["key"]))
        self.assertEqual(FAKE_IV, base64.b64decode(actual["body_key"]["iv"]))
        self.assertEqual(fetch_crypto_keys()["id"], actual["key_id"])

        # verify etag
        self.assertEqual(ciphertext_etag, req_hdrs["Etag"])

        encrypted_etag, _junk, etag_meta = req_hdrs["X-Object-Sysmeta-Crypto-Etag"].partition("; swift_meta=")
        # verify crypto_meta was appended to this etag
        self.assertTrue(etag_meta)
        actual_meta = json.loads(urllib.unquote_plus(etag_meta))
        self.assertEqual(Crypto().cipher, actual_meta["cipher"])

        # verify encrypted version of plaintext etag
        actual = base64.b64decode(encrypted_etag)
        etag_iv = base64.b64decode(actual_meta["iv"])
        enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
        self.assertEqual(enc_etag, actual)

        # verify etag MAC for conditional requests
        actual_hmac = base64.b64decode(req_hdrs["X-Object-Sysmeta-Crypto-Etag-Mac"])
        self.assertEqual(actual_hmac, hmac.new(object_key, plaintext_etag, hashlib.sha256).digest())

        # verify encrypted etag for container update
        self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
        parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
        self.assertEqual(2, len(parts))

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = "swift_meta="
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
        self.assertEqual(Crypto().cipher, actual_meta["cipher"])
        self.assertEqual(fetch_crypto_keys()["id"], actual_meta["key_id"])

        cont_key = fetch_crypto_keys()["container"]
        cont_etag_iv = base64.b64decode(actual_meta["iv"])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv), base64.b64decode(parts[0]))

        # content-type is not encrypted
        self.assertEqual("text/plain", req_hdrs["Content-Type"])

        # user meta is encrypted
        self._verify_user_metadata(req_hdrs, "Test", "encrypt me", object_key)
        self._verify_user_metadata(req_hdrs, "Etag", "not to be confused with the Etag!", object_key)

        # sysmeta is not encrypted
        self.assertEqual("do not encrypt me", req_hdrs["X-Object-Sysmeta-Test"])

        # verify object is encrypted by getting direct from the app
        get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
        resp = get_req.get_response(self.app)
        self.assertEqual(ciphertext, resp.body)
        self.assertEqual(ciphertext_etag, resp.headers["Etag"])

    def test_PUT_zero_size_object(self):
        # object body encryption should be skipped for zero sized object body
        object_key = fetch_crypto_keys()["object"]
        plaintext_etag = EMPTY_ETAG

        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {
            "etag": EMPTY_ETAG,
            "content-type": "text/plain",
            "content-length": "0",
            "x-object-meta-etag": "not to be confused with the Etag!",
            "x-object-meta-test": "encrypt me",
            "x-object-sysmeta-test": "do not encrypt me",
        }
        req = Request.blank("/v1/a/c/o", environ=env, body="", headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})

        resp = req.get_response(self.encrypter)

        self.assertEqual("201 Created", resp.status)
        self.assertEqual(plaintext_etag, resp.headers["Etag"])
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual("PUT", self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify that there is no body crypto meta
        self.assertNotIn("X-Object-Sysmeta-Crypto-Meta", req_hdrs)
        # verify etag is md5 of plaintext
        self.assertEqual(EMPTY_ETAG, req_hdrs["Etag"])
        # verify there is no etag crypto meta
        self.assertNotIn("X-Object-Sysmeta-Crypto-Etag", req_hdrs)
        # verify there is no container update override for etag
        self.assertNotIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)

        # user meta is still encrypted
        self._verify_user_metadata(req_hdrs, "Test", "encrypt me", object_key)
        self._verify_user_metadata(req_hdrs, "Etag", "not to be confused with the Etag!", object_key)

        # sysmeta is not encrypted
        self.assertEqual("do not encrypt me", req_hdrs["X-Object-Sysmeta-Test"])

        # verify object is empty by getting direct from the app
        get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
        resp = get_req.get_response(self.app)
        self.assertEqual("", resp.body)
        self.assertEqual(EMPTY_ETAG, resp.headers["Etag"])

    def test_PUT_with_other_footers(self):
        # verify handling of another middleware's footer callback
        cont_key = fetch_crypto_keys()["container"]
        body_key = os.urandom(32)
        object_key = fetch_crypto_keys()["object"]
        plaintext = "FAKE APP"
        plaintext_etag = md5hex(plaintext)
        ciphertext = encrypt(plaintext, body_key, FAKE_IV)
        ciphertext_etag = md5hex(ciphertext)
        other_footers = {
            "Etag": plaintext_etag,
            "X-Object-Sysmeta-Other": "other sysmeta",
            "X-Object-Sysmeta-Container-Update-Override-Size": "other override",
            "X-Object-Sysmeta-Container-Update-Override-Etag": "final etag",
        }

        env = {
            "REQUEST_METHOD": "PUT",
            CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
            "swift.callback.update_footers": lambda footers: footers.update(other_footers),
        }
        hdrs = {
            "content-type": "text/plain",
            "content-length": str(len(plaintext)),
            "Etag": "correct etag is in footers",
        }
        req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})

        with mock.patch(
            "swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", lambda *args: body_key
        ):
            resp = req.get_response(self.encrypter)

        self.assertEqual("201 Created", resp.status)
        self.assertEqual(plaintext_etag, resp.headers["Etag"])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual("PUT", self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # verify that other middleware's footers made it to app, including any
        # container update overrides but nothing Etag-related
        other_footers.pop("Etag")
        other_footers.pop("X-Object-Sysmeta-Container-Update-Override-Etag")
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify encryption footers are ok
        encrypted_etag, _junk, etag_meta = req_hdrs["X-Object-Sysmeta-Crypto-Etag"].partition("; swift_meta=")
        self.assertTrue(etag_meta)
        actual_meta = json.loads(urllib.unquote_plus(etag_meta))
        self.assertEqual(Crypto().cipher, actual_meta["cipher"])

        self.assertEqual(ciphertext_etag, req_hdrs["Etag"])
        actual = base64.b64decode(encrypted_etag)
        etag_iv = base64.b64decode(actual_meta["iv"])
        self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)

        # verify encrypted etag for container update
        self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
        parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
        self.assertEqual(2, len(parts))

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = "swift_meta="
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
        self.assertEqual(Crypto().cipher, actual_meta["cipher"])

        cont_key = fetch_crypto_keys()["container"]
        cont_etag_iv = base64.b64decode(actual_meta["iv"])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt("final etag", cont_key, cont_etag_iv), base64.b64decode(parts[0]))

        # verify body crypto meta
        actual = req_hdrs["X-Object-Sysmeta-Crypto-Body-Meta"]
        actual = json.loads(urllib.unquote_plus(actual))
        self.assertEqual(Crypto().cipher, actual["cipher"])
        self.assertEqual(FAKE_IV, base64.b64decode(actual["iv"]))

        # verify wrapped body key
        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
        self.assertEqual(expected_wrapped_key, base64.b64decode(actual["body_key"]["key"]))
        self.assertEqual(FAKE_IV, base64.b64decode(actual["body_key"]["iv"]))
        self.assertEqual(fetch_crypto_keys()["id"], actual["key_id"])

    def test_PUT_with_etag_override_in_headers(self):
        # verify handling of another middleware's
        # container-update-override-etag in headers
        plaintext = "FAKE APP"
        plaintext_etag = md5hex(plaintext)

        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {
            "content-type": "text/plain",
            "content-length": str(len(plaintext)),
            "Etag": plaintext_etag,
            "X-Object-Sysmeta-Container-Update-Override-Etag": "final etag",
        }
        req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        resp = req.get_response(self.encrypter)

        self.assertEqual("201 Created", resp.status)
        self.assertEqual(plaintext_etag, resp.headers["Etag"])

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(("PUT", "/v1/a/c/o"), self.app.calls[0])
        req_hdrs = self.app.headers[0]

        # verify encrypted etag for container update
        self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
        parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
        self.assertEqual(2, len(parts))
        cont_key = fetch_crypto_keys()["container"]

        # extract crypto_meta from end of etag for container update
        param = parts[1].strip()
        crypto_meta_tag = "swift_meta="
        self.assertTrue(param.startswith(crypto_meta_tag), param)
        actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
        self.assertEqual(Crypto().cipher, actual_meta["cipher"])
        self.assertEqual(fetch_crypto_keys()["id"], actual_meta["key_id"])

        cont_etag_iv = base64.b64decode(actual_meta["iv"])
        self.assertEqual(FAKE_IV, cont_etag_iv)
        self.assertEqual(encrypt("final etag", cont_key, cont_etag_iv), base64.b64decode(parts[0]))

    def test_PUT_with_bad_etag_in_other_footers(self):
        # verify that etag supplied in footers from other middleware overrides
        # header etag when validating inbound plaintext etags
        plaintext = "FAKE APP"
        plaintext_etag = md5hex(plaintext)
        other_footers = {
            "Etag": "bad etag",
            "X-Object-Sysmeta-Other": "other sysmeta",
            "X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
        }

        env = {
            "REQUEST_METHOD": "PUT",
            CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
            "swift.callback.update_footers": lambda footers: footers.update(other_footers),
        }
        hdrs = {"content-type": "text/plain", "content-length": str(len(plaintext)), "Etag": plaintext_etag}
        req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("422 Unprocessable Entity", resp.status)
        self.assertNotIn("Etag", resp.headers)

    def test_PUT_with_bad_etag_in_headers_and_other_footers(self):
        # verify that etag supplied in headers from other middleware is used if
        # none is supplied in footers when validating inbound plaintext etags
        plaintext = "FAKE APP"
        other_footers = {
            "X-Object-Sysmeta-Other": "other sysmeta",
            "X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
        }

        env = {
            "REQUEST_METHOD": "PUT",
            CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
            "swift.callback.update_footers": lambda footers: footers.update(other_footers),
        }
        hdrs = {"content-type": "text/plain", "content-length": str(len(plaintext)), "Etag": "bad etag"}
        req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("422 Unprocessable Entity", resp.status)
        self.assertNotIn("Etag", resp.headers)

    def test_PUT_nothing_read(self):
        # simulate an artificial scenario of a downstream filter/app not
        # actually reading the input stream from encrypter.
        class NonReadingApp(object):
            def __call__(self, env, start_response):
                # note: no read from wsgi.input
                req = Request(env)
                env["swift.callback.update_footers"](req.headers)
                call_headers.append(req.headers)
                resp = HTTPCreated(req=req, headers={"Etag": "response etag"})
                return resp(env, start_response)

        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {"content-type": "text/plain", "content-length": 0, "etag": "etag from client"}
        req = Request.blank("/v1/a/c/o", environ=env, body="", headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
        self.assertEqual("201 Created", resp.status)
        self.assertEqual("response etag", resp.headers["Etag"])
        self.assertEqual(1, len(call_headers))
        self.assertEqual("etag from client", call_headers[0]["etag"])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith("x-object-sysmeta-crypto-"))

        # check that an upstream footer callback gets called
        other_footers = {
            "Etag": "other etag",
            "X-Object-Sysmeta-Other": "other sysmeta",
            "X-Backend-Container-Update-Override-Etag": "other override",
        }
        env.update({"swift.callback.update_footers": lambda footers: footers.update(other_footers)})
        req = Request.blank("/v1/a/c/o", environ=env, body="", headers=hdrs)

        call_headers = []
        resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))

        self.assertEqual("201 Created", resp.status)
        self.assertEqual("response etag", resp.headers["Etag"])
        self.assertEqual(1, len(call_headers))
        # verify that other middleware's footers made it to app
        for k, v in other_footers.items():
            self.assertEqual(v, call_headers[0][k])
        # verify no encryption footers
        for k in call_headers[0]:
            self.assertFalse(k.lower().startswith("x-object-sysmeta-crypto-"))

    def test_POST_req(self):
        body = "FAKE APP"
        env = {"REQUEST_METHOD": "POST", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {"x-object-meta-test": "encrypt me", "x-object-sysmeta-test": "do not encrypt me"}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        key = fetch_crypto_keys()["object"]
        self.app.register("POST", "/v1/a/c/o", HTTPAccepted, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("202 Accepted", resp.status)
        self.assertNotIn("Etag", resp.headers)

        # verify metadata items
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual("POST", self.app.calls[0][0])
        req_hdrs = self.app.headers[0]

        # user meta is encrypted
        self._verify_user_metadata(req_hdrs, "Test", "encrypt me", key)

        # sysmeta is not encrypted
        self.assertEqual("do not encrypt me", req_hdrs["X-Object-Sysmeta-Test"])

    def _test_no_user_metadata(self, method):
        # verify that x-object-transient-sysmeta-crypto-meta is not set when
        # there is no user metadata
        env = {"REQUEST_METHOD": method, CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank("/v1/a/c/o", environ=env, body="body")
        self.app.register(method, "/v1/a/c/o", HTTPAccepted, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("202 Accepted", resp.status)
        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(method, self.app.calls[0][0])
        self.assertNotIn("x-object-transient-sysmeta-crypto-meta", self.app.headers[0])

    def test_PUT_no_user_metadata(self):
        self._test_no_user_metadata("PUT")

    def test_POST_no_user_metadata(self):
        self._test_no_user_metadata("POST")

    def _test_if_match(self, method, match_header_name):
        def do_test(method, plain_etags, expected_plain_etags=None):
            env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
            match_header_value = ", ".join(plain_etags)
            req = Request.blank(
                "/v1/a/c/o", environ=env, method=method, headers={match_header_name: match_header_value}
            )
            app = FakeSwift()
            app.register(method, "/v1/a/c/o", HTTPOk, {})
            resp = req.get_response(encrypter.Encrypter(app, {}))
            self.assertEqual("200 OK", resp.status)

            self.assertEqual(1, len(app.calls), app.calls)
            self.assertEqual(method, app.calls[0][0])
            actual_headers = app.headers[0]

            # verify the alternate etag location has been specified
            if match_header_value and match_header_value != "*":
                self.assertIn("X-Backend-Etag-Is-At", actual_headers)
                self.assertEqual("X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"])

            # verify etags have been supplemented with masked values
            self.assertIn(match_header_name, actual_headers)
            actual_etags = set(actual_headers[match_header_name].split(", "))
            key = fetch_crypto_keys()["object"]
            masked_etags = [
                '"%s"' % base64.b64encode(hmac.new(key, etag.strip('"'), hashlib.sha256).digest())
                for etag in plain_etags
                if etag not in ("*", "")
            ]
            expected_etags = set((expected_plain_etags or plain_etags) + masked_etags)
            self.assertEqual(expected_etags, actual_etags)
            # check that the request environ was returned to original state
            self.assertEqual(set(plain_etags), set(req.headers[match_header_name].split(", ")))

        do_test(method, [""])
        do_test(method, ['"an etag"'])
        do_test(method, ['"an etag"', '"another_etag"'])
        do_test(method, ["*"])
        # rfc2616 does not allow wildcard *and* etag but test it anyway
        do_test(method, ["*", '"an etag"'])
        # etags should be quoted but check we can cope if they are not
        do_test(method, ["*", "an etag", "another_etag"], expected_plain_etags=["*", '"an etag"', '"another_etag"'])

    def test_GET_if_match(self):
        self._test_if_match("GET", "If-Match")

    def test_HEAD_if_match(self):
        self._test_if_match("HEAD", "If-Match")

    def test_GET_if_none_match(self):
        self._test_if_match("GET", "If-None-Match")

    def test_HEAD_if_none_match(self):
        self._test_if_match("HEAD", "If-None-Match")

    def _test_existing_etag_is_at_header(self, method, match_header_name):
        # if another middleware has already set X-Backend-Etag-Is-At then
        # encrypter should not override that value
        env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank(
            "/v1/a/c/o",
            environ=env,
            method=method,
            headers={match_header_name: "an etag", "X-Backend-Etag-Is-At": "X-Object-Sysmeta-Other-Etag"},
        )
        self.app.register(method, "/v1/a/c/o", HTTPOk, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("200 OK", resp.status)

        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(method, self.app.calls[0][0])
        actual_headers = self.app.headers[0]
        self.assertIn("X-Backend-Etag-Is-At", actual_headers)
        self.assertEqual(
            "X-Object-Sysmeta-Other-Etag,X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"]
        )
        actual_etags = set(actual_headers[match_header_name].split(", "))
        self.assertIn('"an etag"', actual_etags)

    def test_GET_if_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header("GET", "If-Match")

    def test_HEAD_if_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header("HEAD", "If-Match")

    def test_GET_if_none_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header("GET", "If-None-Match")

    def test_HEAD_if_none_match_with_existing_etag_is_at_header(self):
        self._test_existing_etag_is_at_header("HEAD", "If-None-Match")

    def _test_etag_is_at_not_duplicated(self, method):
        # verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
        # X-Backend-Etag-Is-At
        key = fetch_crypto_keys()["object"]
        env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        req = Request.blank(
            "/v1/a/c/o",
            environ=env,
            method=method,
            headers={"If-Match": '"an etag"', "If-None-Match": '"another etag"'},
        )
        self.app.register(method, "/v1/a/c/o", HTTPOk, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("200 OK", resp.status)

        self.assertEqual(1, len(self.app.calls), self.app.calls)
        self.assertEqual(method, self.app.calls[0][0])
        actual_headers = self.app.headers[0]
        self.assertIn("X-Backend-Etag-Is-At", actual_headers)
        self.assertEqual("X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"])

        self.assertIn(
            '"%s"' % base64.b64encode(hmac.new(key, "an etag", hashlib.sha256).digest()), actual_headers["If-Match"]
        )
        self.assertIn('"another etag"', actual_headers["If-None-Match"])
        self.assertIn(
            '"%s"' % base64.b64encode(hmac.new(key, "another etag", hashlib.sha256).digest()),
            actual_headers["If-None-Match"],
        )

    def test_GET_etag_is_at_not_duplicated(self):
        self._test_etag_is_at_not_duplicated("GET")

    def test_HEAD_etag_is_at_not_duplicated(self):
        self._test_etag_is_at_not_duplicated("HEAD")

    def test_PUT_response_inconsistent_etag_is_not_replaced(self):
        # if response is success but etag does not match the ciphertext md5
        # then verify that we do *not* replace it with the plaintext etag
        body = "FAKE APP"
        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {"Etag": "not the ciphertext etag"})
        resp = req.get_response(self.encrypter)
        self.assertEqual("201 Created", resp.status)
        self.assertEqual("not the ciphertext etag", resp.headers["Etag"])

    def test_PUT_multiseg_no_client_etag(self):
        body_key = os.urandom(32)
        chunks = ["some", "chunks", "of data"]
        body = "".join(chunks)
        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys, "wsgi.input": FileLikeIter(chunks)}
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})

        with mock.patch(
            "swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", lambda *args: body_key
        ):
            resp = req.get_response(self.encrypter)

        self.assertEqual("201 Created", resp.status)
        # verify object is encrypted by getting direct from the app
        get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
        self.assertEqual(encrypt(body, body_key, FAKE_IV), get_req.get_response(self.app).body)

    def test_PUT_multiseg_good_client_etag(self):
        body_key = os.urandom(32)
        chunks = ["some", "chunks", "of data"]
        body = "".join(chunks)
        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys, "wsgi.input": FileLikeIter(chunks)}
        hdrs = {"content-type": "text/plain", "content-length": str(len(body)), "Etag": md5hex(body)}
        req = Request.blank("/v1/a/c/o", environ=env, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})

        with mock.patch(
            "swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", lambda *args: body_key
        ):
            resp = req.get_response(self.encrypter)

        self.assertEqual("201 Created", resp.status)
        # verify object is encrypted by getting direct from the app
        get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
        self.assertEqual(encrypt(body, body_key, FAKE_IV), get_req.get_response(self.app).body)

    def test_PUT_multiseg_bad_client_etag(self):
        chunks = ["some", "chunks", "of data"]
        body = "".join(chunks)
        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys, "wsgi.input": FileLikeIter(chunks)}
        hdrs = {"content-type": "text/plain", "content-length": str(len(body)), "Etag": "badclientetag"}
        req = Request.blank("/v1/a/c/o", environ=env, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("422 Unprocessable Entity", resp.status)

    def test_PUT_missing_key_callback(self):
        body = "FAKE APP"
        env = {"REQUEST_METHOD": "PUT"}
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        resp = req.get_response(self.encrypter)
        self.assertEqual("500 Internal Error", resp.status)
        self.assertIn("missing callback", self.encrypter.logger.get_lines_for_level("error")[0])
        self.assertEqual("Unable to retrieve encryption keys.", resp.body)

    def test_PUT_error_in_key_callback(self):
        def raise_exc():
            raise Exception("Testing")

        body = "FAKE APP"
        env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: raise_exc}
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        resp = req.get_response(self.encrypter)
        self.assertEqual("500 Internal Error", resp.status)
        self.assertIn("from callback: Testing", self.encrypter.logger.get_lines_for_level("error")[0])
        self.assertEqual("Unable to retrieve encryption keys.", resp.body)

    def test_PUT_encryption_override(self):
        # set crypto override to disable encryption.
        # simulate another middleware wanting to set footers
        other_footers = {
            "Etag": "other etag",
            "X-Object-Sysmeta-Other": "other sysmeta",
            "X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
        }
        body = "FAKE APP"
        env = {
            "REQUEST_METHOD": "PUT",
            "swift.crypto.override": True,
            "swift.callback.update_footers": lambda footers: footers.update(other_footers),
        }
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
        resp = req.get_response(self.encrypter)
        self.assertEqual("201 Created", resp.status)

        # verify that other middleware's footers made it to app
        req_hdrs = self.app.headers[0]
        for k, v in other_footers.items():
            self.assertEqual(v, req_hdrs[k])

        # verify object is NOT encrypted by getting direct from the app
        get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
        self.assertEqual(body, get_req.get_response(self.app).body)

    def _test_constraints_checking(self, method):
        # verify that the check_metadata function is called on PUT and POST
        body = "FAKE APP"
        env = {"REQUEST_METHOD": method, CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
        hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
        req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
        mocked_func = "swift.common.middleware.crypto.encrypter.check_metadata"
        with mock.patch(mocked_func) as mocked:
            mocked.side_effect = [HTTPBadRequest("testing")]
            resp = req.get_response(self.encrypter)
        self.assertEqual("400 Bad Request", resp.status)
        self.assertEqual(1, mocked.call_count)
        mocked.assert_called_once_with(mock.ANY, "object")
        self.assertEqual(req.headers, mocked.call_args_list[0][0][0].headers)

    def test_PUT_constraints_checking(self):
        self._test_constraints_checking("PUT")

    def test_POST_constraints_checking(self):
        self._test_constraints_checking("POST")

    def test_config_true_value_on_disable_encryption(self):
        app = FakeSwift()
        self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption)
        for val in ("true", "1", "yes", "on", "t", "y"):
            app = encrypter.Encrypter(app, {"disable_encryption": val})
            self.assertTrue(app.disable_encryption)

    def test_PUT_app_exception(self):
        app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {})
        req = Request.blank("/", environ={"REQUEST_METHOD": "PUT"})
        with self.assertRaises(HTTPException) as catcher:
            req.get_response(app)
        self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)
예제 #15
0
class TestOioServerSideCopyMiddleware(TestServerSideCopyMiddleware):

    def setUp(self):
        self.app = FakeSwift()
        self.ssc = copy.filter_factory({
            'object_post_as_copy': 'yes',
        })(self.app)
        self.ssc.logger = self.app.logger

    def tearDown(self):
        # get_object_info() does not close response iterator,
        # thus we have to disable the unclosed_requests test.
        pass

    def test_basic_put_with_x_copy_from(self):
        self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {})
        self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {})
        req = Request.blank('/v1/a/c/o2', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '0',
                                     'X-Copy-From': 'c/o'})
        status, headers, body = self.call_ssc(req)
        self.assertEqual(status, '201 Created')
        self.assertTrue(('X-Copied-From', 'c/o') in headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertEqual('PUT', self.authorized[0].method)
        self.assertEqual('/v1/a/c/o2', self.authorized[0].path)
        self.assertEqual(self.app.swift_sources[0], 'SSC')
        # For basic test cases, assert orig_req_method behavior
        self.assertNotIn('swift.orig_req_method', req.environ)

    def test_static_large_object_manifest(self):
        self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
                          {'X-Static-Large-Object': 'True',
                           'Etag': 'should not be sent'})
        self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
                          {'X-Static-Large-Object': 'True',
                           'Etag': 'should not be sent'}, 'passed')
        self.app.register('PUT', '/v1/a/c/o2?multipart-manifest=put',
                          swob.HTTPCreated, {})
        req = Request.blank('/v1/a/c/o2?multipart-manifest=get',
                            environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '0',
                                     'X-Copy-From': 'c/o'})
        status, headers, body = self.call_ssc(req)
        self.assertEqual(status, '201 Created')
        self.assertTrue(('X-Copied-From', 'c/o') in headers)
        self.assertEqual(3, len(self.app.calls))
        self.assertEqual('HEAD', self.app.calls[0][0])
        self.assertEqual('GET', self.app.calls[1][0])
        get_path, qs = self.app.calls[1][1].split('?')
        params = urllib.parse.parse_qs(qs)
        self.assertDictEqual(
            {'format': ['raw'], 'multipart-manifest': ['get']}, params)
        self.assertEqual(get_path, '/v1/a/c/o')
        self.assertEqual(self.app.calls[2],
                         ('PUT', '/v1/a/c/o2?multipart-manifest=put'))
        req_headers = self.app.headers[2]
        self.assertNotIn('X-Static-Large-Object', req_headers)
        self.assertNotIn('Etag', req_headers)
        self.assertEqual(len(self.authorized), 2)
        self.assertEqual('GET', self.authorized[0].method)
        self.assertEqual('/v1/a/c/o', self.authorized[0].path)
        self.assertEqual('PUT', self.authorized[1].method)
        self.assertEqual('/v1/a/c/o2', self.authorized[1].path)

    def test_static_large_object(self):
        # Compared to the original copy middleware, we do an extra HEAD request
        self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
                          {'X-Static-Large-Object': 'True',
                           'Etag': 'should not be sent'}, 'passed')
        self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
                          {'X-Static-Large-Object': 'True',
                           'Etag': 'should not be sent'}, 'passed')
        self.app.register('PUT', '/v1/a/c/o2',
                          swob.HTTPCreated, {})
        req = Request.blank('/v1/a/c/o2',
                            environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '0',
                                     'X-Copy-From': 'c/o'})
        status, headers, body = self.call_ssc(req)
        self.assertEqual(status, '201 Created')
        self.assertTrue(('X-Copied-From', 'c/o') in headers)
        self.assertEqual(self.app.calls, [
            ('HEAD', '/v1/a/c/o'),
            ('GET', '/v1/a/c/o'),
            ('PUT', '/v1/a/c/o2')])
        req_headers = self.app.headers[1]
        self.assertNotIn('X-Static-Large-Object', req_headers)
        self.assertNotIn('Etag', req_headers)
        self.assertEqual(len(self.authorized), 2)
        self.assertEqual('GET', self.authorized[0].method)
        self.assertEqual('/v1/a/c/o', self.authorized[0].path)
        self.assertEqual('PUT', self.authorized[1].method)
        self.assertEqual('/v1/a/c/o2', self.authorized[1].path)

    def test_basic_put_with_x_copy_from_across_container(self):
        self.app.register('HEAD', '/v1/a/c1/o1', swob.HTTPOk, {})
        self.app.register('PUT', '/v1/a/c2/o2', swob.HTTPCreated, {})
        req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '0',
                                     'X-Copy-From': 'c1/o1'})
        status, headers, body = self.call_ssc(req)
        self.assertEqual(status, '201 Created')
        self.assertTrue(('X-Copied-From', 'c1/o1') in headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertEqual('PUT', self.authorized[0].method)
        self.assertEqual('/v1/a/c2/o2', self.authorized[0].path)

    def test_basic_put_with_x_copy_from_across_container_and_account(self):
        self.app.register('HEAD', '/v1/a1/c1/o1', swob.HTTPOk, {})
        self.app.register('PUT', '/v1/a2/c2/o2', swob.HTTPCreated, {},
                          'passed')
        req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '0',
                                     'X-Copy-From': 'c1/o1',
                                     'X-Copy-From-Account': 'a1'})
        status, headers, body = self.call_ssc(req)
        self.assertEqual(status, '201 Created')
        self.assertTrue(('X-Copied-From', 'c1/o1') in headers)
        self.assertTrue(('X-Copied-From-Account', 'a1') in headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertEqual('PUT', self.authorized[0].method)
        self.assertEqual('/v1/a2/c2/o2', self.authorized[0].path)

    def test_copy_not_found_reading_source(self):
        self.skipTest('To be fixed')

    def test_copy_not_found_reading_source_and_account(self):
        self.skipTest('To be fixed')

    def test_copy_server_error_reading_source(self):
        self.skipTest('To be fixed')

    def test_copy_server_error_reading_source_and_account(self):
        self.skipTest('To be fixed')

    def test_copy_source_larger_than_max_file_size(self):
        self.skipTest('To be fixed')

    def test_COPY_source_metadata(self):
        self.skipTest('To be fixed')

    def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self):
        self.skipTest('To be fixed')

    def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self):
        self.skipTest('To be fixed')

    def test_copy_with_leading_slash_in_x_copy_from(self):
        self.skipTest('To be fixed')

    def test_copy_with_leading_slash_in_x_copy_from_and_account(self):
        self.skipTest('To be fixed')

    def test_copy_with_object_metadata(self):
        self.skipTest('To be fixed')

    def test_copy_with_object_metadata_and_account(self):
        self.skipTest('To be fixed')

    def test_copy_with_slashes_in_x_copy_from(self):
        self.skipTest('To be fixed')

    def test_copy_with_slashes_in_x_copy_from_and_account(self):
        self.skipTest('To be fixed')

    def test_copy_with_spaces_in_x_copy_from(self):
        self.skipTest('To be fixed')

    def test_copy_with_spaces_in_x_copy_from_and_account(self):
        self.skipTest('To be fixed')
예제 #16
0
class TestUntarMetadata(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        self.bulk = bulk.filter_factory({})(self.app)
        self.testdir = mkdtemp(suffix='tmp_test_bulk')

    def tearDown(self):
        rmtree(self.testdir, ignore_errors=1)

    def test_extract_metadata(self):
        self.app.register('HEAD', '/v1/a/c?extract-archive=tar',
                          HTTPNoContent, {}, None)
        self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar',
                          HTTPCreated, {}, None)
        self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar',
                          HTTPCreated, {}, None)

        # It's a real pain to instantiate TarInfo objects directly; they
        # really want to come from a file on disk or a tarball. So, we write
        # out some files and add pax headers to them as they get placed into
        # the tarball.
        with open(os.path.join(self.testdir, "obj1"), "w") as fh1:
            fh1.write("obj1 contents\n")
        with open(os.path.join(self.testdir, "obj2"), "w") as fh2:
            fh2.write("obj2 contents\n")

        tar_ball = StringIO()
        tar_file = tarfile.TarFile.open(fileobj=tar_ball, mode="w",
                                        format=tarfile.PAX_FORMAT)

        # With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with
        # extended attribute user.thingy = dingy gets put into the tarfile
        # with pax_headers containing key/value pair
        # (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type
        # unicode, not type str).
        #
        # With BSD tar (libarchive), you get key/value pair
        # (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as
        # gratuitous incompatibility.
        #
        # Still, we'll support uploads with both. Just heap more code on the
        # problem until you can forget it's under there.
        with open(os.path.join(self.testdir, "obj1")) as fh1:
            tar_info1 = tar_file.gettarinfo(fileobj=fh1,
                                            arcname="obj1")
            tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \
                u'application/food-diary'
            tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \
                u'sopa de albóndigas'
            tar_info1.pax_headers[
                u'SCHILY.xattr.user.meta.afternoon-snack'] = \
                u'gigantic bucket of coffee'
            tar_file.addfile(tar_info1, fh1)

        with open(os.path.join(self.testdir, "obj2")) as fh2:
            tar_info2 = tar_file.gettarinfo(fileobj=fh2,
                                            arcname="obj2")
            tar_info2.pax_headers[
                u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert'
            tar_info2.pax_headers[
                u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy'
            tar_info2.pax_headers[
                u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped'
            tar_file.addfile(tar_info2, fh2)

        tar_ball.seek(0)

        req = Request.blank('/v1/a/c?extract-archive=tar')
        req.environ['REQUEST_METHOD'] = 'PUT'
        req.environ['wsgi.input'] = tar_ball
        req.headers['transfer-encoding'] = 'chunked'
        req.headers['accept'] = 'application/json;q=1.0'

        resp = req.get_response(self.bulk)
        self.assertEqual(resp.status_int, 200)

        # sanity check to make sure the upload worked
        upload_status = utils.json.loads(resp.body)
        self.assertEqual(upload_status['Number Files Created'], 2)

        put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2])
        self.assertEqual(
            put1_headers.get('Content-Type'),
            'application/food-diary')
        self.assertEqual(
            put1_headers.get('X-Object-Meta-Lunch'),
            'sopa de alb\xc3\xb3ndigas')
        self.assertEqual(
            put1_headers.get('X-Object-Meta-Afternoon-Snack'),
            'gigantic bucket of coffee')

        put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
        self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
        self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
        self.assertEqual(put2_headers.get('Content-Type'), None)
        self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)
예제 #17
0
 def setUp(self):
     self.app = FakeSwift()
     self.bulk = bulk.filter_factory({})(self.app)
     self.testdir = mkdtemp(suffix='tmp_test_bulk')
예제 #18
0
class ContainerQuotaCopyingTestCases(unittest.TestCase):

    def setUp(self):
        self.app = FakeSwift()
        self.cq_filter = container_quotas.filter_factory({})(self.app)
        self.copy_filter = copy.filter_factory({})(self.cq_filter)

    def test_exceed_bytes_quota_copy_verb(self):
        cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
                          {'Content-Length': '10'}, 'passed')

        req = Request.blank('/v1/a/c2/o2',
                            environ={'REQUEST_METHOD': 'COPY',
                                     'swift.cache': cache},
                            headers={'Destination': '/c/o'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, 'Upload exceeds quota.')

    def test_not_exceed_bytes_quota_copy_verb(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
                          {'Content-Length': '10'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
        cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
        req = Request.blank('/v1/a/c2/o2',
                            environ={'REQUEST_METHOD': 'COPY',
                                     'swift.cache': cache},
                            headers={'Destination': '/c/o'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_exceed_counts_quota_copy_verb(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
        cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
        req = Request.blank('/v1/a/c2/o2',
                            environ={'REQUEST_METHOD': 'COPY',
                                     'swift.cache': cache},
                            headers={'Destination': '/c/o'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, 'Upload exceeds quota.')

    def test_exceed_counts_quota_copy_cross_account_verb(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
        a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
                     'status': 200, 'object_count': 1}
        a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
                      'status': 200, 'object_count': 1}
        req = Request.blank('/v1/a/c2/o2',
                            environ={'REQUEST_METHOD': 'COPY',
                                     'swift.infocache': {
                                         'swift.container/a/c': a_c_cache,
                                         'swift.container/a2/c': a2_c_cache}},
                            headers={'Destination': '/c/o',
                                     'Destination-Account': 'a2'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, 'Upload exceeds quota.')

    def test_exceed_counts_quota_copy_cross_account_PUT_verb(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
        a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
                     'status': 200, 'object_count': 1}
        a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
                      'status': 200, 'object_count': 1}
        req = Request.blank('/v1/a2/c/o',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'swift.infocache': {
                                         'swift.container/a/c': a_c_cache,
                                         'swift.container/a2/c': a2_c_cache}},
                            headers={'X-Copy-From': '/c2/o2',
                                     'X-Copy-From-Account': 'a'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, 'Upload exceeds quota.')

    def test_exceed_bytes_quota_copy_from(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
                          {'Content-Length': '10'}, 'passed')
        cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})

        req = Request.blank('/v1/a/c/o',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'swift.cache': cache},
                            headers={'x-copy-from': '/c2/o2'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, 'Upload exceeds quota.')

    def test_not_exceed_bytes_quota_copy_from(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
                          {'Content-Length': '10'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
        cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
        req = Request.blank('/v1/a/c/o',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'swift.cache': cache},
                            headers={'x-copy-from': '/c2/o2'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_bytes_quota_copy_from_no_src(self):
        self.app.register('GET', '/v1/a/c2/o3', HTTPOk, {}, 'passed')
        self.app.register(
            'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
        cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
        req = Request.blank('/v1/a/c/o',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'swift.cache': cache},
                            headers={'x-copy-from': '/c2/o3'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_bytes_quota_copy_from_bad_src(self):
        cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
        req = Request.blank('/v1/a/c/o',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'swift.cache': cache},
                            headers={'x-copy-from': 'bad_path'})
        with self.assertRaises(HTTPException) as catcher:
            req.get_response(self.copy_filter)
        self.assertEqual(412, catcher.exception.status_int)

    def test_exceed_counts_quota_copy_from(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
                          {'Content-Length': '10'}, 'passed')
        cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
        req = Request.blank('/v1/a/c/o',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'swift.cache': cache},
                            headers={'x-copy-from': '/c2/o2'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, 'Upload exceeds quota.')

    def test_not_exceed_counts_quota_copy_from(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
                          {'Content-Length': '10'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
        cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
        req = Request.blank('/v1/a/c/o',
                            environ={'REQUEST_METHOD': 'PUT',
                                     'swift.cache': cache},
                            headers={'x-copy-from': '/c2/o2'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_not_exceed_counts_quota_copy_verb(self):
        self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
                          {'Content-Length': '10'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
        cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
        req = Request.blank('/v1/a/c2/o2',
                            environ={'REQUEST_METHOD': 'COPY',
                                     'swift.cache': cache},
                            headers={'Destination': '/c/o'})
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)
예제 #19
0
 def setUp(self):
     self.app = FakeSwift()
     self.decrypter = decrypter.Decrypter(self.app, {})
     self.decrypter.logger = FakeLogger()
예제 #20
0
class TestDecrypterContainerRequests(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        self.decrypter = decrypter.Decrypter(self.app, {})
        self.decrypter.logger = FakeLogger()

    def _make_cont_get_req(self, resp_body, format, override=False,
                           callback=fetch_crypto_keys):
        path = '/v1/a/c'
        content_type = 'text/plain'
        if format:
            path = '%s/?format=%s' % (path, format)
            content_type = 'application/' + format
        env = {'REQUEST_METHOD': 'GET',
               CRYPTO_KEY_CALLBACK: callback}
        if override:
            env['swift.crypto.override'] = True
        req = Request.blank(path, environ=env)
        hdrs = {'content-type': content_type}
        self.app.register('GET', path, HTTPOk, body=resp_body, headers=hdrs)
        return req.get_response(self.decrypter)

    def test_GET_container_success(self):
        # no format requested, listing has names only
        fake_body = 'testfile1\ntestfile2\n'
        calls = [0]

        def wrapped_fetch_crypto_keys():
            calls[0] += 1
            return fetch_crypto_keys()

        resp = self._make_cont_get_req(fake_body, None,
                                       callback=wrapped_fetch_crypto_keys)

        self.assertEqual('200 OK', resp.status)
        names = resp.body.split('\n')
        self.assertEqual(3, len(names))
        self.assertIn('testfile1', names)
        self.assertIn('testfile2', names)
        self.assertIn('', names)
        self.assertEqual(0, calls[0])

    def test_GET_container_json(self):
        content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
        content_type_2 = 'text/plain; param=foo'
        pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
        pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
        key = fetch_crypto_keys()['container']

        obj_dict_1 = {"bytes": 16,
                      "last_modified": "2015-04-14T23:33:06.439040",
                      "hash": encrypt_and_append_meta(
                          pt_etag1.encode('utf-8'), key),
                      "name": "testfile",
                      "content_type": content_type_1}

        obj_dict_2 = {"bytes": 24,
                      "last_modified": "2015-04-14T23:33:06.519020",
                      "hash": encrypt_and_append_meta(
                          pt_etag2.encode('utf-8'), key),
                      "name": "testfile2",
                      "content_type": content_type_2}

        listing = [obj_dict_1, obj_dict_2]
        fake_body = json.dumps(listing)

        resp = self._make_cont_get_req(fake_body, 'json')

        self.assertEqual('200 OK', resp.status)
        body = resp.body
        self.assertEqual(len(body), int(resp.headers['Content-Length']))
        body_json = json.loads(body)
        self.assertEqual(2, len(body_json))
        obj_dict_1['hash'] = pt_etag1
        self.assertDictEqual(obj_dict_1, body_json[0])
        obj_dict_2['hash'] = pt_etag2
        self.assertDictEqual(obj_dict_2, body_json[1])

    def test_GET_container_json_with_crypto_override(self):
        content_type_1 = 'image/jpeg'
        content_type_2 = 'text/plain; param=foo'
        pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
        pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'

        obj_dict_1 = {"bytes": 16,
                      "last_modified": "2015-04-14T23:33:06.439040",
                      "hash": pt_etag1,
                      "name": "testfile",
                      "content_type": content_type_1}

        obj_dict_2 = {"bytes": 24,
                      "last_modified": "2015-04-14T23:33:06.519020",
                      "hash": pt_etag2,
                      "name": "testfile2",
                      "content_type": content_type_2}

        listing = [obj_dict_1, obj_dict_2]
        fake_body = json.dumps(listing)

        resp = self._make_cont_get_req(fake_body, 'json', override=True)

        self.assertEqual('200 OK', resp.status)
        body = resp.body
        self.assertEqual(len(body), int(resp.headers['Content-Length']))
        body_json = json.loads(body)
        self.assertEqual(2, len(body_json))
        self.assertDictEqual(obj_dict_1, body_json[0])
        self.assertDictEqual(obj_dict_2, body_json[1])

    def test_cont_get_json_req_with_cipher_mismatch(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['cipher'] = 'unknown_cipher'
        key = fetch_crypto_keys()['container']
        pt_etag = 'c6e8196d7f0fff6444b90861fe8d609d'
        ct_etag = encrypt_and_append_meta(pt_etag, key,
                                          crypto_meta=bad_crypto_meta)

        obj_dict_1 = {"bytes": 16,
                      "last_modified": "2015-04-14T23:33:06.439040",
                      "hash": ct_etag,
                      "name": "testfile",
                      "content_type": "image/jpeg"}

        listing = [obj_dict_1]
        fake_body = json.dumps(listing)

        resp = self._make_cont_get_req(fake_body, 'json')

        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Error decrypting container listing', resp.body)
        self.assertIn("Cipher must be AES_CTR_256",
                      self.decrypter.logger.get_lines_for_level('error')[0])

    def _assert_element_contains_dict(self, expected, element):
        for k, v in expected.items():
            entry = element.getElementsByTagName(k)
            self.assertIsNotNone(entry, 'Key %s not found' % k)
            actual = entry[0].childNodes[0].nodeValue
            self.assertEqual(v, actual,
                             "Expected %s but got %s for key %s"
                             % (v, actual, k))

    def test_GET_container_xml(self):
        content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
        content_type_2 = 'text/plain; param=foo'
        pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
        pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
        key = fetch_crypto_keys()['container']

        fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>\
''' + encrypt_and_append_meta(pt_etag1.encode('utf8'), key) + '''\
</hash><content_type>\
''' + content_type_1 + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
<object><hash>\
''' + encrypt_and_append_meta(pt_etag2.encode('utf8'), key) + '''\
</hash><content_type>\
''' + content_type_2 + '''\
</content_type><name>testfile2</name><bytes>24</bytes>\
<last_modified>2015-04-19T02:37:39.684740</last_modified></object>\
</container>'''

        resp = self._make_cont_get_req(fake_body, 'xml')
        self.assertEqual('200 OK', resp.status)
        body = resp.body
        self.assertEqual(len(body), int(resp.headers['Content-Length']))

        tree = minidom.parseString(body)
        containers = tree.getElementsByTagName('container')
        self.assertEqual(1, len(containers))
        self.assertEqual('testc',
                         containers[0].attributes.getNamedItem("name").value)

        objs = tree.getElementsByTagName('object')
        self.assertEqual(2, len(objs))

        obj_dict_1 = {"bytes": "16",
                      "last_modified": "2015-04-19T02:37:39.601660",
                      "hash": pt_etag1,
                      "name": "testfile",
                      "content_type": content_type_1}
        self._assert_element_contains_dict(obj_dict_1, objs[0])
        obj_dict_2 = {"bytes": "24",
                      "last_modified": "2015-04-19T02:37:39.684740",
                      "hash": pt_etag2,
                      "name": "testfile2",
                      "content_type": content_type_2}
        self._assert_element_contains_dict(obj_dict_2, objs[1])

    def test_GET_container_xml_with_crypto_override(self):
        content_type_1 = 'image/jpeg'
        content_type_2 = 'text/plain; param=foo'

        fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>c6e8196d7f0fff6444b90861fe8d609d</hash>\
<content_type>''' + content_type_1 + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
<object><hash>ac0374ed4d43635f803c82469d0b5a10</hash>\
<content_type>''' + content_type_2 + '''\
</content_type><name>testfile2</name><bytes>24</bytes>\
<last_modified>2015-04-19T02:37:39.684740</last_modified></object>\
</container>'''

        resp = self._make_cont_get_req(fake_body, 'xml', override=True)

        self.assertEqual('200 OK', resp.status)
        body = resp.body
        self.assertEqual(len(body), int(resp.headers['Content-Length']))

        tree = minidom.parseString(body)
        containers = tree.getElementsByTagName('container')
        self.assertEqual(1, len(containers))
        self.assertEqual('testc',
                         containers[0].attributes.getNamedItem("name").value)

        objs = tree.getElementsByTagName('object')
        self.assertEqual(2, len(objs))

        obj_dict_1 = {"bytes": "16",
                      "last_modified": "2015-04-19T02:37:39.601660",
                      "hash": "c6e8196d7f0fff6444b90861fe8d609d",
                      "name": "testfile",
                      "content_type": content_type_1}
        self._assert_element_contains_dict(obj_dict_1, objs[0])
        obj_dict_2 = {"bytes": "24",
                      "last_modified": "2015-04-19T02:37:39.684740",
                      "hash": "ac0374ed4d43635f803c82469d0b5a10",
                      "name": "testfile2",
                      "content_type": content_type_2}
        self._assert_element_contains_dict(obj_dict_2, objs[1])

    def test_cont_get_xml_req_with_cipher_mismatch(self):
        bad_crypto_meta = fake_get_crypto_meta()
        bad_crypto_meta['cipher'] = 'unknown_cipher'

        fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc"><object>\
<hash>''' + encrypt_and_append_meta('c6e8196d7f0fff6444b90861fe8d609d',
                                    fetch_crypto_keys()['container'],
                                    crypto_meta=bad_crypto_meta) + '''\
</hash>\
<content_type>image/jpeg</content_type>\
<name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
</container>'''

        resp = self._make_cont_get_req(fake_body, 'xml')

        self.assertEqual('500 Internal Error', resp.status)
        self.assertEqual('Error decrypting container listing', resp.body)
        self.assertIn("Cipher must be AES_CTR_256",
                      self.decrypter.logger.get_lines_for_level('error')[0])
예제 #21
0
class ContainerQuotaCopyingTestCases(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        self.cq_filter = container_quotas.filter_factory({})(self.app)
        self.copy_filter = copy.filter_factory({})(self.cq_filter)

    def test_exceed_bytes_quota_copy_verb(self):
        cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "2"}})
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")

        req = Request.blank(
            "/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, "Upload exceeds quota.")

    def test_not_exceed_bytes_quota_copy_verb(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
        self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
        cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
        req = Request.blank(
            "/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_exceed_counts_quota_copy_verb(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {}, "passed")
        cache = FakeCache({"object_count": 1, "meta": {"quota-count": "1"}})
        req = Request.blank(
            "/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, "Upload exceeds quota.")

    def test_exceed_counts_quota_copy_cross_account_verb(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {}, "passed")
        a_c_cache = {"storage_policy": "0", "meta": {"quota-count": "2"}, "status": 200, "object_count": 1}
        a2_c_cache = {"storage_policy": "0", "meta": {"quota-count": "1"}, "status": 200, "object_count": 1}
        req = Request.blank(
            "/v1/a/c2/o2",
            environ={
                "REQUEST_METHOD": "COPY",
                "swift.infocache": {"container/a/c": a_c_cache, "container/a2/c": a2_c_cache},
            },
            headers={"Destination": "/c/o", "Destination-Account": "a2"},
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, "Upload exceeds quota.")

    def test_exceed_counts_quota_copy_cross_account_PUT_verb(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {}, "passed")
        a_c_cache = {"storage_policy": "0", "meta": {"quota-count": "2"}, "status": 200, "object_count": 1}
        a2_c_cache = {"storage_policy": "0", "meta": {"quota-count": "1"}, "status": 200, "object_count": 1}
        req = Request.blank(
            "/v1/a2/c/o",
            environ={
                "REQUEST_METHOD": "PUT",
                "swift.infocache": {"container/a/c": a_c_cache, "container/a2/c": a2_c_cache},
            },
            headers={"X-Copy-From": "/c2/o2", "X-Copy-From-Account": "a"},
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, "Upload exceeds quota.")

    def test_exceed_bytes_quota_copy_from(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
        cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "2"}})

        req = Request.blank(
            "/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, "Upload exceeds quota.")

    def test_not_exceed_bytes_quota_copy_from(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
        self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
        cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
        req = Request.blank(
            "/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_bytes_quota_copy_from_no_src(self):
        self.app.register("GET", "/v1/a/c2/o3", HTTPOk, {}, "passed")
        self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
        cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
        req = Request.blank(
            "/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o3"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_bytes_quota_copy_from_bad_src(self):
        cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
        req = Request.blank(
            "/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "bad_path"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 412)

    def test_exceed_counts_quota_copy_from(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
        cache = FakeCache({"object_count": 1, "meta": {"quota-count": "1"}})
        req = Request.blank(
            "/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 413)
        self.assertEqual(res.body, "Upload exceeds quota.")

    def test_not_exceed_counts_quota_copy_from(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
        self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
        cache = FakeCache({"object_count": 1, "meta": {"quota-count": "2"}})
        req = Request.blank(
            "/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)

    def test_not_exceed_counts_quota_copy_verb(self):
        self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
        self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
        cache = FakeCache({"object_count": 1, "meta": {"quota-count": "2"}})
        req = Request.blank(
            "/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
        )
        res = req.get_response(self.copy_filter)
        self.assertEqual(res.status_int, 200)
예제 #22
0
파일: test_dlo.py 프로젝트: charint/swift
class DloTestCase(unittest.TestCase):
    def call_dlo(self, req, app=None, expect_exception=False):
        if app is None:
            app = self.dlo

        req.headers.setdefault("User-Agent", "Soap Opera")

        status = [None]
        headers = [None]

        def start_response(s, h, ei=None):
            status[0] = s
            headers[0] = h

        body_iter = app(req.environ, start_response)
        body = ''
        caught_exc = None
        try:
            for chunk in body_iter:
                body += chunk
        except Exception as exc:
            if expect_exception:
                caught_exc = exc
            else:
                raise

        if expect_exception:
            return status[0], headers[0], body, caught_exc
        else:
            return status[0], headers[0], body

    def setUp(self):
        self.app = FakeSwift()
        self.dlo = dlo.filter_factory({
            # don't slow down tests with rate limiting
            'rate_limit_after_segment': '1000000',
        })(self.app)

        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_01',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg01-etag'},
            'aaaaa')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_02',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg02-etag'},
            'bbbbb')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg03-etag'},
            'ccccc')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_04',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg04-etag'},
            'ddddd')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_05',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg05-etag'},
            'eeeee')

        # an unrelated object (not seg*) to test the prefix matching
        self.app.register(
            'GET', '/v1/AUTH_test/c/catpicture.jpg',
            swob.HTTPOk, {'Content-Length': '9', 'Etag': 'cats-etag'},
            'meow meow meow meow')

        self.app.register(
            'GET', '/v1/AUTH_test/mancon/manifest',
            swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
                          'X-Object-Manifest': 'c/seg'},
            'manifest-contents')

        lm = '2013-11-22T02:42:13.781760'
        ct = 'application/octet-stream'
        segs = [{"hash": "seg01-etag", "bytes": 5, "name": "seg_01",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg02-etag", "bytes": 5, "name": "seg_02",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg03-etag", "bytes": 5, "name": "seg_03",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg04-etag", "bytes": 5, "name": "seg_04",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg05-etag", "bytes": 5, "name": "seg_05",
                 "last_modified": lm, "content_type": ct}]

        full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9,
                                          "name": "catpicture.jpg",
                                          "last_modified": lm,
                                          "content_type": "application/png"}]
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(full_container_listing))
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=seg',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(segs))

        # This is to let us test multi-page container listings; we use the
        # trailing underscore to send small (pagesize=3) listings.
        #
        # If you're testing against this, be sure to mock out
        # CONTAINER_LISTING_LIMIT to 3 in your test.
        self.app.register(
            'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
            swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
                          'X-Object-Manifest': 'c/seg_'},
            'manyseg')
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=seg_',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(segs[:3]))
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(segs[3:]))

        # Here's a manifest with 0 segments
        self.app.register(
            'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
            swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
                          'X-Object-Manifest': 'c/noseg_'},
            'noseg')
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=noseg_',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps([]))
예제 #23
0
 def setUp(self):
     self.app = FakeSwift()
     self.cq_filter = container_quotas.filter_factory({})(self.app)
     self.copy_filter = copy.filter_factory({})(self.cq_filter)
예제 #24
0
class TestListingFormats(unittest.TestCase):
    def setUp(self):
        self.fake_swift = FakeSwift()
        self.app = listing_formats.ListingFilter(self.fake_swift)
        self.fake_account_listing = json.dumps([
            {'name': 'bar', 'bytes': 0, 'count': 0,
             'last_modified': '1970-01-01T00:00:00.000000'},
            {'subdir': 'foo_'},
        ])
        self.fake_container_listing = json.dumps([
            {'name': 'bar', 'hash': 'etag', 'bytes': 0,
             'content_type': 'text/plain',
             'last_modified': '1970-01-01T00:00:00.000000'},
            {'subdir': 'foo/'},
        ])

    def test_valid_account(self):
        self.fake_swift.register('GET', '/v1/a', HTTPOk, {
            'Content-Length': str(len(self.fake_account_listing)),
            'Content-Type': 'application/json'}, self.fake_account_listing)

        req = Request.blank('/v1/a')
        resp = req.get_response(self.app)
        self.assertEqual(resp.body, 'bar\nfoo_\n')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

        req = Request.blank('/v1/a?format=txt')
        resp = req.get_response(self.app)
        self.assertEqual(resp.body, 'bar\nfoo_\n')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

        req = Request.blank('/v1/a?format=json')
        resp = req.get_response(self.app)
        self.assertEqual(resp.body, self.fake_account_listing)
        self.assertEqual(resp.headers['Content-Type'],
                         'application/json; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

        req = Request.blank('/v1/a?format=xml')
        resp = req.get_response(self.app)
        self.assertEqual(resp.body.split('\n'), [
            '<?xml version="1.0" encoding="UTF-8"?>',
            '<account name="a">',
            '<container><name>bar</name><count>0</count><bytes>0</bytes>'
            '<last_modified>1970-01-01T00:00:00.000000</last_modified>'
            '</container>',
            '<subdir name="foo_" />',
            '</account>',
        ])
        self.assertEqual(resp.headers['Content-Type'],
                         'application/xml; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

    def test_valid_container(self):
        self.fake_swift.register('GET', '/v1/a/c', HTTPOk, {
            'Content-Length': str(len(self.fake_container_listing)),
            'Content-Type': 'application/json'}, self.fake_container_listing)

        req = Request.blank('/v1/a/c')
        resp = req.get_response(self.app)
        self.assertEqual(resp.body, 'bar\nfoo/\n')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

        req = Request.blank('/v1/a/c?format=txt')
        resp = req.get_response(self.app)
        self.assertEqual(resp.body, 'bar\nfoo/\n')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

        req = Request.blank('/v1/a/c?format=json')
        resp = req.get_response(self.app)
        self.assertEqual(resp.body, self.fake_container_listing)
        self.assertEqual(resp.headers['Content-Type'],
                         'application/json; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

        req = Request.blank('/v1/a/c?format=xml')
        resp = req.get_response(self.app)
        self.assertEqual(
            resp.body,
            '<?xml version="1.0" encoding="UTF-8"?>\n'
            '<container name="c">'
            '<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
            '<content_type>text/plain</content_type>'
            '<last_modified>1970-01-01T00:00:00.000000</last_modified>'
            '</object>'
            '<subdir name="foo/"><name>foo/</name></subdir>'
            '</container>'
        )
        self.assertEqual(resp.headers['Content-Type'],
                         'application/xml; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

    def test_blank_account(self):
        self.fake_swift.register('GET', '/v1/a', HTTPOk, {
            'Content-Length': '2', 'Content-Type': 'application/json'}, '[]')

        req = Request.blank('/v1/a')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '204 No Content')
        self.assertEqual(resp.body, '')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

        req = Request.blank('/v1/a?format=txt')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '204 No Content')
        self.assertEqual(resp.body, '')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

        req = Request.blank('/v1/a?format=json')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '200 OK')
        self.assertEqual(resp.body, '[]')
        self.assertEqual(resp.headers['Content-Type'],
                         'application/json; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

        req = Request.blank('/v1/a?format=xml')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '200 OK')
        self.assertEqual(resp.body.split('\n'), [
            '<?xml version="1.0" encoding="UTF-8"?>',
            '<account name="a">',
            '</account>',
        ])
        self.assertEqual(resp.headers['Content-Type'],
                         'application/xml; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a?format=json'))

    def test_blank_container(self):
        self.fake_swift.register('GET', '/v1/a/c', HTTPOk, {
            'Content-Length': '2', 'Content-Type': 'application/json'}, '[]')

        req = Request.blank('/v1/a/c')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '204 No Content')
        self.assertEqual(resp.body, '')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

        req = Request.blank('/v1/a/c?format=txt')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '204 No Content')
        self.assertEqual(resp.body, '')
        self.assertEqual(resp.headers['Content-Type'],
                         'text/plain; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

        req = Request.blank('/v1/a/c?format=json')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '200 OK')
        self.assertEqual(resp.body, '[]')
        self.assertEqual(resp.headers['Content-Type'],
                         'application/json; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

        req = Request.blank('/v1/a/c?format=xml')
        resp = req.get_response(self.app)
        self.assertEqual(resp.status, '200 OK')
        self.assertEqual(resp.body.split('\n'), [
            '<?xml version="1.0" encoding="UTF-8"?>',
            '<container name="c" />',
        ])
        self.assertEqual(resp.headers['Content-Type'],
                         'application/xml; charset=utf-8')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/a/c?format=json'))

    def test_pass_through(self):
        def do_test(path):
            self.fake_swift.register(
                'GET', path, HTTPOk, {
                    'Content-Length': str(len(self.fake_container_listing)),
                    'Content-Type': 'application/json'},
                self.fake_container_listing)
            req = Request.blank(path + '?format=xml')
            resp = req.get_response(self.app)
            self.assertEqual(resp.body, self.fake_container_listing)
            self.assertEqual(resp.headers['Content-Type'], 'application/json')
            self.assertEqual(self.fake_swift.calls[-1], (
                'GET', path + '?format=xml'))  # query param is unchanged

        do_test('/')
        do_test('/v1')
        do_test('/auth/v1.0')
        do_test('/v1/a/c/o')

    def test_static_web_not_json(self):
        body = 'doesnt matter'
        self.fake_swift.register(
            'GET', '/v1/staticweb/not-json', HTTPOk,
            {'Content-Length': str(len(body)),
             'Content-Type': 'text/plain'},
            body)

        resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
        self.assertEqual(resp.body, body)
        self.assertEqual(resp.headers['Content-Type'], 'text/plain')
        # We *did* try, though
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/staticweb/not-json?format=json'))
        # TODO: add a similar test that has *no* content-type
        # FakeSwift seems to make this hard to do

    def test_static_web_not_really_json(self):
        body = 'raises ValueError'
        self.fake_swift.register(
            'GET', '/v1/staticweb/not-json', HTTPOk,
            {'Content-Length': str(len(body)),
             'Content-Type': 'application/json'},
            body)

        resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
        self.assertEqual(resp.body, body)
        self.assertEqual(resp.headers['Content-Type'], 'application/json')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/staticweb/not-json?format=json'))

    def test_static_web_pretend_to_be_giant_json(self):
        body = json.dumps(self.fake_container_listing * 1000000)
        self.assertGreater(  # sanity
            len(body), listing_formats.MAX_CONTAINER_LISTING_CONTENT_LENGTH)

        self.fake_swift.register(
            'GET', '/v1/staticweb/not-json', HTTPOk,
            {'Content-Type': 'application/json'},
            body)

        resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
        self.assertEqual(resp.body, body)
        self.assertEqual(resp.headers['Content-Type'], 'application/json')
        self.assertEqual(self.fake_swift.calls[-1], (
            'GET', '/v1/staticweb/not-json?format=json'))
        # TODO: add a similar test for chunked transfers
        # (staticweb referencing a DLO that doesn't fit in a single listing?)

    def test_static_web_bad_json(self):
        def do_test(body_obj):
            body = json.dumps(body_obj)
            self.fake_swift.register(
                'GET', '/v1/staticweb/bad-json', HTTPOk,
                {'Content-Length': str(len(body)),
                 'Content-Type': 'application/json'},
                body)

            def do_sub_test(path):
                resp = Request.blank(path).get_response(self.app)
                self.assertEqual(resp.body, body)
                # NB: no charset is added; we pass through whatever we got
                self.assertEqual(resp.headers['Content-Type'],
                                 'application/json')
                self.assertEqual(self.fake_swift.calls[-1], (
                    'GET', '/v1/staticweb/bad-json?format=json'))

            do_sub_test('/v1/staticweb/bad-json')
            do_sub_test('/v1/staticweb/bad-json?format=txt')
            do_sub_test('/v1/staticweb/bad-json?format=xml')
            do_sub_test('/v1/staticweb/bad-json?format=json')

        do_test({})
        do_test({'non-empty': 'hash'})
        do_test(None)
        do_test(0)
        do_test('some string')
        do_test([None])
        do_test([0])
        do_test(['some string'])

    def test_static_web_bad_but_not_terrible_json(self):
        body = json.dumps([{'no name': 'nor subdir'}])
        self.fake_swift.register(
            'GET', '/v1/staticweb/bad-json', HTTPOk,
            {'Content-Length': str(len(body)),
             'Content-Type': 'application/json'},
            body)

        def do_test(path, expect_charset=False):
            resp = Request.blank(path).get_response(self.app)
            self.assertEqual(resp.body, body)
            if expect_charset:
                self.assertEqual(resp.headers['Content-Type'],
                                 'application/json; charset=utf-8')
            else:
                self.assertEqual(resp.headers['Content-Type'],
                                 'application/json')
            self.assertEqual(self.fake_swift.calls[-1], (
                'GET', '/v1/staticweb/bad-json?format=json'))

        do_test('/v1/staticweb/bad-json')
        do_test('/v1/staticweb/bad-json?format=txt')
        do_test('/v1/staticweb/bad-json?format=xml')
        # The response we get is *just close enough* to being valid that we
        # assume it is and slap on the missing charset. If you set up staticweb
        # to serve back such responses, your clients are already hosed.
        do_test('/v1/staticweb/bad-json?format=json', expect_charset=True)
예제 #25
0
 def setUp(self):
     self.app = FakeSwift()
     self.ssc = copy.filter_factory({
         'object_post_as_copy': 'yes',
     })(self.app)
     self.ssc.logger = self.app.logger
예제 #26
0
class TestKeymaster(unittest.TestCase):

    def setUp(self):
        super(TestKeymaster, self).setUp()
        self.swift = FakeSwift()
        self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)

    def test_object_path(self):
        self.verify_keys_for_path(
            '/a/c/o', expected_keys=('object', 'container'))

    def test_container_path(self):
        self.verify_keys_for_path(
            '/a/c', expected_keys=('container',))

    def verify_keys_for_path(self, path, expected_keys, key_id=None):
        put_keys = None
        for method, resp_class, status in (
                ('PUT', swob.HTTPCreated, '201'),
                ('POST', swob.HTTPAccepted, '202'),
                ('GET', swob.HTTPOk, '200'),
                ('HEAD', swob.HTTPNoContent, '204')):
            resp_headers = {}
            self.swift.register(
                method, '/v1' + path, resp_class, resp_headers, '')
            req = Request.blank(
                '/v1' + path, environ={'REQUEST_METHOD': method})
            start_response, calls = capture_start_response()
            self.app(req.environ, start_response)
            self.assertEqual(1, len(calls))
            self.assertTrue(calls[0][0].startswith(status))
            self.assertNotIn('swift.crypto.override', req.environ)
            self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
                          '%s not set in env' % CRYPTO_KEY_CALLBACK)
            keys = req.environ.get(CRYPTO_KEY_CALLBACK)(key_id=key_id)
            self.assertIn('id', keys)
            id = keys.pop('id')
            self.assertEqual(path, id['path'])
            self.assertEqual('1', id['v'])
            keys.pop('all_ids')
            self.assertListEqual(sorted(expected_keys), sorted(keys.keys()),
                                 '%s %s got keys %r, but expected %r'
                                 % (method, path, keys.keys(), expected_keys))
            if put_keys is not None:
                # check all key sets were consistent for this path
                self.assertDictEqual(put_keys, keys)
            else:
                put_keys = keys
        return put_keys

    def test_key_uniqueness(self):
        # a rudimentary check that different keys are made for different paths
        ref_path_parts = ('a1', 'c1', 'o1')
        path = '/' + '/'.join(ref_path_parts)
        ref_keys = self.verify_keys_for_path(
            path, expected_keys=('object', 'container'))

        # for same path and for each differing path check that keys are unique
        # when path to object or container is unique and vice-versa
        for path_parts in [(a, c, o) for a in ('a1', 'a2')
                           for c in ('c1', 'c2')
                           for o in ('o1', 'o2')]:
            path = '/' + '/'.join(path_parts)
            keys = self.verify_keys_for_path(
                path, expected_keys=('object', 'container'))
            # object keys should only be equal when complete paths are equal
            self.assertEqual(path_parts == ref_path_parts,
                             keys['object'] == ref_keys['object'],
                             'Path %s keys:\n%s\npath %s keys\n%s' %
                             (ref_path_parts, ref_keys, path_parts, keys))
            # container keys should only be equal when paths to container are
            # equal
            self.assertEqual(path_parts[:2] == ref_path_parts[:2],
                             keys['container'] == ref_keys['container'],
                             'Path %s keys:\n%s\npath %s keys\n%s' %
                             (ref_path_parts, ref_keys, path_parts, keys))

    def test_filter(self):
        factory = keymaster.filter_factory(TEST_KEYMASTER_CONF)
        self.assertTrue(callable(factory))
        self.assertTrue(callable(factory(self.swift)))

    def test_app_exception(self):
        app = keymaster.KeyMaster(
            FakeAppThatExcepts(), TEST_KEYMASTER_CONF)
        req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
        start_response, _ = capture_start_response()
        self.assertRaises(Exception, app, req.environ, start_response)

    def test_missing_conf_section(self):
        sample_conf = "[default]\nuser = %s\n" % getuser()
        with tmpfile(sample_conf) as conf_file:
            self.assertRaisesRegexp(
                ValueError, 'Unable to find keymaster config section in.*',
                keymaster.KeyMaster, self.swift, {
                    'keymaster_config_path': conf_file})

    def test_root_secret(self):
        def do_test(dflt_id):
            for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
                encoded_secret = base64.b64encode(secret)
                for conf_val in (
                        bytes(encoded_secret),
                        unicode(encoded_secret),
                        encoded_secret[:30] + '\n' + encoded_secret[30:]):
                    try:
                        app = keymaster.KeyMaster(
                            self.swift, {'encryption_root_secret': conf_val,
                                         'active_root_secret_id': dflt_id,
                                         'keymaster_config_path': ''})
                        self.assertEqual(secret, app.root_secret)
                    except AssertionError as err:
                        self.fail(str(err) + ' for secret %r' % conf_val)
        do_test(None)
        do_test('')

    def test_no_root_secret(self):
        with self.assertRaises(ValueError) as cm:
            keymaster.KeyMaster(self.swift, {})
        self.assertEqual('No secret loaded for active_root_secret_id None',
                         str(cm.exception))

    def test_multiple_root_secrets(self):
        secrets = {None: os.urandom(32),
                   '22': os.urandom(33),
                   'my_secret_id': os.urandom(50)}

        conf = {}
        for secret_id, secret in secrets.items():
            opt = ('encryption_root_secret%s' %
                   (('_%s' % secret_id) if secret_id else ''))
            conf[opt] = base64.b64encode(secret)
        app = keymaster.KeyMaster(self.swift, conf)
        self.assertEqual(secrets, app._root_secrets)
        self.assertEqual([None, '22', 'my_secret_id'], app.root_secret_ids)

    def test_chained_keymasters(self):
        conf_inner = {'active_root_secret_id': '22'}
        conf_inner.update(
            ('encryption_root_secret_%s' % secret_id, base64.b64encode(secret))
            for secret_id, secret in [('22', os.urandom(33)),
                                      ('my_secret_id', os.urandom(50))])
        conf_outer = {'encryption_root_secret': base64.b64encode(
            os.urandom(32))}
        app = keymaster.KeyMaster(
            keymaster.KeyMaster(self.swift, conf_inner),
            conf_outer)

        self.swift.register('GET', '/v1/a/c', swob.HTTPOk, {}, b'')
        req = Request.blank('/v1/a/c')
        start_response, calls = capture_start_response()
        app(req.environ, start_response)
        self.assertEqual(1, len(calls))
        self.assertNotIn('swift.crypto.override', req.environ)
        self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
                      '%s not set in env' % CRYPTO_KEY_CALLBACK)
        keys = copy.deepcopy(req.environ[CRYPTO_KEY_CALLBACK](key_id=None))
        self.assertIn('id', keys)
        self.assertEqual(keys.pop('id'), {
            'v': '1',
            'path': '/a/c',
            'secret_id': '22',
        })
        # Inner-most active root secret wins
        root_key = base64.b64decode(conf_inner['encryption_root_secret_22'])
        self.assertIn('container', keys)
        self.assertEqual(keys.pop('container'),
                         hmac.new(root_key, '/a/c',
                                  digestmod=hashlib.sha256).digest())
        self.assertIn('all_ids', keys)
        all_keys = set()
        at_least_one_old_style_id = False
        for key_id in keys.pop('all_ids'):
            # Can get key material for each key_id
            all_keys.add(req.environ[CRYPTO_KEY_CALLBACK](
                key_id=key_id)['container'])

            if 'secret_id' in key_id:
                self.assertIn(key_id.pop('secret_id'), {'22', 'my_secret_id'})
            else:
                at_least_one_old_style_id = True
            self.assertEqual(key_id, {
                'path': '/a/c',
                'v': '1',
            })
        self.assertTrue(at_least_one_old_style_id)
        self.assertEqual(len(all_keys), 3)
        self.assertFalse(keys)

        # Also all works for objects
        self.swift.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, b'')
        req = Request.blank('/v1/a/c/o')
        start_response, calls = capture_start_response()
        app(req.environ, start_response)
        self.assertEqual(1, len(calls))
        self.assertNotIn('swift.crypto.override', req.environ)
        self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
                      '%s not set in env' % CRYPTO_KEY_CALLBACK)
        keys = req.environ.get(CRYPTO_KEY_CALLBACK)(key_id=None)
        self.assertIn('id', keys)
        self.assertEqual(keys.pop('id'), {
            'v': '1',
            'path': '/a/c/o',
            'secret_id': '22',
        })
        root_key = base64.b64decode(conf_inner['encryption_root_secret_22'])
        self.assertIn('container', keys)
        self.assertEqual(keys.pop('container'),
                         hmac.new(root_key, '/a/c',
                                  digestmod=hashlib.sha256).digest())
        self.assertIn('object', keys)
        self.assertEqual(keys.pop('object'),
                         hmac.new(root_key, '/a/c/o',
                                  digestmod=hashlib.sha256).digest())
        self.assertIn('all_ids', keys)
        at_least_one_old_style_id = False
        for key_id in keys.pop('all_ids'):
            if 'secret_id' not in key_id:
                at_least_one_old_style_id = True
            else:
                self.assertIn(key_id.pop('secret_id'), {'22', 'my_secret_id'})
            self.assertEqual(key_id, {
                'path': '/a/c/o',
                'v': '1',
            })
        self.assertTrue(at_least_one_old_style_id)
        self.assertEqual(len(all_keys), 3)
        self.assertFalse(keys)

    def test_multiple_root_secrets_with_invalid_secret(self):
        conf = {'encryption_root_secret': base64.b64encode(os.urandom(32)),
                # too short...
                'encryption_root_secret_22': base64.b64encode(os.urandom(31))}
        with self.assertRaises(ValueError) as err:
            keymaster.KeyMaster(self.swift, conf)
        self.assertEqual(
            'encryption_root_secret_22 option in proxy-server.conf '
            'must be a base64 encoding of at least 32 raw bytes',
            str(err.exception))

    def test_multiple_root_secrets_with_invalid_id(self):
        def do_test(bad_option):
            conf = {'encryption_root_secret': base64.b64encode(os.urandom(32)),
                    bad_option: base64.b64encode(os.urandom(32))}
            with self.assertRaises(ValueError) as err:
                keymaster.KeyMaster(self.swift, conf)
            self.assertEqual(
                'Malformed root secret option name %s' % bad_option,
                str(err.exception))
        do_test('encryption_root_secret1')
        do_test('encryption_root_secret123')
        do_test('encryption_root_secret_')

    def test_multiple_root_secrets_missing_active_root_secret_id(self):
        conf = {'encryption_root_secret_22': base64.b64encode(os.urandom(32))}
        with self.assertRaises(ValueError) as err:
            keymaster.KeyMaster(self.swift, conf)
        self.assertEqual(
            'No secret loaded for active_root_secret_id None',
            str(err.exception))

        conf = {'encryption_root_secret_22': base64.b64encode(os.urandom(32)),
                'active_root_secret_id': 'missing'}
        with self.assertRaises(ValueError) as err:
            keymaster.KeyMaster(self.swift, conf)
        self.assertEqual(
            'No secret loaded for active_root_secret_id missing',
            str(err.exception))

    def test_correct_root_secret_used(self):
        secrets = {None: os.urandom(32),
                   '22': os.urandom(33),
                   'my_secret_id': os.urandom(50)}

        # no active_root_secret_id configured
        conf = {}
        for secret_id, secret in secrets.items():
            opt = ('encryption_root_secret%s' %
                   (('_%s' % secret_id) if secret_id else ''))
            conf[opt] = base64.b64encode(secret)
        self.app = keymaster.KeyMaster(self.swift, conf)
        keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'))
        expected_keys = {
            'container': hmac.new(secrets[None], '/a/c',
                                  digestmod=hashlib.sha256).digest(),
            'object': hmac.new(secrets[None], '/a/c/o',
                               digestmod=hashlib.sha256).digest()}
        self.assertEqual(expected_keys, keys)

        # active_root_secret_id configured
        conf['active_root_secret_id'] = '22'
        self.app = keymaster.KeyMaster(self.swift, conf)
        keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'))
        expected_keys = {
            'container': hmac.new(secrets['22'], '/a/c',
                                  digestmod=hashlib.sha256).digest(),
            'object': hmac.new(secrets['22'], '/a/c/o',
                               digestmod=hashlib.sha256).digest()}
        self.assertEqual(expected_keys, keys)

        # secret_id passed to fetch_crypto_keys callback
        for secret_id in ('my_secret_id', None):
            keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'),
                                             key_id={'secret_id': secret_id})
            expected_keys = {
                'container': hmac.new(secrets[secret_id], '/a/c',
                                      digestmod=hashlib.sha256).digest(),
                'object': hmac.new(secrets[secret_id], '/a/c/o',
                                   digestmod=hashlib.sha256).digest()}
            self.assertEqual(expected_keys, keys)

    def test_keys_cached(self):
        secrets = {None: os.urandom(32),
                   '22': os.urandom(33),
                   'my_secret_id': os.urandom(50)}
        conf = {}
        for secret_id, secret in secrets.items():
            opt = ('encryption_root_secret%s' %
                   (('_%s' % secret_id) if secret_id else ''))
            conf[opt] = base64.b64encode(secret)
        conf['active_root_secret_id'] = '22'
        self.app = keymaster.KeyMaster(self.swift, conf)
        orig_create_key = self.app.create_key
        calls = []

        def mock_create_key(path, secret_id=None):
            calls.append((path, secret_id))
            return orig_create_key(path, secret_id)

        context = keymaster.KeyMasterContext(self.app, 'a', 'c', 'o')
        with mock.patch.object(self.app, 'create_key', mock_create_key):
            keys = context.fetch_crypto_keys()
        expected_keys = {
            'container': hmac.new(secrets['22'], '/a/c',
                                  digestmod=hashlib.sha256).digest(),
            'object': hmac.new(secrets['22'], '/a/c/o',
                               digestmod=hashlib.sha256).digest(),
            'id': {'path': '/a/c/o', 'secret_id': '22', 'v': '1'},
            'all_ids': [
                {'path': '/a/c/o', 'v': '1'},
                {'path': '/a/c/o', 'secret_id': '22', 'v': '1'},
                {'path': '/a/c/o', 'secret_id': 'my_secret_id', 'v': '1'}]}
        self.assertEqual(expected_keys, keys)
        self.assertEqual([('/a/c', '22'), ('/a/c/o', '22')], calls)
        with mock.patch.object(self.app, 'create_key', mock_create_key):
            keys = context.fetch_crypto_keys()
        # no more calls to create_key
        self.assertEqual([('/a/c', '22'), ('/a/c/o', '22')], calls)
        self.assertEqual(expected_keys, keys)
        with mock.patch.object(self.app, 'create_key', mock_create_key):
            keys = context.fetch_crypto_keys(key_id={'secret_id': None})
        expected_keys = {
            'container': hmac.new(secrets[None], '/a/c',
                                  digestmod=hashlib.sha256).digest(),
            'object': hmac.new(secrets[None], '/a/c/o',
                               digestmod=hashlib.sha256).digest(),
            'id': {'path': '/a/c/o', 'v': '1'},
            'all_ids': [
                {'path': '/a/c/o', 'v': '1'},
                {'path': '/a/c/o', 'secret_id': '22', 'v': '1'},
                {'path': '/a/c/o', 'secret_id': 'my_secret_id', 'v': '1'}]}
        self.assertEqual(expected_keys, keys)
        self.assertEqual([('/a/c', '22'), ('/a/c/o', '22'),
                          ('/a/c', None), ('/a/c/o', None)],
                         calls)

    @mock.patch('swift.common.middleware.crypto.keymaster.readconf')
    def test_keymaster_config_path(self, mock_readconf):
        for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
            enc_secret = base64.b64encode(secret)
            for conf_val in (bytes(enc_secret), unicode(enc_secret),
                             enc_secret[:30] + '\n' + enc_secret[30:],
                             enc_secret[:30] + '\r\n' + enc_secret[30:]):
                mock_readconf.reset_mock()
                mock_readconf.return_value = {
                    'encryption_root_secret': conf_val}

                app = keymaster.KeyMaster(self.swift, {
                    'keymaster_config_path': '/some/path'})
                try:
                    self.assertEqual(secret, app.root_secret)
                    self.assertEqual(mock_readconf.mock_calls, [
                        mock.call('/some/path', 'keymaster')])
                except AssertionError as err:
                    self.fail(str(err) + ' for secret %r' % secret)

    def test_invalid_root_secret(self):
        for secret in (bytes(base64.b64encode(os.urandom(31))),  # too short
                       unicode(base64.b64encode(os.urandom(31))),
                       u'a' * 44 + u'????', b'a' * 44 + b'????',  # not base64
                       u'a' * 45, b'a' * 45,  # bad padding
                       99, None):
            conf = {'encryption_root_secret': secret}
            try:
                with self.assertRaises(ValueError) as err:
                    keymaster.KeyMaster(self.swift, conf)
                self.assertEqual(
                    'encryption_root_secret option in proxy-server.conf '
                    'must be a base64 encoding of at least 32 raw bytes',
                    str(err.exception))
            except AssertionError as err:
                self.fail(str(err) + ' for conf %s' % str(conf))

    @mock.patch('swift.common.middleware.crypto.keymaster.readconf')
    def test_root_secret_path_invalid_secret(self, mock_readconf):
        for secret in (bytes(base64.b64encode(os.urandom(31))),  # too short
                       unicode(base64.b64encode(os.urandom(31))),
                       u'a' * 44 + u'????', b'a' * 44 + b'????',  # not base64
                       u'a' * 45, b'a' * 45,  # bad padding
                       99, None):
            mock_readconf.reset_mock()
            mock_readconf.return_value = {'encryption_root_secret': secret}

            try:
                with self.assertRaises(ValueError) as err:
                    keymaster.KeyMaster(self.swift, {
                        'keymaster_config_path': '/some/other/path'})
                self.assertEqual(
                    'encryption_root_secret option in /some/other/path '
                    'must be a base64 encoding of at least 32 raw bytes',
                    str(err.exception))
                self.assertEqual(mock_readconf.mock_calls, [
                    mock.call('/some/other/path', 'keymaster')])
            except AssertionError as err:
                self.fail(str(err) + ' for secret %r' % secret)

    def test_can_only_configure_secret_in_one_place(self):
        def do_test(conf):
            with self.assertRaises(ValueError) as err:
                keymaster.KeyMaster(self.swift, conf)
            expected_message = ('keymaster_config_path is set, but there are '
                                'other config options specified:')
            self.assertTrue(str(err.exception).startswith(expected_message),
                            "Error message does not start with '%s'" %
                            expected_message)

        conf = {'encryption_root_secret': 'a' * 44,
                'keymaster_config_path': '/etc/swift/keymaster.conf'}
        do_test(conf)
        conf = {'encryption_root_secret_1': 'a' * 44,
                'keymaster_config_path': '/etc/swift/keymaster.conf'}
        do_test(conf)
        conf = {'encryption_root_secret_': 'a' * 44,
                'keymaster_config_path': '/etc/swift/keymaster.conf'}
        do_test(conf)
        conf = {'active_root_secret_id': '1',
                'keymaster_config_path': '/etc/swift/keymaster.conf'}
        do_test(conf)
예제 #27
0
 def setUp(self):
     self.app = FakeSwift()
     self.encrypter = encrypter.Encrypter(self.app, {})
     self.encrypter.logger = FakeLogger()
class OioContainerHierarchy(unittest.TestCase):
    def setUp(self):
        conf = {'sds_default_account': 'OPENIO'}
        self.filter_conf = {
            'strip_v1': 'true',
            'swift3_compat': 'true',
            'account_first': 'true'
        }
        self.app = FakeSwift()
        self.ch = container_hierarchy.filter_factory(
            conf,
            **self.filter_conf)(self.app)

    def mock(self):
        self.ch._create_key = mock.MagicMock(return_value=None)
        self.ch._remove_key = mock.MagicMock(return_value=None)

    def call_app(self, req, app=None):
        if app is None:
            app = self.app

        self.authorized = []

        def authorize(req):
            self.authorized.append(req)

        if 'swift.authorize' not in req.environ:
            req.environ['swift.authorize'] = authorize

        req.headers.setdefault("User-Agent", "Melted Cheddar")

        status = [None]
        headers = [None]

        def start_response(s, h, ei=None):
            status[0] = s
            headers[0] = h

        body_iter = app(req.environ, start_response)
        with utils.closing_if_possible(body_iter):
            body = b''.join(body_iter)

        return status[0], headers[0], body

    def call_ch(self, req):
        return self.call_app(req, app=self.ch)

    def test_simple_put(self):
        """check number of request generated by Container Hierarchy"""
        self.mock()
        self.app.register(
            'PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated, {})

        req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
        resp = self.call_ch(req)

        self.assertEqual(resp[0], '201 Created')
        self.ch._create_key.assert_called_with(mock.ANY,
                                               'a', 'c', 'cnt', 'd1/d2/d3/')

    def test_fake_directory_put(self):
        self.mock()
        req = Request.blank('/v1/a/c/d1/d2/d3/', method='PUT')
        resp = self.call_ch(req)

        self.assertEqual(resp[0], '201 Created')
        self.ch._create_key.assert_called_with(mock.ANY,
                                               'a', 'c', 'obj', 'd1/d2/d3/')

    def test_get(self):
        self.app.register(
            'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPOk, {})
        req = Request.blank('/v1/a/c/d1/d2/d3/o', method='GET')
        resp = self.call_ch(req)
        self.assertEqual(resp[0], '200 OK')

    def test_recursive_listing(self):
        self.ch.conn.keys = mock.MagicMock(return_value=['CS:a:cnt:d1/d2/d3/'])
        self.app.register(
            'GET',
            '/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&format=json',
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": "o",
                         "content_type": "application/octet-stream"}]))

        req = Request.blank('/v1/a/c?prefix=d1%2Fd2%2F', method='GET')
        resp = self.call_ch(req)

        data = json.loads(resp[2])
        self.assertEqual(data[0]['name'], 'd1/d2/d3/o')

    def test_listing_with_space(self):
        self.ch.conn.keys = mock.MagicMock(return_value=['CS:a:cnt:d 1/d2/'])
        self.app.register(
            'GET',
            '/v1/a/c%2Fd 1%2Fd2?prefix=&limit=10000&format=json',
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": "o",
                         "content_type": "application/octet-stream"}]))

        req = Request.blank('/v1/a/c?prefix=d%201%2Fd2%2F', method='GET')
        resp = self.call_ch(req)

        data = json.loads(resp[2])
        self.assertEqual(data[0]['name'], 'd 1/d2/o')

    def test_global_listing(self):
        self.app.register(
            'GET', '/v1/a', swob.HTTPOk, {})

        req = Request.blank('/v1/a', method='GET')
        resp = self.call_ch(req)
        self.assertEqual(resp[0], '200 OK')

    def test_delete_object(self):
        self.app.register(
            'PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated, {})

        req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
        resp = self.call_ch(req)
        self.assertEqual(resp[0], '201 Created')
        self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)

        self.app.register(
            'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=1&format=json',
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": "o",
                         "content_type": "application/octet-stream"}]))
        self.app.register(
            'DELETE', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPNoContent, {})

        req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
        resp = self.call_ch(req)

        self.assertEqual(resp[0], '204 No Content')
        self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)

        self.app.register(
            'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=1&format=json',
            swob.HTTPOk, {}, json.dumps([]))

        req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
        resp = self.call_ch(req)
        self.assertEqual(resp[0], '204 No Content')

        self.assertNotIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)

    def test_fake_directory(self):
        req = Request.blank('/v1/a/container/d2/d3/', method='PUT')
        resp = self.call_ch(req)
        self.assertIn('CS:a:container:obj:d2/d3/', self.ch.conn._keys)
        req = Request.blank('/v1/a/container/d2/d3/', method='DELETE')
        resp = self.call_ch(req)
        self.assertEqual(resp[0], "204 No Content")
        self.assertNotIn('CS:a:container:obj:d2/d3/', self.ch.conn._keys)

    def _listing(self, is_recursive):
        self.ch.conn.keys = mock.MagicMock(
            return_value=['CS:a:bucket:cnt:d1/', 'CS:a:bucket:cnt:d1/d2/'])
        self.ch.conn.exist = mock.MagicMock(return_value=True)
        self.app.register(
            'GET',
            '/v1/a/bucket%2Fd1?prefix=d&limit=10000&format=json',
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": "o1",
                         "content_type": "application/octet-stream"}]))
        if is_recursive:
            self.app.register(
                'GET',
                '/v1/a/bucket%2Fd1%2Fd2?prefix=&limit=10000&format=json',
                swob.HTTPOk, {},
                json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                             "last_modified": "2018-04-20T09:40:59.000000",
                             "bytes": 0, "name": "o2",
                             "content_type": "application/octet-stream"}]))
        recursive = '' if is_recursive else '&delimiter=%2F'
        req = Request.blank('/v1/a/bucket?prefix=d1/d&limit=10%s' % recursive,
                            method='GET')
        resp = self.call_ch(req)

        names = [item.get('name', item.get('subdir'))
                 for item in json.loads(resp[2])]
        return names

    def test_listing_with_prefix(self):
        names = self._listing(False)
        self.assertIn('d1/o1', names)
        self.assertIn('d1/d2/', names)

    def test_listing_with_prefix_recursive(self):
        names = self._listing(True)
        self.assertIn('d1/o1', names)
        self.assertIn('d1/d2/o2', names)

    def test_listing_root_container(self):
        self.ch.conn.keys = mock.MagicMock(
            return_value=['CS:a:bucket:cnt:d1/'])
        self.app.register(
            'GET',
            '/v1/a/bucket?prefix=d&limit=10000&format=json',
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": "d0",
                         "content_type": "application/octet-stream"}]))
        req = Request.blank('/v1/a/bucket?prefix=d&limit=10&delimiter=%2F',
                            method='GET')
        resp = self.call_ch(req)
        names = [item.get('name', item.get('subdir'))
                 for item in json.loads(resp[2])]
        self.assertIn("d0", names)
        self.assertIn("d1/", names)

    def test_listing_with_marker(self):
        self.ch.conn.keys = mock.MagicMock(
            return_value=['CS:a:bucket:cnt:d1/',
                          'CS:a:bucket:cnt:d2/',
                          ])
        req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
                            method='GET')
        resp = self.call_ch(req)
        names = [item.get('name', item.get('subdir'))
                 for item in json.loads(resp[2])]
        self.assertNotIn('d1/', names)
        self.assertIn('d2/', names)

    def test_listing_with_marker_multi_container(self):
        self.ch.conn.keys = mock.MagicMock(
            return_value=['CS:a:bucket:cnt:d1/',
                          'CS:a:bucket:cnt:d2/',
                          ])

        # with marker aa (as we inspect d1/)
        self.app.register(
            'GET',
            '/v1/a/bucket%2Fd1?marker=aa&prefix=&limit=10000&format=json', # noqa
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": "d0",
                         "content_type": "application/octet-stream"}]))
        # without marker on second container
        self.app.register(
            'GET',
            '/v1/a/bucket%2Fd2?prefix=&limit=10000&format=json',
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": "d0",
                         "content_type": "application/octet-stream"}]))
        req = Request.blank('/v1/a/bucket?limit=10&marker=d1/aa',
                            method='GET')
        resp = self.call_ch(req)
        names = [item.get('name', item.get('subdir'))
                 for item in json.loads(resp[2])]
        self.assertIn('d1/d0', names)
        self.assertIn('d2/d0', names)

    def test_duplicate_obj_cnt(self):
        self.ch.conn.keys = mock.MagicMock(
            return_value=['CS:a:bucket:cnt:d1/cnt/',
                          'CS:a:bucket:obj:d1/obj/',
                          ])
        req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
                            method='GET')
        resp = self.call_ch(req)
        names = [item.get('name', item.get('subdir'))
                 for item in json.loads(resp[2])]
        self.assertIn('d1/', names)
        self.assertEqual(1, len(names))

    def test_remove_bucket(self):
        self.app.register(
            'DELETE',
            '/v1/a/bucket',
            swob.HTTPNoContent, {},
            "")
        req = Request.blank('/v1/a/bucket',
                            method='DELETE')
        resp = self.call_ch(req)
        self.assertEqual(resp[0], "204 No Content")

    def test_invalid_path(self):
        req = Request.blank('/v1/a/',
                            method='GET')
        with self.assertRaises(HTTPException) as cm:
            self.call_ch(req)
        self.assertEqual(cm.exception.status, "400 Bad Request")

    def test_path(self):
        cont = 'bucket'

        path = 'dir1/dir2/object'
        res = self.ch._fake_container_and_obj(cont, path.split('/'))
        self.assertEqual(res, (cont + '%2Fdir1%2Fdir2', 'object'))

        path = 'object'
        res = self.ch._fake_container_and_obj(cont, path.split('/'))
        self.assertEqual(res, (cont, 'object'))

    def test_mpu_path(self):
        cont = 'bucket+segments'
        uploadid = 'MzNkYWZlNjItNjg3Yy00ZmIyLWIwOGYtOTA2OGVlZTA2MzA5'

        path = ('dir1/dir2/object/%s/1' % uploadid).split('/')
        res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
        self.assertEqual(res, (cont + '%2Fdir1%2Fdir2',
                               'object/%s/1' % uploadid))

        path = ('dir1/dir2/object/' + uploadid).split('/')
        res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
        self.assertEqual(res, (cont + '%2Fdir1%2Fdir2',
                               'object/' + uploadid))

        path = ('object/%s/1' % uploadid).split('/')
        res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
        self.assertEqual(res, (cont,
                               'object/%s/1' % uploadid))

        path = ('object/' + uploadid).split('/')
        res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
        self.assertEqual(res, (cont,
                               'object/' + uploadid))

    def test_upload_in_progress(self):
        self.ch.conn.keys = mock.MagicMock(
            return_value=['CS:a:bucket+segments:cnt:d1/d2/d3/'])
        upload = ["obj/YmYwY2I1ZDYtNjMyYi00OGNiLWEzMzEtZDdhYTk0ODZkNWU2",
                  "root/MzNkYWZlNjItNjg3Yy00ZmIyLWIwOGYtOTA2OGVlZTA2MzA5"]
        self.app.register(
            'GET',
            '/v1/a/bucket+segments%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&format=json',  # noqa
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": upload[0],
                         "content_type": "application/octet-stream"},
                        {"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 400, "name": upload[0] + '/1',
                         "content_type": "application/octet-stream"}]))
        self.app.register(
            'GET',
            '/v1/a/bucket+segments?prefix=&limit=10000&format=json',
            swob.HTTPOk, {},
            json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 0, "name": upload[1],
                         "content_type": "application/octet-stream"},
                        {"hash": "d41d8cd98f00b204e9800998ecf8427e",
                         "last_modified": "2018-04-20T09:40:59.000000",
                         "bytes": 400, "name": upload[1] + '/1',
                         "content_type": "application/octet-stream"}]))
        req = Request.blank('/v1/a/bucket+segments',
                            method='GET')
        resp = self.call_ch(req)

        names = [item.get('name', item.get('subdir'))
                 for item in json.loads(resp[2])]
        self.assertEqual(names,
                         ['d1/d2/d3/' + upload[0],
                          'd1/d2/d3/' + upload[0] + '/1',
                          upload[1],
                          upload[1] + '/1'])

    def test_copy_headers(self):
        self.app.register(
            'PUT', '/v1/a/bucket%2Fdir1/target',
            swob.HTTPNoContent, {},
        )
        req = Request.blank(
            '/v1/a/bucket/dir1/target',
            method='PUT',
            headers={'Oio-Copy-From': '/v1/a/bucket/sub1/source'})

        resp = self.call_ch(req)
        self.assertEqual(resp[0], '204 No Content')
        self.assertEqual(self.app.headers[0]['Oio-Copy-From'],
                         "/v1%2Fa%2Fbucket%2Fsub1/source")
예제 #29
0
 def setUp(self):
     super(TestKeymaster, self).setUp()
     self.swift = FakeSwift()
     self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)
예제 #30
0
class SymlinkVersioningTestCase(TestSymlinkMiddlewareBase):
    # verify interaction of versioned_writes and symlink middlewares

    def setUp(self):
        self.app = FakeSwift()
        conf = {'symloop_max': '2'}
        self.sym = symlink.filter_factory(conf)(self.app)
        self.sym.logger = self.app.logger
        vw_conf = {'allow_versioned_writes': 'true'}
        self.vw = versioned_writes.filter_factory(vw_conf)(self.sym)

    def call_vw(self, req, **kwargs):
        return self.call_app(req, app=self.vw, **kwargs)

    def assertRequestEqual(self, req, other):
        self.assertEqual(req.method, other.method)
        self.assertEqual(req.path, other.path)

    def test_new_symlink_version_success(self):
        self.app.register(
            'PUT', '/v1/a/c/symlink', swob.HTTPCreated,
            {'X-Symlink-Target': 'new_cont/new_tgt',
             'X-Symlink-Target-Account': 'a'}, None)
        self.app.register(
            'GET', '/v1/a/c/symlink', swob.HTTPOk,
            {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT',
             'X-Object-Sysmeta-Symlink-Target': 'old_cont/old_tgt',
             'X-Object-Sysmeta-Symlink-Target-Account': 'a'},
            '')
        self.app.register(
            'PUT', '/v1/a/ver_cont/007symlink/0000000001.00000',
            swob.HTTPCreated,
            {'X-Symlink-Target': 'old_cont/old_tgt',
             'X-Symlink-Target-Account': 'a'}, None)
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/symlink',
            headers={'X-Symlink-Target': 'new_cont/new_tgt'},
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0',
                     'swift.trans_id': 'fake_trans_id'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        # authorized twice now because versioned_writes now makes a check on
        # PUT
        self.assertEqual(len(self.authorized), 2)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(['VW', 'VW', None], self.app.swift_sources)
        self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[2]
        self.assertEqual('PUT', method)
        self.assertEqual('/v1/a/c/symlink', path)
        self.assertEqual(
            'new_cont/new_tgt',
            req_headers['X-Object-Sysmeta-Symlink-Target'])

    def test_delete_latest_version_no_marker_success(self):
        self.app.register(
            'GET',
            '/v1/a/ver_cont?prefix=003sym/&marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 0, '
            '"name": "003sym/2", '
            '"content_type": "text/plain"}, '
            '{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 0, '
            '"name": "003sym/1", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'GET', '/v1/a/ver_cont/003sym/2', swob.HTTPCreated,
            {'content-length': '0',
             'X-Object-Sysmeta-Symlink-Target': 'c/tgt'}, None)
        self.app.register(
            'PUT', '/v1/a/c/sym', swob.HTTPCreated,
            {'X-Symlink-Target': 'c/tgt', 'X-Symlink-Target-Account': 'a'},
            None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/003sym/2', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/sym',
            headers={'X-If-Delete-At': 1},
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0', 'swift.trans_id': 'fake_trans_id'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(4, self.app.call_count)
        self.assertEqual(['VW', 'VW', 'VW', 'VW'], self.app.swift_sources)
        self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[2]
        self.assertEqual('PUT', method)
        self.assertEqual('/v1/a/c/sym', path)
        self.assertEqual(
            'c/tgt',
            req_headers['X-Object-Sysmeta-Symlink-Target'])
예제 #31
0
파일: test_dlo.py 프로젝트: charint/swift
    def setUp(self):
        self.app = FakeSwift()
        self.dlo = dlo.filter_factory({
            # don't slow down tests with rate limiting
            'rate_limit_after_segment': '1000000',
        })(self.app)

        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_01',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg01-etag'},
            'aaaaa')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_02',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg02-etag'},
            'bbbbb')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg03-etag'},
            'ccccc')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_04',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg04-etag'},
            'ddddd')
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_05',
            swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg05-etag'},
            'eeeee')

        # an unrelated object (not seg*) to test the prefix matching
        self.app.register(
            'GET', '/v1/AUTH_test/c/catpicture.jpg',
            swob.HTTPOk, {'Content-Length': '9', 'Etag': 'cats-etag'},
            'meow meow meow meow')

        self.app.register(
            'GET', '/v1/AUTH_test/mancon/manifest',
            swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
                          'X-Object-Manifest': 'c/seg'},
            'manifest-contents')

        lm = '2013-11-22T02:42:13.781760'
        ct = 'application/octet-stream'
        segs = [{"hash": "seg01-etag", "bytes": 5, "name": "seg_01",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg02-etag", "bytes": 5, "name": "seg_02",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg03-etag", "bytes": 5, "name": "seg_03",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg04-etag", "bytes": 5, "name": "seg_04",
                 "last_modified": lm, "content_type": ct},
                {"hash": "seg05-etag", "bytes": 5, "name": "seg_05",
                 "last_modified": lm, "content_type": ct}]

        full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9,
                                          "name": "catpicture.jpg",
                                          "last_modified": lm,
                                          "content_type": "application/png"}]
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(full_container_listing))
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=seg',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(segs))

        # This is to let us test multi-page container listings; we use the
        # trailing underscore to send small (pagesize=3) listings.
        #
        # If you're testing against this, be sure to mock out
        # CONTAINER_LISTING_LIMIT to 3 in your test.
        self.app.register(
            'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
            swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
                          'X-Object-Manifest': 'c/seg_'},
            'manyseg')
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=seg_',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(segs[:3]))
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(segs[3:]))

        # Here's a manifest with 0 segments
        self.app.register(
            'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
            swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
                          'X-Object-Manifest': 'c/noseg_'},
            'noseg')
        self.app.register(
            'GET', '/v1/AUTH_test/c?format=json&prefix=noseg_',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps([]))
예제 #32
0
class TestKeymaster(unittest.TestCase):

    def setUp(self):
        super(TestKeymaster, self).setUp()
        self.swift = FakeSwift()
        self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)

    def test_object_path(self):
        self.verify_keys_for_path(
            '/a/c/o', expected_keys=('object', 'container'))

    def test_container_path(self):
        self.verify_keys_for_path(
            '/a/c', expected_keys=('container',))

    def verify_keys_for_path(self, path, expected_keys):
        put_keys = None
        for method, resp_class, status in (
                ('PUT', swob.HTTPCreated, '201'),
                ('POST', swob.HTTPAccepted, '202'),
                ('GET', swob.HTTPOk, '200'),
                ('HEAD', swob.HTTPNoContent, '204')):
            resp_headers = {}
            self.swift.register(
                method, '/v1' + path, resp_class, resp_headers, '')
            req = Request.blank(
                '/v1' + path, environ={'REQUEST_METHOD': method})
            start_response, calls = capture_start_response()
            self.app(req.environ, start_response)
            self.assertEqual(1, len(calls))
            self.assertTrue(calls[0][0].startswith(status))
            self.assertNotIn('swift.crypto.override', req.environ)
            self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
                          '%s not set in env' % CRYPTO_KEY_CALLBACK)
            keys = req.environ.get(CRYPTO_KEY_CALLBACK)()
            self.assertIn('id', keys)
            id = keys.pop('id')
            self.assertEqual(path, id['path'])
            self.assertEqual('1', id['v'])
            self.assertListEqual(sorted(expected_keys), sorted(keys.keys()),
                                 '%s %s got keys %r, but expected %r'
                                 % (method, path, keys.keys(), expected_keys))
            if put_keys is not None:
                # check all key sets were consistent for this path
                self.assertDictEqual(put_keys, keys)
            else:
                put_keys = keys
        return put_keys

    def test_key_uniqueness(self):
        # a rudimentary check that different keys are made for different paths
        ref_path_parts = ('a1', 'c1', 'o1')
        path = '/' + '/'.join(ref_path_parts)
        ref_keys = self.verify_keys_for_path(
            path, expected_keys=('object', 'container'))

        # for same path and for each differing path check that keys are unique
        # when path to object or container is unique and vice-versa
        for path_parts in [(a, c, o) for a in ('a1', 'a2')
                           for c in ('c1', 'c2')
                           for o in ('o1', 'o2')]:
            path = '/' + '/'.join(path_parts)
            keys = self.verify_keys_for_path(
                path, expected_keys=('object', 'container'))
            # object keys should only be equal when complete paths are equal
            self.assertEqual(path_parts == ref_path_parts,
                             keys['object'] == ref_keys['object'],
                             'Path %s keys:\n%s\npath %s keys\n%s' %
                             (ref_path_parts, ref_keys, path_parts, keys))
            # container keys should only be equal when paths to container are
            # equal
            self.assertEqual(path_parts[:2] == ref_path_parts[:2],
                             keys['container'] == ref_keys['container'],
                             'Path %s keys:\n%s\npath %s keys\n%s' %
                             (ref_path_parts, ref_keys, path_parts, keys))

    def test_filter(self):
        factory = keymaster.filter_factory(TEST_KEYMASTER_CONF)
        self.assertTrue(callable(factory))
        self.assertTrue(callable(factory(self.swift)))

    def test_app_exception(self):
        app = keymaster.KeyMaster(
            FakeAppThatExcepts(), TEST_KEYMASTER_CONF)
        req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
        start_response, _ = capture_start_response()
        self.assertRaises(Exception, app, req.environ, start_response)

    def test_missing_conf_section(self):
        sample_conf = "[default]\nuser = %s\n" % getuser()
        with tmpfile(sample_conf) as conf_file:
            self.assertRaisesRegexp(
                ValueError, 'Unable to find keymaster config section in.*',
                keymaster.KeyMaster, self.swift, {
                    'keymaster_config_path': conf_file})

    def test_root_secret(self):
        for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
            encoded_secret = base64.b64encode(secret)
            for conf_val in (bytes(encoded_secret), unicode(encoded_secret),
                             encoded_secret[:30] + '\n' + encoded_secret[30:]):
                try:
                    app = keymaster.KeyMaster(
                        self.swift, {'encryption_root_secret': conf_val,
                                     'encryption_root_secret_path': ''})
                    self.assertEqual(secret, app.root_secret)
                except AssertionError as err:
                    self.fail(str(err) + ' for secret %r' % conf_val)

    @mock.patch('swift.common.middleware.crypto.keymaster.readconf')
    def test_keymaster_config_path(self, mock_readconf):
        for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
            enc_secret = base64.b64encode(secret)
            for conf_val in (bytes(enc_secret), unicode(enc_secret),
                             enc_secret[:30] + '\n' + enc_secret[30:],
                             enc_secret[:30] + '\r\n' + enc_secret[30:]):
                for ignored_secret in ('invalid! but ignored!',
                                       'xValidButIgnored' * 10):
                    mock_readconf.reset_mock()
                    mock_readconf.return_value = {
                        'encryption_root_secret': conf_val}

                    app = keymaster.KeyMaster(self.swift, {
                        'keymaster_config_path': '/some/path'})
                    try:
                        self.assertEqual(secret, app.root_secret)
                        self.assertEqual(mock_readconf.mock_calls, [
                            mock.call('/some/path', 'keymaster')])
                    except AssertionError as err:
                        self.fail(str(err) + ' for secret %r' % secret)

    def test_invalid_root_secret(self):
        for secret in (bytes(base64.b64encode(os.urandom(31))),  # too short
                       unicode(base64.b64encode(os.urandom(31))),
                       u'a' * 44 + u'????', b'a' * 44 + b'????',  # not base64
                       u'a' * 45, b'a' * 45,  # bad padding
                       99, None):
            conf = {'encryption_root_secret': secret}
            try:
                with self.assertRaises(ValueError) as err:
                    keymaster.KeyMaster(self.swift, conf)
                self.assertEqual(
                    'encryption_root_secret option in proxy-server.conf '
                    'must be a base64 encoding of at least 32 raw bytes',
                    err.exception.message)
            except AssertionError as err:
                self.fail(str(err) + ' for conf %s' % str(conf))

    @mock.patch('swift.common.middleware.crypto.keymaster.readconf')
    def test_root_secret_path_invalid_secret(self, mock_readconf):
        for secret in (bytes(base64.b64encode(os.urandom(31))),  # too short
                       unicode(base64.b64encode(os.urandom(31))),
                       u'a' * 44 + u'????', b'a' * 44 + b'????',  # not base64
                       u'a' * 45, b'a' * 45,  # bad padding
                       99, None):
            mock_readconf.reset_mock()
            mock_readconf.return_value = {'encryption_root_secret': secret}

            try:
                with self.assertRaises(ValueError) as err:
                    keymaster.KeyMaster(self.swift, {
                        'keymaster_config_path': '/some/other/path'})
                self.assertEqual(
                    'encryption_root_secret option in /some/other/path '
                    'must be a base64 encoding of at least 32 raw bytes',
                    err.exception.message)
                self.assertEqual(mock_readconf.mock_calls, [
                    mock.call('/some/other/path', 'keymaster')])
            except AssertionError as err:
                self.fail(str(err) + ' for secret %r' % secret)

    def test_can_only_configure_secret_in_one_place(self):
        conf = {'encryption_root_secret': 'a' * 44,
                'keymaster_config_path': '/ets/swift/keymaster.conf'}
        with self.assertRaises(ValueError) as err:
            keymaster.KeyMaster(self.swift, conf)
        self.assertEqual('keymaster_config_path is set, but there are '
                         'other config options specified!',
                         err.exception.message)