class TestHashedContainer(unittest.TestCase):
GLOBAL_CONF = {
'sds_namespace': 'OPENIO',
'sds_default_account': 'OPENIO',
'sds_proxy_url': '127.0.0.1:666'
}
def setUp(self):
self.app = FakeSwift()
# prevent a call to oio-proxy
clientmanager.ClientManager.nsinfo = {
'options': {'flat_bitlength': '17'}}
self.hc = hashedcontainer.filter_factory(self.GLOBAL_CONF)(self.app)
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
def _check_conversion(self, path_in, path_out):
self.app.register('PUT', path_out, swob.HTTPCreated, {})
req = Request.blank(path_in, method='PUT')
resp = self.call_app(req, app=self.hc)
self.assertEqual(resp[0], "201 Created")
self.assertEqual(self.app.calls, [('PUT', path_out)])
def test_default_config(self):
self._check_conversion(
'/prefix/229/358493922_something',
'/v1/OPENIO/6C800/prefix/229/358493922_something')
def test_custom_bits(self):
self.hc = hashedcontainer.filter_factory(
self.GLOBAL_CONF, bits=12)(self.app)
self._check_conversion(
'/prefix/229/358493922_something',
'/v1/OPENIO/6C8/prefix/229/358493922_something')
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ", ".join(plain_etags)
req = Request.blank(
"/v1/a/c/o", environ=env, method=method, headers={match_header_name: match_header_value}
)
app = FakeSwift()
app.register(method, "/v1/a/c/o", HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual("200 OK", resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != "*":
self.assertIn("X-Backend-Etag-Is-At", actual_headers)
self.assertEqual("X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(", "))
key = fetch_crypto_keys()["object"]
masked_etags = [
'"%s"' % base64.b64encode(hmac.new(key, etag.strip('"'), hashlib.sha256).digest())
for etag in plain_etags
if etag not in ("*", "")
]
expected_etags = set((expected_plain_etags or plain_etags) + masked_etags)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags), set(req.headers[match_header_name].split(", ")))
def get_mw(self, conf, etag='unquoted-etag', path=None):
if path is None:
path = '/v1/AUTH_acc/con/some/path/to/obj'
app = FakeSwift()
hdrs = {} if etag is None else {'ETag': etag}
app.register('GET', path, swob.HTTPOk, hdrs)
return etag_quoter.filter_factory({}, **conf)(app)
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ', '.join(plain_etags)
req = Request.blank(
'/v1/a/c/o',
environ=env,
method=method,
headers={match_header_name: match_header_value})
app = FakeSwift()
app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != '*':
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(', '))
# masked values for secret_id None
key = fetch_crypto_keys()['object']
masked_etags = [
'"%s"' % bytes_to_wsgi(
base64.b64encode(
hmac.new(key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')
]
# masked values for secret_id myid
key = fetch_crypto_keys(key_id={'secret_id': 'myid'})['object']
masked_etags_myid = [
'"%s"' % bytes_to_wsgi(
base64.b64encode(
hmac.new(key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')
]
expected_etags = set((expected_plain_etags or plain_etags) +
masked_etags + masked_etags_myid)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags),
set(req.headers[match_header_name].split(', ')))
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ', '.join(plain_etags)
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={match_header_name: match_header_value})
app = FakeSwift()
app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != '*':
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(', '))
# masked values for secret_id None
key = fetch_crypto_keys()['object']
masked_etags = [
'"%s"' % bytes_to_wsgi(base64.b64encode(hmac.new(
key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')]
# masked values for secret_id myid
key = fetch_crypto_keys(key_id={'secret_id': 'myid'})['object']
masked_etags_myid = [
'"%s"' % bytes_to_wsgi(base64.b64encode(hmac.new(
key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')]
expected_etags = set((expected_plain_etags or plain_etags) +
masked_etags + masked_etags_myid)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags),
set(req.headers[match_header_name].split(', ')))
class TestEncrypter(OrigTestEncrypter):
def setUp(self):
self.app = FakeSwift()
self.encrypter = encrypter.Encrypter(self.app, {})
self.encrypter.logger = FakeLogger()
def test_PUT_missing_key_in_header(self):
def raise_exc():
raise HTTPBadRequest(
'Missing X-Amz-Server-Side-Encryption-Customer-Key header')
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: raise_exc}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
# If it does not find a key, no problem,
# oioswift's encrypter let the request pass through,
# and does not encrypt the object.
self.assertEqual('201 Created', resp.status)
class VersionedWritesTestCase(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
conf = {'allow_versioned_writes': 'true'}
self.vw = versioned_writes.filter_factory(conf)(self.app)
def call_app(self, req, app=None, expect_exception=False):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Marula Kruger")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
body = ''
caught_exc = None
try:
for chunk in body_iter:
body += chunk
except Exception as exc:
if expect_exception:
caught_exc = exc
else:
raise
if expect_exception:
return status[0], headers[0], body, caught_exc
else:
return status[0], headers[0], body
def call_vw(self, req, **kwargs):
return self.call_app(req, app=self.vw, **kwargs)
def assertRequestEqual(self, req, other):
self.assertEqual(req.method, other.method)
self.assertEqual(req.path, other.path)
def test_put_container(self):
self.app.register('PUT', '/v1/a/c', swob.HTTPOk, {}, 'passed')
req = Request.blank('/v1/a/c',
headers={'X-Versions-Location': 'ver_cont'},
environ={'REQUEST_METHOD': 'PUT'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
# check for sysmeta header
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('PUT', method)
self.assertEqual('/v1/a/c', path)
self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_container_allow_versioned_writes_false(self):
self.vw.conf = {'allow_versioned_writes': 'false'}
# PUT/POST container must fail as 412 when allow_versioned_writes
# set to false
for method in ('PUT', 'POST'):
req = Request.blank('/v1/a/c',
headers={'X-Versions-Location': 'ver_cont'},
environ={'REQUEST_METHOD': method})
status, headers, body = self.call_vw(req)
self.assertEqual(status, "412 Precondition Failed")
# GET/HEAD performs as normal
self.app.register('GET', '/v1/a/c', swob.HTTPOk, {}, 'passed')
self.app.register('HEAD', '/v1/a/c', swob.HTTPOk, {}, 'passed')
for method in ('GET', 'HEAD'):
req = Request.blank('/v1/a/c',
headers={'X-Versions-Location': 'ver_cont'},
environ={'REQUEST_METHOD': method})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
def test_remove_versions_location(self):
self.app.register('POST', '/v1/a/c', swob.HTTPOk, {}, 'passed')
req = Request.blank('/v1/a/c',
headers={'X-Remove-Versions-Location': 'x'},
environ={'REQUEST_METHOD': 'POST'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
# check for sysmeta header
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('POST', method)
self.assertEqual('/v1/a/c', path)
self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
self.assertTrue('x-versions-location' in req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_remove_add_versions_precedence(self):
self.app.register(
'POST', '/v1/a/c', swob.HTTPOk,
{'x-container-sysmeta-versions-location': 'ver_cont'}, 'passed')
req = Request.blank('/v1/a/c',
headers={
'X-Remove-Versions-Location': 'x',
'X-Versions-Location': 'ver_cont'
},
environ={'REQUEST_METHOD': 'POST'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)
# check for sysmeta header
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('POST', method)
self.assertEqual('/v1/a/c', path)
self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
self.assertTrue('x-remove-versions-location' not in req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_get_container(self):
self.app.register(
'GET', '/v1/a/c', swob.HTTPOk,
{'x-container-sysmeta-versions-location': 'ver_cont'}, None)
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_get_head(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {}, None)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'HEAD'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_put_object_no_versioning(self):
self.app.register('PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
cache = FakeCache({})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_put_first_object_success(self):
self.app.register('PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_PUT_versioning_with_nonzero_default_policy(self):
self.app.register('PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None)
cache = FakeCache({'versions': 'ver_cont', 'storage_policy': '2'})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
# check for 'X-Backend-Storage-Policy-Index' in HEAD request
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('HEAD', method)
self.assertEqual('/v1/a/c/o', path)
self.assertTrue('X-Backend-Storage-Policy-Index' in req_headers)
self.assertEqual('2',
req_headers.get('X-Backend-Storage-Policy-Index'))
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_put_object_no_versioning_with_container_config_true(self):
# set False to versions_write obsously and expect no COPY occurred
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'},
'passed')
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
called_method = [method for (method, path, hdrs) in self.app._calls]
self.assertTrue('COPY' not in called_method)
def test_delete_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no GET versioning
# container and COPY called (just delete object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register('DELETE', '/v1/a/c/o', swob.HTTPNoContent, {},
'passed')
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'swift.cache': cache
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '204 No Content')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
called_method = \
[method for (method, path, rheaders) in self.app._calls]
self.assertTrue('COPY' not in called_method)
self.assertTrue('GET' not in called_method)
def test_copy_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no extra
# COPY called (just copy object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register('COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
called_method = \
[method for (method, path, rheaders) in self.app._calls]
self.assertTrue('COPY' in called_method)
self.assertEqual(called_method.count('COPY'), 1)
def test_new_version_success(self):
self.app.register('PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'},
'passed')
self.app.register('COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_new_version_sysmeta_precedence(self):
self.app.register('PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'},
'passed')
self.app.register('COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
# fill cache with two different values for versions location
# new middleware should use sysmeta first
cache = FakeCache({
'versions': 'old_ver_cont',
'sysmeta': {
'versions-location': 'ver_cont'
}
})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
# check that sysmeta header was used
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('COPY', method)
self.assertEqual('/v1/a/c/o', path)
self.assertTrue(req_headers['Destination'].startswith('ver_cont/'))
def test_copy_first_version(self):
self.app.register('COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {},
'passed')
self.app.register('HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound,
{}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_copy_new_version(self):
self.app.register('COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {},
'passed')
self.app.register('HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'},
'passed')
self.app.register('COPY', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_copy_new_version_different_account(self):
self.app.register('COPY', '/v1/src_a/src_cont/src_obj', swob.HTTPOk,
{}, 'passed')
self.app.register('HEAD', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'},
'passed')
self.app.register('COPY', '/v1/tgt_a/tgt_cont/tgt_obj',
swob.HTTPCreated, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/src_a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
},
headers={
'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': 'tgt_a'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_copy_new_version_bogus_account(self):
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/src_a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
},
headers={
'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': '/im/on/a/boat'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '412 Precondition Failed')
def test_delete_first_object_success(self):
self.app.register('DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPNotFound, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'swift.cache': cache,
'CONTENT_LENGTH': '0'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_delete_latest_version_success(self):
self.app.register('DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPOk, {}, '[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 3, '
'"name": "001o/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 3, '
'"name": "001o/1", '
'"content_type": "text/plain"}]')
self.app.register('COPY', '/v1/a/ver_cont/001o/2', swob.HTTPCreated,
{}, None)
self.app.register('DELETE', '/v1/a/ver_cont/001o/2', swob.HTTPOk, {},
None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/o',
headers={'X-If-Delete-At': 1},
environ={
'REQUEST_METHOD': 'DELETE',
'swift.cache': cache,
'CONTENT_LENGTH': '0'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
# check that X-If-Delete-At was removed from DELETE request
calls = self.app.calls_with_headers
method, path, req_headers = calls.pop()
self.assertEqual('DELETE', method)
self.assertTrue(path.startswith('/v1/a/ver_cont/001o/2'))
self.assertFalse('x-if-delete-at' in req_headers
or 'X-If-Delete-At' in req_headers)
def test_DELETE_on_expired_versioned_object(self):
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPOk, {}, '[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 3, '
'"name": "001o/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 3, '
'"name": "001o/1", '
'"content_type": "text/plain"}]')
# expired object
self.app.register('COPY', '/v1/a/ver_cont/001o/2', swob.HTTPNotFound,
{}, None)
self.app.register('COPY', '/v1/a/ver_cont/001o/1', swob.HTTPCreated,
{}, None)
self.app.register('DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPOk, {},
None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'swift.cache': cache,
'CONTENT_LENGTH': '0'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_denied_DELETE_of_versioned_object(self):
authorize_call = []
self.app.register('DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPOk, {}, '[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 3, '
'"name": "001o/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 3, '
'"name": "001o/1", '
'"content_type": "text/plain"}]')
self.app.register('DELETE', '/v1/a/c/o', swob.HTTPForbidden, {}, None)
def fake_authorize(req):
authorize_call.append(req)
return swob.HTTPForbidden()
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'swift.cache': cache,
'swift.authorize': fake_authorize,
'CONTENT_LENGTH': '0'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '403 Forbidden')
self.assertEqual(len(authorize_call), 1)
self.assertRequestEqual(req, authorize_call[0])
class ContainerQuotaCopyingTestCases(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.cq_filter = container_quotas.filter_factory({})(self.app)
self.copy_filter = copy.filter_factory({})(self.cq_filter)
def test_exceed_bytes_quota_copy_verb(self):
cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "2"}})
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
req = Request.blank(
"/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, "Upload exceeds quota.")
def test_not_exceed_bytes_quota_copy_verb(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
req = Request.blank(
"/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_exceed_counts_quota_copy_verb(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {}, "passed")
cache = FakeCache({"object_count": 1, "meta": {"quota-count": "1"}})
req = Request.blank(
"/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, "Upload exceeds quota.")
def test_exceed_counts_quota_copy_cross_account_verb(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {}, "passed")
a_c_cache = {"storage_policy": "0", "meta": {"quota-count": "2"}, "status": 200, "object_count": 1}
a2_c_cache = {"storage_policy": "0", "meta": {"quota-count": "1"}, "status": 200, "object_count": 1}
req = Request.blank(
"/v1/a/c2/o2",
environ={
"REQUEST_METHOD": "COPY",
"swift.infocache": {"container/a/c": a_c_cache, "container/a2/c": a2_c_cache},
},
headers={"Destination": "/c/o", "Destination-Account": "a2"},
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, "Upload exceeds quota.")
def test_exceed_counts_quota_copy_cross_account_PUT_verb(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {}, "passed")
a_c_cache = {"storage_policy": "0", "meta": {"quota-count": "2"}, "status": 200, "object_count": 1}
a2_c_cache = {"storage_policy": "0", "meta": {"quota-count": "1"}, "status": 200, "object_count": 1}
req = Request.blank(
"/v1/a2/c/o",
environ={
"REQUEST_METHOD": "PUT",
"swift.infocache": {"container/a/c": a_c_cache, "container/a2/c": a2_c_cache},
},
headers={"X-Copy-From": "/c2/o2", "X-Copy-From-Account": "a"},
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, "Upload exceeds quota.")
def test_exceed_bytes_quota_copy_from(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "2"}})
req = Request.blank(
"/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, "Upload exceeds quota.")
def test_not_exceed_bytes_quota_copy_from(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
req = Request.blank(
"/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_no_src(self):
self.app.register("GET", "/v1/a/c2/o3", HTTPOk, {}, "passed")
self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
req = Request.blank(
"/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o3"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_bad_src(self):
cache = FakeCache({"bytes": 0, "meta": {"quota-bytes": "100"}})
req = Request.blank(
"/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "bad_path"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 412)
def test_exceed_counts_quota_copy_from(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
cache = FakeCache({"object_count": 1, "meta": {"quota-count": "1"}})
req = Request.blank(
"/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, "Upload exceeds quota.")
def test_not_exceed_counts_quota_copy_from(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
cache = FakeCache({"object_count": 1, "meta": {"quota-count": "2"}})
req = Request.blank(
"/v1/a/c/o", environ={"REQUEST_METHOD": "PUT", "swift.cache": cache}, headers={"x-copy-from": "/c2/o2"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_not_exceed_counts_quota_copy_verb(self):
self.app.register("GET", "/v1/a/c2/o2", HTTPOk, {"Content-Length": "10"}, "passed")
self.app.register("PUT", "/v1/a/c/o", HTTPOk, {}, "passed")
cache = FakeCache({"object_count": 1, "meta": {"quota-count": "2"}})
req = Request.blank(
"/v1/a/c2/o2", environ={"REQUEST_METHOD": "COPY", "swift.cache": cache}, headers={"Destination": "/c/o"}
)
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
class VersionedWritesTestCase(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
conf = {'allow_versioned_writes': 'true'}
self.vw = versioned_writes.filter_factory(conf)(self.app)
def call_app(self, req, app=None, expect_exception=False):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Marula Kruger")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
body = ''
caught_exc = None
try:
for chunk in body_iter:
body += chunk
except Exception as exc:
if expect_exception:
caught_exc = exc
else:
raise
if expect_exception:
return status[0], headers[0], body, caught_exc
else:
return status[0], headers[0], body
def call_vw(self, req, **kwargs):
return self.call_app(req, app=self.vw, **kwargs)
def assertRequestEqual(self, req, other):
self.assertEqual(req.method, other.method)
self.assertEqual(req.path, other.path)
def test_put_container(self):
self.app.register('PUT', '/v1/a/c', swob.HTTPOk, {}, 'passed')
req = Request.blank('/v1/a/c',
headers={'X-Versions-Location': 'ver_cont'},
environ={'REQUEST_METHOD': 'PUT'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
# check for sysmeta header
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('PUT', method)
self.assertEqual('/v1/a/c', path)
self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_container_allow_versioned_writes_false(self):
self.vw.conf = {'allow_versioned_writes': 'false'}
# PUT/POST container must fail as 412 when allow_versioned_writes
# set to false
for method in ('PUT', 'POST'):
req = Request.blank('/v1/a/c',
headers={'X-Versions-Location': 'ver_cont'},
environ={'REQUEST_METHOD': method})
status, headers, body = self.call_vw(req)
self.assertEqual(status, "412 Precondition Failed")
# GET/HEAD performs as normal
self.app.register('GET', '/v1/a/c', swob.HTTPOk, {}, 'passed')
self.app.register('HEAD', '/v1/a/c', swob.HTTPOk, {}, 'passed')
for method in ('GET', 'HEAD'):
req = Request.blank('/v1/a/c',
headers={'X-Versions-Location': 'ver_cont'},
environ={'REQUEST_METHOD': method})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
def test_remove_versions_location(self):
self.app.register('POST', '/v1/a/c', swob.HTTPOk, {}, 'passed')
req = Request.blank('/v1/a/c',
headers={'X-Remove-Versions-Location': 'x'},
environ={'REQUEST_METHOD': 'POST'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
# check for sysmeta header
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('POST', method)
self.assertEqual('/v1/a/c', path)
self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
self.assertTrue('x-versions-location' in req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_remove_add_versions_precedence(self):
self.app.register(
'POST', '/v1/a/c', swob.HTTPOk,
{'x-container-sysmeta-versions-location': 'ver_cont'},
'passed')
req = Request.blank('/v1/a/c',
headers={'X-Remove-Versions-Location': 'x',
'X-Versions-Location': 'ver_cont'},
environ={'REQUEST_METHOD': 'POST'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)
# check for sysmeta header
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('POST', method)
self.assertEqual('/v1/a/c', path)
self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
self.assertTrue('x-remove-versions-location' not in req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_get_container(self):
self.app.register(
'GET', '/v1/a/c', swob.HTTPOk,
{'x-container-sysmeta-versions-location': 'ver_cont'}, None)
req = Request.blank(
'/v1/a/c',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_get_head(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None)
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {}, None)
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_put_object_no_versioning(self):
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
cache = FakeCache({})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_put_first_object_success(self):
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_PUT_versioning_with_nonzero_default_policy(self):
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'HEAD', '/v1/a/c/o', swob.HTTPNotFound, {}, None)
cache = FakeCache({'versions': 'ver_cont', 'storage_policy': '2'})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
# check for 'X-Backend-Storage-Policy-Index' in HEAD request
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('HEAD', method)
self.assertEqual('/v1/a/c/o', path)
self.assertTrue('X-Backend-Storage-Policy-Index' in req_headers)
self.assertEqual('2',
req_headers.get('X-Backend-Storage-Policy-Index'))
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_put_object_no_versioning_with_container_config_true(self):
# set False to versions_write obsously and expect no COPY occurred
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
self.app.register(
'HEAD', '/v1/a/c/o', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
called_method = [method for (method, path, hdrs) in self.app._calls]
self.assertTrue('COPY' not in called_method)
def test_delete_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no GET versioning
# container and COPY called (just delete object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register(
'DELETE', '/v1/a/c/o', swob.HTTPNoContent, {}, 'passed')
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '204 No Content')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
called_method = \
[method for (method, path, rheaders) in self.app._calls]
self.assertTrue('COPY' not in called_method)
self.assertTrue('GET' not in called_method)
def test_copy_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no extra
# COPY called (just copy object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register(
'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
called_method = \
[method for (method, path, rheaders) in self.app._calls]
self.assertTrue('COPY' in called_method)
self.assertEqual(called_method.count('COPY'), 1)
def test_new_version_success(self):
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'HEAD', '/v1/a/c/o', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
self.app.register(
'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
@local_tz
def test_new_version_sysmeta_precedence(self):
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'HEAD', '/v1/a/c/o', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:00 GMT'}, 'passed')
self.app.register(
'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
# fill cache with two different values for versions location
# new middleware should use sysmeta first
cache = FakeCache({'versions': 'old_ver_cont',
'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
# check that sysmeta header was used
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('COPY', method)
self.assertEqual('/v1/a/c/o', path)
self.assertEqual('ver_cont/001o/0000000000.00000',
req_headers['Destination'])
def test_copy_first_version(self):
self.app.register(
'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_copy_new_version(self):
self.app.register(
'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'HEAD', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
self.app.register(
'COPY', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_copy_new_version_different_account(self):
self.app.register(
'COPY', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'HEAD', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Wed, 19 Nov 2014 18:19:02 GMT'}, 'passed')
self.app.register(
'COPY', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/src_a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': 'tgt_a'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_copy_new_version_bogus_account(self):
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/src_a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': '/im/on/a/boat'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '412 Precondition Failed')
def test_delete_first_object_success(self):
self.app.register(
'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPNotFound, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
'CONTENT_LENGTH': '0'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_delete_latest_version_success(self):
self.app.register(
'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPOk, {},
'[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 3, '
'"name": "001o/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 3, '
'"name": "001o/1", '
'"content_type": "text/plain"}]')
self.app.register(
'COPY', '/v1/a/ver_cont/001o/2', swob.HTTPCreated,
{}, None)
self.app.register(
'DELETE', '/v1/a/ver_cont/001o/2', swob.HTTPOk,
{}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
headers={'X-If-Delete-At': 1},
environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
'CONTENT_LENGTH': '0'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
# check that X-If-Delete-At was removed from DELETE request
calls = self.app.calls_with_headers
method, path, req_headers = calls.pop()
self.assertEqual('DELETE', method)
self.assertTrue(path.startswith('/v1/a/ver_cont/001o/2'))
self.assertFalse('x-if-delete-at' in req_headers or
'X-If-Delete-At' in req_headers)
def test_DELETE_on_expired_versioned_object(self):
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPOk, {},
'[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 3, '
'"name": "001o/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 3, '
'"name": "001o/1", '
'"content_type": "text/plain"}]')
# expired object
self.app.register(
'COPY', '/v1/a/ver_cont/001o/2', swob.HTTPNotFound,
{}, None)
self.app.register(
'COPY', '/v1/a/ver_cont/001o/1', swob.HTTPCreated,
{}, None)
self.app.register(
'DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPOk,
{}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
'CONTENT_LENGTH': '0'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_denied_DELETE_of_versioned_object(self):
authorize_call = []
self.app.register(
'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET',
'/v1/a/ver_cont?format=json&prefix=001o/&reverse=on&marker=',
swob.HTTPOk, {},
'[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 3, '
'"name": "001o/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 3, '
'"name": "001o/1", '
'"content_type": "text/plain"}]')
self.app.register(
'DELETE', '/v1/a/c/o', swob.HTTPForbidden,
{}, None)
def fake_authorize(req):
authorize_call.append(req)
return swob.HTTPForbidden()
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
'swift.authorize': fake_authorize,
'CONTENT_LENGTH': '0'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '403 Forbidden')
self.assertEqual(len(authorize_call), 1)
self.assertRequestEqual(req, authorize_call[0])
class TestKeymaster(unittest.TestCase):
def setUp(self):
super(TestKeymaster, self).setUp()
self.swift = FakeSwift()
self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)
def test_object_path(self):
self.verify_keys_for_path(
'/a/c/o', expected_keys=('object', 'container'))
def test_container_path(self):
self.verify_keys_for_path(
'/a/c', expected_keys=('container',))
def verify_keys_for_path(self, path, expected_keys):
put_keys = None
for method, resp_class, status in (
('PUT', swob.HTTPCreated, '201'),
('POST', swob.HTTPAccepted, '202'),
('GET', swob.HTTPOk, '200'),
('HEAD', swob.HTTPNoContent, '204')):
resp_headers = {}
self.swift.register(
method, '/v1' + path, resp_class, resp_headers, '')
req = Request.blank(
'/v1' + path, environ={'REQUEST_METHOD': method})
start_response, calls = capture_start_response()
self.app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertTrue(calls[0][0].startswith(status))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)()
self.assertIn('id', keys)
id = keys.pop('id')
self.assertEqual(path, id['path'])
self.assertEqual('1', id['v'])
self.assertListEqual(sorted(expected_keys), sorted(keys.keys()),
'%s %s got keys %r, but expected %r'
% (method, path, keys.keys(), expected_keys))
if put_keys is not None:
# check all key sets were consistent for this path
self.assertDictEqual(put_keys, keys)
else:
put_keys = keys
return put_keys
def test_key_uniqueness(self):
# a rudimentary check that different keys are made for different paths
ref_path_parts = ('a1', 'c1', 'o1')
path = '/' + '/'.join(ref_path_parts)
ref_keys = self.verify_keys_for_path(
path, expected_keys=('object', 'container'))
# for same path and for each differing path check that keys are unique
# when path to object or container is unique and vice-versa
for path_parts in [(a, c, o) for a in ('a1', 'a2')
for c in ('c1', 'c2')
for o in ('o1', 'o2')]:
path = '/' + '/'.join(path_parts)
keys = self.verify_keys_for_path(
path, expected_keys=('object', 'container'))
# object keys should only be equal when complete paths are equal
self.assertEqual(path_parts == ref_path_parts,
keys['object'] == ref_keys['object'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
# container keys should only be equal when paths to container are
# equal
self.assertEqual(path_parts[:2] == ref_path_parts[:2],
keys['container'] == ref_keys['container'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
def test_filter(self):
factory = keymaster.filter_factory(TEST_KEYMASTER_CONF)
self.assertTrue(callable(factory))
self.assertTrue(callable(factory(self.swift)))
def test_app_exception(self):
app = keymaster.KeyMaster(
FakeAppThatExcepts(), TEST_KEYMASTER_CONF)
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
start_response, _ = capture_start_response()
self.assertRaises(Exception, app, req.environ, start_response)
def test_missing_conf_section(self):
sample_conf = "[default]\nuser = %s\n" % getuser()
with tmpfile(sample_conf) as conf_file:
self.assertRaisesRegexp(
ValueError, 'Unable to find keymaster config section in.*',
keymaster.KeyMaster, self.swift, {
'keymaster_config_path': conf_file})
def test_root_secret(self):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
encoded_secret = base64.b64encode(secret)
for conf_val in (bytes(encoded_secret), unicode(encoded_secret),
encoded_secret[:30] + '\n' + encoded_secret[30:]):
try:
app = keymaster.KeyMaster(
self.swift, {'encryption_root_secret': conf_val,
'encryption_root_secret_path': ''})
self.assertEqual(secret, app.root_secret)
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % conf_val)
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_keymaster_config_path(self, mock_readconf):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
enc_secret = base64.b64encode(secret)
for conf_val in (bytes(enc_secret), unicode(enc_secret),
enc_secret[:30] + '\n' + enc_secret[30:],
enc_secret[:30] + '\r\n' + enc_secret[30:]):
for ignored_secret in ('invalid! but ignored!',
'xValidButIgnored' * 10):
mock_readconf.reset_mock()
mock_readconf.return_value = {
'encryption_root_secret': conf_val}
app = keymaster.KeyMaster(self.swift, {
'keymaster_config_path': '/some/path'})
try:
self.assertEqual(secret, app.root_secret)
self.assertEqual(mock_readconf.mock_calls, [
mock.call('/some/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_invalid_root_secret(self):
for secret in (bytes(base64.b64encode(os.urandom(31))), # too short
unicode(base64.b64encode(os.urandom(31))),
u'a' * 44 + u'????', b'a' * 44 + b'????', # not base64
u'a' * 45, b'a' * 45, # bad padding
99, None):
conf = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'encryption_root_secret option in proxy-server.conf '
'must be a base64 encoding of at least 32 raw bytes',
err.exception.message)
except AssertionError as err:
self.fail(str(err) + ' for conf %s' % str(conf))
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_root_secret_path_invalid_secret(self, mock_readconf):
for secret in (bytes(base64.b64encode(os.urandom(31))), # too short
unicode(base64.b64encode(os.urandom(31))),
u'a' * 44 + u'????', b'a' * 44 + b'????', # not base64
u'a' * 45, b'a' * 45, # bad padding
99, None):
mock_readconf.reset_mock()
mock_readconf.return_value = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, {
'keymaster_config_path': '/some/other/path'})
self.assertEqual(
'encryption_root_secret option in /some/other/path '
'must be a base64 encoding of at least 32 raw bytes',
err.exception.message)
self.assertEqual(mock_readconf.mock_calls, [
mock.call('/some/other/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_can_only_configure_secret_in_one_place(self):
conf = {'encryption_root_secret': 'a' * 44,
'keymaster_config_path': '/ets/swift/keymaster.conf'}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual('keymaster_config_path is set, but there are '
'other config options specified!',
err.exception.message)
class TestEncrypter(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.encrypter = encrypter.Encrypter(self.app, {})
self.encrypter.logger = FakeLogger()
def _verify_user_metadata(self, req_hdrs, name, value, key):
# verify encrypted version of user metadata
self.assertNotIn("X-Object-Meta-" + name, req_hdrs)
expected_hdr = "X-Object-Transient-Sysmeta-Crypto-Meta-" + name
self.assertIn(expected_hdr, req_hdrs)
enc_val, param = req_hdrs[expected_hdr].split(";")
param = param.strip()
self.assertTrue(param.startswith("swift_meta="))
actual_meta = json.loads(urllib.unquote_plus(param[len("swift_meta=") :]))
self.assertEqual(Crypto.cipher, actual_meta["cipher"])
meta_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(FAKE_IV, meta_iv)
self.assertEqual(base64.b64encode(encrypt(value, key, meta_iv)), enc_val)
# if there is any encrypted user metadata then this header should exist
self.assertIn("X-Object-Transient-Sysmeta-Crypto-Meta", req_hdrs)
common_meta = json.loads(urllib.unquote_plus(req_hdrs["X-Object-Transient-Sysmeta-Crypto-Meta"]))
self.assertDictEqual({"cipher": Crypto.cipher, "key_id": {"v": "fake", "path": "/a/c/fake"}}, common_meta)
def test_PUT_req(self):
body_key = os.urandom(32)
object_key = fetch_crypto_keys()["object"]
plaintext = "FAKE APP"
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
"etag": plaintext_etag,
"content-type": "text/plain",
"content-length": str(len(plaintext)),
"x-object-meta-etag": "not to be confused with the Etag!",
"x-object-meta-test": "encrypt me",
"x-object-sysmeta-test": "do not encrypt me",
}
req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
with mock.patch(
"swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", return_value=body_key
):
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
self.assertEqual(plaintext_etag, resp.headers["Etag"])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual("PUT", self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify body crypto meta
actual = req_hdrs["X-Object-Sysmeta-Crypto-Body-Meta"]
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual["cipher"])
self.assertEqual(FAKE_IV, base64.b64decode(actual["iv"]))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key, base64.b64decode(actual["body_key"]["key"]))
self.assertEqual(FAKE_IV, base64.b64decode(actual["body_key"]["iv"]))
self.assertEqual(fetch_crypto_keys()["id"], actual["key_id"])
# verify etag
self.assertEqual(ciphertext_etag, req_hdrs["Etag"])
encrypted_etag, _junk, etag_meta = req_hdrs["X-Object-Sysmeta-Crypto-Etag"].partition("; swift_meta=")
# verify crypto_meta was appended to this etag
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta["cipher"])
# verify encrypted version of plaintext etag
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta["iv"])
enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
self.assertEqual(enc_etag, actual)
# verify etag MAC for conditional requests
actual_hmac = base64.b64decode(req_hdrs["X-Object-Sysmeta-Crypto-Etag-Mac"])
self.assertEqual(actual_hmac, hmac.new(object_key, plaintext_etag, hashlib.sha256).digest())
# verify encrypted etag for container update
self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = "swift_meta="
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
self.assertEqual(Crypto().cipher, actual_meta["cipher"])
self.assertEqual(fetch_crypto_keys()["id"], actual_meta["key_id"])
cont_key = fetch_crypto_keys()["container"]
cont_etag_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv), base64.b64decode(parts[0]))
# content-type is not encrypted
self.assertEqual("text/plain", req_hdrs["Content-Type"])
# user meta is encrypted
self._verify_user_metadata(req_hdrs, "Test", "encrypt me", object_key)
self._verify_user_metadata(req_hdrs, "Etag", "not to be confused with the Etag!", object_key)
# sysmeta is not encrypted
self.assertEqual("do not encrypt me", req_hdrs["X-Object-Sysmeta-Test"])
# verify object is encrypted by getting direct from the app
get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
resp = get_req.get_response(self.app)
self.assertEqual(ciphertext, resp.body)
self.assertEqual(ciphertext_etag, resp.headers["Etag"])
def test_PUT_zero_size_object(self):
# object body encryption should be skipped for zero sized object body
object_key = fetch_crypto_keys()["object"]
plaintext_etag = EMPTY_ETAG
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
"etag": EMPTY_ETAG,
"content-type": "text/plain",
"content-length": "0",
"x-object-meta-etag": "not to be confused with the Etag!",
"x-object-meta-test": "encrypt me",
"x-object-sysmeta-test": "do not encrypt me",
}
req = Request.blank("/v1/a/c/o", environ=env, body="", headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
self.assertEqual(plaintext_etag, resp.headers["Etag"])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual("PUT", self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that there is no body crypto meta
self.assertNotIn("X-Object-Sysmeta-Crypto-Meta", req_hdrs)
# verify etag is md5 of plaintext
self.assertEqual(EMPTY_ETAG, req_hdrs["Etag"])
# verify there is no etag crypto meta
self.assertNotIn("X-Object-Sysmeta-Crypto-Etag", req_hdrs)
# verify there is no container update override for etag
self.assertNotIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
# user meta is still encrypted
self._verify_user_metadata(req_hdrs, "Test", "encrypt me", object_key)
self._verify_user_metadata(req_hdrs, "Etag", "not to be confused with the Etag!", object_key)
# sysmeta is not encrypted
self.assertEqual("do not encrypt me", req_hdrs["X-Object-Sysmeta-Test"])
# verify object is empty by getting direct from the app
get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
resp = get_req.get_response(self.app)
self.assertEqual("", resp.body)
self.assertEqual(EMPTY_ETAG, resp.headers["Etag"])
def test_PUT_with_other_footers(self):
# verify handling of another middleware's footer callback
cont_key = fetch_crypto_keys()["container"]
body_key = os.urandom(32)
object_key = fetch_crypto_keys()["object"]
plaintext = "FAKE APP"
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
other_footers = {
"Etag": plaintext_etag,
"X-Object-Sysmeta-Other": "other sysmeta",
"X-Object-Sysmeta-Container-Update-Override-Size": "other override",
"X-Object-Sysmeta-Container-Update-Override-Etag": "final etag",
}
env = {
"REQUEST_METHOD": "PUT",
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
"swift.callback.update_footers": lambda footers: footers.update(other_footers),
}
hdrs = {
"content-type": "text/plain",
"content-length": str(len(plaintext)),
"Etag": "correct etag is in footers",
}
req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
with mock.patch(
"swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", lambda *args: body_key
):
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
self.assertEqual(plaintext_etag, resp.headers["Etag"])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual("PUT", self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that other middleware's footers made it to app, including any
# container update overrides but nothing Etag-related
other_footers.pop("Etag")
other_footers.pop("X-Object-Sysmeta-Container-Update-Override-Etag")
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify encryption footers are ok
encrypted_etag, _junk, etag_meta = req_hdrs["X-Object-Sysmeta-Crypto-Etag"].partition("; swift_meta=")
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta["cipher"])
self.assertEqual(ciphertext_etag, req_hdrs["Etag"])
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)
# verify encrypted etag for container update
self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = "swift_meta="
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
self.assertEqual(Crypto().cipher, actual_meta["cipher"])
cont_key = fetch_crypto_keys()["container"]
cont_etag_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt("final etag", cont_key, cont_etag_iv), base64.b64decode(parts[0]))
# verify body crypto meta
actual = req_hdrs["X-Object-Sysmeta-Crypto-Body-Meta"]
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual["cipher"])
self.assertEqual(FAKE_IV, base64.b64decode(actual["iv"]))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key, base64.b64decode(actual["body_key"]["key"]))
self.assertEqual(FAKE_IV, base64.b64decode(actual["body_key"]["iv"]))
self.assertEqual(fetch_crypto_keys()["id"], actual["key_id"])
def test_PUT_with_etag_override_in_headers(self):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = "FAKE APP"
plaintext_etag = md5hex(plaintext)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
"content-type": "text/plain",
"content-length": str(len(plaintext)),
"Etag": plaintext_etag,
"X-Object-Sysmeta-Container-Update-Override-Etag": "final etag",
}
req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
self.assertEqual(plaintext_etag, resp.headers["Etag"])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(("PUT", "/v1/a/c/o"), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()["container"]
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = "swift_meta="
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
self.assertEqual(Crypto().cipher, actual_meta["cipher"])
self.assertEqual(fetch_crypto_keys()["id"], actual_meta["key_id"])
cont_etag_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt("final etag", cont_key, cont_etag_iv), base64.b64decode(parts[0]))
def test_PUT_with_bad_etag_in_other_footers(self):
# verify that etag supplied in footers from other middleware overrides
# header etag when validating inbound plaintext etags
plaintext = "FAKE APP"
plaintext_etag = md5hex(plaintext)
other_footers = {
"Etag": "bad etag",
"X-Object-Sysmeta-Other": "other sysmeta",
"X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
}
env = {
"REQUEST_METHOD": "PUT",
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
"swift.callback.update_footers": lambda footers: footers.update(other_footers),
}
hdrs = {"content-type": "text/plain", "content-length": str(len(plaintext)), "Etag": plaintext_etag}
req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("422 Unprocessable Entity", resp.status)
self.assertNotIn("Etag", resp.headers)
def test_PUT_with_bad_etag_in_headers_and_other_footers(self):
# verify that etag supplied in headers from other middleware is used if
# none is supplied in footers when validating inbound plaintext etags
plaintext = "FAKE APP"
other_footers = {
"X-Object-Sysmeta-Other": "other sysmeta",
"X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
}
env = {
"REQUEST_METHOD": "PUT",
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
"swift.callback.update_footers": lambda footers: footers.update(other_footers),
}
hdrs = {"content-type": "text/plain", "content-length": str(len(plaintext)), "Etag": "bad etag"}
req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("422 Unprocessable Entity", resp.status)
self.assertNotIn("Etag", resp.headers)
def test_PUT_nothing_read(self):
# simulate an artificial scenario of a downstream filter/app not
# actually reading the input stream from encrypter.
class NonReadingApp(object):
def __call__(self, env, start_response):
# note: no read from wsgi.input
req = Request(env)
env["swift.callback.update_footers"](req.headers)
call_headers.append(req.headers)
resp = HTTPCreated(req=req, headers={"Etag": "response etag"})
return resp(env, start_response)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {"content-type": "text/plain", "content-length": 0, "etag": "etag from client"}
req = Request.blank("/v1/a/c/o", environ=env, body="", headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual("201 Created", resp.status)
self.assertEqual("response etag", resp.headers["Etag"])
self.assertEqual(1, len(call_headers))
self.assertEqual("etag from client", call_headers[0]["etag"])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith("x-object-sysmeta-crypto-"))
# check that an upstream footer callback gets called
other_footers = {
"Etag": "other etag",
"X-Object-Sysmeta-Other": "other sysmeta",
"X-Backend-Container-Update-Override-Etag": "other override",
}
env.update({"swift.callback.update_footers": lambda footers: footers.update(other_footers)})
req = Request.blank("/v1/a/c/o", environ=env, body="", headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual("201 Created", resp.status)
self.assertEqual("response etag", resp.headers["Etag"])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith("x-object-sysmeta-crypto-"))
def test_POST_req(self):
body = "FAKE APP"
env = {"REQUEST_METHOD": "POST", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {"x-object-meta-test": "encrypt me", "x-object-sysmeta-test": "do not encrypt me"}
req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
key = fetch_crypto_keys()["object"]
self.app.register("POST", "/v1/a/c/o", HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual("202 Accepted", resp.status)
self.assertNotIn("Etag", resp.headers)
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual("POST", self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# user meta is encrypted
self._verify_user_metadata(req_hdrs, "Test", "encrypt me", key)
# sysmeta is not encrypted
self.assertEqual("do not encrypt me", req_hdrs["X-Object-Sysmeta-Test"])
def _test_no_user_metadata(self, method):
# verify that x-object-transient-sysmeta-crypto-meta is not set when
# there is no user metadata
env = {"REQUEST_METHOD": method, CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank("/v1/a/c/o", environ=env, body="body")
self.app.register(method, "/v1/a/c/o", HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual("202 Accepted", resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
self.assertNotIn("x-object-transient-sysmeta-crypto-meta", self.app.headers[0])
def test_PUT_no_user_metadata(self):
self._test_no_user_metadata("PUT")
def test_POST_no_user_metadata(self):
self._test_no_user_metadata("POST")
def _test_if_match(self, method, match_header_name):
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ", ".join(plain_etags)
req = Request.blank(
"/v1/a/c/o", environ=env, method=method, headers={match_header_name: match_header_value}
)
app = FakeSwift()
app.register(method, "/v1/a/c/o", HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual("200 OK", resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != "*":
self.assertIn("X-Backend-Etag-Is-At", actual_headers)
self.assertEqual("X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(", "))
key = fetch_crypto_keys()["object"]
masked_etags = [
'"%s"' % base64.b64encode(hmac.new(key, etag.strip('"'), hashlib.sha256).digest())
for etag in plain_etags
if etag not in ("*", "")
]
expected_etags = set((expected_plain_etags or plain_etags) + masked_etags)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags), set(req.headers[match_header_name].split(", ")))
do_test(method, [""])
do_test(method, ['"an etag"'])
do_test(method, ['"an etag"', '"another_etag"'])
do_test(method, ["*"])
# rfc2616 does not allow wildcard *and* etag but test it anyway
do_test(method, ["*", '"an etag"'])
# etags should be quoted but check we can cope if they are not
do_test(method, ["*", "an etag", "another_etag"], expected_plain_etags=["*", '"an etag"', '"another_etag"'])
def test_GET_if_match(self):
self._test_if_match("GET", "If-Match")
def test_HEAD_if_match(self):
self._test_if_match("HEAD", "If-Match")
def test_GET_if_none_match(self):
self._test_if_match("GET", "If-None-Match")
def test_HEAD_if_none_match(self):
self._test_if_match("HEAD", "If-None-Match")
def _test_existing_etag_is_at_header(self, method, match_header_name):
# if another middleware has already set X-Backend-Etag-Is-At then
# encrypter should not override that value
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
"/v1/a/c/o",
environ=env,
method=method,
headers={match_header_name: "an etag", "X-Backend-Etag-Is-At": "X-Object-Sysmeta-Other-Etag"},
)
self.app.register(method, "/v1/a/c/o", HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual("200 OK", resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn("X-Backend-Etag-Is-At", actual_headers)
self.assertEqual(
"X-Object-Sysmeta-Other-Etag,X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"]
)
actual_etags = set(actual_headers[match_header_name].split(", "))
self.assertIn('"an etag"', actual_etags)
def test_GET_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header("GET", "If-Match")
def test_HEAD_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header("HEAD", "If-Match")
def test_GET_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header("GET", "If-None-Match")
def test_HEAD_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header("HEAD", "If-None-Match")
def _test_etag_is_at_not_duplicated(self, method):
# verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
# X-Backend-Etag-Is-At
key = fetch_crypto_keys()["object"]
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
"/v1/a/c/o",
environ=env,
method=method,
headers={"If-Match": '"an etag"', "If-None-Match": '"another etag"'},
)
self.app.register(method, "/v1/a/c/o", HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual("200 OK", resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn("X-Backend-Etag-Is-At", actual_headers)
self.assertEqual("X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"])
self.assertIn(
'"%s"' % base64.b64encode(hmac.new(key, "an etag", hashlib.sha256).digest()), actual_headers["If-Match"]
)
self.assertIn('"another etag"', actual_headers["If-None-Match"])
self.assertIn(
'"%s"' % base64.b64encode(hmac.new(key, "another etag", hashlib.sha256).digest()),
actual_headers["If-None-Match"],
)
def test_GET_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated("GET")
def test_HEAD_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated("HEAD")
def test_PUT_response_inconsistent_etag_is_not_replaced(self):
# if response is success but etag does not match the ciphertext md5
# then verify that we do *not* replace it with the plaintext etag
body = "FAKE APP"
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {"Etag": "not the ciphertext etag"})
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
self.assertEqual("not the ciphertext etag", resp.headers["Etag"])
def test_PUT_multiseg_no_client_etag(self):
body_key = os.urandom(32)
chunks = ["some", "chunks", "of data"]
body = "".join(chunks)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys, "wsgi.input": FileLikeIter(chunks)}
hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
req = Request.blank("/v1/a/c/o", environ=env, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
with mock.patch(
"swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", lambda *args: body_key
):
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
self.assertEqual(encrypt(body, body_key, FAKE_IV), get_req.get_response(self.app).body)
def test_PUT_multiseg_good_client_etag(self):
body_key = os.urandom(32)
chunks = ["some", "chunks", "of data"]
body = "".join(chunks)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys, "wsgi.input": FileLikeIter(chunks)}
hdrs = {"content-type": "text/plain", "content-length": str(len(body)), "Etag": md5hex(body)}
req = Request.blank("/v1/a/c/o", environ=env, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
with mock.patch(
"swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", lambda *args: body_key
):
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
self.assertEqual(encrypt(body, body_key, FAKE_IV), get_req.get_response(self.app).body)
def test_PUT_multiseg_bad_client_etag(self):
chunks = ["some", "chunks", "of data"]
body = "".join(chunks)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys, "wsgi.input": FileLikeIter(chunks)}
hdrs = {"content-type": "text/plain", "content-length": str(len(body)), "Etag": "badclientetag"}
req = Request.blank("/v1/a/c/o", environ=env, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("422 Unprocessable Entity", resp.status)
def test_PUT_missing_key_callback(self):
body = "FAKE APP"
env = {"REQUEST_METHOD": "PUT"}
hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual("500 Internal Error", resp.status)
self.assertIn("missing callback", self.encrypter.logger.get_lines_for_level("error")[0])
self.assertEqual("Unable to retrieve encryption keys.", resp.body)
def test_PUT_error_in_key_callback(self):
def raise_exc():
raise Exception("Testing")
body = "FAKE APP"
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: raise_exc}
hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual("500 Internal Error", resp.status)
self.assertIn("from callback: Testing", self.encrypter.logger.get_lines_for_level("error")[0])
self.assertEqual("Unable to retrieve encryption keys.", resp.body)
def test_PUT_encryption_override(self):
# set crypto override to disable encryption.
# simulate another middleware wanting to set footers
other_footers = {
"Etag": "other etag",
"X-Object-Sysmeta-Other": "other sysmeta",
"X-Object-Sysmeta-Container-Update-Override-Etag": "other override",
}
body = "FAKE APP"
env = {
"REQUEST_METHOD": "PUT",
"swift.crypto.override": True,
"swift.callback.update_footers": lambda footers: footers.update(other_footers),
}
hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
# verify that other middleware's footers made it to app
req_hdrs = self.app.headers[0]
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify object is NOT encrypted by getting direct from the app
get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
self.assertEqual(body, get_req.get_response(self.app).body)
def _test_constraints_checking(self, method):
# verify that the check_metadata function is called on PUT and POST
body = "FAKE APP"
env = {"REQUEST_METHOD": method, CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {"content-type": "text/plain", "content-length": str(len(body))}
req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs)
mocked_func = "swift.common.middleware.crypto.encrypter.check_metadata"
with mock.patch(mocked_func) as mocked:
mocked.side_effect = [HTTPBadRequest("testing")]
resp = req.get_response(self.encrypter)
self.assertEqual("400 Bad Request", resp.status)
self.assertEqual(1, mocked.call_count)
mocked.assert_called_once_with(mock.ANY, "object")
self.assertEqual(req.headers, mocked.call_args_list[0][0][0].headers)
def test_PUT_constraints_checking(self):
self._test_constraints_checking("PUT")
def test_POST_constraints_checking(self):
self._test_constraints_checking("POST")
def test_config_true_value_on_disable_encryption(self):
app = FakeSwift()
self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption)
for val in ("true", "1", "yes", "on", "t", "y"):
app = encrypter.Encrypter(app, {"disable_encryption": val})
self.assertTrue(app.disable_encryption)
def test_PUT_app_exception(self):
app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {})
req = Request.blank("/", environ={"REQUEST_METHOD": "PUT"})
with self.assertRaises(HTTPException) as catcher:
req.get_response(app)
self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)
class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase):
# verify interaction of copy and versioned_writes middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'allow_versioned_writes': 'true'}
self.vw = versioned_writes.filter_factory(conf)(self.app)
self.filter = copy.filter_factory({})(self.vw)
def call_filter(self, req, **kwargs):
return self.call_app(req, app=self.filter, **kwargs)
def test_copy_first_version(self):
# no existing object to move to the versions container
self.app.register(
'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None)
self.app.register(
'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
# note the GET on tgt_cont/tgt_obj is pre-authed
self.assertEqual(3, self.app.call_count, self.app.calls)
def test_copy_new_version(self):
# existing object should be moved to versions container
self.app.register(
'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
self.app.register(
'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk,
{}, None)
self.app.register(
'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(4, self.app.call_count)
def test_copy_new_version_different_account(self):
self.app.register(
'GET', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
self.app.register(
'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000',
swob.HTTPOk, {}, None)
self.app.register(
'PUT', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {},
'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/src_a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': 'tgt_a'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/src_a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/tgt_a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(4, self.app.call_count)
def test_copy_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no extra
# COPY called (just copy object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register(
'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache},
headers={'Destination': '/tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(2, self.app.call_count)
class TestUntarMetadata(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.bulk = bulk.filter_factory({})(self.app)
self.testdir = mkdtemp(suffix='tmp_test_bulk')
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
def test_extract_metadata(self):
self.app.register('HEAD', '/v1/a/c?extract-archive=tar', HTTPNoContent,
{}, None)
self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar',
HTTPCreated, {}, None)
self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar',
HTTPCreated, {}, None)
# It's a real pain to instantiate TarInfo objects directly; they
# really want to come from a file on disk or a tarball. So, we write
# out some files and add pax headers to them as they get placed into
# the tarball.
with open(os.path.join(self.testdir, "obj1"), "w") as fh1:
fh1.write("obj1 contents\n")
with open(os.path.join(self.testdir, "obj2"), "w") as fh2:
fh2.write("obj2 contents\n")
tar_ball = BytesIO()
tar_file = tarfile.TarFile.open(fileobj=tar_ball,
mode="w",
format=tarfile.PAX_FORMAT)
# With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with
# extended attribute user.thingy = dingy gets put into the tarfile
# with pax_headers containing key/value pair
# (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type
# unicode, not type str).
#
# With BSD tar (libarchive), you get key/value pair
# (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as
# gratuitous incompatibility.
#
# Still, we'll support uploads with both. Just heap more code on the
# problem until you can forget it's under there.
with open(os.path.join(self.testdir, "obj1")) as fh1:
tar_info1 = tar_file.gettarinfo(fileobj=fh1, arcname="obj1")
tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \
u'application/food-diary'
tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \
u'sopa de albóndigas'
tar_info1.pax_headers[
u'SCHILY.xattr.user.meta.afternoon-snack'] = \
u'gigantic bucket of coffee'
tar_file.addfile(tar_info1, fh1)
with open(os.path.join(self.testdir, "obj2")) as fh2:
tar_info2 = tar_file.gettarinfo(fileobj=fh2, arcname="obj2")
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped'
tar_file.addfile(tar_info2, fh2)
tar_ball.seek(0)
req = Request.blank('/v1/a/c?extract-archive=tar')
req.environ['REQUEST_METHOD'] = 'PUT'
req.environ['wsgi.input'] = tar_ball
req.headers['transfer-encoding'] = 'chunked'
req.headers['accept'] = 'application/json;q=1.0'
resp = req.get_response(self.bulk)
self.assertEqual(resp.status_int, 200)
# sanity check to make sure the upload worked
upload_status = utils.json.loads(resp.body)
self.assertEqual(upload_status['Number Files Created'], 2)
put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2])
self.assertEqual(put1_headers.get('Content-Type'),
'application/food-diary')
self.assertEqual(put1_headers.get('X-Object-Meta-Lunch'),
'sopa de alb\xc3\xb3ndigas')
self.assertEqual(put1_headers.get('X-Object-Meta-Afternoon-Snack'),
'gigantic bucket of coffee')
put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
self.assertEqual(put2_headers.get('Content-Type'), None)
self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)
class OioContainerHierarchy(unittest.TestCase):
def setUp(self):
conf = {'sds_default_account': 'OPENIO'}
self.filter_conf = {
'strip_v1': 'true',
'swift3_compat': 'true',
'account_first': 'true'
}
self.app = FakeSwift()
self.ch = container_hierarchy.filter_factory(
conf,
**self.filter_conf)(self.app)
def mock(self):
self.ch._create_key = mock.MagicMock(return_value=None)
self.ch._remove_key = mock.MagicMock(return_value=None)
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
def call_ch(self, req):
return self.call_app(req, app=self.ch)
def test_simple_put(self):
"""check number of request generated by Container Hierarchy"""
self.mock()
self.app.register(
'PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.ch._create_key.assert_called_with(mock.ANY,
'a', 'c', 'cnt', 'd1/d2/d3/')
def test_fake_directory_put(self):
self.mock()
req = Request.blank('/v1/a/c/d1/d2/d3/', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.ch._create_key.assert_called_with(mock.ANY,
'a', 'c', 'obj', 'd1/d2/d3/')
def test_get(self):
self.app.register(
'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPOk, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_recursive_listing(self):
self.ch.conn.keys = mock.MagicMock(return_value=['CS:a:cnt:d1/d2/d3/'])
self.app.register(
'GET',
'/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/c?prefix=d1%2Fd2%2F', method='GET')
resp = self.call_ch(req)
data = json.loads(resp[2])
self.assertEqual(data[0]['name'], 'd1/d2/d3/o')
def test_listing_with_space(self):
self.ch.conn.keys = mock.MagicMock(return_value=['CS:a:cnt:d 1/d2/'])
self.app.register(
'GET',
'/v1/a/c%2Fd 1%2Fd2?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/c?prefix=d%201%2Fd2%2F', method='GET')
resp = self.call_ch(req)
data = json.loads(resp[2])
self.assertEqual(data[0]['name'], 'd 1/d2/o')
def test_global_listing(self):
self.app.register(
'GET', '/v1/a', swob.HTTPOk, {})
req = Request.blank('/v1/a', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_delete_object(self):
self.app.register(
'PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
self.app.register(
'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=1&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o",
"content_type": "application/octet-stream"}]))
self.app.register(
'DELETE', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPNoContent, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
self.app.register(
'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=1&format=json',
swob.HTTPOk, {}, json.dumps([]))
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
self.assertNotIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
def test_fake_directory(self):
req = Request.blank('/v1/a/container/d2/d3/', method='PUT')
resp = self.call_ch(req)
self.assertIn('CS:a:container:obj:d2/d3/', self.ch.conn._keys)
req = Request.blank('/v1/a/container/d2/d3/', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], "204 No Content")
self.assertNotIn('CS:a:container:obj:d2/d3/', self.ch.conn._keys)
def _listing(self, is_recursive):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/', 'CS:a:bucket:cnt:d1/d2/'])
self.ch.conn.exist = mock.MagicMock(return_value=True)
self.app.register(
'GET',
'/v1/a/bucket%2Fd1?prefix=d&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o1",
"content_type": "application/octet-stream"}]))
if is_recursive:
self.app.register(
'GET',
'/v1/a/bucket%2Fd1%2Fd2?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o2",
"content_type": "application/octet-stream"}]))
recursive = '' if is_recursive else '&delimiter=%2F'
req = Request.blank('/v1/a/bucket?prefix=d1/d&limit=10%s' % recursive,
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
return names
def test_listing_with_prefix(self):
names = self._listing(False)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/', names)
def test_listing_with_prefix_recursive(self):
names = self._listing(True)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/o2', names)
def test_listing_root_container(self):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/'])
self.app.register(
'GET',
'/v1/a/bucket?prefix=d&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "d0",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/bucket?prefix=d&limit=10&delimiter=%2F',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertIn("d0", names)
self.assertIn("d1/", names)
def test_listing_with_marker(self):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/',
'CS:a:bucket:cnt:d2/',
])
req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertNotIn('d1/', names)
self.assertIn('d2/', names)
def test_listing_with_marker_multi_container(self):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/',
'CS:a:bucket:cnt:d2/',
])
# with marker aa (as we inspect d1/)
self.app.register(
'GET',
'/v1/a/bucket%2Fd1?marker=aa&prefix=&limit=10000&format=json', # noqa
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "d0",
"content_type": "application/octet-stream"}]))
# without marker on second container
self.app.register(
'GET',
'/v1/a/bucket%2Fd2?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "d0",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/bucket?limit=10&marker=d1/aa',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertIn('d1/d0', names)
self.assertIn('d2/d0', names)
def test_duplicate_obj_cnt(self):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/cnt/',
'CS:a:bucket:obj:d1/obj/',
])
req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertIn('d1/', names)
self.assertEqual(1, len(names))
def test_remove_bucket(self):
self.app.register(
'DELETE',
'/v1/a/bucket',
swob.HTTPNoContent, {},
"")
req = Request.blank('/v1/a/bucket',
method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], "204 No Content")
def test_invalid_path(self):
req = Request.blank('/v1/a/',
method='GET')
with self.assertRaises(HTTPException) as cm:
self.call_ch(req)
self.assertEqual(cm.exception.status, "400 Bad Request")
def test_path(self):
cont = 'bucket'
path = 'dir1/dir2/object'
res = self.ch._fake_container_and_obj(cont, path.split('/'))
self.assertEqual(res, (cont + '%2Fdir1%2Fdir2', 'object'))
path = 'object'
res = self.ch._fake_container_and_obj(cont, path.split('/'))
self.assertEqual(res, (cont, 'object'))
def test_mpu_path(self):
cont = 'bucket+segments'
uploadid = 'MzNkYWZlNjItNjg3Yy00ZmIyLWIwOGYtOTA2OGVlZTA2MzA5'
path = ('dir1/dir2/object/%s/1' % uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont + '%2Fdir1%2Fdir2',
'object/%s/1' % uploadid))
path = ('dir1/dir2/object/' + uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont + '%2Fdir1%2Fdir2',
'object/' + uploadid))
path = ('object/%s/1' % uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont,
'object/%s/1' % uploadid))
path = ('object/' + uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont,
'object/' + uploadid))
def test_upload_in_progress(self):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket+segments:cnt:d1/d2/d3/'])
upload = ["obj/YmYwY2I1ZDYtNjMyYi00OGNiLWEzMzEtZDdhYTk0ODZkNWU2",
"root/MzNkYWZlNjItNjg3Yy00ZmIyLWIwOGYtOTA2OGVlZTA2MzA5"]
self.app.register(
'GET',
'/v1/a/bucket+segments%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&format=json', # noqa
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": upload[0],
"content_type": "application/octet-stream"},
{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 400, "name": upload[0] + '/1',
"content_type": "application/octet-stream"}]))
self.app.register(
'GET',
'/v1/a/bucket+segments?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": upload[1],
"content_type": "application/octet-stream"},
{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 400, "name": upload[1] + '/1',
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/bucket+segments',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertEqual(names,
['d1/d2/d3/' + upload[0],
'd1/d2/d3/' + upload[0] + '/1',
upload[1],
upload[1] + '/1'])
def test_copy_headers(self):
self.app.register(
'PUT', '/v1/a/bucket%2Fdir1/target',
swob.HTTPNoContent, {},
)
req = Request.blank(
'/v1/a/bucket/dir1/target',
method='PUT',
headers={'Oio-Copy-From': '/v1/a/bucket/sub1/source'})
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
self.assertEqual(self.app.headers[0]['Oio-Copy-From'],
"/v1%2Fa%2Fbucket%2Fsub1/source")
class TestKeymaster(unittest.TestCase):
def setUp(self):
super(TestKeymaster, self).setUp()
self.swift = FakeSwift()
self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)
def test_object_path(self):
self.verify_keys_for_path(
'/a/c/o', expected_keys=('object', 'container'))
def test_container_path(self):
self.verify_keys_for_path(
'/a/c', expected_keys=('container',))
def verify_keys_for_path(self, path, expected_keys, key_id=None):
put_keys = None
for method, resp_class, status in (
('PUT', swob.HTTPCreated, '201'),
('POST', swob.HTTPAccepted, '202'),
('GET', swob.HTTPOk, '200'),
('HEAD', swob.HTTPNoContent, '204')):
resp_headers = {}
self.swift.register(
method, '/v1' + path, resp_class, resp_headers, '')
req = Request.blank(
'/v1' + path, environ={'REQUEST_METHOD': method})
start_response, calls = capture_start_response()
self.app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertTrue(calls[0][0].startswith(status))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)(key_id=key_id)
self.assertIn('id', keys)
id = keys.pop('id')
self.assertEqual(path, id['path'])
self.assertEqual('1', id['v'])
keys.pop('all_ids')
self.assertListEqual(sorted(expected_keys), sorted(keys.keys()),
'%s %s got keys %r, but expected %r'
% (method, path, keys.keys(), expected_keys))
if put_keys is not None:
# check all key sets were consistent for this path
self.assertDictEqual(put_keys, keys)
else:
put_keys = keys
return put_keys
def test_key_uniqueness(self):
# a rudimentary check that different keys are made for different paths
ref_path_parts = ('a1', 'c1', 'o1')
path = '/' + '/'.join(ref_path_parts)
ref_keys = self.verify_keys_for_path(
path, expected_keys=('object', 'container'))
# for same path and for each differing path check that keys are unique
# when path to object or container is unique and vice-versa
for path_parts in [(a, c, o) for a in ('a1', 'a2')
for c in ('c1', 'c2')
for o in ('o1', 'o2')]:
path = '/' + '/'.join(path_parts)
keys = self.verify_keys_for_path(
path, expected_keys=('object', 'container'))
# object keys should only be equal when complete paths are equal
self.assertEqual(path_parts == ref_path_parts,
keys['object'] == ref_keys['object'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
# container keys should only be equal when paths to container are
# equal
self.assertEqual(path_parts[:2] == ref_path_parts[:2],
keys['container'] == ref_keys['container'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
def test_filter(self):
factory = keymaster.filter_factory(TEST_KEYMASTER_CONF)
self.assertTrue(callable(factory))
self.assertTrue(callable(factory(self.swift)))
def test_app_exception(self):
app = keymaster.KeyMaster(
FakeAppThatExcepts(), TEST_KEYMASTER_CONF)
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
start_response, _ = capture_start_response()
self.assertRaises(Exception, app, req.environ, start_response)
def test_missing_conf_section(self):
sample_conf = "[default]\nuser = %s\n" % getuser()
with tmpfile(sample_conf) as conf_file:
self.assertRaisesRegexp(
ValueError, 'Unable to find keymaster config section in.*',
keymaster.KeyMaster, self.swift, {
'keymaster_config_path': conf_file})
def test_root_secret(self):
def do_test(dflt_id):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
encoded_secret = base64.b64encode(secret)
for conf_val in (
bytes(encoded_secret),
unicode(encoded_secret),
encoded_secret[:30] + '\n' + encoded_secret[30:]):
try:
app = keymaster.KeyMaster(
self.swift, {'encryption_root_secret': conf_val,
'active_root_secret_id': dflt_id,
'keymaster_config_path': ''})
self.assertEqual(secret, app.root_secret)
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % conf_val)
do_test(None)
do_test('')
def test_no_root_secret(self):
with self.assertRaises(ValueError) as cm:
keymaster.KeyMaster(self.swift, {})
self.assertEqual('No secret loaded for active_root_secret_id None',
str(cm.exception))
def test_multiple_root_secrets(self):
secrets = {None: os.urandom(32),
'22': os.urandom(33),
'my_secret_id': os.urandom(50)}
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
app = keymaster.KeyMaster(self.swift, conf)
self.assertEqual(secrets, app._root_secrets)
self.assertEqual([None, '22', 'my_secret_id'], app.root_secret_ids)
def test_chained_keymasters(self):
conf_inner = {'active_root_secret_id': '22'}
conf_inner.update(
('encryption_root_secret_%s' % secret_id, base64.b64encode(secret))
for secret_id, secret in [('22', os.urandom(33)),
('my_secret_id', os.urandom(50))])
conf_outer = {'encryption_root_secret': base64.b64encode(
os.urandom(32))}
app = keymaster.KeyMaster(
keymaster.KeyMaster(self.swift, conf_inner),
conf_outer)
self.swift.register('GET', '/v1/a/c', swob.HTTPOk, {}, b'')
req = Request.blank('/v1/a/c')
start_response, calls = capture_start_response()
app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = copy.deepcopy(req.environ[CRYPTO_KEY_CALLBACK](key_id=None))
self.assertIn('id', keys)
self.assertEqual(keys.pop('id'), {
'v': '1',
'path': '/a/c',
'secret_id': '22',
})
# Inner-most active root secret wins
root_key = base64.b64decode(conf_inner['encryption_root_secret_22'])
self.assertIn('container', keys)
self.assertEqual(keys.pop('container'),
hmac.new(root_key, '/a/c',
digestmod=hashlib.sha256).digest())
self.assertIn('all_ids', keys)
all_keys = set()
at_least_one_old_style_id = False
for key_id in keys.pop('all_ids'):
# Can get key material for each key_id
all_keys.add(req.environ[CRYPTO_KEY_CALLBACK](
key_id=key_id)['container'])
if 'secret_id' in key_id:
self.assertIn(key_id.pop('secret_id'), {'22', 'my_secret_id'})
else:
at_least_one_old_style_id = True
self.assertEqual(key_id, {
'path': '/a/c',
'v': '1',
})
self.assertTrue(at_least_one_old_style_id)
self.assertEqual(len(all_keys), 3)
self.assertFalse(keys)
# Also all works for objects
self.swift.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, b'')
req = Request.blank('/v1/a/c/o')
start_response, calls = capture_start_response()
app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)(key_id=None)
self.assertIn('id', keys)
self.assertEqual(keys.pop('id'), {
'v': '1',
'path': '/a/c/o',
'secret_id': '22',
})
root_key = base64.b64decode(conf_inner['encryption_root_secret_22'])
self.assertIn('container', keys)
self.assertEqual(keys.pop('container'),
hmac.new(root_key, '/a/c',
digestmod=hashlib.sha256).digest())
self.assertIn('object', keys)
self.assertEqual(keys.pop('object'),
hmac.new(root_key, '/a/c/o',
digestmod=hashlib.sha256).digest())
self.assertIn('all_ids', keys)
at_least_one_old_style_id = False
for key_id in keys.pop('all_ids'):
if 'secret_id' not in key_id:
at_least_one_old_style_id = True
else:
self.assertIn(key_id.pop('secret_id'), {'22', 'my_secret_id'})
self.assertEqual(key_id, {
'path': '/a/c/o',
'v': '1',
})
self.assertTrue(at_least_one_old_style_id)
self.assertEqual(len(all_keys), 3)
self.assertFalse(keys)
def test_multiple_root_secrets_with_invalid_secret(self):
conf = {'encryption_root_secret': base64.b64encode(os.urandom(32)),
# too short...
'encryption_root_secret_22': base64.b64encode(os.urandom(31))}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'encryption_root_secret_22 option in proxy-server.conf '
'must be a base64 encoding of at least 32 raw bytes',
str(err.exception))
def test_multiple_root_secrets_with_invalid_id(self):
def do_test(bad_option):
conf = {'encryption_root_secret': base64.b64encode(os.urandom(32)),
bad_option: base64.b64encode(os.urandom(32))}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'Malformed root secret option name %s' % bad_option,
str(err.exception))
do_test('encryption_root_secret1')
do_test('encryption_root_secret123')
do_test('encryption_root_secret_')
def test_multiple_root_secrets_missing_active_root_secret_id(self):
conf = {'encryption_root_secret_22': base64.b64encode(os.urandom(32))}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'No secret loaded for active_root_secret_id None',
str(err.exception))
conf = {'encryption_root_secret_22': base64.b64encode(os.urandom(32)),
'active_root_secret_id': 'missing'}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'No secret loaded for active_root_secret_id missing',
str(err.exception))
def test_correct_root_secret_used(self):
secrets = {None: os.urandom(32),
'22': os.urandom(33),
'my_secret_id': os.urandom(50)}
# no active_root_secret_id configured
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
self.app = keymaster.KeyMaster(self.swift, conf)
keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'))
expected_keys = {
'container': hmac.new(secrets[None], '/a/c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets[None], '/a/c/o',
digestmod=hashlib.sha256).digest()}
self.assertEqual(expected_keys, keys)
# active_root_secret_id configured
conf['active_root_secret_id'] = '22'
self.app = keymaster.KeyMaster(self.swift, conf)
keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'))
expected_keys = {
'container': hmac.new(secrets['22'], '/a/c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets['22'], '/a/c/o',
digestmod=hashlib.sha256).digest()}
self.assertEqual(expected_keys, keys)
# secret_id passed to fetch_crypto_keys callback
for secret_id in ('my_secret_id', None):
keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'),
key_id={'secret_id': secret_id})
expected_keys = {
'container': hmac.new(secrets[secret_id], '/a/c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets[secret_id], '/a/c/o',
digestmod=hashlib.sha256).digest()}
self.assertEqual(expected_keys, keys)
def test_keys_cached(self):
secrets = {None: os.urandom(32),
'22': os.urandom(33),
'my_secret_id': os.urandom(50)}
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
conf['active_root_secret_id'] = '22'
self.app = keymaster.KeyMaster(self.swift, conf)
orig_create_key = self.app.create_key
calls = []
def mock_create_key(path, secret_id=None):
calls.append((path, secret_id))
return orig_create_key(path, secret_id)
context = keymaster.KeyMasterContext(self.app, 'a', 'c', 'o')
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys()
expected_keys = {
'container': hmac.new(secrets['22'], '/a/c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets['22'], '/a/c/o',
digestmod=hashlib.sha256).digest(),
'id': {'path': '/a/c/o', 'secret_id': '22', 'v': '1'},
'all_ids': [
{'path': '/a/c/o', 'v': '1'},
{'path': '/a/c/o', 'secret_id': '22', 'v': '1'},
{'path': '/a/c/o', 'secret_id': 'my_secret_id', 'v': '1'}]}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', '22'), ('/a/c/o', '22')], calls)
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys()
# no more calls to create_key
self.assertEqual([('/a/c', '22'), ('/a/c/o', '22')], calls)
self.assertEqual(expected_keys, keys)
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={'secret_id': None})
expected_keys = {
'container': hmac.new(secrets[None], '/a/c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets[None], '/a/c/o',
digestmod=hashlib.sha256).digest(),
'id': {'path': '/a/c/o', 'v': '1'},
'all_ids': [
{'path': '/a/c/o', 'v': '1'},
{'path': '/a/c/o', 'secret_id': '22', 'v': '1'},
{'path': '/a/c/o', 'secret_id': 'my_secret_id', 'v': '1'}]}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', '22'), ('/a/c/o', '22'),
('/a/c', None), ('/a/c/o', None)],
calls)
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_keymaster_config_path(self, mock_readconf):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
enc_secret = base64.b64encode(secret)
for conf_val in (bytes(enc_secret), unicode(enc_secret),
enc_secret[:30] + '\n' + enc_secret[30:],
enc_secret[:30] + '\r\n' + enc_secret[30:]):
mock_readconf.reset_mock()
mock_readconf.return_value = {
'encryption_root_secret': conf_val}
app = keymaster.KeyMaster(self.swift, {
'keymaster_config_path': '/some/path'})
try:
self.assertEqual(secret, app.root_secret)
self.assertEqual(mock_readconf.mock_calls, [
mock.call('/some/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_invalid_root_secret(self):
for secret in (bytes(base64.b64encode(os.urandom(31))), # too short
unicode(base64.b64encode(os.urandom(31))),
u'a' * 44 + u'????', b'a' * 44 + b'????', # not base64
u'a' * 45, b'a' * 45, # bad padding
99, None):
conf = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'encryption_root_secret option in proxy-server.conf '
'must be a base64 encoding of at least 32 raw bytes',
str(err.exception))
except AssertionError as err:
self.fail(str(err) + ' for conf %s' % str(conf))
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_root_secret_path_invalid_secret(self, mock_readconf):
for secret in (bytes(base64.b64encode(os.urandom(31))), # too short
unicode(base64.b64encode(os.urandom(31))),
u'a' * 44 + u'????', b'a' * 44 + b'????', # not base64
u'a' * 45, b'a' * 45, # bad padding
99, None):
mock_readconf.reset_mock()
mock_readconf.return_value = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, {
'keymaster_config_path': '/some/other/path'})
self.assertEqual(
'encryption_root_secret option in /some/other/path '
'must be a base64 encoding of at least 32 raw bytes',
str(err.exception))
self.assertEqual(mock_readconf.mock_calls, [
mock.call('/some/other/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_can_only_configure_secret_in_one_place(self):
def do_test(conf):
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
expected_message = ('keymaster_config_path is set, but there are '
'other config options specified:')
self.assertTrue(str(err.exception).startswith(expected_message),
"Error message does not start with '%s'" %
expected_message)
conf = {'encryption_root_secret': 'a' * 44,
'keymaster_config_path': '/etc/swift/keymaster.conf'}
do_test(conf)
conf = {'encryption_root_secret_1': 'a' * 44,
'keymaster_config_path': '/etc/swift/keymaster.conf'}
do_test(conf)
conf = {'encryption_root_secret_': 'a' * 44,
'keymaster_config_path': '/etc/swift/keymaster.conf'}
do_test(conf)
conf = {'active_root_secret_id': '1',
'keymaster_config_path': '/etc/swift/keymaster.conf'}
do_test(conf)
class SymlinkVersioningTestCase(TestSymlinkMiddlewareBase):
# verify interaction of versioned_writes and symlink middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'symloop_max': '2'}
self.sym = symlink.filter_factory(conf)(self.app)
self.sym.logger = self.app.logger
vw_conf = {'allow_versioned_writes': 'true'}
self.vw = versioned_writes.filter_factory(vw_conf)(self.sym)
def call_vw(self, req, **kwargs):
return self.call_app(req, app=self.vw, **kwargs)
def assertRequestEqual(self, req, other):
self.assertEqual(req.method, other.method)
self.assertEqual(req.path, other.path)
def test_new_symlink_version_success(self):
self.app.register(
'PUT', '/v1/a/c/symlink', swob.HTTPCreated,
{'X-Symlink-Target': 'new_cont/new_tgt',
'X-Symlink-Target-Account': 'a'}, None)
self.app.register(
'GET', '/v1/a/c/symlink', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT',
'X-Object-Sysmeta-Symlink-Target': 'old_cont/old_tgt',
'X-Object-Sysmeta-Symlink-Target-Account': 'a'},
'')
self.app.register(
'PUT', '/v1/a/ver_cont/007symlink/0000000001.00000',
swob.HTTPCreated,
{'X-Symlink-Target': 'old_cont/old_tgt',
'X-Symlink-Target-Account': 'a'}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/symlink',
headers={'X-Symlink-Target': 'new_cont/new_tgt'},
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '0',
'swift.trans_id': 'fake_trans_id'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
# authorized twice now because versioned_writes now makes a check on
# PUT
self.assertEqual(len(self.authorized), 2)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(['VW', 'VW', None], self.app.swift_sources)
self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))
calls = self.app.calls_with_headers
method, path, req_headers = calls[2]
self.assertEqual('PUT', method)
self.assertEqual('/v1/a/c/symlink', path)
self.assertEqual(
'new_cont/new_tgt',
req_headers['X-Object-Sysmeta-Symlink-Target'])
def test_delete_latest_version_no_marker_success(self):
self.app.register(
'GET',
'/v1/a/ver_cont?prefix=003sym/&marker=&reverse=on',
swob.HTTPOk, {},
'[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 0, '
'"name": "003sym/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 0, '
'"name": "003sym/1", '
'"content_type": "text/plain"}]')
self.app.register(
'GET', '/v1/a/ver_cont/003sym/2', swob.HTTPCreated,
{'content-length': '0',
'X-Object-Sysmeta-Symlink-Target': 'c/tgt'}, None)
self.app.register(
'PUT', '/v1/a/c/sym', swob.HTTPCreated,
{'X-Symlink-Target': 'c/tgt', 'X-Symlink-Target-Account': 'a'},
None)
self.app.register(
'DELETE', '/v1/a/ver_cont/003sym/2', swob.HTTPOk,
{}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/sym',
headers={'X-If-Delete-At': 1},
environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
'CONTENT_LENGTH': '0', 'swift.trans_id': 'fake_trans_id'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(4, self.app.call_count)
self.assertEqual(['VW', 'VW', 'VW', 'VW'], self.app.swift_sources)
self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))
calls = self.app.calls_with_headers
method, path, req_headers = calls[2]
self.assertEqual('PUT', method)
self.assertEqual('/v1/a/c/sym', path)
self.assertEqual(
'c/tgt',
req_headers['X-Object-Sysmeta-Symlink-Target'])
class SymlinkCopyingTestCase(TestSymlinkMiddlewareBase):
# verify interaction of copy and symlink middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'symloop_max': '2'}
self.sym = symlink.filter_factory(conf)(self.app)
self.sym.logger = self.app.logger
self.copy = copy.filter_factory({})(self.sym)
def call_copy(self, req, **kwargs):
return self.call_app(req, app=self.copy, **kwargs)
def test_copy_symlink_target(self):
req = Request.blank('/v1/a/src_cont/symlink', method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj'})
self._test_copy_symlink_target(req)
req = Request.blank('/v1/a/tgt_cont/tgt_obj', method='PUT',
headers={'X-Copy-From': 'src_cont/symlink'})
self._test_copy_symlink_target(req)
def _test_copy_symlink_target(self, req):
self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'})
self.app.register('GET', '/v1/a2/c1/o', swob.HTTPOk, {}, 'resp_body')
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{}, 'resp_body')
status, headers, body = self.call_copy(req)
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a/src_cont/symlink')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a2/c1/o')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[2]
self.assertEqual(method, 'PUT')
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
# this is raw object copy
self.assertEqual(val, None)
self.assertEqual(status, '201 Created')
def test_copy_symlink(self):
req = Request.blank(
'/v1/a/src_cont/symlink?symlink=get', method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj'})
self._test_copy_symlink(req)
req = Request.blank(
'/v1/a/tgt_cont/tgt_obj?symlink=get', method='PUT',
headers={'X-Copy-From': 'src_cont/symlink'})
self._test_copy_symlink(req)
def _test_copy_symlink(self, req):
self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a/src_cont/symlink?symlink=get')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertEqual(
hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'), 'a2')
def test_copy_symlink_new_target(self):
req = Request.blank(
'/v1/a/src_cont/symlink?symlink=get', method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj',
'X-Symlink-Target': 'new_cont/new_obj',
'X-Symlink-Target-Account': 'new_acct'})
self._test_copy_symlink_new_target(req)
req = Request.blank(
'/v1/a/tgt_cont/tgt_obj?symlink=get', method='PUT',
headers={'X-Copy-From': 'src_cont/symlink',
'X-Symlink-Target': 'new_cont/new_obj',
'X-Symlink-Target-Account': 'new_acct'})
self._test_copy_symlink_new_target(req)
def _test_copy_symlink_new_target(self, req):
self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a/src_cont/symlink?symlink=get')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
self.assertEqual(method, 'PUT')
self.assertEqual(path, '/v1/a/tgt_cont/tgt_obj?symlink=get')
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'new_cont/new_obj')
self.assertEqual(hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'),
'new_acct')
def test_copy_symlink_with_slo_query(self):
req = Request.blank(
'/v1/a/src_cont/symlink?multipart-manifest=get&symlink=get',
method='COPY', headers={'Destination': 'tgt_cont/tgt_obj'})
self._test_copy_symlink_with_slo_query(req)
req = Request.blank(
'/v1/a/tgt_cont/tgt_obj?multipart-manifest=get&symlink=get',
method='PUT', headers={'X-Copy-From': 'src_cont/symlink'})
self._test_copy_symlink_with_slo_query(req)
def _test_copy_symlink_with_slo_query(self, req):
self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
path, query = path.split('?')
query_dict = parse_qs(query)
self.assertEqual(
path, '/v1/a/src_cont/symlink')
self.assertEqual(
query_dict,
{'multipart-manifest': ['get'], 'symlink': ['get'],
'format': ['raw']})
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertEqual(
hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'), 'a2')
class DloTestCase(unittest.TestCase):
def call_dlo(self, req, app=None, expect_exception=False):
if app is None:
app = self.dlo
req.headers.setdefault("User-Agent", "Soap Opera")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
body = ""
caught_exc = None
try:
for chunk in body_iter:
body += chunk
except Exception as exc:
if expect_exception:
caught_exc = exc
else:
raise
if expect_exception:
return status[0], headers[0], body, caught_exc
else:
return status[0], headers[0], body
def setUp(self):
self.app = FakeSwift()
self.dlo = dlo.filter_factory(
{
# don't slow down tests with rate limiting
"rate_limit_after_segment": "1000000"
}
)(self.app)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_01", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg01-etag"}, "aaaaa"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_02", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg02-etag"}, "bbbbb"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_03", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg03-etag"}, "ccccc"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_04", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg04-etag"}, "ddddd"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_05", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg05-etag"}, "eeeee"
)
# an unrelated object (not seg*) to test the prefix matching
self.app.register(
"GET",
"/v1/AUTH_test/c/catpicture.jpg",
swob.HTTPOk,
{"Content-Length": "9", "Etag": "cats-etag"},
"meow meow meow meow",
)
self.app.register(
"GET",
"/v1/AUTH_test/mancon/manifest",
swob.HTTPOk,
{"Content-Length": "17", "Etag": "manifest-etag", "X-Object-Manifest": "c/seg"},
"manifest-contents",
)
lm = "2013-11-22T02:42:13.781760"
ct = "application/octet-stream"
segs = [
{"hash": "seg01-etag", "bytes": 5, "name": "seg_01", "last_modified": lm, "content_type": ct},
{"hash": "seg02-etag", "bytes": 5, "name": "seg_02", "last_modified": lm, "content_type": ct},
{"hash": "seg03-etag", "bytes": 5, "name": "seg_03", "last_modified": lm, "content_type": ct},
{"hash": "seg04-etag", "bytes": 5, "name": "seg_04", "last_modified": lm, "content_type": ct},
{"hash": "seg05-etag", "bytes": 5, "name": "seg_05", "last_modified": lm, "content_type": ct},
]
full_container_listing = segs + [
{
"hash": "cats-etag",
"bytes": 9,
"name": "catpicture.jpg",
"last_modified": lm,
"content_type": "application/png",
}
]
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(full_container_listing),
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=seg",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(segs),
)
# This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings.
#
# If you're testing against this, be sure to mock out
# CONTAINER_LISTING_LIMIT to 3 in your test.
self.app.register(
"GET",
"/v1/AUTH_test/mancon/manifest-many-segments",
swob.HTTPOk,
{"Content-Length": "7", "Etag": "etag-manyseg", "X-Object-Manifest": "c/seg_"},
"manyseg",
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=seg_",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(segs[:3]),
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(segs[3:]),
)
# Here's a manifest with 0 segments
self.app.register(
"GET",
"/v1/AUTH_test/mancon/manifest-no-segments",
swob.HTTPOk,
{"Content-Length": "7", "Etag": "noseg", "X-Object-Manifest": "c/noseg_"},
"noseg",
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=noseg_",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps([]),
)
class TestListingFormats(unittest.TestCase):
def setUp(self):
self.fake_swift = FakeSwift()
self.app = listing_formats.ListingFilter(self.fake_swift)
self.fake_account_listing = json.dumps([
{
'name': 'bar',
'bytes': 0,
'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo_'
},
]).encode('ascii')
self.fake_container_listing = json.dumps([
{
'name': 'bar',
'hash': 'etag',
'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo/'
},
]).encode('ascii')
def test_valid_account(self):
self.fake_swift.register(
'GET', '/v1/a', HTTPOk, {
'Content-Length': str(len(self.fake_account_listing)),
'Content-Type': 'application/json'
}, self.fake_account_listing)
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_account_listing)
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<subdir name="foo_" />',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
def test_valid_container(self):
self.fake_swift.register(
'GET', '/v1/a/c', HTTPOk, {
'Content-Length': str(len(self.fake_container_listing)),
'Content-Type': 'application/json'
}, self.fake_container_listing)
req = Request.blank('/v1/a/c')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_container_listing)
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=xml')
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<container name="c">'
b'<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<subdir name="foo/"><name>foo/</name></subdir>'
b'</container>')
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
def test_blank_account(self):
self.fake_swift.register('GET', '/v1/a', HTTPOk, {
'Content-Length': '2',
'Content-Type': 'application/json'
}, b'[]')
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, b'[]')
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a">',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
def test_blank_container(self):
self.fake_swift.register('GET', '/v1/a/c', HTTPOk, {
'Content-Length': '2',
'Content-Type': 'application/json'
}, b'[]')
req = Request.blank('/v1/a/c')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, b'[]')
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<container name="c" />',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
def test_pass_through(self):
def do_test(path):
self.fake_swift.register(
'GET', path, HTTPOk, {
'Content-Length': str(len(self.fake_container_listing)),
'Content-Type': 'application/json'
}, self.fake_container_listing)
req = Request.blank(path + '?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_container_listing)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(
self.fake_swift.calls[-1],
('GET', path + '?format=xml')) # query param is unchanged
do_test('/')
do_test('/v1')
do_test('/auth/v1.0')
do_test('/v1/a/c/o')
def test_static_web_not_json(self):
body = b'doesnt matter'
self.fake_swift.register('GET', '/v1/staticweb/not-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'text/plain'
}, body)
resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
self.assertEqual(resp.body, body)
self.assertEqual(resp.headers['Content-Type'], 'text/plain')
# We *did* try, though
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/not-json?format=json'))
# TODO: add a similar test that has *no* content-type
# FakeSwift seems to make this hard to do
def test_static_web_not_really_json(self):
body = b'raises ValueError'
self.fake_swift.register('GET', '/v1/staticweb/not-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'application/json'
}, body)
resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
self.assertEqual(resp.body, body)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/not-json?format=json'))
def test_static_web_pretend_to_be_giant_json(self):
body = json.dumps([
{
'name': 'bar',
'hash': 'etag',
'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo/'
},
] * 160000).encode('ascii')
self.assertGreater( # sanity
len(body), listing_formats.MAX_CONTAINER_LISTING_CONTENT_LENGTH)
self.fake_swift.register('GET', '/v1/staticweb/long-json', HTTPOk,
{'Content-Type': 'application/json'}, body)
resp = Request.blank('/v1/staticweb/long-json').get_response(self.app)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(resp.body, body)
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/long-json?format=json'))
# TODO: add a similar test for chunked transfers
# (staticweb referencing a DLO that doesn't fit in a single listing?)
def test_static_web_bad_json(self):
def do_test(body_obj):
body = json.dumps(body_obj).encode('ascii')
self.fake_swift.register(
'GET', '/v1/staticweb/bad-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'application/json'
}, body)
def do_sub_test(path):
resp = Request.blank(path).get_response(self.app)
self.assertEqual(resp.body, body)
# NB: no charset is added; we pass through whatever we got
self.assertEqual(resp.headers['Content-Type'],
'application/json')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/bad-json?format=json'))
do_sub_test('/v1/staticweb/bad-json')
do_sub_test('/v1/staticweb/bad-json?format=txt')
do_sub_test('/v1/staticweb/bad-json?format=xml')
do_sub_test('/v1/staticweb/bad-json?format=json')
do_test({})
do_test({'non-empty': 'hash'})
do_test(None)
do_test(0)
do_test('some string')
do_test([None])
do_test([0])
do_test(['some string'])
def test_static_web_bad_but_not_terrible_json(self):
body = json.dumps([{'no name': 'nor subdir'}]).encode('ascii')
self.fake_swift.register('GET', '/v1/staticweb/bad-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'application/json'
}, body)
def do_test(path, expect_charset=False):
resp = Request.blank(path).get_response(self.app)
self.assertEqual(resp.body, body)
if expect_charset:
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
else:
self.assertEqual(resp.headers['Content-Type'],
'application/json')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/bad-json?format=json'))
do_test('/v1/staticweb/bad-json')
do_test('/v1/staticweb/bad-json?format=txt')
do_test('/v1/staticweb/bad-json?format=xml')
# The response we get is *just close enough* to being valid that we
# assume it is and slap on the missing charset. If you set up staticweb
# to serve back such responses, your clients are already hosed.
do_test('/v1/staticweb/bad-json?format=json', expect_charset=True)
class OioContainerHierarchy(unittest.TestCase):
def setUp(self):
conf = {'sds_default_account': 'OPENIO'}
self.filter_conf = {
'strip_v1': 'true',
'swift3_compat': 'true',
'account_first': 'true'
}
self.app = FakeSwift()
self.ch = container_hierarchy.filter_factory(conf, **self.filter_conf)(
self.app)
def mock(self):
self.ch._create_key = mock.MagicMock(return_value=None)
self.ch._remove_key = mock.MagicMock(return_value=None)
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
def call_ch(self, req):
return self.call_app(req, app=self.ch)
def test_simple_put(self):
"""check number of request generated by Container Hierarchy"""
self.mock()
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated,
{})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.ch._create_key.assert_called_with(mock.ANY, 'a', 'c', 'cnt',
'd1/d2/d3/')
def test_fake_directory_put(self):
self.mock()
req = Request.blank('/v1/a/c/d1/d2/d3/', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.ch._create_key.assert_called_with(mock.ANY, 'a', 'c', 'obj',
'd1/d2/d3/')
def test_get(self):
self.app.register('GET', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPOk, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_recursive_listing(self):
self.ch.conn.keys = mock.MagicMock(return_value=['CS:a:cnt:d1/d2/d3/'])
self.app.register(
'GET',
'/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&delimiter=%2F&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "o",
"content_type": "application/octet-stream"
}]))
req = Request.blank('/v1/a/c?prefix=d1%2Fd2%2F', method='GET')
resp = self.call_ch(req)
data = json.loads(resp[2])
self.assertEqual(data[0]['name'], 'd1/d2/d3/o')
def test_global_listing(self):
self.app.register('GET', '/v1/a', swob.HTTPOk, {})
req = Request.blank('/v1/a', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_delete_object(self):
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated,
{})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
self.app.register(
'GET',
'/v1/a/c%2Fd1%2Fd2%2Fd3?delimiter=%2F&limit=1&prefix=&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "o",
"content_type": "application/octet-stream"
}]))
self.app.register('DELETE', '/v1/a/c%2Fd1%2Fd2%2Fd3/o',
swob.HTTPNoContent, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
self.app.register(
'GET',
'/v1/a/c%2Fd1%2Fd2%2Fd3?delimiter=%2F&limit=1&prefix=&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([]))
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
self.assertNotIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
def test_fake_directory(self):
req = Request.blank('/v1/a/d1/d2/d3/', method='PUT')
resp = self.call_ch(req)
self.assertIn('CS:a:d1:obj:d2/d3/', self.ch.conn._keys)
req = Request.blank('/v1/a/d1/d2/d3/', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], "204 No Content")
self.assertNotIn('CS:a:d1:obj:d2/d3/', self.ch.conn._keys)
def _listing(self, is_recursive):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/', 'CS:a:bucket:cnt:d1/d2/'])
self.ch.conn.exist = mock.MagicMock(return_value=True)
self.app.register(
'GET',
'/v1/a/bucket%2Fd1?prefix=d&limit=10000&delimiter=%2F&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "o1",
"content_type": "application/octet-stream"
}]))
if is_recursive:
self.app.register(
'GET',
'/v1/a/bucket%2Fd1%2Fd2?prefix=&limit=10000&delimiter=%2F&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "o2",
"content_type": "application/octet-stream"
}]))
recursive = '' if is_recursive else '&delimiter=%2F'
req = Request.blank('/v1/a/bucket?prefix=d1/d&limit=10%s' % recursive,
method='GET')
resp = self.call_ch(req)
names = [
item.get('name', item.get('subdir'))
for item in json.loads(resp[2])
]
return names
def test_listing_with_prefix(self):
names = self._listing(False)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/', names)
def test_listing_with_prefix_recursive(self):
names = self._listing(True)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/o2', names)
def test_listing_root_container(self):
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/'])
self.app.register(
'GET',
'/v1/a/bucket?prefix=d&limit=10000&delimiter=%2F&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "d0",
"content_type": "application/octet-stream"
}]))
req = Request.blank('/v1/a/bucket?prefix=d&limit=10&delimiter=%2F',
method='GET')
resp = self.call_ch(req)
names = [
item.get('name', item.get('subdir'))
for item in json.loads(resp[2])
]
self.assertIn("d0", names)
self.assertIn("d1/", names)
def test_listing_with_marker(self):
self.ch.conn.keys = mock.MagicMock(return_value=[
'CS:a:bucket:cnt:d1/',
'CS:a:bucket:cnt:d2/',
])
req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
method='GET')
resp = self.call_ch(req)
names = [
item.get('name', item.get('subdir'))
for item in json.loads(resp[2])
]
self.assertNotIn('d1/', names)
self.assertIn('d2/', names)
def test_listing_with_marker_multi_container(self):
self.ch.conn.keys = mock.MagicMock(return_value=[
'CS:a:bucket:cnt:d1/',
'CS:a:bucket:cnt:d2/',
])
# with marker aa (as we inspect d1/)
self.app.register(
'GET',
'/v1/a/bucket%2Fd1?marker=aa&delimiter=%2F&limit=10000&prefix=&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "d0",
"content_type": "application/octet-stream"
}]))
# without marker on second container
self.app.register(
'GET',
'/v1/a/bucket%2Fd2?delimiter=%2F&limit=10000&prefix=&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "d0",
"content_type": "application/octet-stream"
}]))
req = Request.blank('/v1/a/bucket?limit=10&marker=d1/aa', method='GET')
resp = self.call_ch(req)
names = [
item.get('name', item.get('subdir'))
for item in json.loads(resp[2])
]
self.assertIn('d1/d0', names)
self.assertIn('d2/d0', names)
def test_duplicate_obj_cnt(self):
self.ch.conn.keys = mock.MagicMock(return_value=[
'CS:a:bucket:cnt:d1/cnt/',
'CS:a:bucket:obj:d1/obj/',
])
req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
method='GET')
resp = self.call_ch(req)
names = [
item.get('name', item.get('subdir'))
for item in json.loads(resp[2])
]
self.assertIn('d1/', names)
self.assertEqual(1, len(names))
class TestEncrypter(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.encrypter = encrypter.Encrypter(self.app, {})
self.encrypter.logger = FakeLogger()
def _verify_user_metadata(self, req_hdrs, name, value, key):
# verify encrypted version of user metadata
self.assertNotIn('X-Object-Meta-' + name, req_hdrs)
expected_hdr = 'X-Object-Transient-Sysmeta-Crypto-Meta-' + name
self.assertIn(expected_hdr, req_hdrs)
enc_val, param = req_hdrs[expected_hdr].split(';')
param = param.strip()
self.assertTrue(param.startswith('swift_meta='))
actual_meta = json.loads(
urlparse.unquote_plus(param[len('swift_meta='):]))
self.assertEqual(Crypto.cipher, actual_meta['cipher'])
meta_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, meta_iv)
self.assertEqual(
base64.b64encode(encrypt(value, key, meta_iv)),
enc_val)
# if there is any encrypted user metadata then this header should exist
self.assertIn('X-Object-Transient-Sysmeta-Crypto-Meta', req_hdrs)
common_meta = json.loads(urlparse.unquote_plus(
req_hdrs['X-Object-Transient-Sysmeta-Crypto-Meta']))
self.assertDictEqual({'cipher': Crypto.cipher,
'key_id': {'v': 'fake', 'path': '/a/c/fake'}},
common_meta)
def test_PUT_req(self):
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'etag': plaintext_etag,
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
return_value=body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urlparse.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV,
base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
# verify etag
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
# verify crypto_meta was appended to this etag
self.assertTrue(etag_meta)
actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
# verify encrypted version of plaintext etag
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
self.assertEqual(enc_etag, actual)
# verify etag MAC for conditional requests
actual_hmac = base64.b64decode(
req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac'])
self.assertEqual(actual_hmac, hmac.new(
object_key, plaintext_etag, hashlib.sha256).digest())
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# content-type is not encrypted
self.assertEqual('text/plain', req_hdrs['Content-Type'])
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(
req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual(ciphertext, resp.body)
self.assertEqual(ciphertext_etag, resp.headers['Etag'])
def test_PUT_zero_size_object(self):
# object body encryption should be skipped for zero sized object body
object_key = fetch_crypto_keys()['object']
plaintext_etag = EMPTY_ETAG
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'etag': EMPTY_ETAG,
'content-type': 'text/plain',
'content-length': '0',
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank(
'/v1/a/c/o', environ=env, body='', headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that there is no body crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs)
# verify etag is md5 of plaintext
self.assertEqual(EMPTY_ETAG, req_hdrs['Etag'])
# verify there is no etag crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs)
# verify there is no container update override for etag
self.assertNotIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
# user meta is still encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(
req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is empty by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual('', resp.body)
self.assertEqual(EMPTY_ETAG, resp.headers['Etag'])
def _test_PUT_with_other_footers(self, override_etag):
# verify handling of another middleware's footer callback
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
other_footers = {
'Etag': plaintext_etag,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Size':
'other override',
'X-Object-Sysmeta-Container-Update-Override-Etag':
override_etag}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'correct etag is in footers'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that other middleware's footers made it to app, including any
# container update overrides but nothing Etag-related
other_footers.pop('Etag')
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify encryption footers are ok
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
self.assertTrue(etag_meta)
actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urlparse.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV,
base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
def test_PUT_with_other_footers(self):
self._test_PUT_with_other_footers('override etag')
def test_PUT_with_other_footers_and_etag_of_empty_body(self):
# verify that an override etag value of EMPTY_ETAG will be encrypted
# when there was a non-zero body length
self._test_PUT_with_other_footers(EMPTY_ETAG)
def _test_PUT_with_etag_override_in_headers(self, override_etag):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag':
override_etag}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
def test_PUT_with_etag_override_in_headers(self):
self._test_PUT_with_etag_override_in_headers('override_etag')
def test_PUT_with_etag_of_empty_body_override_in_headers(self):
# verify that an override etag value of EMPTY_ETAG will be encrypted
# when there was a non-zero body length
self._test_PUT_with_etag_override_in_headers(EMPTY_ETAG)
def _test_PUT_with_empty_etag_override_in_headers(self, plaintext):
# verify that an override etag value of '' from other middleware is
# passed through unencrypted
plaintext_etag = md5hex(plaintext)
override_etag = ''
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag':
override_etag}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
self.assertEqual(
override_etag,
req_hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'])
def test_PUT_with_empty_etag_override_in_headers(self):
self._test_PUT_with_empty_etag_override_in_headers('body')
def test_PUT_with_empty_etag_override_in_headers_no_body(self):
self._test_PUT_with_empty_etag_override_in_headers('')
def _test_PUT_with_empty_etag_override_in_footers(self, plaintext):
# verify that an override etag value of '' from other middleware is
# passed through unencrypted
plaintext_etag = md5hex(plaintext)
override_etag = ''
other_footers = {
'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
self.assertEqual(
override_etag,
req_hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'])
def test_PUT_with_empty_etag_override_in_footers(self):
self._test_PUT_with_empty_etag_override_in_footers('body')
def test_PUT_with_empty_etag_override_in_footers_no_body(self):
self._test_PUT_with_empty_etag_override_in_footers('')
def test_PUT_with_bad_etag_in_other_footers(self):
# verify that etag supplied in footers from other middleware overrides
# header etag when validating inbound plaintext etags
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
other_footers = {
'Etag': 'bad etag',
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
self.assertNotIn('Etag', resp.headers)
def test_PUT_with_bad_etag_in_headers_and_other_footers(self):
# verify that etag supplied in headers from other middleware is used if
# none is supplied in footers when validating inbound plaintext etags
plaintext = 'FAKE APP'
other_footers = {
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'bad etag'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
self.assertNotIn('Etag', resp.headers)
def test_PUT_nothing_read(self):
# simulate an artificial scenario of a downstream filter/app not
# actually reading the input stream from encrypter.
class NonReadingApp(object):
def __call__(self, env, start_response):
# note: no read from wsgi.input
req = Request(env)
env['swift.callback.update_footers'](req.headers)
call_headers.append(req.headers)
resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
return resp(env, start_response)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': 0,
'etag': 'etag from client'}
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
self.assertEqual('etag from client', call_headers[0]['etag'])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# check that an upstream footer callback gets called
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify encrypted override etag for container update.
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', call_headers[0])
parts = call_headers[0][
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt('other override', cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify that other middleware's footers made it to app
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# if upstream footer override etag is for an empty body then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# if upstream footer override etag is an empty string then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Container-Update-Override-Etag': ''}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
def test_POST_req(self):
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'POST',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'x-object-meta-test': 'encrypt me',
'x-object-meta-test2': '',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
key = fetch_crypto_keys()['object']
self.app.register('POST', '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertNotIn('Etag', resp.headers)
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('POST', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', key)
# unless it had no value
self.assertEqual('', req_hdrs['X-Object-Meta-Test2'])
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
def _test_no_user_metadata(self, method):
# verify that x-object-transient-sysmeta-crypto-meta is not set when
# there is no user metadata
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env, body='body')
self.app.register(method, '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
self.assertNotIn('x-object-transient-sysmeta-crypto-meta',
self.app.headers[0])
def test_PUT_no_user_metadata(self):
self._test_no_user_metadata('PUT')
def test_POST_no_user_metadata(self):
self._test_no_user_metadata('POST')
def _test_if_match(self, method, match_header_name):
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ', '.join(plain_etags)
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={match_header_name: match_header_value})
app = FakeSwift()
app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != '*':
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(', '))
# masked values for secret_id None
key = fetch_crypto_keys()['object']
masked_etags = [
'"%s"' % base64.b64encode(hmac.new(
key, etag.strip('"'), hashlib.sha256).digest())
for etag in plain_etags if etag not in ('*', '')]
# masked values for secret_id myid
key = fetch_crypto_keys(key_id={'secret_id': 'myid'})['object']
masked_etags_myid = [
'"%s"' % base64.b64encode(hmac.new(
key, etag.strip('"'), hashlib.sha256).digest())
for etag in plain_etags if etag not in ('*', '')]
expected_etags = set((expected_plain_etags or plain_etags) +
masked_etags + masked_etags_myid)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags),
set(req.headers[match_header_name].split(', ')))
do_test(method, [''])
do_test(method, ['"an etag"'])
do_test(method, ['"an etag"', '"another_etag"'])
do_test(method, ['*'])
# rfc2616 does not allow wildcard *and* etag but test it anyway
do_test(method, ['*', '"an etag"'])
# etags should be quoted but check we can cope if they are not
do_test(
method, ['*', 'an etag', 'another_etag'],
expected_plain_etags=['*', '"an etag"', '"another_etag"'])
def test_GET_if_match(self):
self._test_if_match('GET', 'If-Match')
def test_HEAD_if_match(self):
self._test_if_match('HEAD', 'If-Match')
def test_GET_if_none_match(self):
self._test_if_match('GET', 'If-None-Match')
def test_HEAD_if_none_match(self):
self._test_if_match('HEAD', 'If-None-Match')
def _test_existing_etag_is_at_header(self, method, match_header_name):
# if another middleware has already set X-Backend-Etag-Is-At then
# encrypter should not override that value
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={match_header_name: "an etag",
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Other-Etag'})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual(
'X-Object-Sysmeta-Other-Etag,X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
actual_etags = set(actual_headers[match_header_name].split(', '))
self.assertIn('"an etag"', actual_etags)
def test_GET_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('GET', 'If-Match')
def test_HEAD_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('HEAD', 'If-Match')
def test_GET_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('GET', 'If-None-Match')
def test_HEAD_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('HEAD', 'If-None-Match')
def _test_etag_is_at_not_duplicated(self, method):
# verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
# X-Backend-Etag-Is-At
key = fetch_crypto_keys()['object']
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={'If-Match': '"an etag"',
'If-None-Match': '"another etag"'})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'an etag', hashlib.sha256).digest()),
actual_headers['If-Match'])
self.assertIn('"another etag"', actual_headers['If-None-Match'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'another etag', hashlib.sha256).digest()),
actual_headers['If-None-Match'])
def test_GET_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated('GET')
def test_HEAD_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated('HEAD')
def test_PUT_response_inconsistent_etag_is_not_replaced(self):
# if response is success but etag does not match the ciphertext md5
# then verify that we do *not* replace it with the plaintext etag
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated,
{'Etag': 'not the ciphertext etag'})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual('not the ciphertext etag', resp.headers['Etag'])
def test_PUT_multiseg_no_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def test_PUT_multiseg_good_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': md5hex(body)}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def test_PUT_multiseg_bad_client_etag(self):
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': 'badclientetag'}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
def test_PUT_missing_key_callback(self):
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT'}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('missing callback',
self.encrypter.logger.get_lines_for_level('error')[0])
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
def test_PUT_error_in_key_callback(self):
def raise_exc(*args, **kwargs):
raise Exception('Testing')
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: raise_exc}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('from callback: Testing',
self.encrypter.logger.get_lines_for_level('error')[0])
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
def test_PUT_encryption_override(self):
# set crypto override to disable encryption.
# simulate another middleware wanting to set footers
other_footers = {
'Etag': 'other etag',
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT',
'swift.crypto.override': True,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify that other middleware's footers made it to app
req_hdrs = self.app.headers[0]
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify object is NOT encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(body, get_req.get_response(self.app).body)
def _test_constraints_checking(self, method):
# verify that the check_metadata function is called on PUT and POST
body = 'FAKE APP'
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
mocked_func = 'swift.common.middleware.crypto.encrypter.check_metadata'
with mock.patch(mocked_func) as mocked:
mocked.side_effect = [HTTPBadRequest('testing')]
resp = req.get_response(self.encrypter)
self.assertEqual('400 Bad Request', resp.status)
self.assertEqual(1, mocked.call_count)
mocked.assert_called_once_with(mock.ANY, 'object')
self.assertEqual(req.headers,
mocked.call_args_list[0][0][0].headers)
def test_PUT_constraints_checking(self):
self._test_constraints_checking('PUT')
def test_POST_constraints_checking(self):
self._test_constraints_checking('POST')
def test_config_true_value_on_disable_encryption(self):
app = FakeSwift()
self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption)
for val in ('true', '1', 'yes', 'on', 't', 'y'):
app = encrypter.Encrypter(app,
{'disable_encryption': val})
self.assertTrue(app.disable_encryption)
def test_PUT_app_exception(self):
app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {})
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
with self.assertRaises(HTTPException) as catcher:
req.get_response(app)
self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)
def test_encrypt_header_val(self):
# Prepare key and Crypto instance
object_key = fetch_crypto_keys()['object']
# - Normal string can be crypted
encrypted = encrypter.encrypt_header_val(Crypto(), 'aaa', object_key)
# sanity: return value is 2 item tuple
self.assertEqual(2, len(encrypted))
crypted_val, crypt_info = encrypted
expected_crypt_val = base64.b64encode(
encrypt('aaa', object_key, FAKE_IV))
expected_crypt_info = {
'cipher': 'AES_CTR_256', 'iv': 'This is an IV123'}
self.assertEqual(expected_crypt_val, crypted_val)
self.assertEqual(expected_crypt_info, crypt_info)
# - Empty string raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), '', object_key)
self.assertEqual('empty value is not acceptable',
cm.exception.message)
# - None also raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), None, object_key)
self.assertEqual('empty value is not acceptable',
cm.exception.message)
class TestDecrypterContainerRequests(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.decrypter = decrypter.Decrypter(self.app, {})
self.decrypter.logger = FakeLogger()
def _make_cont_get_req(self,
resp_body,
format,
override=False,
callback=fetch_crypto_keys):
path = '/v1/a/c'
content_type = 'text/plain'
if format:
path = '%s/?format=%s' % (path, format)
content_type = 'application/' + format
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: callback}
if override:
env['swift.crypto.override'] = True
req = Request.blank(path, environ=env)
hdrs = {'content-type': content_type}
self.app.register('GET', path, HTTPOk, body=resp_body, headers=hdrs)
return req.get_response(self.decrypter)
def test_GET_container_success(self):
# no format requested, listing has names only
fake_body = 'testfile1\ntestfile2\n'
calls = [0]
def wrapped_fetch_crypto_keys():
calls[0] += 1
return fetch_crypto_keys()
resp = self._make_cont_get_req(fake_body,
None,
callback=wrapped_fetch_crypto_keys)
self.assertEqual('200 OK', resp.status)
names = resp.body.split('\n')
self.assertEqual(3, len(names))
self.assertIn('testfile1', names)
self.assertIn('testfile2', names)
self.assertIn('', names)
self.assertEqual(0, calls[0])
def test_GET_container_json(self):
content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
content_type_2 = 'text/plain; param=foo'
pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
key = fetch_crypto_keys()['container']
subdir = {"subdir": "pseudo-dir/"}
obj_dict_1 = {
"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": encrypt_and_append_meta(pt_etag1.encode('utf-8'), key),
"name": "testfile",
"content_type": content_type_1
}
obj_dict_2 = {
"bytes": 24,
"last_modified": "2015-04-14T23:33:06.519020",
"hash": encrypt_and_append_meta(pt_etag2.encode('utf-8'), key),
"name": "testfile2",
"content_type": content_type_2
}
listing = [subdir, obj_dict_1, obj_dict_2]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(3, len(body_json))
self.assertDictEqual(subdir, body_json[0])
obj_dict_1['hash'] = pt_etag1
self.assertDictEqual(obj_dict_1, body_json[1])
obj_dict_2['hash'] = pt_etag2
self.assertDictEqual(obj_dict_2, body_json[2])
def test_GET_container_json_with_crypto_override(self):
content_type_1 = 'image/jpeg'
content_type_2 = 'text/plain; param=foo'
pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
obj_dict_1 = {
"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": pt_etag1,
"name": "testfile",
"content_type": content_type_1
}
obj_dict_2 = {
"bytes": 24,
"last_modified": "2015-04-14T23:33:06.519020",
"hash": pt_etag2,
"name": "testfile2",
"content_type": content_type_2
}
listing = [obj_dict_1, obj_dict_2]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json', override=True)
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(2, len(body_json))
self.assertDictEqual(obj_dict_1, body_json[0])
self.assertDictEqual(obj_dict_2, body_json[1])
def test_cont_get_json_req_with_cipher_mismatch(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
key = fetch_crypto_keys()['container']
pt_etag = 'c6e8196d7f0fff6444b90861fe8d609d'
ct_etag = encrypt_and_append_meta(pt_etag,
key,
crypto_meta=bad_crypto_meta)
obj_dict_1 = {
"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": ct_etag,
"name": "testfile",
"content_type": "image/jpeg"
}
listing = [obj_dict_1]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting container listing', resp.body)
self.assertIn("Cipher must be AES_CTR_256",
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_container_json_not_encrypted_obj(self):
pt_etag = '%s; symlink_path=/a/c/o' % MD5_OF_EMPTY_STRING
obj_dict = {
"bytes": 0,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": pt_etag,
"name": "symlink",
"content_type": 'application/symlink'
}
listing = [obj_dict]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(1, len(body_json))
self.assertEqual(pt_etag, body_json[0]['hash'])
class TestDecrypterObjectRequests(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.decrypter = decrypter.Decrypter(self.app, {})
self.decrypter.logger = FakeLogger()
def _make_response_headers(self, content_length, plaintext_etag, keys,
body_key):
# helper method to make a typical set of response headers for a GET or
# HEAD request
cont_key = keys['container']
object_key = keys['object']
body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
'iv': FAKE_IV}
body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
return HeaderKeyDict({
'Etag': 'hashOfCiphertext',
'content-type': 'text/plain',
'content-length': content_length,
'X-Object-Sysmeta-Crypto-Etag': '%s; swift_meta=%s' % (
base64.b64encode(encrypt(plaintext_etag, object_key, FAKE_IV)),
get_crypto_meta_header()),
'X-Object-Sysmeta-Crypto-Body-Meta':
get_crypto_meta_header(body_crypto_meta),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', object_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-container-update-override-etag':
encrypt_and_append_meta('encrypt me, too', cont_key),
'x-object-sysmeta-test': 'do not encrypt me',
})
def _test_request_success(self, method, body):
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)
# there shouldn't be any x-object-meta- headers, but if there are
# then the decrypted header will win where there is a name clash...
hdrs.update({
'x-object-meta-test': 'unexpected, overwritten by decrypted value',
'x-object-meta-distinct': 'unexpected but distinct from encrypted'
})
self.app.register(
method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
self.assertEqual('unexpected but distinct from encrypted',
resp.headers['x-object-meta-distinct'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
self.assertEqual(
'encrypt me, too',
resp.headers['X-Object-Sysmeta-Container-Update-Override-Etag'])
self.assertNotIn('X-Object-Sysmeta-Crypto-Body-Meta', resp.headers)
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', resp.headers)
return resp
def test_GET_success(self):
body = 'FAKE APP'
resp = self._test_request_success('GET', body)
self.assertEqual(body, resp.body)
def test_HEAD_success(self):
body = 'FAKE APP'
resp = self._test_request_success('HEAD', body)
self.assertEqual('', resp.body)
def test_headers_case(self):
body = 'fAkE ApP'
req = Request.blank('/v1/a/c/o', body='FaKe')
req.environ[CRYPTO_KEY_CALLBACK] = fetch_crypto_keys
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs.update({
'x-Object-mEta-ignoRes-caSe': 'thIs pArt WilL bE cOol',
})
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
status, headers, app_iter = req.call_application(self.decrypter)
self.assertEqual(status, '200 OK')
expected = {
'Etag': '7f7837924188f7b511a9e3881a9f77a8',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'encrypt me, too',
'X-Object-Meta-Test': 'encrypt me',
'Content-Length': '8',
'X-Object-Meta-Ignores-Case': 'thIs pArt WilL bE cOol',
'X-Object-Sysmeta-Test': 'do not encrypt me',
'Content-Type': 'text/plain',
}
self.assertEqual(dict(headers), expected)
self.assertEqual('fAkE ApP', ''.join(app_iter))
def _test_412_response(self, method):
# simulate a 412 response to a conditional GET which has an Etag header
data = 'the object content'
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env, method=method)
resp_body = 'I am sorry, you have failed to meet a precondition'
hdrs = self._make_response_headers(
len(resp_body), md5hex(data), fetch_crypto_keys(), 'not used')
self.app.register(method, '/v1/a/c/o', HTTPPreconditionFailed,
body=resp_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('412 Precondition Failed', resp.status)
# the response body should not be decrypted, it is already plaintext
self.assertEqual(resp_body if method == 'GET' else '', resp.body)
# whereas the Etag and other headers should be decrypted
self.assertEqual(md5hex(data), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
def test_GET_412_response(self):
self._test_412_response('GET')
def test_HEAD_412_response(self):
self._test_412_response('HEAD')
def _test_404_response(self, method):
# simulate a 404 response, sanity check response headers
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env, method=method)
resp_body = 'You still have not found what you are looking for'
hdrs = {'content-type': 'text/plain',
'content-length': len(resp_body)}
self.app.register(method, '/v1/a/c/o', HTTPNotFound,
body=resp_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('404 Not Found', resp.status)
# the response body should not be decrypted, it is already plaintext
self.assertEqual(resp_body if method == 'GET' else '', resp.body)
# there should be no etag header inserted by decrypter
self.assertNotIn('Etag', resp.headers)
self.assertEqual('text/plain', resp.headers['Content-Type'])
def test_GET_404_response(self):
self._test_404_response('GET')
def test_HEAD_404_response(self):
self._test_404_response('HEAD')
def test_GET_missing_etag_crypto_meta(self):
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted etag
hdrs['X-Object-Sysmeta-Crypto-Etag'] = \
base64.b64encode(encrypt(md5hex(body), key, FAKE_IV))
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header', resp.body)
self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_override_etag_bad_meta(self, method, bad_crypto_meta):
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted override etag
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
encrypt_and_append_meta(
md5hex(body), key, crypto_meta=bad_crypto_meta)
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header '
'X-Object-Sysmeta-Container-Update-Override-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def test_GET_override_etag_bad_iv(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta)
self.assertIn('Error decrypting header', resp.body)
def test_HEAD_override_etag_bad_iv(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta)
self.assertEqual('', resp.body)
def test_GET_override_etag_bad_cipher(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown cipher'
resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta)
self.assertIn('Error decrypting header', resp.body)
def test_HEAD_override_etag_bad_cipher(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown cipher'
resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta)
self.assertEqual('', resp.body)
def _test_bad_key(self, method):
# use bad key
def bad_fetch_crypto_keys():
keys = fetch_crypto_keys()
keys['object'] = 'bad key'
return keys
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
return req.get_response(self.decrypter)
def test_HEAD_with_bad_key(self):
resp = self._test_bad_key('HEAD')
self.assertEqual('500 Internal Error', resp.status)
self.assertIn("Bad key for 'object'",
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_key(self):
resp = self._test_bad_key('GET')
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.',
resp.body)
self.assertIn("Bad key for 'object'",
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_bad_crypto_meta_for_user_metadata(self, method, bad_crypto_meta):
# use bad iv for metadata headers
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
enc_val = base64.b64encode(encrypt('encrypt me', key, FAKE_IV))
if bad_crypto_meta:
enc_val += ';swift_meta=' + get_crypto_meta_header(
crypto_meta=bad_crypto_meta)
hdrs['x-object-transient-sysmeta-crypto-meta-test'] = enc_val
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test', self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def test_HEAD_with_missing_crypto_meta_for_user_metadata(self):
self._test_bad_crypto_meta_for_user_metadata('HEAD', None)
self.assertIn('Missing crypto meta in value',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_crypto_meta_for_user_metadata(self):
self._test_bad_crypto_meta_for_user_metadata('GET', None)
self.assertIn('Missing crypto meta in value',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_HEAD_with_bad_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta)
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_HEAD_with_missing_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta.pop('iv')
self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta)
self.assertIn(
'iv', self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
resp = self._test_bad_crypto_meta_for_user_metadata(
'GET', bad_crypto_meta)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta.pop('iv')
resp = self._test_bad_crypto_meta_for_user_metadata(
'GET', bad_crypto_meta)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn(
'iv', self.decrypter.logger.get_lines_for_level('error')[0])
def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta):
# use bad iv for object body
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_iv_for_object_body(self):
bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32))
bad_crypto_meta['iv'] = 'bad_iv'
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_iv_for_object_body(self):
bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32))
bad_crypto_meta.pop('iv')
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn("Missing 'iv'",
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_body_key_for_object_body(self):
body_key_meta = {'key': 'wrapped too short key', 'iv': FAKE_IV}
bad_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn('Key must be length 32',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_body_key_for_object_body(self):
bad_crypto_meta = fake_get_crypto_meta() # no key by default
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn("Missing 'body_key'",
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_req_metadata_not_encrypted(self, method):
# check that metadata is not decrypted if it does not have crypto meta;
# testing for case of an unencrypted POST to an object.
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs.pop('x-object-transient-sysmeta-crypto-meta-test')
hdrs['x-object-meta-test'] = 'plaintext not encrypted'
self.app.register(
method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('plaintext not encrypted',
resp.headers['x-object-meta-test'])
def test_HEAD_metadata_not_encrypted(self):
self._test_req_metadata_not_encrypted('HEAD')
def test_GET_metadata_not_encrypted(self):
self._test_req_metadata_not_encrypted('GET')
def test_GET_unencrypted_data(self):
# testing case of an unencrypted object with encrypted metadata from
# a later POST
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
obj_key = fetch_crypto_keys()['object']
hdrs = {'Etag': md5hex(body),
'content-type': 'text/plain',
'content-length': len(body),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', obj_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-test': 'do not encrypt me'}
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(md5hex(body), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
# POSTed user meta was encrypted
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
# PUT sysmeta was not encrypted
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
def test_GET_multiseg(self):
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
hdrs = self._make_response_headers(
sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(),
body_key)
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
def test_GET_multiseg_with_range(self):
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
req.headers['Content-Range'] = 'bytes 3-10/17'
chunks = ['0123', '45678', '9abcdef']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]]
hdrs = self._make_response_headers(
sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(),
body_key)
hdrs['content-range'] = req.headers['Content-Range']
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('3456789a', resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
# Force the decrypter context updates to be less than one of our range
# sizes to check that the decrypt context offset is setup correctly with
# offset to first byte of range for first update and then re-used.
# Do mocking here to have the mocked value have effect in the generator
# function.
@mock.patch.object(decrypter, 'DECRYPT_CHUNK_SIZE', 4)
def test_GET_multipart_ciphertext(self):
# build fake multipart response body
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
parts = ((0, 3, 'text/plain'),
(4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart headers could be re-ordered, so parse response body to
# verify expected content
resp_lines = resp.body.split('\r\n')
resp_lines.reverse()
for start, end, ctype in parts:
self.assertEqual('--multipartboundary', resp_lines.pop())
expected_header_lines = {
'Content-Type: %s' % ctype,
'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)}
resp_header_lines = {resp_lines.pop(), resp_lines.pop()}
self.assertEqual(expected_header_lines, resp_header_lines)
self.assertEqual('', resp_lines.pop())
self.assertEqual(plaintext[start:end], resp_lines.pop())
self.assertEqual('--multipartboundary--', resp_lines.pop())
# we should have consumed the whole response body
self.assertFalse(resp_lines)
def test_GET_multipart_content_type(self):
# *just* having multipart content type shouldn't trigger the mime doc
# code path
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
# register request with fake swift
hdrs = self._make_response_headers(
len(ciphertext), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=ciphertext,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(plaintext), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
self.assertEqual(plaintext, resp.body)
def test_GET_multipart_no_body_crypto_meta(self):
# build fake multipart response body
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
parts = ((0, 3, 'text/plain'),
(4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(plaintext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + plaintext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = {
'Etag': plaintext_etag,
'content-type': 'multipart/byteranges;boundary=multipartboundary',
'content-length': len(body)}
self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart response body should be unchanged
self.assertEqual(body, resp.body)
def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta):
# build fake multipart response body
key = fetch_crypto_keys()['object']
ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, ctxt=ctxt)
parts = ((0, 3, 'text/plain'),
(4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), 'not used')
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_bad_body_cipher(self):
self._test_GET_multipart_bad_body_crypto_meta(
{'cipher': 'Mystery cipher', 'iv': '1234567887654321'})
self.assertIn('Cipher must be AES_CTR_256',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_missing_body_cipher(self):
self._test_GET_multipart_bad_body_crypto_meta(
{'iv': '1234567887654321'})
self.assertIn('cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_too_short_body_iv(self):
self._test_GET_multipart_bad_body_crypto_meta(
{'cipher': 'AES_CTR_256', 'iv': 'too short'})
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_too_long_body_iv(self):
self._test_GET_multipart_bad_body_crypto_meta(
{'cipher': 'AES_CTR_256', 'iv': 'a little too long'})
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_missing_body_iv(self):
self._test_GET_multipart_bad_body_crypto_meta(
{'cipher': 'AES_CTR_256'})
self.assertIn('iv',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_missing_key_callback(self):
# Do not provide keys, and do not set override flag
env = {'REQUEST_METHOD': 'GET'}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex('not the body'), fetch_crypto_keys(), 'not used')
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.',
resp.body)
self.assertIn('missing callback',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: raise_exc}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.',
resp.body)
self.assertIn('from callback: Testing',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_cipher_mismatch_for_body(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(
len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
self.assertIn('Bad crypto meta: Cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_cipher_mismatch_for_metadata(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(
len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs.update({'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', key, FAKE_IV)) +
';swift_meta=' +
get_crypto_meta_header(crypto_meta=bad_crypto_meta)})
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test', self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_decryption_override(self):
# This covers the case of an old un-encrypted object
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.crypto.override': True}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
hdrs = {'Etag': md5hex(body),
'content-type': 'text/plain',
'content-length': len(body),
'x-object-meta-test': 'do not encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(md5hex(body), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-meta-test'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
class DloTestCase(unittest.TestCase):
def call_dlo(self, req, app=None, expect_exception=False):
if app is None:
app = self.dlo
req.headers.setdefault("User-Agent", "Soap Opera")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
body = ''
caught_exc = None
try:
for chunk in body_iter:
body += chunk
except Exception as exc:
if expect_exception:
caught_exc = exc
else:
raise
if expect_exception:
return status[0], headers[0], body, caught_exc
else:
return status[0], headers[0], body
def setUp(self):
self.app = FakeSwift()
self.dlo = dlo.filter_factory({
# don't slow down tests with rate limiting
'rate_limit_after_segment': '1000000',
})(self.app)
self.app.register(
'GET', '/v1/AUTH_test/c/seg_01',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg01-etag'},
'aaaaa')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_02',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg02-etag'},
'bbbbb')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg03-etag'},
'ccccc')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_04',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg04-etag'},
'ddddd')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_05',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg05-etag'},
'eeeee')
# an unrelated object (not seg*) to test the prefix matching
self.app.register(
'GET', '/v1/AUTH_test/c/catpicture.jpg',
swob.HTTPOk, {'Content-Length': '9', 'Etag': 'cats-etag'},
'meow meow meow meow')
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest',
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'},
'manifest-contents')
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
segs = [{"hash": "seg01-etag", "bytes": 5, "name": "seg_01",
"last_modified": lm, "content_type": ct},
{"hash": "seg02-etag", "bytes": 5, "name": "seg_02",
"last_modified": lm, "content_type": ct},
{"hash": "seg03-etag", "bytes": 5, "name": "seg_03",
"last_modified": lm, "content_type": ct},
{"hash": "seg04-etag", "bytes": 5, "name": "seg_04",
"last_modified": lm, "content_type": ct},
{"hash": "seg05-etag", "bytes": 5, "name": "seg_05",
"last_modified": lm, "content_type": ct}]
full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9,
"name": "catpicture.jpg",
"last_modified": lm,
"content_type": "application/png"}]
self.app.register(
'GET', '/v1/AUTH_test/c?format=json',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(full_container_listing))
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=seg',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs))
# This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings.
#
# If you're testing against this, be sure to mock out
# CONTAINER_LISTING_LIMIT to 3 in your test.
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
'X-Object-Manifest': 'c/seg_'},
'manyseg')
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=seg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[:3]))
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[3:]))
# Here's a manifest with 0 segments
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
'X-Object-Manifest': 'c/noseg_'},
'noseg')
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=noseg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps([]))
class TestDecrypterContainerRequests(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.decrypter = decrypter.Decrypter(self.app, {})
self.decrypter.logger = FakeLogger()
def _make_cont_get_req(self, resp_body, format, override=False,
callback=fetch_crypto_keys):
path = '/v1/a/c'
content_type = 'text/plain'
if format:
path = '%s/?format=%s' % (path, format)
content_type = 'application/' + format
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: callback}
if override:
env['swift.crypto.override'] = True
req = Request.blank(path, environ=env)
hdrs = {'content-type': content_type}
self.app.register('GET', path, HTTPOk, body=resp_body, headers=hdrs)
return req.get_response(self.decrypter)
def test_GET_container_success(self):
# no format requested, listing has names only
fake_body = 'testfile1\ntestfile2\n'
calls = [0]
def wrapped_fetch_crypto_keys():
calls[0] += 1
return fetch_crypto_keys()
resp = self._make_cont_get_req(fake_body, None,
callback=wrapped_fetch_crypto_keys)
self.assertEqual('200 OK', resp.status)
names = resp.body.split('\n')
self.assertEqual(3, len(names))
self.assertIn('testfile1', names)
self.assertIn('testfile2', names)
self.assertIn('', names)
self.assertEqual(0, calls[0])
def test_GET_container_json(self):
content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
content_type_2 = 'text/plain; param=foo'
pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
key = fetch_crypto_keys()['container']
obj_dict_1 = {"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": encrypt_and_append_meta(
pt_etag1.encode('utf-8'), key),
"name": "testfile",
"content_type": content_type_1}
obj_dict_2 = {"bytes": 24,
"last_modified": "2015-04-14T23:33:06.519020",
"hash": encrypt_and_append_meta(
pt_etag2.encode('utf-8'), key),
"name": "testfile2",
"content_type": content_type_2}
listing = [obj_dict_1, obj_dict_2]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(2, len(body_json))
obj_dict_1['hash'] = pt_etag1
self.assertDictEqual(obj_dict_1, body_json[0])
obj_dict_2['hash'] = pt_etag2
self.assertDictEqual(obj_dict_2, body_json[1])
def test_GET_container_json_with_crypto_override(self):
content_type_1 = 'image/jpeg'
content_type_2 = 'text/plain; param=foo'
pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
obj_dict_1 = {"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": pt_etag1,
"name": "testfile",
"content_type": content_type_1}
obj_dict_2 = {"bytes": 24,
"last_modified": "2015-04-14T23:33:06.519020",
"hash": pt_etag2,
"name": "testfile2",
"content_type": content_type_2}
listing = [obj_dict_1, obj_dict_2]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json', override=True)
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(2, len(body_json))
self.assertDictEqual(obj_dict_1, body_json[0])
self.assertDictEqual(obj_dict_2, body_json[1])
def test_cont_get_json_req_with_cipher_mismatch(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
key = fetch_crypto_keys()['container']
pt_etag = 'c6e8196d7f0fff6444b90861fe8d609d'
ct_etag = encrypt_and_append_meta(pt_etag, key,
crypto_meta=bad_crypto_meta)
obj_dict_1 = {"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": ct_etag,
"name": "testfile",
"content_type": "image/jpeg"}
listing = [obj_dict_1]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting container listing', resp.body)
self.assertIn("Cipher must be AES_CTR_256",
self.decrypter.logger.get_lines_for_level('error')[0])
def _assert_element_contains_dict(self, expected, element):
for k, v in expected.items():
entry = element.getElementsByTagName(k)
self.assertIsNotNone(entry, 'Key %s not found' % k)
actual = entry[0].childNodes[0].nodeValue
self.assertEqual(v, actual,
"Expected %s but got %s for key %s"
% (v, actual, k))
def test_GET_container_xml(self):
content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
content_type_2 = 'text/plain; param=foo'
pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
key = fetch_crypto_keys()['container']
fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>\
''' + encrypt_and_append_meta(pt_etag1.encode('utf8'), key) + '''\
</hash><content_type>\
''' + content_type_1 + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
<object><hash>\
''' + encrypt_and_append_meta(pt_etag2.encode('utf8'), key) + '''\
</hash><content_type>\
''' + content_type_2 + '''\
</content_type><name>testfile2</name><bytes>24</bytes>\
<last_modified>2015-04-19T02:37:39.684740</last_modified></object>\
</container>'''
resp = self._make_cont_get_req(fake_body, 'xml')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
tree = minidom.parseString(body)
containers = tree.getElementsByTagName('container')
self.assertEqual(1, len(containers))
self.assertEqual('testc',
containers[0].attributes.getNamedItem("name").value)
objs = tree.getElementsByTagName('object')
self.assertEqual(2, len(objs))
obj_dict_1 = {"bytes": "16",
"last_modified": "2015-04-19T02:37:39.601660",
"hash": pt_etag1,
"name": "testfile",
"content_type": content_type_1}
self._assert_element_contains_dict(obj_dict_1, objs[0])
obj_dict_2 = {"bytes": "24",
"last_modified": "2015-04-19T02:37:39.684740",
"hash": pt_etag2,
"name": "testfile2",
"content_type": content_type_2}
self._assert_element_contains_dict(obj_dict_2, objs[1])
def test_GET_container_xml_with_crypto_override(self):
content_type_1 = 'image/jpeg'
content_type_2 = 'text/plain; param=foo'
fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>c6e8196d7f0fff6444b90861fe8d609d</hash>\
<content_type>''' + content_type_1 + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
<object><hash>ac0374ed4d43635f803c82469d0b5a10</hash>\
<content_type>''' + content_type_2 + '''\
</content_type><name>testfile2</name><bytes>24</bytes>\
<last_modified>2015-04-19T02:37:39.684740</last_modified></object>\
</container>'''
resp = self._make_cont_get_req(fake_body, 'xml', override=True)
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
tree = minidom.parseString(body)
containers = tree.getElementsByTagName('container')
self.assertEqual(1, len(containers))
self.assertEqual('testc',
containers[0].attributes.getNamedItem("name").value)
objs = tree.getElementsByTagName('object')
self.assertEqual(2, len(objs))
obj_dict_1 = {"bytes": "16",
"last_modified": "2015-04-19T02:37:39.601660",
"hash": "c6e8196d7f0fff6444b90861fe8d609d",
"name": "testfile",
"content_type": content_type_1}
self._assert_element_contains_dict(obj_dict_1, objs[0])
obj_dict_2 = {"bytes": "24",
"last_modified": "2015-04-19T02:37:39.684740",
"hash": "ac0374ed4d43635f803c82469d0b5a10",
"name": "testfile2",
"content_type": content_type_2}
self._assert_element_contains_dict(obj_dict_2, objs[1])
def test_cont_get_xml_req_with_cipher_mismatch(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc"><object>\
<hash>''' + encrypt_and_append_meta('c6e8196d7f0fff6444b90861fe8d609d',
fetch_crypto_keys()['container'],
crypto_meta=bad_crypto_meta) + '''\
</hash>\
<content_type>image/jpeg</content_type>\
<name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
</container>'''
resp = self._make_cont_get_req(fake_body, 'xml')
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting container listing', resp.body)
self.assertIn("Cipher must be AES_CTR_256",
self.decrypter.logger.get_lines_for_level('error')[0])
class TestKeymaster(unittest.TestCase):
def setUp(self):
super(TestKeymaster, self).setUp()
self.swift = FakeSwift()
self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)
def test_object_path(self):
self.verify_keys_for_path('/a/c/o',
expected_keys=('object', 'container'))
def test_container_path(self):
self.verify_keys_for_path('/a/c', expected_keys=('container', ))
def verify_keys_for_path(self, path, expected_keys):
put_keys = None
for method, resp_class, status in (('PUT', swob.HTTPCreated, '201'),
('POST', swob.HTTPAccepted, '202'),
('GET', swob.HTTPOk, '200'),
('HEAD', swob.HTTPNoContent,
'204')):
resp_headers = {}
self.swift.register(method, '/v1' + path, resp_class, resp_headers,
'')
req = Request.blank('/v1' + path,
environ={'REQUEST_METHOD': method})
start_response, calls = capture_start_response()
self.app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertTrue(calls[0][0].startswith(status))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)()
self.assertIn('id', keys)
id = keys.pop('id')
self.assertEqual(path, id['path'])
self.assertEqual('1', id['v'])
self.assertListEqual(
sorted(expected_keys), sorted(keys.keys()),
'%s %s got keys %r, but expected %r' %
(method, path, keys.keys(), expected_keys))
if put_keys is not None:
# check all key sets were consistent for this path
self.assertDictEqual(put_keys, keys)
else:
put_keys = keys
return put_keys
def test_key_uniqueness(self):
# a rudimentary check that different keys are made for different paths
ref_path_parts = ('a1', 'c1', 'o1')
path = '/' + '/'.join(ref_path_parts)
ref_keys = self.verify_keys_for_path(path,
expected_keys=('object',
'container'))
# for same path and for each differing path check that keys are unique
# when path to object or container is unique and vice-versa
for path_parts in [(a, c, o) for a in ('a1', 'a2')
for c in ('c1', 'c2') for o in ('o1', 'o2')]:
path = '/' + '/'.join(path_parts)
keys = self.verify_keys_for_path(path,
expected_keys=('object',
'container'))
# object keys should only be equal when complete paths are equal
self.assertEqual(
path_parts == ref_path_parts,
keys['object'] == ref_keys['object'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
# container keys should only be equal when paths to container are
# equal
self.assertEqual(
path_parts[:2] == ref_path_parts[:2],
keys['container'] == ref_keys['container'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
def test_filter(self):
factory = keymaster.filter_factory(TEST_KEYMASTER_CONF)
self.assertTrue(callable(factory))
self.assertTrue(callable(factory(self.swift)))
def test_app_exception(self):
app = keymaster.KeyMaster(FakeAppThatExcepts(), TEST_KEYMASTER_CONF)
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
start_response, _ = capture_start_response()
self.assertRaises(Exception, app, req.environ, start_response)
def test_missing_conf_section(self):
sample_conf = "[default]\nuser = %s\n" % getuser()
with tmpfile(sample_conf) as conf_file:
self.assertRaisesRegexp(
ValueError, 'Unable to find keymaster config section in.*',
keymaster.KeyMaster, self.swift,
{'keymaster_config_path': conf_file})
def test_root_secret(self):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
encoded_secret = base64.b64encode(secret)
for conf_val in (bytes(encoded_secret), unicode(encoded_secret),
encoded_secret[:30] + '\n' + encoded_secret[30:]):
try:
app = keymaster.KeyMaster(
self.swift, {
'encryption_root_secret': conf_val,
'encryption_root_secret_path': ''
})
self.assertEqual(secret, app.root_secret)
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % conf_val)
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_keymaster_config_path(self, mock_readconf):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
enc_secret = base64.b64encode(secret)
for conf_val in (bytes(enc_secret), unicode(enc_secret),
enc_secret[:30] + '\n' + enc_secret[30:],
enc_secret[:30] + '\r\n' + enc_secret[30:]):
for ignored_secret in ('invalid! but ignored!',
'xValidButIgnored' * 10):
mock_readconf.reset_mock()
mock_readconf.return_value = {
'encryption_root_secret': conf_val
}
app = keymaster.KeyMaster(
self.swift, {'keymaster_config_path': '/some/path'})
try:
self.assertEqual(secret, app.root_secret)
self.assertEqual(
mock_readconf.mock_calls,
[mock.call('/some/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_invalid_root_secret(self):
for secret in (
bytes(base64.b64encode(os.urandom(31))), # too short
unicode(base64.b64encode(os.urandom(31))),
u'a' * 44 + u'????',
b'a' * 44 + b'????', # not base64
u'a' * 45,
b'a' * 45, # bad padding
99,
None):
conf = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'encryption_root_secret option in proxy-server.conf '
'must be a base64 encoding of at least 32 raw bytes',
err.exception.message)
except AssertionError as err:
self.fail(str(err) + ' for conf %s' % str(conf))
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_root_secret_path_invalid_secret(self, mock_readconf):
for secret in (
bytes(base64.b64encode(os.urandom(31))), # too short
unicode(base64.b64encode(os.urandom(31))),
u'a' * 44 + u'????',
b'a' * 44 + b'????', # not base64
u'a' * 45,
b'a' * 45, # bad padding
99,
None):
mock_readconf.reset_mock()
mock_readconf.return_value = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(
self.swift,
{'keymaster_config_path': '/some/other/path'})
self.assertEqual(
'encryption_root_secret option in /some/other/path '
'must be a base64 encoding of at least 32 raw bytes',
err.exception.message)
self.assertEqual(mock_readconf.mock_calls,
[mock.call('/some/other/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_can_only_configure_secret_in_one_place(self):
conf = {
'encryption_root_secret': 'a' * 44,
'keymaster_config_path': '/ets/swift/keymaster.conf'
}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
expected_message = ('keymaster_config_path is set, but there are '
'other config options specified:')
self.assertTrue(
err.exception.message.startswith(expected_message),
"Error message does not start with '%s'" % expected_message)
class TestListingFormats(unittest.TestCase):
def setUp(self):
self.fake_swift = FakeSwift()
self.logger = debug_logger('test-listing')
self.app = listing_formats.ListingFilter(self.fake_swift, {},
logger=self.logger)
self.fake_account_listing = json.dumps([
{
'name': 'bar',
'bytes': 0,
'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo_'
},
]).encode('ascii')
self.fake_container_listing = json.dumps([
{
'name': 'bar',
'hash': 'etag',
'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo/'
},
]).encode('ascii')
self.fake_account_listing_with_reserved = json.dumps([
{
'name': 'bar',
'bytes': 0,
'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'name': get_reserved_name('bar', 'versions'),
'bytes': 0,
'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo_'
},
{
'subdir': get_reserved_name('foo_')
},
]).encode('ascii')
self.fake_container_listing_with_reserved = json.dumps([
{
'name': 'bar',
'hash': 'etag',
'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'name': get_reserved_name('bar', 'extra data'),
'hash': 'etag',
'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo/'
},
{
'subdir': get_reserved_name('foo/')
},
]).encode('ascii')
def test_valid_account(self):
self.fake_swift.register(
'GET', '/v1/a', HTTPOk, {
'Content-Length': str(len(self.fake_account_listing)),
'Content-Type': 'application/json'
}, self.fake_account_listing)
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=json')
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_account_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<subdir name="foo_" />',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
def test_valid_content_type_on_txt_head(self):
self.fake_swift.register(
'HEAD', '/v1/a', HTTPNoContent, {
'Content-Length': str(len(self.fake_account_listing)),
'Content-Type': 'application/json'
}, self.fake_account_listing)
req = Request.blank('/v1/a', method='HEAD')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertIn('Vary', resp.headers)
# Even though the client didn't send an Accept header, the response
# could change *if a subsequent request does*, so include Vary: Accept
self.assertEqual(resp.headers['Vary'], 'Accept')
self.assertEqual(self.fake_swift.calls[-1],
('HEAD', '/v1/a?format=json'))
def test_valid_content_type_on_xml_head(self):
self.fake_swift.register(
'HEAD', '/v1/a', HTTPNoContent, {
'Content-Length': str(len(self.fake_account_listing)),
'Content-Type': 'application/json'
}, self.fake_account_listing)
req = Request.blank('/v1/a?format=xml', method='HEAD')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
# query param overrides header, so it won't vary
self.assertNotIn('Vary', resp.headers)
self.assertEqual(self.fake_swift.calls[-1],
('HEAD', '/v1/a?format=json'))
def test_update_vary_if_present(self):
self.fake_swift.register(
'HEAD', '/v1/a', HTTPNoContent, {
'Content-Length': str(len(self.fake_account_listing)),
'Content-Type': 'application/json',
'Vary': 'Origin'
}, self.fake_account_listing)
req = Request.blank('/v1/a', method='HEAD')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(resp.headers['Vary'], 'Origin, Accept')
self.assertEqual(self.fake_swift.calls[-1],
('HEAD', '/v1/a?format=json'))
def test_update_vary_does_not_duplicate(self):
self.fake_swift.register(
'HEAD', '/v1/a', HTTPNoContent, {
'Content-Length': str(len(self.fake_account_listing)),
'Content-Type': 'application/json',
'Vary': 'Accept'
}, self.fake_account_listing)
req = Request.blank('/v1/a', method='HEAD')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(resp.headers['Vary'], 'Accept')
self.assertEqual(self.fake_swift.calls[-1],
('HEAD', '/v1/a?format=json'))
def test_valid_account_with_reserved(self):
body_len = len(self.fake_account_listing_with_reserved)
self.fake_swift.register('GET', '/v1/a\xe2\x98\x83', HTTPOk, {
'Content-Length': str(body_len),
'Content-Type': 'application/json',
}, self.fake_account_listing_with_reserved)
req = Request.blank('/v1/a\xe2\x98\x83')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
self.assertEqual(self.logger.get_lines_for_level('warning'), [
"Account listing for a%E2%98%83 had reserved byte in name: "
"'\\x00bar\\x00versions'",
"Account listing for a%E2%98%83 had reserved byte in subdir: "
"'\\x00foo_'",
])
req = Request.blank('/v1/a\xe2\x98\x83',
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'bar\n%s\nfoo_\n%s\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=txt',
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'bar\n%s\nfoo_\n%s\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=json')
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_account_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=json',
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_account_listing_with_reserved))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a\xe2\x98\x83">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<subdir name="foo_" />',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=xml',
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a\xe2\x98\x83">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<container><name>%s</name>'
b'<count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>' %
get_reserved_name('bar', 'versions').encode('ascii'),
b'<subdir name="foo_" />',
b'<subdir name="%s" />' %
get_reserved_name('foo_').encode('ascii'),
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a\xe2\x98\x83?format=json'))
def test_valid_container(self):
self.fake_swift.register(
'GET', '/v1/a/c', HTTPOk, {
'Content-Length': str(len(self.fake_container_listing)),
'Content-Type': 'application/json'
}, self.fake_container_listing)
req = Request.blank('/v1/a/c')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=json')
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_container_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=xml')
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<container name="c">'
b'<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<subdir name="foo/"><name>foo/</name></subdir>'
b'</container>')
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
def test_valid_container_with_reserved(self):
path = '/v1/a\xe2\x98\x83/c\xf0\x9f\x8c\xb4'
body_len = len(self.fake_container_listing_with_reserved)
self.fake_swift.register('GET', path, HTTPOk, {
'Content-Length': str(body_len),
'Content-Type': 'application/json',
}, self.fake_container_listing_with_reserved)
req = Request.blank(path)
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
self.assertEqual(self.logger.get_lines_for_level('warning'), [
"Container listing for a%E2%98%83/c%F0%9F%8C%B4 had reserved byte "
"in name: '\\x00bar\\x00extra data'",
"Container listing for a%E2%98%83/c%F0%9F%8C%B4 had reserved byte "
"in subdir: '\\x00foo/'",
])
req = Request.blank(path,
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'bar\n%s\nfoo/\n%s\n' % (
get_reserved_name('bar', 'extra data').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
req = Request.blank(path + '?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
req = Request.blank(path + '?format=txt',
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'bar\n%s\nfoo/\n%s\n' % (
get_reserved_name('bar', 'extra data').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
req = Request.blank(path + '?format=json')
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_container_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
req = Request.blank(path + '?format=json',
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_container_listing_with_reserved))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
req = Request.blank(path + '?format=xml')
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<container name="c\xf0\x9f\x8c\xb4">'
b'<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<subdir name="foo/"><name>foo/</name></subdir>'
b'</container>')
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
req = Request.blank(path + '?format=xml',
headers={'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(
resp.body, b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<container name="c\xf0\x9f\x8c\xb4">'
b'<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<object><name>%s</name>'
b'<hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<subdir name="foo/"><name>foo/</name></subdir>'
b'<subdir name="%s"><name>%s</name></subdir>'
b'</container>' % (
get_reserved_name('bar', 'extra data').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', path + '?format=json'))
def test_blank_account(self):
self.fake_swift.register('GET', '/v1/a', HTTPOk, {
'Content-Length': '2',
'Content-Type': 'application/json'
}, b'[]')
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, b'[]')
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a">',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a?format=json'))
def test_blank_container(self):
self.fake_swift.register('GET', '/v1/a/c', HTTPOk, {
'Content-Length': '2',
'Content-Type': 'application/json'
}, b'[]')
req = Request.blank('/v1/a/c')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, b'')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, b'[]')
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<container name="c" />',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/a/c?format=json'))
def test_pass_through(self):
def do_test(path):
self.fake_swift.register(
'GET', path, HTTPOk, {
'Content-Length': str(len(self.fake_container_listing)),
'Content-Type': 'application/json'
}, self.fake_container_listing)
req = Request.blank(path + '?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_container_listing)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(
self.fake_swift.calls[-1],
('GET', path + '?format=xml')) # query param is unchanged
do_test('/')
do_test('/v1')
do_test('/auth/v1.0')
do_test('/v1/a/c/o')
def test_static_web_not_json(self):
body = b'doesnt matter'
self.fake_swift.register('GET', '/v1/staticweb/not-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'text/plain'
}, body)
resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
self.assertEqual(resp.body, body)
self.assertEqual(resp.headers['Content-Type'], 'text/plain')
# We *did* try, though
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/not-json?format=json'))
# TODO: add a similar test that has *no* content-type
# FakeSwift seems to make this hard to do
def test_static_web_not_really_json(self):
body = b'raises ValueError'
self.fake_swift.register('GET', '/v1/staticweb/not-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'application/json'
}, body)
resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
self.assertEqual(resp.body, body)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/not-json?format=json'))
def test_static_web_pretend_to_be_giant_json(self):
body = json.dumps([
{
'name': 'bar',
'hash': 'etag',
'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'
},
{
'subdir': 'foo/'
},
] * 160000).encode('ascii')
self.assertGreater( # sanity
len(body), listing_formats.MAX_CONTAINER_LISTING_CONTENT_LENGTH)
self.fake_swift.register('GET', '/v1/staticweb/long-json', HTTPOk,
{'Content-Type': 'application/json'}, body)
resp = Request.blank('/v1/staticweb/long-json').get_response(self.app)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(resp.body, body)
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/long-json?format=json'))
# TODO: add a similar test for chunked transfers
# (staticweb referencing a DLO that doesn't fit in a single listing?)
def test_static_web_bad_json(self):
def do_test(body_obj):
body = json.dumps(body_obj).encode('ascii')
self.fake_swift.register(
'GET', '/v1/staticweb/bad-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'application/json'
}, body)
def do_sub_test(path):
resp = Request.blank(path).get_response(self.app)
self.assertEqual(resp.body, body)
# NB: no charset is added; we pass through whatever we got
self.assertEqual(resp.headers['Content-Type'],
'application/json')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/bad-json?format=json'))
do_sub_test('/v1/staticweb/bad-json')
do_sub_test('/v1/staticweb/bad-json?format=txt')
do_sub_test('/v1/staticweb/bad-json?format=xml')
do_sub_test('/v1/staticweb/bad-json?format=json')
do_test({})
do_test({'non-empty': 'hash'})
do_test(None)
do_test(0)
do_test('some string')
do_test([None])
do_test([0])
do_test(['some string'])
def test_static_web_bad_but_not_terrible_json(self):
body = json.dumps([{'no name': 'nor subdir'}]).encode('ascii')
self.fake_swift.register('GET', '/v1/staticweb/bad-json', HTTPOk, {
'Content-Length': str(len(body)),
'Content-Type': 'application/json'
}, body)
def do_test(path, expect_charset=False):
resp = Request.blank(path).get_response(self.app)
self.assertEqual(resp.body, body)
if expect_charset:
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
else:
self.assertEqual(resp.headers['Content-Type'],
'application/json')
self.assertEqual(self.fake_swift.calls[-1],
('GET', '/v1/staticweb/bad-json?format=json'))
do_test('/v1/staticweb/bad-json')
do_test('/v1/staticweb/bad-json?format=txt')
do_test('/v1/staticweb/bad-json?format=xml')
# The response we get is *just close enough* to being valid that we
# assume it is and slap on the missing charset. If you set up staticweb
# to serve back such responses, your clients are already hosed.
do_test('/v1/staticweb/bad-json?format=json', expect_charset=True)
class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase):
# verify interaction of copy and versioned_writes middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'allow_versioned_writes': 'true'}
self.vw = versioned_writes.filter_factory(conf)(self.app)
self.filter = copy.filter_factory({})(self.vw)
def call_filter(self, req, **kwargs):
return self.call_app(req, app=self.filter, **kwargs)
def test_copy_first_version(self):
# no existing object to move to the versions container
self.app.register('GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound,
{}, None)
self.app.register('GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {},
'passed')
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
# note the GET on tgt_cont/tgt_obj is pre-authed
self.assertEqual(3, self.app.call_count, self.app.calls)
def test_copy_new_version(self):
# existing object should be moved to versions container
self.app.register('GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {},
'passed')
self.app.register('GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'},
'passed')
self.app.register('PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000',
swob.HTTPOk, {}, None)
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(4, self.app.call_count)
def test_copy_new_version_different_account(self):
self.app.register('GET', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {},
'passed')
self.app.register('GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'},
'passed')
self.app.register('PUT',
'/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000',
swob.HTTPOk, {}, None)
self.app.register('PUT', '/v1/tgt_a/tgt_cont/tgt_obj',
swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/src_a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'CONTENT_LENGTH': '100'
},
headers={
'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': 'tgt_a'
})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/src_a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/tgt_a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(4, self.app.call_count)
def test_copy_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no extra
# COPY called (just copy object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register('GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {},
'passed')
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{}, 'passed')
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank('/v1/a/src_cont/src_obj',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache
},
headers={'Destination': '/tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(2, self.app.call_count)
class ContainerQuotaCopyingTestCases(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.cq_filter = container_quotas.filter_factory({})(self.app)
self.copy_filter = copy.filter_factory({})(self.cq_filter)
def test_exceed_bytes_quota_copy_verb(self):
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_exceed_counts_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
'status': 200, 'object_count': 1}
a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.infocache': {
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache}},
headers={'Destination': '/c/o',
'Destination-Account': 'a2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_PUT_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
'status': 200, 'object_count': 1}
a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a2/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.infocache': {
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache}},
headers={'X-Copy-From': '/c2/o2',
'X-Copy-From-Account': 'a'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_bytes_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_no_src(self):
self.app.register('GET', '/v1/a/c2/o3', HTTPOk, {}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o3'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_bad_src(self):
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': 'bad_path'})
with self.assertRaises(HTTPException) as catcher:
req.get_response(self.copy_filter)
self.assertEqual(412, catcher.exception.status_int)
def test_exceed_counts_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_counts_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_not_exceed_counts_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
class OioRegexContainer(unittest.TestCase):
def setUp(self):
conf = {'sds_default_account': 'OPENIO'}
self.filter_conf = {
'strip_v1': 'true',
'swift3_compat': 'true',
'account_first': 'true',
'stop_at_first_match': 'true',
'pattern1': r'(\d{3})/(\d{3})/(\d)\d\d/\d\d(\d)/',
'pattern2': r'(\d{3})/(\d)\d\d/\d\d(\d)/',
'pattern3': r'^(cloud)/([0-9a-f][0-9a-f])',
'pattern4': r'^(cloud)/([0-9a-f])',
'pattern9': r'^/?([^/]+)',
}
if hasattr(ContainerBuilder, 'alternatives'):
self.filter_conf['stop_at_first_match'] = 'false'
self.app = FakeSwift()
self.ch = regexcontainer.filter_factory(
conf,
**self.filter_conf)(self.app)
def tearDown(self):
pass
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
def call_rc(self, req):
return self.call_app(req, app=self.ch)
def _check_pattern(self, path_in, path_out):
self.app.register('PUT', path_out, swob.HTTPCreated, {})
req = Request.blank(path_in, method='PUT')
resp = self.call_rc(req)
self.assertEqual(resp[0], "201 Created")
self.assertEqual(self.app.calls, [('PUT', path_out)])
def test_pattern1(self):
self._check_pattern(
'/v1/a/c/111/222/456/789/o',
'/v1/a/11122249/111/222/456/789/o')
def test_pattern2(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET', '/v1/a/11122249/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/11126/111/222/456/789/o',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], "200 OK")
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/11122249/111/222/456/789/o'),
('GET', '/v1/a/11126/111/222/456/789/o')])
def test_pattern3(self):
self._check_pattern('/v1/a/c/cloud/ff_object',
'/v1/a/cloudff/cloud/ff_object')
def test_pattern4(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET', '/v1/a/cloudff/cloud/ff_object',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/cloudf/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c/cloud/ff_object', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], "200 OK")
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/cloudff/cloud/ff_object'),
('GET', '/v1/a/cloudf/cloud/ff_object')])
def test_pattern9(self):
self._check_pattern('/v1/a/c/gc_regex/path/ob',
'/v1/a/gc_regex/gc_regex/path/ob')
def test_get_without_matching_pattern(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET', '/v1/a/11122249/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/11126/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/111/111/222/456/789/o',
swob.HTTPNotFound, {})
req = Request.blank('/v1/a/c/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], "404 Not Found")
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/11122249/111/222/456/789/o'),
('GET', '/v1/a/11126/111/222/456/789/o'),
('GET', '/v1/a/111/111/222/456/789/o')])
def test_simple_listing(self):
self._check_pattern('/v1/a/c/111/222/456/789/o',
'/v1/a/11122249/111/222/456/789/o')
self.app.register('GET',
'/v1/a/11122249?prefix=/111/222/456/789/o',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c?prefix=/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
def test_fallback_listing(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET',
'/v1/a/11122249?prefix=/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET',
'/v1/a/11126?prefix=/111/222/456/789/o',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c?prefix=/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/11122249?prefix=/111/222/456/789/o'),
('GET', '/v1/a/11126?prefix=/111/222/456/789/o')])
def test_swift3_mpu(self):
self.app.register('PUT',
'/v1/a/cloudff+segments/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank(
'/v1/a/c+segments/cloud/ff_object', method='PUT')
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
def test_copy(self):
self.app.register('PUT',
'/v1/a/cloudff/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank(
'/v1/a/c/cloud/ff_object', method='PUT',
headers={'X-Copy-From': 'container/path/dir1/object'})
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
self.assertEqual(self.app.headers[0]['X-Copy-From'],
"/dir1/dir1/object")
def test_fastcopy(self):
self.app.register('PUT',
'/v1/a/cloudff/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c/cloud/ff_object', method='PUT',
headers={'Oio-Copy-From': 'container/path/object'})
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
self.assertEqual(self.app.headers[0]['Oio-Copy-From'],
"/object/object")
class TestUntarMetadata(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.bulk = bulk.filter_factory({})(self.app)
self.testdir = mkdtemp(suffix='tmp_test_bulk')
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
def test_extract_metadata(self):
self.app.register('HEAD', '/v1/a/c?extract-archive=tar',
HTTPNoContent, {}, None)
self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar',
HTTPCreated, {}, None)
self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar',
HTTPCreated, {}, None)
# It's a real pain to instantiate TarInfo objects directly; they
# really want to come from a file on disk or a tarball. So, we write
# out some files and add pax headers to them as they get placed into
# the tarball.
with open(os.path.join(self.testdir, "obj1"), "w") as fh1:
fh1.write("obj1 contents\n")
with open(os.path.join(self.testdir, "obj2"), "w") as fh2:
fh2.write("obj2 contents\n")
tar_ball = StringIO()
tar_file = tarfile.TarFile.open(fileobj=tar_ball, mode="w",
format=tarfile.PAX_FORMAT)
# With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with
# extended attribute user.thingy = dingy gets put into the tarfile
# with pax_headers containing key/value pair
# (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type
# unicode, not type str).
#
# With BSD tar (libarchive), you get key/value pair
# (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as
# gratuitous incompatibility.
#
# Still, we'll support uploads with both. Just heap more code on the
# problem until you can forget it's under there.
with open(os.path.join(self.testdir, "obj1")) as fh1:
tar_info1 = tar_file.gettarinfo(fileobj=fh1,
arcname="obj1")
tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \
u'application/food-diary'
tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \
u'sopa de albóndigas'
tar_info1.pax_headers[
u'SCHILY.xattr.user.meta.afternoon-snack'] = \
u'gigantic bucket of coffee'
tar_file.addfile(tar_info1, fh1)
with open(os.path.join(self.testdir, "obj2")) as fh2:
tar_info2 = tar_file.gettarinfo(fileobj=fh2,
arcname="obj2")
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped'
tar_file.addfile(tar_info2, fh2)
tar_ball.seek(0)
req = Request.blank('/v1/a/c?extract-archive=tar')
req.environ['REQUEST_METHOD'] = 'PUT'
req.environ['wsgi.input'] = tar_ball
req.headers['transfer-encoding'] = 'chunked'
req.headers['accept'] = 'application/json;q=1.0'
resp = req.get_response(self.bulk)
self.assertEqual(resp.status_int, 200)
# sanity check to make sure the upload worked
upload_status = utils.json.loads(resp.body)
self.assertEqual(upload_status['Number Files Created'], 2)
put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2])
self.assertEqual(
put1_headers.get('Content-Type'),
'application/food-diary')
self.assertEqual(
put1_headers.get('X-Object-Meta-Lunch'),
'sopa de alb\xc3\xb3ndigas')
self.assertEqual(
put1_headers.get('X-Object-Meta-Afternoon-Snack'),
'gigantic bucket of coffee')
put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
self.assertEqual(put2_headers.get('Content-Type'), None)
self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)
class TestKeymaster(unittest.TestCase):
def setUp(self):
super(TestKeymaster, self).setUp()
self.swift = FakeSwift()
self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)
def test_object_path(self):
self.verify_keys_for_path('/a/c/o',
expected_keys=('object', 'container'))
self.verify_keys_for_path('/a/c//o',
expected_keys=('object', 'container'))
def test_container_path(self):
self.verify_keys_for_path('/a/c', expected_keys=('container', ))
def verify_keys_for_path(self, path, expected_keys, key_id=None):
put_keys = None
for method, resp_class, status in (('PUT', swob.HTTPCreated, '201'),
('POST', swob.HTTPAccepted, '202'),
('GET', swob.HTTPOk, '200'),
('HEAD', swob.HTTPNoContent,
'204')):
resp_headers = {}
self.swift.register(method, '/v1' + path, resp_class, resp_headers,
b'')
req = Request.blank('/v1' + path,
environ={'REQUEST_METHOD': method})
start_response, calls = capture_start_response()
self.app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertTrue(calls[0][0].startswith(status))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)(key_id=key_id)
self.assertIn('id', keys)
id = keys.pop('id')
self.assertEqual(path, id['path'])
self.assertEqual('2', id['v'])
keys.pop('all_ids')
self.assertListEqual(
sorted(expected_keys), sorted(keys.keys()),
'%s %s got keys %r, but expected %r' %
(method, path, keys.keys(), expected_keys))
if put_keys is not None:
# check all key sets were consistent for this path
self.assertDictEqual(put_keys, keys)
else:
put_keys = keys
return put_keys
def test_key_uniqueness(self):
# a rudimentary check that different keys are made for different paths
ref_path_parts = ('a1', 'c1', 'o1')
path = '/' + '/'.join(ref_path_parts)
ref_keys = self.verify_keys_for_path(path,
expected_keys=('object',
'container'))
# for same path and for each differing path check that keys are unique
# when path to object or container is unique and vice-versa
for path_parts in [(a, c, o) for a in ('a1', 'a2')
for c in ('c1', 'c2') for o in ('o1', 'o2')]:
path = '/' + '/'.join(path_parts)
keys = self.verify_keys_for_path(path,
expected_keys=('object',
'container'))
# object keys should only be equal when complete paths are equal
self.assertEqual(
path_parts == ref_path_parts,
keys['object'] == ref_keys['object'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
# container keys should only be equal when paths to container are
# equal
self.assertEqual(
path_parts[:2] == ref_path_parts[:2],
keys['container'] == ref_keys['container'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
def test_filter(self):
factory = keymaster.filter_factory(TEST_KEYMASTER_CONF)
self.assertTrue(callable(factory))
self.assertTrue(callable(factory(self.swift)))
def test_app_exception(self):
app = keymaster.KeyMaster(FakeAppThatExcepts(), TEST_KEYMASTER_CONF)
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
start_response, _ = capture_start_response()
self.assertRaises(Exception, app, req.environ, start_response)
def test_missing_conf_section(self):
sample_conf = "[default]\nuser = %s\n" % getuser()
with tmpfile(sample_conf) as conf_file:
self.assertRaisesRegexp(
ValueError, 'Unable to find keymaster config section in.*',
keymaster.KeyMaster, self.swift,
{'keymaster_config_path': conf_file})
def test_root_secret(self):
def do_test(dflt_id):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
encoded_secret = base64.b64encode(secret)
self.assertIsInstance(encoded_secret, bytes)
for conf_val in (encoded_secret,
encoded_secret.decode('ascii'),
encoded_secret[:30] + b'\n' +
encoded_secret[30:],
(encoded_secret[:30] + b'\n' +
encoded_secret[30:]).decode('ascii')):
try:
app = keymaster.KeyMaster(
self.swift, {
'encryption_root_secret': conf_val,
'active_root_secret_id': dflt_id,
'keymaster_config_path': ''
})
self.assertEqual(secret, app.root_secret)
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % conf_val)
do_test(None)
do_test('')
def test_no_root_secret(self):
with self.assertRaises(ValueError) as cm:
keymaster.KeyMaster(self.swift, {})
self.assertEqual('No secret loaded for active_root_secret_id None',
str(cm.exception))
def test_multiple_root_secrets(self):
secrets = {
None: os.urandom(32),
'22': os.urandom(33),
'my_secret_id': os.urandom(50)
}
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
app = keymaster.KeyMaster(self.swift, conf)
self.assertEqual(secrets, app._root_secrets)
self.assertEqual([None, '22', 'my_secret_id'], app.root_secret_ids)
def test_chained_keymasters(self):
conf_inner = {'active_root_secret_id': '22'}
conf_inner.update(
('encryption_root_secret_%s' % secret_id, base64.b64encode(secret))
for secret_id, secret in [(
'22', os.urandom(33)), ('my_secret_id', os.urandom(50))])
conf_outer = {
'encryption_root_secret': base64.b64encode(os.urandom(32))
}
app = keymaster.KeyMaster(keymaster.KeyMaster(self.swift, conf_inner),
conf_outer)
self.swift.register('GET', '/v1/a/c', swob.HTTPOk, {}, b'')
req = Request.blank('/v1/a/c')
start_response, calls = capture_start_response()
app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = copy.deepcopy(req.environ[CRYPTO_KEY_CALLBACK](key_id=None))
self.assertIn('id', keys)
self.assertEqual(keys.pop('id'), {
'v': '2',
'path': '/a/c',
'secret_id': '22',
})
# Inner-most active root secret wins
root_key = base64.b64decode(conf_inner['encryption_root_secret_22'])
self.assertIn('container', keys)
self.assertEqual(
keys.pop('container'),
hmac.new(root_key, b'/a/c', digestmod=hashlib.sha256).digest())
self.assertIn('all_ids', keys)
all_keys = set()
at_least_one_old_style_id = False
for key_id in keys.pop('all_ids'):
# Can get key material for each key_id
all_keys.add(
req.environ[CRYPTO_KEY_CALLBACK](key_id=key_id)['container'])
if 'secret_id' in key_id:
self.assertIn(key_id.pop('secret_id'), {'22', 'my_secret_id'})
else:
at_least_one_old_style_id = True
self.assertEqual(key_id, {
'path': '/a/c',
'v': '2',
})
self.assertTrue(at_least_one_old_style_id)
self.assertEqual(len(all_keys), 3)
self.assertFalse(keys)
# Also all works for objects
self.swift.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, b'')
req = Request.blank('/v1/a/c/o')
start_response, calls = capture_start_response()
app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)(key_id=None)
self.assertIn('id', keys)
self.assertEqual(keys.pop('id'), {
'v': '2',
'path': '/a/c/o',
'secret_id': '22',
})
root_key = base64.b64decode(conf_inner['encryption_root_secret_22'])
self.assertIn('container', keys)
self.assertEqual(
keys.pop('container'),
hmac.new(root_key, b'/a/c', digestmod=hashlib.sha256).digest())
self.assertIn('object', keys)
self.assertEqual(
keys.pop('object'),
hmac.new(root_key, b'/a/c/o', digestmod=hashlib.sha256).digest())
self.assertIn('all_ids', keys)
at_least_one_old_style_id = False
for key_id in keys.pop('all_ids'):
if 'secret_id' not in key_id:
at_least_one_old_style_id = True
else:
self.assertIn(key_id.pop('secret_id'), {'22', 'my_secret_id'})
self.assertEqual(key_id, {
'path': '/a/c/o',
'v': '2',
})
self.assertTrue(at_least_one_old_style_id)
self.assertEqual(len(all_keys), 3)
self.assertFalse(keys)
def test_multiple_root_secrets_with_invalid_secret(self):
conf = {
'encryption_root_secret': base64.b64encode(os.urandom(32)),
# too short...
'encryption_root_secret_22': base64.b64encode(os.urandom(31))
}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'encryption_root_secret_22 option in proxy-server.conf '
'must be a base64 encoding of at least 32 raw bytes',
str(err.exception))
def test_multiple_root_secrets_with_invalid_id(self):
def do_test(bad_option):
conf = {
'encryption_root_secret': base64.b64encode(os.urandom(32)),
bad_option: base64.b64encode(os.urandom(32))
}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'Malformed root secret option name %s' % bad_option,
str(err.exception))
do_test('encryption_root_secret1')
do_test('encryption_root_secret123')
do_test('encryption_root_secret_')
def test_multiple_root_secrets_missing_active_root_secret_id(self):
conf = {'encryption_root_secret_22': base64.b64encode(os.urandom(32))}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual('No secret loaded for active_root_secret_id None',
str(err.exception))
conf = {
'encryption_root_secret_22': base64.b64encode(os.urandom(32)),
'active_root_secret_id': 'missing'
}
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual('No secret loaded for active_root_secret_id missing',
str(err.exception))
def test_correct_root_secret_used(self):
secrets = {
None: os.urandom(32),
'22': os.urandom(33),
'my_secret_id': os.urandom(50)
}
# no active_root_secret_id configured
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
self.app = keymaster.KeyMaster(self.swift, conf)
keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'))
expected_keys = {
'container':
hmac.new(secrets[None], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets[None], b'/a/c/o',
digestmod=hashlib.sha256).digest()
}
self.assertEqual(expected_keys, keys)
# active_root_secret_id configured
conf['active_root_secret_id'] = '22'
self.app = keymaster.KeyMaster(self.swift, conf)
keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'))
expected_keys = {
'container':
hmac.new(secrets['22'], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets['22'], b'/a/c/o',
digestmod=hashlib.sha256).digest()
}
self.assertEqual(expected_keys, keys)
# secret_id passed to fetch_crypto_keys callback
for secret_id in ('my_secret_id', None):
keys = self.verify_keys_for_path('/a/c/o', ('container', 'object'),
key_id={
'secret_id': secret_id,
'v': '2',
'path': '/a/c/o'
})
expected_keys = {
'container':
hmac.new(secrets[secret_id], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets[secret_id],
b'/a/c/o',
digestmod=hashlib.sha256).digest()
}
self.assertEqual(expected_keys, keys)
def test_keys_cached(self):
secrets = {
None: os.urandom(32),
'22': os.urandom(33),
'my_secret_id': os.urandom(50)
}
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
conf['active_root_secret_id'] = '22'
self.app = keymaster.KeyMaster(self.swift, conf)
orig_create_key = self.app.create_key
calls = []
def mock_create_key(path, secret_id=None):
calls.append((path, secret_id))
return orig_create_key(path, secret_id)
context = keymaster.KeyMasterContext(self.app, 'a', 'c', 'o')
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys()
expected_keys = {
'container':
hmac.new(secrets['22'], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets['22'], b'/a/c/o',
digestmod=hashlib.sha256).digest(),
'id': {
'path': '/a/c/o',
'secret_id': '22',
'v': '2'
},
'all_ids': [{
'path': '/a/c/o',
'v': '2'
}, {
'path': '/a/c/o',
'secret_id': '22',
'v': '2'
}, {
'path': '/a/c/o',
'secret_id': 'my_secret_id',
'v': '2'
}]
}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', '22'), ('/a/c/o', '22')], calls)
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys()
# no more calls to create_key
self.assertEqual([('/a/c', '22'), ('/a/c/o', '22')], calls)
self.assertEqual(expected_keys, keys)
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={
'secret_id': None,
'v': '2',
'path': '/a/c/o'
})
expected_keys = {
'container':
hmac.new(secrets[None], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets[None], b'/a/c/o',
digestmod=hashlib.sha256).digest(),
'id': {
'path': '/a/c/o',
'v': '2'
},
'all_ids': [{
'path': '/a/c/o',
'v': '2'
}, {
'path': '/a/c/o',
'secret_id': '22',
'v': '2'
}, {
'path': '/a/c/o',
'secret_id': 'my_secret_id',
'v': '2'
}]
}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', '22'), ('/a/c/o', '22'), ('/a/c', None),
('/a/c/o', None)], calls)
def test_v1_keys(self):
secrets = {None: os.urandom(32), '22': os.urandom(33)}
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
conf['active_root_secret_id'] = '22'
self.app = keymaster.KeyMaster(self.swift, conf)
orig_create_key = self.app.create_key
calls = []
def mock_create_key(path, secret_id=None):
calls.append((path, secret_id))
return orig_create_key(path, secret_id)
context = keymaster.KeyMasterContext(self.app, 'a', 'c', 'o')
for version in ('1', '2'):
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={
'v': version,
'path': '/a/c/o'
})
expected_keys = {
'container':
hmac.new(secrets[None], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets[None], b'/a/c/o',
digestmod=hashlib.sha256).digest(),
'id': {
'path': '/a/c/o',
'v': version
},
'all_ids': [{
'path': '/a/c/o',
'v': version
}, {
'path': '/a/c/o',
'secret_id': '22',
'v': version
}]
}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', None), ('/a/c/o', None)], calls)
del calls[:]
context = keymaster.KeyMasterContext(self.app, 'a', 'c', '/o')
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={'v': '1', 'path': '/o'})
expected_keys = {
'container':
hmac.new(secrets[None], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets[None], b'/o', digestmod=hashlib.sha256).digest(),
'id': {
'path': '/o',
'v': '1'
},
'all_ids': [{
'path': '/o',
'v': '1'
}, {
'path': '/o',
'secret_id': '22',
'v': '1'
}]
}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', None), ('/o', None)], calls)
del calls[:]
context = keymaster.KeyMasterContext(self.app, 'a', 'c', '/o')
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={
'v': '2',
'path': '/a/c//o'
})
expected_keys = {
'container':
hmac.new(secrets[None], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object':
hmac.new(secrets[None], b'/a/c//o',
digestmod=hashlib.sha256).digest(),
'id': {
'path': '/a/c//o',
'v': '2'
},
'all_ids': [{
'path': '/a/c//o',
'v': '2'
}, {
'path': '/a/c//o',
'secret_id': '22',
'v': '2'
}]
}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', None), ('/a/c//o', None)], calls)
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_keymaster_config_path(self, mock_readconf):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
enc_secret = base64.b64encode(secret)
self.assertIsInstance(enc_secret, bytes)
for conf_val in (enc_secret, enc_secret.decode('ascii'),
enc_secret[:30] + b'\n' + enc_secret[30:],
enc_secret[:30] + b'\r\n' + enc_secret[30:],
(enc_secret[:30] + b'\n' +
enc_secret[30:]).decode('ascii'),
(enc_secret[:30] + b'\r\n' +
enc_secret[30:]).decode('ascii')):
mock_readconf.reset_mock()
mock_readconf.return_value = {
'encryption_root_secret': conf_val
}
app = keymaster.KeyMaster(
self.swift, {'keymaster_config_path': '/some/path'})
try:
self.assertEqual(secret, app.root_secret)
self.assertEqual(mock_readconf.mock_calls,
[mock.call('/some/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_invalid_root_secret(self):
for secret in (
base64.b64encode(os.urandom(31)), # too short
base64.b64encode(os.urandom(31)).decode('ascii'),
u'a' * 44 + u'????',
b'a' * 44 + b'????', # not base64
u'a' * 45,
b'a' * 45, # bad padding
99,
None):
conf = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'encryption_root_secret option in proxy-server.conf '
'must be a base64 encoding of at least 32 raw bytes',
str(err.exception))
except AssertionError as err:
self.fail(str(err) + ' for conf %s' % str(conf))
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_root_secret_path_invalid_secret(self, mock_readconf):
for secret in (
base64.b64encode(os.urandom(31)), # too short
base64.b64encode(os.urandom(31)).decode('ascii'),
u'a' * 44 + u'????',
b'a' * 44 + b'????', # not base64
u'a' * 45,
b'a' * 45, # bad padding
99,
None):
mock_readconf.reset_mock()
mock_readconf.return_value = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(
self.swift,
{'keymaster_config_path': '/some/other/path'})
self.assertEqual(
'encryption_root_secret option in /some/other/path '
'must be a base64 encoding of at least 32 raw bytes',
str(err.exception))
self.assertEqual(mock_readconf.mock_calls,
[mock.call('/some/other/path', 'keymaster')])
except AssertionError as err:
self.fail(str(err) + ' for secret %r' % secret)
def test_can_only_configure_secret_in_one_place(self):
def do_test(conf):
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
expected_message = ('keymaster_config_path is set, but there are '
'other config options specified:')
self.assertTrue(
str(err.exception).startswith(expected_message),
"Error message does not start with '%s'" % expected_message)
conf = {
'encryption_root_secret': 'a' * 44,
'keymaster_config_path': '/etc/swift/keymaster.conf'
}
do_test(conf)
conf = {
'encryption_root_secret_1': 'a' * 44,
'keymaster_config_path': '/etc/swift/keymaster.conf'
}
do_test(conf)
conf = {
'encryption_root_secret_': 'a' * 44,
'keymaster_config_path': '/etc/swift/keymaster.conf'
}
do_test(conf)
conf = {
'active_root_secret_id': '1',
'keymaster_config_path': '/etc/swift/keymaster.conf'
}
do_test(conf)
class SymlinkCopyingTestCase(TestSymlinkMiddlewareBase):
# verify interaction of copy and symlink middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'symloop_max': '2'}
self.sym = symlink.filter_factory(conf)(self.app)
self.sym.logger = self.app.logger
self.copy = copy.filter_factory({})(self.sym)
def call_copy(self, req, **kwargs):
return self.call_app(req, app=self.copy, **kwargs)
def test_copy_symlink_target(self):
self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'})
self.app.register('GET', '/v1/a2/c1/o', swob.HTTPOk, {}, 'resp_body')
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{}, 'resp_body')
req = Request.blank('/v1/a/src_cont/symlink', method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
def test_copy_symlink(self):
self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'})
req = Request.blank('/v1/a/src_cont/symlink?symlink=get',
method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertEqual(
hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'), 'a2')
def test_copy_symlink_new_target(self):
self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'})
req = Request.blank('/v1/a/src_cont/symlink?symlink=get',
method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj',
'X-Symlink-Target': 'new_cont/new_obj',
'X-Symlink-Target-Account': 'new_acct'})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[1]
self.assertEqual(method, 'PUT')
self.assertEqual(path, '/v1/a/tgt_cont/tgt_obj?symlink=get')
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'new_cont/new_obj')
self.assertEqual(hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'),
'new_acct')
class DloTestCase(unittest.TestCase):
def call_dlo(self, req, app=None):
if app is None:
app = self.dlo
req.headers.setdefault("User-Agent", "Soap Opera")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
body = b''
# appease the close-checker
with closing_if_possible(body_iter):
for chunk in body_iter:
body += chunk
return status[0], headers[0], body
def setUp(self):
self.app = FakeSwift()
self.dlo = dlo.filter_factory({
# don't slow down tests with rate limiting
'rate_limit_after_segment': '1000000',
})(self.app)
self.dlo.logger = self.app.logger
self.app.register('GET', '/v1/AUTH_test/c/seg_01', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("aaaaa")
}, b'aaaaa')
self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("bbbbb")
}, b'bbbbb')
self.app.register('GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("ccccc")
}, b'ccccc')
self.app.register('GET', '/v1/AUTH_test/c/seg_04', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("ddddd")
}, b'ddddd')
self.app.register('GET', '/v1/AUTH_test/c/seg_05', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("eeeee")
}, b'eeeee')
# an unrelated object (not seg*) to test the prefix matching
self.app.register('GET', '/v1/AUTH_test/c/catpicture.jpg', swob.HTTPOk,
{
'Content-Length': '9',
'Etag': md5hex("meow meow meow meow")
}, b'meow meow meow meow')
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest', swob.HTTPOk, {
'Content-Length': '17',
'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'
}, b'manifest-contents')
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
segs = [{
"hash": md5hex("aaaaa"),
"bytes": 5,
"name": "seg_01",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("bbbbb"),
"bytes": 5,
"name": "seg_02",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("ccccc"),
"bytes": 5,
"name": "seg_03",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("ddddd"),
"bytes": 5,
"name": "seg_04",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("eeeee"),
"bytes": 5,
"name": "seg_05",
"last_modified": lm,
"content_type": ct
}]
full_container_listing = segs + [{
"hash": "cats-etag",
"bytes": 9,
"name": "catpicture.jpg",
"last_modified": lm,
"content_type": "application/png"
}]
self.app.register('GET', '/v1/AUTH_test/c', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(full_container_listing).encode('ascii'))
self.app.register('GET', '/v1/AUTH_test/c?prefix=seg', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs).encode('ascii'))
# This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings.
#
# If you're testing against this, be sure to mock out
# CONTAINER_LISTING_LIMIT to 3 in your test.
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-many-segments', swob.HTTPOk,
{
'Content-Length': '7',
'Etag': 'etag-manyseg',
'X-Object-Manifest': 'c/seg_'
}, b'manyseg')
self.app.register('GET', '/v1/AUTH_test/c?prefix=seg_', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[:3]).encode('ascii'))
self.app.register('GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[3:]).encode('ascii'))
# Here's a manifest with 0 segments
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-no-segments', swob.HTTPOk, {
'Content-Length': '7',
'Etag': 'noseg',
'X-Object-Manifest': 'c/noseg_'
}, b'noseg')
self.app.register('GET', '/v1/AUTH_test/c?prefix=noseg_', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps([]).encode('ascii'))
class OioContainerHierarchy(unittest.TestCase):
def setUp(self):
conf = {'sds_default_account': 'OPENIO'}
self.filter_conf = {
'strip_v1': 'true',
'swift3_compat': 'true',
'account_first': 'true',
'create_dir_placeholders': 'true',
'recursive_placeholders': 'true',
'sds_default_account': 'OPENIO'
}
self.app = FakeSwift()
self.ch = container_hierarchy.filter_factory(conf, **self.filter_conf)(
self.app)
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
def call_ch(self, req):
return self.call_app(req, app=self.ch)
def test_simple_put(self):
"""check number of request generated by Container Hierarchy"""
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated,
{})
# placeholders
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2/d3/', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a/c%2Fd1/d2/', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a/c/d1/', swob.HTTPOk, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.assertEqual([('PUT', '/v1/a/c%2Fd1%2Fd2/d3/'),
('PUT', '/v1/a/c%2Fd1/d2/'), ('PUT', '/v1/a/c/d1/'),
('PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o')], self.app.calls)
def test_placeholder_put(self):
"""stop at first placeholder already present"""
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated,
{})
# placeholder already set
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2/d3/',
swob.HTTPPreconditionFailed, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.assertEqual(len(self.app.calls), 2)
def test_placeholder_failed_put(self):
"""simulate issue when creating placeholder"""
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated,
{})
# issue on placeholder
self.app.register('PUT', '/v1/a/c%2Fd1%2Fd2/d3/',
swob.HTTPRequestTimeout, {})
# other placeholders are ok
self.app.register('PUT', '/v1/a/c%2Fd1/d2/', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a/c/d1/', swob.HTTPOk, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
def test_get(self):
self.app.register('GET', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPOk, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_recursive_listing(self):
self.app.register(
'GET',
'/v1/a/c%2Fd1%2Fd2?prefix=&limit=10000&delimiter=%2F&format=json',
swob.HTTPOk, {}, json.dumps([{
"subdir": "d3/"
}]))
self.app.register(
'GET',
'/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&delimiter=%2F&format=json', # noqa
swob.HTTPOk,
{},
json.dumps([{
"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0,
"name": "o",
"content_type": "application/octet-stream"
}]))
req = Request.blank('/v1/a/c?prefix=d1%2Fd2%2F', method='GET')
resp = self.call_ch(req)
data = json.loads(resp[2])
self.assertEqual(data[0]['name'], 'd1/d2/d3/o')
def test_global_listing(self):
self.app.register('GET', '/v1/a', swob.HTTPOk, {})
req = Request.blank('/v1/a', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_delete_object(self):
self.app._responses = {}
self.app.register('DELETE', '/v1/a/d1%2Fd2%2Fd3/o', swob.HTTPOk, {})
req = Request.blank('/v1/a/d1/d2/d3/o', method='DELETE')
resp = self.call_ch(req)
self.app.register(
'GET',
'/v1/a/d1%2Fd2%2Fd3?delimiter=%2F&limit=1&prefix=&format=json', # noqa
swob.HTTPOk,
{})
self.app.register('DELETE', '/v1/a/d1%2Fd2/d3/', swob.HTTPNoContent,
{})
# distcp job create and remove its own placeholder and we wan
# to refuse them if any object are still available
req = Request.blank('/v1/a/d1/d2/d3/', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
class TestOioServerSideCopyMiddleware(TestServerSideCopyMiddleware):
def setUp(self):
self.app = FakeSwift()
self.ssc = copy.filter_factory({
'object_post_as_copy': 'yes',
})(self.app)
self.ssc.logger = self.app.logger
def tearDown(self):
# get_object_info() does not close response iterator,
# thus we have to disable the unclosed_requests test.
pass
def test_basic_put_with_x_copy_from(self):
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o2', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c/o'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c/o') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertEqual('PUT', self.authorized[0].method)
self.assertEqual('/v1/a/c/o2', self.authorized[0].path)
self.assertEqual(self.app.swift_sources[0], 'SSC')
# For basic test cases, assert orig_req_method behavior
self.assertNotIn('swift.orig_req_method', req.environ)
def test_static_large_object_manifest(self):
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
{'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'})
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
{'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'}, 'passed')
self.app.register('PUT', '/v1/a/c/o2?multipart-manifest=put',
swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o2?multipart-manifest=get',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c/o'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c/o') in headers)
self.assertEqual(3, len(self.app.calls))
self.assertEqual('HEAD', self.app.calls[0][0])
self.assertEqual('GET', self.app.calls[1][0])
get_path, qs = self.app.calls[1][1].split('?')
params = urllib.parse.parse_qs(qs)
self.assertDictEqual(
{'format': ['raw'], 'multipart-manifest': ['get']}, params)
self.assertEqual(get_path, '/v1/a/c/o')
self.assertEqual(self.app.calls[2],
('PUT', '/v1/a/c/o2?multipart-manifest=put'))
req_headers = self.app.headers[2]
self.assertNotIn('X-Static-Large-Object', req_headers)
self.assertNotIn('Etag', req_headers)
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/c/o', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/c/o2', self.authorized[1].path)
def test_static_large_object(self):
# Compared to the original copy middleware, we do an extra HEAD request
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
{'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'}, 'passed')
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
{'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'}, 'passed')
self.app.register('PUT', '/v1/a/c/o2',
swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o2',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c/o'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c/o') in headers)
self.assertEqual(self.app.calls, [
('HEAD', '/v1/a/c/o'),
('GET', '/v1/a/c/o'),
('PUT', '/v1/a/c/o2')])
req_headers = self.app.headers[1]
self.assertNotIn('X-Static-Large-Object', req_headers)
self.assertNotIn('Etag', req_headers)
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/c/o', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/c/o2', self.authorized[1].path)
def test_basic_put_with_x_copy_from_across_container(self):
self.app.register('HEAD', '/v1/a/c1/o1', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a/c2/o2', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c1/o1'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c1/o1') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertEqual('PUT', self.authorized[0].method)
self.assertEqual('/v1/a/c2/o2', self.authorized[0].path)
def test_basic_put_with_x_copy_from_across_container_and_account(self):
self.app.register('HEAD', '/v1/a1/c1/o1', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a2/c2/o2', swob.HTTPCreated, {},
'passed')
req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c1/o1',
'X-Copy-From-Account': 'a1'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c1/o1') in headers)
self.assertTrue(('X-Copied-From-Account', 'a1') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertEqual('PUT', self.authorized[0].method)
self.assertEqual('/v1/a2/c2/o2', self.authorized[0].path)
def test_copy_not_found_reading_source(self):
self.skipTest('To be fixed')
def test_copy_not_found_reading_source_and_account(self):
self.skipTest('To be fixed')
def test_copy_server_error_reading_source(self):
self.skipTest('To be fixed')
def test_copy_server_error_reading_source_and_account(self):
self.skipTest('To be fixed')
def test_copy_source_larger_than_max_file_size(self):
self.skipTest('To be fixed')
def test_COPY_source_metadata(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_in_x_copy_from_and_account(self):
self.skipTest('To be fixed')
def test_copy_with_object_metadata(self):
self.skipTest('To be fixed')
def test_copy_with_object_metadata_and_account(self):
self.skipTest('To be fixed')
def test_copy_with_slashes_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_slashes_in_x_copy_from_and_account(self):
self.skipTest('To be fixed')
def test_copy_with_spaces_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_spaces_in_x_copy_from_and_account(self):
self.skipTest('To be fixed')
class ContainerQuotaCopyingTestCases(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.cq_filter = container_quotas.filter_factory({})(self.app)
self.copy_filter = copy.filter_factory({})(self.cq_filter)
def test_exceed_bytes_quota_copy_verb(self):
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
req = Request.blank('/v1/a/c2/o2',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache
},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c2/o2',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache
},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_exceed_counts_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c2/o2',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache
},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
a_c_cache = {
'storage_policy': '0',
'meta': {
'quota-count': '2'
},
'status': 200,
'object_count': 1
}
a2_c_cache = {
'storage_policy': '0',
'meta': {
'quota-count': '1'
},
'status': 200,
'object_count': 1
}
req = Request.blank('/v1/a/c2/o2',
environ={
'REQUEST_METHOD': 'COPY',
'swift.infocache': {
'container/a/c': a_c_cache,
'container/a2/c': a2_c_cache
}
},
headers={
'Destination': '/c/o',
'Destination-Account': 'a2'
})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_PUT_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
a_c_cache = {
'storage_policy': '0',
'meta': {
'quota-count': '2'
},
'status': 200,
'object_count': 1
}
a2_c_cache = {
'storage_policy': '0',
'meta': {
'quota-count': '1'
},
'status': 200,
'object_count': 1
}
req = Request.blank('/v1/a2/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.infocache': {
'container/a/c': a_c_cache,
'container/a2/c': a2_c_cache
}
},
headers={
'X-Copy-From': '/c2/o2',
'X-Copy-From-Account': 'a'
})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_bytes_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache
},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache
},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_no_src(self):
self.app.register('GET', '/v1/a/c2/o3', HTTPOk, {}, 'passed')
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache
},
headers={'x-copy-from': '/c2/o3'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_bad_src(self):
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache
},
headers={'x-copy-from': 'bad_path'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 412)
def test_exceed_counts_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache
},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_counts_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache
},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_not_exceed_counts_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c2/o2',
environ={
'REQUEST_METHOD': 'COPY',
'swift.cache': cache
},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
class TestDecrypterObjectRequests(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.decrypter = decrypter.Decrypter(self.app, {})
self.decrypter.logger = FakeLogger()
def _make_response_headers(self, content_length, plaintext_etag, keys,
body_key):
# helper method to make a typical set of response headers for a GET or
# HEAD request
cont_key = keys['container']
object_key = keys['object']
body_key_meta = {
'key': encrypt(body_key, object_key, FAKE_IV),
'iv': FAKE_IV
}
body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
return HeaderKeyDict({
'Etag':
'hashOfCiphertext',
'content-type':
'text/plain',
'content-length':
content_length,
'X-Object-Sysmeta-Crypto-Etag':
'%s; swift_meta=%s' %
(base64.b64encode(encrypt(plaintext_etag, object_key,
FAKE_IV)), get_crypto_meta_header()),
'X-Object-Sysmeta-Crypto-Body-Meta':
get_crypto_meta_header(body_crypto_meta),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', object_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-container-update-override-etag':
encrypt_and_append_meta('encrypt me, too', cont_key),
'x-object-sysmeta-test':
'do not encrypt me',
})
def _test_request_success(self, method, body):
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(len(enc_body), plaintext_etag,
fetch_crypto_keys(), body_key)
# there shouldn't be any x-object-meta- headers, but if there are
# then the decrypted header will win where there is a name clash...
hdrs.update({
'x-object-meta-test':
'unexpected, overwritten by decrypted value',
'x-object-meta-distinct':
'unexpected but distinct from encrypted'
})
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
self.assertEqual('unexpected but distinct from encrypted',
resp.headers['x-object-meta-distinct'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
self.assertEqual(
'encrypt me, too',
resp.headers['X-Object-Sysmeta-Container-Update-Override-Etag'])
self.assertNotIn('X-Object-Sysmeta-Crypto-Body-Meta', resp.headers)
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', resp.headers)
return resp
def test_GET_success(self):
body = 'FAKE APP'
resp = self._test_request_success('GET', body)
self.assertEqual(body, resp.body)
def test_HEAD_success(self):
body = 'FAKE APP'
resp = self._test_request_success('HEAD', body)
self.assertEqual('', resp.body)
def test_headers_case(self):
body = 'fAkE ApP'
req = Request.blank('/v1/a/c/o', body='FaKe')
req.environ[CRYPTO_KEY_CALLBACK] = fetch_crypto_keys
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(len(enc_body), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs.update({
'x-Object-mEta-ignoRes-caSe': 'thIs pArt WilL bE cOol',
})
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
status, headers, app_iter = req.call_application(self.decrypter)
self.assertEqual(status, '200 OK')
expected = {
'Etag': '7f7837924188f7b511a9e3881a9f77a8',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'encrypt me, too',
'X-Object-Meta-Test': 'encrypt me',
'Content-Length': '8',
'X-Object-Meta-Ignores-Case': 'thIs pArt WilL bE cOol',
'X-Object-Sysmeta-Test': 'do not encrypt me',
'Content-Type': 'text/plain',
}
self.assertEqual(dict(headers), expected)
self.assertEqual('fAkE ApP', ''.join(app_iter))
def _test_412_response(self, method):
# simulate a 412 response to a conditional GET which has an Etag header
data = 'the object content'
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env, method=method)
resp_body = 'I am sorry, you have failed to meet a precondition'
hdrs = self._make_response_headers(len(resp_body), md5hex(data),
fetch_crypto_keys(), 'not used')
self.app.register(method,
'/v1/a/c/o',
HTTPPreconditionFailed,
body=resp_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('412 Precondition Failed', resp.status)
# the response body should not be decrypted, it is already plaintext
self.assertEqual(resp_body if method == 'GET' else '', resp.body)
# whereas the Etag and other headers should be decrypted
self.assertEqual(md5hex(data), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
def test_GET_412_response(self):
self._test_412_response('GET')
def test_HEAD_412_response(self):
self._test_412_response('HEAD')
def _test_404_response(self, method):
# simulate a 404 response, sanity check response headers
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env, method=method)
resp_body = 'You still have not found what you are looking for'
hdrs = {'content-type': 'text/plain', 'content-length': len(resp_body)}
self.app.register(method,
'/v1/a/c/o',
HTTPNotFound,
body=resp_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('404 Not Found', resp.status)
# the response body should not be decrypted, it is already plaintext
self.assertEqual(resp_body if method == 'GET' else '', resp.body)
# there should be no etag header inserted by decrypter
self.assertNotIn('Etag', resp.headers)
self.assertEqual('text/plain', resp.headers['Content-Type'])
def test_GET_404_response(self):
self._test_404_response('GET')
def test_HEAD_404_response(self):
self._test_404_response('HEAD')
def test_GET_missing_etag_crypto_meta(self):
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted etag
hdrs['X-Object-Sysmeta-Crypto-Etag'] = \
base64.b64encode(encrypt(md5hex(body), key, FAKE_IV))
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header', resp.body)
self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_override_etag_bad_meta(self, method, bad_crypto_meta):
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted override etag
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
encrypt_and_append_meta(
md5hex(body), key, crypto_meta=bad_crypto_meta)
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn(
'Error decrypting header '
'X-Object-Sysmeta-Container-Update-Override-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def test_GET_override_etag_bad_iv(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta)
self.assertIn('Error decrypting header', resp.body)
def test_HEAD_override_etag_bad_iv(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta)
self.assertEqual('', resp.body)
def test_GET_override_etag_bad_cipher(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown cipher'
resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta)
self.assertIn('Error decrypting header', resp.body)
def test_HEAD_override_etag_bad_cipher(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown cipher'
resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta)
self.assertEqual('', resp.body)
def _test_bad_key(self, method):
# use bad key
def bad_fetch_crypto_keys():
keys = fetch_crypto_keys()
keys['object'] = 'bad key'
return keys
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
return req.get_response(self.decrypter)
def test_HEAD_with_bad_key(self):
resp = self._test_bad_key('HEAD')
self.assertEqual('500 Internal Error', resp.status)
self.assertIn("Bad key for 'object'",
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_key(self):
resp = self._test_bad_key('GET')
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
self.assertIn("Bad key for 'object'",
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_bad_crypto_meta_for_user_metadata(self, method, bad_crypto_meta):
# use bad iv for metadata headers
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
enc_val = base64.b64encode(encrypt('encrypt me', key, FAKE_IV))
if bad_crypto_meta:
enc_val += ';swift_meta=' + get_crypto_meta_header(
crypto_meta=bad_crypto_meta)
hdrs['x-object-transient-sysmeta-crypto-meta-test'] = enc_val
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test',
self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def test_HEAD_with_missing_crypto_meta_for_user_metadata(self):
self._test_bad_crypto_meta_for_user_metadata('HEAD', None)
self.assertIn('Missing crypto meta in value',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_crypto_meta_for_user_metadata(self):
self._test_bad_crypto_meta_for_user_metadata('GET', None)
self.assertIn('Missing crypto meta in value',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_HEAD_with_bad_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta)
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_HEAD_with_missing_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta.pop('iv')
self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta)
self.assertIn('iv',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['iv'] = 'bad_iv'
resp = self._test_bad_crypto_meta_for_user_metadata(
'GET', bad_crypto_meta)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_iv_for_user_metadata(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta.pop('iv')
resp = self._test_bad_crypto_meta_for_user_metadata(
'GET', bad_crypto_meta)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn('iv',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta):
# use bad iv for object body
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_iv_for_object_body(self):
bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32))
bad_crypto_meta['iv'] = 'bad_iv'
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_iv_for_object_body(self):
bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32))
bad_crypto_meta.pop('iv')
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn("Missing 'iv'",
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_bad_body_key_for_object_body(self):
body_key_meta = {'key': 'wrapped too short key', 'iv': FAKE_IV}
bad_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn('Key must be length 32',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_with_missing_body_key_for_object_body(self):
bad_crypto_meta = fake_get_crypto_meta() # no key by default
self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
self.assertIn("Missing 'body_key'",
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_req_metadata_not_encrypted(self, method):
# check that metadata is not decrypted if it does not have crypto meta;
# testing for case of an unencrypted POST to an object.
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs.pop('x-object-transient-sysmeta-crypto-meta-test')
hdrs['x-object-meta-test'] = 'plaintext not encrypted'
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('plaintext not encrypted',
resp.headers['x-object-meta-test'])
def test_HEAD_metadata_not_encrypted(self):
self._test_req_metadata_not_encrypted('HEAD')
def test_GET_metadata_not_encrypted(self):
self._test_req_metadata_not_encrypted('GET')
def test_GET_unencrypted_data(self):
# testing case of an unencrypted object with encrypted metadata from
# a later POST
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
obj_key = fetch_crypto_keys()['object']
hdrs = {
'Etag':
md5hex(body),
'content-type':
'text/plain',
'content-length':
len(body),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', obj_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-test':
'do not encrypt me'
}
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(md5hex(body), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
# POSTed user meta was encrypted
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
# PUT sysmeta was not encrypted
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
def test_GET_multiseg(self):
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
hdrs = self._make_response_headers(sum(map(len,
enc_body)), plaintext_etag,
fetch_crypto_keys(), body_key)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
def test_GET_multiseg_with_range(self):
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
req.headers['Content-Range'] = 'bytes 3-10/17'
chunks = ['0123', '45678', '9abcdef']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]]
hdrs = self._make_response_headers(sum(map(len,
enc_body)), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs['content-range'] = req.headers['Content-Range']
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('3456789a', resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
# Force the decrypter context updates to be less than one of our range
# sizes to check that the decrypt context offset is setup correctly with
# offset to first byte of range for first update and then re-used.
# Do mocking here to have the mocked value have effect in the generator
# function.
@mock.patch.object(decrypter, 'DECRYPT_CHUNK_SIZE', 4)
def test_GET_multipart_ciphertext(self):
# build fake multipart response body
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET',
'/v1/a/c/o',
HTTPPartialContent,
body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart headers could be re-ordered, so parse response body to
# verify expected content
resp_lines = resp.body.split('\r\n')
resp_lines.reverse()
for start, end, ctype in parts:
self.assertEqual('--multipartboundary', resp_lines.pop())
expected_header_lines = {
'Content-Type: %s' % ctype,
'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
}
resp_header_lines = {resp_lines.pop(), resp_lines.pop()}
self.assertEqual(expected_header_lines, resp_header_lines)
self.assertEqual('', resp_lines.pop())
self.assertEqual(plaintext[start:end], resp_lines.pop())
self.assertEqual('--multipartboundary--', resp_lines.pop())
# we should have consumed the whole response body
self.assertFalse(resp_lines)
def test_GET_multipart_content_type(self):
# *just* having multipart content type shouldn't trigger the mime doc
# code path
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
# register request with fake swift
hdrs = self._make_response_headers(len(ciphertext), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=ciphertext,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(plaintext), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
self.assertEqual(plaintext, resp.body)
def test_GET_multipart_no_body_crypto_meta(self):
# build fake multipart response body
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(plaintext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + plaintext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = {
'Etag': plaintext_etag,
'content-type': 'multipart/byteranges;boundary=multipartboundary',
'content-length': len(body)
}
self.app.register('GET',
'/v1/a/c/o',
HTTPPartialContent,
body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart response body should be unchanged
self.assertEqual(body, resp.body)
def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta):
# build fake multipart response body
key = fetch_crypto_keys()['object']
ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, ctxt=ctxt)
parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), 'not used')
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_bad_body_cipher(self):
self._test_GET_multipart_bad_body_crypto_meta({
'cipher': 'Mystery cipher',
'iv': '1234567887654321'
})
self.assertIn('Cipher must be AES_CTR_256',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_missing_body_cipher(self):
self._test_GET_multipart_bad_body_crypto_meta(
{'iv': '1234567887654321'})
self.assertIn('cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_too_short_body_iv(self):
self._test_GET_multipart_bad_body_crypto_meta({
'cipher': 'AES_CTR_256',
'iv': 'too short'
})
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_too_long_body_iv(self):
self._test_GET_multipart_bad_body_crypto_meta({
'cipher': 'AES_CTR_256',
'iv': 'a little too long'
})
self.assertIn('IV must be length 16',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_missing_body_iv(self):
self._test_GET_multipart_bad_body_crypto_meta(
{'cipher': 'AES_CTR_256'})
self.assertIn('iv',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_missing_key_callback(self):
# Do not provide keys, and do not set override flag
env = {'REQUEST_METHOD': 'GET'}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex('not the body'),
fetch_crypto_keys(), 'not used')
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
self.assertIn('missing callback',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: raise_exc}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
self.assertIn('from callback: Testing',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_cipher_mismatch_for_body(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(len(enc_body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
self.assertIn('Bad crypto meta: Cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_cipher_mismatch_for_metadata(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(len(enc_body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs.update({
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', key, FAKE_IV)) +
';swift_meta=' +
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
})
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_decryption_override(self):
# This covers the case of an old un-encrypted object
env = {
'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.crypto.override': True
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
hdrs = {
'Etag': md5hex(body),
'content-type': 'text/plain',
'content-length': len(body),
'x-object-meta-test': 'do not encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'
}
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(md5hex(body), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-meta-test'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
class SymlinkCopyingTestCase(TestSymlinkMiddlewareBase):
# verify interaction of copy and symlink middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'symloop_max': '2'}
self.sym = symlink.filter_factory(conf)(self.app)
self.sym.logger = self.app.logger
self.copy = copy.filter_factory({})(self.sym)
def call_copy(self, req, **kwargs):
return self.call_app(req, app=self.copy, **kwargs)
def test_copy_symlink_target(self):
req = Request.blank('/v1/a/src_cont/symlink',
method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj'})
self._test_copy_symlink_target(req)
req = Request.blank('/v1/a/tgt_cont/tgt_obj',
method='PUT',
headers={'X-Copy-From': 'src_cont/symlink'})
self._test_copy_symlink_target(req)
def _test_copy_symlink_target(self, req):
self.app.register(
'GET', '/v1/a/src_cont/symlink', swob.HTTPOk, {
'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'
})
self.app.register('GET', '/v1/a2/c1/o', swob.HTTPOk, {}, 'resp_body')
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated,
{}, 'resp_body')
status, headers, body = self.call_copy(req)
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a/src_cont/symlink')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a2/c1/o')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[2]
self.assertEqual(method, 'PUT')
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
# this is raw object copy
self.assertEqual(val, None)
self.assertEqual(status, '201 Created')
def test_copy_symlink(self):
req = Request.blank('/v1/a/src_cont/symlink?symlink=get',
method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj'})
self._test_copy_symlink(req)
req = Request.blank('/v1/a/tgt_cont/tgt_obj?symlink=get',
method='PUT',
headers={'X-Copy-From': 'src_cont/symlink'})
self._test_copy_symlink(req)
def _test_copy_symlink(self, req):
self.app.register(
'GET', '/v1/a/src_cont/symlink', swob.HTTPOk, {
'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'
})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'
})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a/src_cont/symlink?symlink=get')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertEqual(hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'),
'a2')
def test_copy_symlink_new_target(self):
req = Request.blank('/v1/a/src_cont/symlink?symlink=get',
method='COPY',
headers={
'Destination': 'tgt_cont/tgt_obj',
'X-Symlink-Target': 'new_cont/new_obj',
'X-Symlink-Target-Account': 'new_acct'
})
self._test_copy_symlink_new_target(req)
req = Request.blank('/v1/a/tgt_cont/tgt_obj?symlink=get',
method='PUT',
headers={
'X-Copy-From': 'src_cont/symlink',
'X-Symlink-Target': 'new_cont/new_obj',
'X-Symlink-Target-Account': 'new_acct'
})
self._test_copy_symlink_new_target(req)
def _test_copy_symlink_new_target(self, req):
self.app.register(
'GET', '/v1/a/src_cont/symlink', swob.HTTPOk, {
'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'
})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'
})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
self.assertEqual(path, '/v1/a/src_cont/symlink?symlink=get')
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
self.assertEqual(method, 'PUT')
self.assertEqual(path, '/v1/a/tgt_cont/tgt_obj?symlink=get')
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'new_cont/new_obj')
self.assertEqual(hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'),
'new_acct')
def test_copy_symlink_with_slo_query(self):
req = Request.blank(
'/v1/a/src_cont/symlink?multipart-manifest=get&symlink=get',
method='COPY',
headers={'Destination': 'tgt_cont/tgt_obj'})
self._test_copy_symlink_with_slo_query(req)
req = Request.blank(
'/v1/a/tgt_cont/tgt_obj?multipart-manifest=get&symlink=get',
method='PUT',
headers={'X-Copy-From': 'src_cont/symlink'})
self._test_copy_symlink_with_slo_query(req)
def _test_copy_symlink_with_slo_query(self, req):
self.app.register(
'GET', '/v1/a/src_cont/symlink', swob.HTTPOk, {
'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Account': 'a2'
})
self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Account': 'a2'
})
status, headers, body = self.call_copy(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
self.assertEqual(method, 'GET')
path, query = path.split('?')
query_dict = parse_qs(query)
self.assertEqual(path, '/v1/a/src_cont/symlink')
self.assertEqual(query_dict, {
'multipart-manifest': ['get'],
'symlink': ['get'],
'format': ['raw']
})
self.assertEqual('/src_cont/symlink', hdrs.get('X-Copy-From'))
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertEqual(hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'),
'a2')
class TestListingFormats(unittest.TestCase):
def setUp(self):
self.fake_swift = FakeSwift()
self.app = listing_formats.ListingFilter(self.fake_swift)
self.fake_account_listing = json.dumps([
{'name': 'bar', 'bytes': 0, 'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'},
{'subdir': 'foo_'},
])
self.fake_container_listing = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'},
{'subdir': 'foo/'},
])
def test_valid_account(self):
self.fake_swift.register('GET', '/v1/a', HTTPOk, {
'Content-Length': str(len(self.fake_account_listing)),
'Content-Type': 'application/json'}, self.fake_account_listing)
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
self.assertEqual(resp.body, 'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, 'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_account_listing)
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body.split('\n'), [
'<?xml version="1.0" encoding="UTF-8"?>',
'<account name="a">',
'<container><name>bar</name><count>0</count><bytes>0</bytes>'
'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
'</container>',
'<subdir name="foo_" />',
'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
def test_valid_container(self):
self.fake_swift.register('GET', '/v1/a/c', HTTPOk, {
'Content-Length': str(len(self.fake_container_listing)),
'Content-Type': 'application/json'}, self.fake_container_listing)
req = Request.blank('/v1/a/c')
resp = req.get_response(self.app)
self.assertEqual(resp.body, 'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, 'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_container_listing)
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=xml')
resp = req.get_response(self.app)
self.assertEqual(
resp.body,
'<?xml version="1.0" encoding="UTF-8"?>\n'
'<container name="c">'
'<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
'<content_type>text/plain</content_type>'
'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
'</object>'
'<subdir name="foo/"><name>foo/</name></subdir>'
'</container>'
)
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
def test_blank_account(self):
self.fake_swift.register('GET', '/v1/a', HTTPOk, {
'Content-Length': '2', 'Content-Type': 'application/json'}, '[]')
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, '')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, '')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, '[]')
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
req = Request.blank('/v1/a?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body.split('\n'), [
'<?xml version="1.0" encoding="UTF-8"?>',
'<account name="a">',
'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
def test_blank_container(self):
self.fake_swift.register('GET', '/v1/a/c', HTTPOk, {
'Content-Length': '2', 'Content-Type': 'application/json'}, '[]')
req = Request.blank('/v1/a/c')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, '')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '204 No Content')
self.assertEqual(resp.body, '')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, '[]')
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
req = Request.blank('/v1/a/c?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body.split('\n'), [
'<?xml version="1.0" encoding="UTF-8"?>',
'<container name="c" />',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
def test_pass_through(self):
def do_test(path):
self.fake_swift.register(
'GET', path, HTTPOk, {
'Content-Length': str(len(self.fake_container_listing)),
'Content-Type': 'application/json'},
self.fake_container_listing)
req = Request.blank(path + '?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_container_listing)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=xml')) # query param is unchanged
do_test('/')
do_test('/v1')
do_test('/auth/v1.0')
do_test('/v1/a/c/o')
def test_static_web_not_json(self):
body = 'doesnt matter'
self.fake_swift.register(
'GET', '/v1/staticweb/not-json', HTTPOk,
{'Content-Length': str(len(body)),
'Content-Type': 'text/plain'},
body)
resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
self.assertEqual(resp.body, body)
self.assertEqual(resp.headers['Content-Type'], 'text/plain')
# We *did* try, though
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/staticweb/not-json?format=json'))
# TODO: add a similar test that has *no* content-type
# FakeSwift seems to make this hard to do
def test_static_web_not_really_json(self):
body = 'raises ValueError'
self.fake_swift.register(
'GET', '/v1/staticweb/not-json', HTTPOk,
{'Content-Length': str(len(body)),
'Content-Type': 'application/json'},
body)
resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
self.assertEqual(resp.body, body)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/staticweb/not-json?format=json'))
def test_static_web_pretend_to_be_giant_json(self):
body = json.dumps(self.fake_container_listing * 1000000)
self.assertGreater( # sanity
len(body), listing_formats.MAX_CONTAINER_LISTING_CONTENT_LENGTH)
self.fake_swift.register(
'GET', '/v1/staticweb/not-json', HTTPOk,
{'Content-Type': 'application/json'},
body)
resp = Request.blank('/v1/staticweb/not-json').get_response(self.app)
self.assertEqual(resp.body, body)
self.assertEqual(resp.headers['Content-Type'], 'application/json')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/staticweb/not-json?format=json'))
# TODO: add a similar test for chunked transfers
# (staticweb referencing a DLO that doesn't fit in a single listing?)
def test_static_web_bad_json(self):
def do_test(body_obj):
body = json.dumps(body_obj)
self.fake_swift.register(
'GET', '/v1/staticweb/bad-json', HTTPOk,
{'Content-Length': str(len(body)),
'Content-Type': 'application/json'},
body)
def do_sub_test(path):
resp = Request.blank(path).get_response(self.app)
self.assertEqual(resp.body, body)
# NB: no charset is added; we pass through whatever we got
self.assertEqual(resp.headers['Content-Type'],
'application/json')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/staticweb/bad-json?format=json'))
do_sub_test('/v1/staticweb/bad-json')
do_sub_test('/v1/staticweb/bad-json?format=txt')
do_sub_test('/v1/staticweb/bad-json?format=xml')
do_sub_test('/v1/staticweb/bad-json?format=json')
do_test({})
do_test({'non-empty': 'hash'})
do_test(None)
do_test(0)
do_test('some string')
do_test([None])
do_test([0])
do_test(['some string'])
def test_static_web_bad_but_not_terrible_json(self):
body = json.dumps([{'no name': 'nor subdir'}])
self.fake_swift.register(
'GET', '/v1/staticweb/bad-json', HTTPOk,
{'Content-Length': str(len(body)),
'Content-Type': 'application/json'},
body)
def do_test(path, expect_charset=False):
resp = Request.blank(path).get_response(self.app)
self.assertEqual(resp.body, body)
if expect_charset:
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
else:
self.assertEqual(resp.headers['Content-Type'],
'application/json')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/staticweb/bad-json?format=json'))
do_test('/v1/staticweb/bad-json')
do_test('/v1/staticweb/bad-json?format=txt')
do_test('/v1/staticweb/bad-json?format=xml')
# The response we get is *just close enough* to being valid that we
# assume it is and slap on the missing charset. If you set up staticweb
# to serve back such responses, your clients are already hosed.
do_test('/v1/staticweb/bad-json?format=json', expect_charset=True)
class SymlinkVersioningTestCase(TestSymlinkMiddlewareBase):
# verify interaction of versioned_writes and symlink middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'symloop_max': '2'}
self.sym = symlink.filter_factory(conf)(self.app)
self.sym.logger = self.app.logger
vw_conf = {'allow_versioned_writes': 'true'}
self.vw = versioned_writes.filter_factory(vw_conf)(self.sym)
def call_vw(self, req, **kwargs):
return self.call_app(req, app=self.vw, **kwargs)
def assertRequestEqual(self, req, other):
self.assertEqual(req.method, other.method)
self.assertEqual(req.path, other.path)
def test_new_symlink_version_success(self):
self.app.register(
'PUT', '/v1/a/c/symlink', swob.HTTPCreated, {
'X-Symlink-Target': 'new_cont/new_tgt',
'X-Symlink-Target-Account': 'a'
}, None)
self.app.register(
'GET', '/v1/a/c/symlink', swob.HTTPOk, {
'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT',
'X-Object-Sysmeta-Symlink-Target': 'old_cont/old_tgt',
'X-Object-Sysmeta-Symlink-Target-Account': 'a'
}, '')
self.app.register(
'PUT', '/v1/a/ver_cont/007symlink/0000000001.00000',
swob.HTTPCreated, {
'X-Symlink-Target': 'old_cont/old_tgt',
'X-Symlink-Target-Account': 'a'
}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/symlink',
headers={'X-Symlink-Target': 'new_cont/new_tgt'},
environ={
'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'CONTENT_LENGTH': '0',
'swift.trans_id': 'fake_trans_id'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
# authorized twice now because versioned_writes now makes a check on
# PUT
self.assertEqual(len(self.authorized), 2)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(['VW', 'VW', None], self.app.swift_sources)
self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))
calls = self.app.calls_with_headers
method, path, req_headers = calls[2]
self.assertEqual('PUT', method)
self.assertEqual('/v1/a/c/symlink', path)
self.assertEqual('new_cont/new_tgt',
req_headers['X-Object-Sysmeta-Symlink-Target'])
def test_delete_latest_version_no_marker_success(self):
self.app.register(
'GET', '/v1/a/ver_cont?prefix=003sym/&marker=&reverse=on',
swob.HTTPOk, {}, '[{"hash": "y", '
'"last_modified": "2014-11-21T14:23:02.206740", '
'"bytes": 0, '
'"name": "003sym/2", '
'"content_type": "text/plain"}, '
'{"hash": "x", '
'"last_modified": "2014-11-21T14:14:27.409100", '
'"bytes": 0, '
'"name": "003sym/1", '
'"content_type": "text/plain"}]')
self.app.register('GET', '/v1/a/ver_cont/003sym/2', swob.HTTPCreated, {
'content-length': '0',
'X-Object-Sysmeta-Symlink-Target': 'c/tgt'
}, None)
self.app.register('PUT', '/v1/a/c/sym', swob.HTTPCreated, {
'X-Symlink-Target': 'c/tgt',
'X-Symlink-Target-Account': 'a'
}, None)
self.app.register('DELETE', '/v1/a/ver_cont/003sym/2', swob.HTTPOk, {},
None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank('/v1/a/c/sym',
headers={'X-If-Delete-At': 1},
environ={
'REQUEST_METHOD': 'DELETE',
'swift.cache': cache,
'CONTENT_LENGTH': '0',
'swift.trans_id': 'fake_trans_id'
})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(4, self.app.call_count)
self.assertEqual(['VW', 'VW', 'VW', 'VW'], self.app.swift_sources)
self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))
calls = self.app.calls_with_headers
method, path, req_headers = calls[2]
self.assertEqual('PUT', method)
self.assertEqual('/v1/a/c/sym', path)
self.assertEqual('c/tgt',
req_headers['X-Object-Sysmeta-Symlink-Target'])
class OioRegexContainer(unittest.TestCase):
def setUp(self):
conf = {'sds_default_account': 'OPENIO'}
self.filter_conf = {
'strip_v1': 'true',
'swift3_compat': 'true',
'account_first': 'true',
'stop_at_first_match': 'true',
'pattern1': '(\d{3})/(\d{3})/(\d)\d\d/\d\d(\d)/',
'pattern2': '(\d{3})/(\d)\d\d/\d\d(\d)/',
'pattern3': '^(cloud)/([0-9a-f][0-9a-f])',
'pattern4': '^(cloud)/([0-9a-f])',
'pattern9': '^/?([^/]+)',
}
if hasattr(ContainerBuilder, 'alternatives'):
self.filter_conf['stop_at_first_match'] = 'false'
self.app = FakeSwift()
self.ch = regexcontainer.filter_factory(
conf,
**self.filter_conf)(self.app)
def tearDown(self):
pass
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
def call_rc(self, req):
return self.call_app(req, app=self.ch)
def _check_pattern(self, path_in, path_out):
self.app.register('PUT', path_out, swob.HTTPCreated, {})
req = Request.blank(path_in, method='PUT')
resp = self.call_rc(req)
self.assertEqual(resp[0], "201 Created")
self.assertEqual(self.app.calls, [('PUT', path_out)])
def test_pattern1(self):
self._check_pattern(
'/v1/a/c/111/222/456/789/o',
'/v1/a/11122249/111/222/456/789/o')
def test_pattern2(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET', '/v1/a/11122249/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/11126/111/222/456/789/o',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], "200 OK")
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/11122249/111/222/456/789/o'),
('GET', '/v1/a/11126/111/222/456/789/o')])
def test_pattern3(self):
self._check_pattern('/v1/a/c/cloud/ff_object',
'/v1/a/cloudff/cloud/ff_object')
def test_pattern4(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET', '/v1/a/cloudff/cloud/ff_object',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/cloudf/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c/cloud/ff_object', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], "200 OK")
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/cloudff/cloud/ff_object'),
('GET', '/v1/a/cloudf/cloud/ff_object')])
def test_pattern9(self):
self._check_pattern('/v1/a/c/gc_regex/path/ob',
'/v1/a/gc_regex/gc_regex/path/ob')
def test_get_without_matching_pattern(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET', '/v1/a/11122249/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/11126/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET', '/v1/a/111/111/222/456/789/o',
swob.HTTPNotFound, {})
req = Request.blank('/v1/a/c/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], "404 Not Found")
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/11122249/111/222/456/789/o'),
('GET', '/v1/a/11126/111/222/456/789/o'),
('GET', '/v1/a/111/111/222/456/789/o')])
def test_simple_listing(self):
self._check_pattern('/v1/a/c/111/222/456/789/o',
'/v1/a/11122249/111/222/456/789/o')
self.app.register('GET',
'/v1/a/11122249?prefix=/111/222/456/789/o',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c?prefix=/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
def test_fallback_listing(self):
if self.filter_conf['stop_at_first_match'] == 'true':
self.skipTest("require openio-sds >= 4.2")
self.app.register('GET',
'/v1/a/11122249?prefix=/111/222/456/789/o',
swob.HTTPNotFound, {})
self.app.register('GET',
'/v1/a/11126?prefix=/111/222/456/789/o',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c?prefix=/111/222/456/789/o', method='GET')
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
self.assertEqual(
self.app.calls,
[('GET', '/v1/a/11122249?prefix=/111/222/456/789/o'),
('GET', '/v1/a/11126?prefix=/111/222/456/789/o')])
def test_swift3_mpu(self):
self.app.register('PUT',
'/v1/a/cloudff+segments/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank(
'/v1/a/c+segments/cloud/ff_object', method='PUT')
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
def test_copy(self):
self.app.register('PUT',
'/v1/a/cloudff/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank(
'/v1/a/c/cloud/ff_object', method='PUT',
headers={'X-Copy-From': 'container/path/dir1/object'})
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
self.assertEqual(self.app.headers[0]['X-Copy-From'],
"/dir1/dir1/object")
def test_fastcopy(self):
self.app.register('PUT',
'/v1/a/cloudff/cloud/ff_object',
swob.HTTPOk, {})
req = Request.blank('/v1/a/c/cloud/ff_object', method='PUT',
headers={'Oio-Copy-From': 'container/path/object'})
resp = self.call_rc(req)
self.assertEqual(resp[0], '200 OK')
self.assertEqual(self.app.headers[0]['Oio-Copy-From'],
"/object/object")
class TestEncrypter(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.encrypter = encrypter.Encrypter(self.app, {})
self.encrypter.logger = FakeLogger()
def _verify_user_metadata(self, req_hdrs, name, value, key):
# verify encrypted version of user metadata
self.assertNotIn('X-Object-Meta-' + name, req_hdrs)
expected_hdr = 'X-Object-Transient-Sysmeta-Crypto-Meta-' + name
self.assertIn(expected_hdr, req_hdrs)
enc_val, param = req_hdrs[expected_hdr].split(';')
param = param.strip()
self.assertTrue(param.startswith('swift_meta='))
actual_meta = json.loads(
urllib.unquote_plus(param[len('swift_meta='):]))
self.assertEqual(Crypto.cipher, actual_meta['cipher'])
meta_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, meta_iv)
self.assertEqual(base64.b64encode(encrypt(value, key, meta_iv)),
enc_val)
# if there is any encrypted user metadata then this header should exist
self.assertIn('X-Object-Transient-Sysmeta-Crypto-Meta', req_hdrs)
common_meta = json.loads(
urllib.unquote_plus(
req_hdrs['X-Object-Transient-Sysmeta-Crypto-Meta']))
self.assertDictEqual(
{
'cipher': Crypto.cipher,
'key_id': {
'v': 'fake',
'path': '/a/c/fake'
}
}, common_meta)
def test_PUT_req(self):
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'etag': plaintext_etag,
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
return_value=body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
# verify etag
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
# verify crypto_meta was appended to this etag
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
# verify encrypted version of plaintext etag
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
self.assertEqual(enc_etag, actual)
# verify etag MAC for conditional requests
actual_hmac = base64.b64decode(
req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac'])
self.assertEqual(
actual_hmac,
hmac.new(object_key, plaintext_etag, hashlib.sha256).digest())
# verify encrypted etag for container update
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# content-type is not encrypted
self.assertEqual('text/plain', req_hdrs['Content-Type'])
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(req_hdrs, 'Etag',
'not to be confused with the Etag!',
object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual(ciphertext, resp.body)
self.assertEqual(ciphertext_etag, resp.headers['Etag'])
def test_PUT_zero_size_object(self):
# object body encryption should be skipped for zero sized object body
object_key = fetch_crypto_keys()['object']
plaintext_etag = EMPTY_ETAG
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'etag': EMPTY_ETAG,
'content-type': 'text/plain',
'content-length': '0',
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'
}
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that there is no body crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs)
# verify etag is md5 of plaintext
self.assertEqual(EMPTY_ETAG, req_hdrs['Etag'])
# verify there is no etag crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs)
# verify there is no container update override for etag
self.assertNotIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
# user meta is still encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(req_hdrs, 'Etag',
'not to be confused with the Etag!',
object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is empty by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual('', resp.body)
self.assertEqual(EMPTY_ETAG, resp.headers['Etag'])
def _test_PUT_with_other_footers(self, override_etag):
# verify handling of another middleware's footer callback
cont_key = fetch_crypto_keys()['container']
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
other_footers = {
'Etag': plaintext_etag,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Size':
'other override',
'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag
}
env = {
'REQUEST_METHOD':
'PUT',
CRYPTO_KEY_CALLBACK:
fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'correct etag is in footers'
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key', lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that other middleware's footers made it to app, including any
# container update overrides but nothing Etag-related
other_footers.pop('Etag')
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify encryption footers are ok
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)
# verify encrypted etag for container update
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
def test_PUT_with_other_footers(self):
self._test_PUT_with_other_footers('override etag')
def test_PUT_with_other_footers_and_empty_etag(self):
# verify that an override etag value of EMPTY_ETAG will be encrypted
# when there was a non-zero body length
self._test_PUT_with_other_footers(EMPTY_ETAG)
def _test_PUT_with_etag_override_in_headers(self, override_etag):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
def test_PUT_with_etag_override_in_headers(self):
self._test_PUT_with_etag_override_in_headers('override_etag')
def test_PUT_with_etag_override_in_headers_and_empty_etag(self):
# verify that an override etag value of EMPTY_ETAG will be encrypted
# when there was a non-zero body length
self._test_PUT_with_etag_override_in_headers(EMPTY_ETAG)
def test_PUT_with_bad_etag_in_other_footers(self):
# verify that etag supplied in footers from other middleware overrides
# header etag when validating inbound plaintext etags
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
other_footers = {
'Etag': 'bad etag',
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'
}
env = {
'REQUEST_METHOD':
'PUT',
CRYPTO_KEY_CALLBACK:
fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
self.assertNotIn('Etag', resp.headers)
def test_PUT_with_bad_etag_in_headers_and_other_footers(self):
# verify that etag supplied in headers from other middleware is used if
# none is supplied in footers when validating inbound plaintext etags
plaintext = 'FAKE APP'
other_footers = {
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'
}
env = {
'REQUEST_METHOD':
'PUT',
CRYPTO_KEY_CALLBACK:
fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'bad etag'
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
self.assertNotIn('Etag', resp.headers)
def test_PUT_nothing_read(self):
# simulate an artificial scenario of a downstream filter/app not
# actually reading the input stream from encrypter.
class NonReadingApp(object):
def __call__(self, env, start_response):
# note: no read from wsgi.input
req = Request(env)
env['swift.callback.update_footers'](req.headers)
call_headers.append(req.headers)
resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
return resp(env, start_response)
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'content-type': 'text/plain',
'content-length': 0,
'etag': 'etag from client'
}
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
self.assertEqual('etag from client', call_headers[0]['etag'])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# check that an upstream footer callback gets called
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'
}
env.update({
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify encrypted override etag for container update.
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
call_headers[0])
parts = call_headers[0][
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt('other override', cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify that other middleware's footers made it to app
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# if upstream footer override etag is for an empty body then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG
}
env.update({
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
def test_POST_req(self):
body = 'FAKE APP'
env = {
'REQUEST_METHOD': 'POST',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
hdrs = {
'x-object-meta-test': 'encrypt me',
'x-object-meta-test2': '',
'x-object-sysmeta-test': 'do not encrypt me'
}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
key = fetch_crypto_keys()['object']
self.app.register('POST', '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertNotIn('Etag', resp.headers)
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('POST', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', key)
# unless it had no value
self.assertEqual('', req_hdrs['X-Object-Meta-Test2'])
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
def _test_no_user_metadata(self, method):
# verify that x-object-transient-sysmeta-crypto-meta is not set when
# there is no user metadata
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env, body='body')
self.app.register(method, '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
self.assertNotIn('x-object-transient-sysmeta-crypto-meta',
self.app.headers[0])
def test_PUT_no_user_metadata(self):
self._test_no_user_metadata('PUT')
def test_POST_no_user_metadata(self):
self._test_no_user_metadata('POST')
def _test_if_match(self, method, match_header_name):
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ', '.join(plain_etags)
req = Request.blank(
'/v1/a/c/o',
environ=env,
method=method,
headers={match_header_name: match_header_value})
app = FakeSwift()
app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != '*':
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(', '))
key = fetch_crypto_keys()['object']
masked_etags = [
'"%s"' % base64.b64encode(
hmac.new(key, etag.strip('"'), hashlib.sha256).digest())
for etag in plain_etags if etag not in ('*', '')
]
expected_etags = set((expected_plain_etags or plain_etags) +
masked_etags)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags),
set(req.headers[match_header_name].split(', ')))
do_test(method, [''])
do_test(method, ['"an etag"'])
do_test(method, ['"an etag"', '"another_etag"'])
do_test(method, ['*'])
# rfc2616 does not allow wildcard *and* etag but test it anyway
do_test(method, ['*', '"an etag"'])
# etags should be quoted but check we can cope if they are not
do_test(method, ['*', 'an etag', 'another_etag'],
expected_plain_etags=['*', '"an etag"', '"another_etag"'])
def test_GET_if_match(self):
self._test_if_match('GET', 'If-Match')
def test_HEAD_if_match(self):
self._test_if_match('HEAD', 'If-Match')
def test_GET_if_none_match(self):
self._test_if_match('GET', 'If-None-Match')
def test_HEAD_if_none_match(self):
self._test_if_match('HEAD', 'If-None-Match')
def _test_existing_etag_is_at_header(self, method, match_header_name):
# if another middleware has already set X-Backend-Etag-Is-At then
# encrypter should not override that value
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o',
environ=env,
method=method,
headers={
match_header_name:
"an etag",
'X-Backend-Etag-Is-At':
'X-Object-Sysmeta-Other-Etag'
})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual(
'X-Object-Sysmeta-Other-Etag,X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
actual_etags = set(actual_headers[match_header_name].split(', '))
self.assertIn('"an etag"', actual_etags)
def test_GET_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('GET', 'If-Match')
def test_HEAD_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('HEAD', 'If-Match')
def test_GET_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('GET', 'If-None-Match')
def test_HEAD_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('HEAD', 'If-None-Match')
def _test_etag_is_at_not_duplicated(self, method):
# verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
# X-Backend-Etag-Is-At
key = fetch_crypto_keys()['object']
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o',
environ=env,
method=method,
headers={
'If-Match': '"an etag"',
'If-None-Match': '"another etag"'
})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
self.assertIn(
'"%s"' % base64.b64encode(
hmac.new(key, 'an etag', hashlib.sha256).digest()),
actual_headers['If-Match'])
self.assertIn('"another etag"', actual_headers['If-None-Match'])
self.assertIn(
'"%s"' % base64.b64encode(
hmac.new(key, 'another etag', hashlib.sha256).digest()),
actual_headers['If-None-Match'])
def test_GET_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated('GET')
def test_HEAD_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated('HEAD')
def test_PUT_response_inconsistent_etag_is_not_replaced(self):
# if response is success but etag does not match the ciphertext md5
# then verify that we do *not* replace it with the plaintext etag
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated,
{'Etag': 'not the ciphertext etag'})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual('not the ciphertext etag', resp.headers['Etag'])
def test_PUT_multiseg_no_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {
'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)
}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key', lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def test_PUT_multiseg_good_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {
'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)
}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': md5hex(body)
}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key', lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def test_PUT_multiseg_bad_client_etag(self):
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {
'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)
}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': 'badclientetag'
}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
def test_PUT_missing_key_callback(self):
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT'}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('missing callback',
self.encrypter.logger.get_lines_for_level('error')[0])
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
def test_PUT_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: raise_exc}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('from callback: Testing',
self.encrypter.logger.get_lines_for_level('error')[0])
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
def test_PUT_encryption_override(self):
# set crypto override to disable encryption.
# simulate another middleware wanting to set footers
other_footers = {
'Etag': 'other etag',
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'
}
body = 'FAKE APP'
env = {
'REQUEST_METHOD':
'PUT',
'swift.crypto.override':
True,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify that other middleware's footers made it to app
req_hdrs = self.app.headers[0]
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify object is NOT encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(body, get_req.get_response(self.app).body)
def _test_constraints_checking(self, method):
# verify that the check_metadata function is called on PUT and POST
body = 'FAKE APP'
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
mocked_func = 'swift.common.middleware.crypto.encrypter.check_metadata'
with mock.patch(mocked_func) as mocked:
mocked.side_effect = [HTTPBadRequest('testing')]
resp = req.get_response(self.encrypter)
self.assertEqual('400 Bad Request', resp.status)
self.assertEqual(1, mocked.call_count)
mocked.assert_called_once_with(mock.ANY, 'object')
self.assertEqual(req.headers, mocked.call_args_list[0][0][0].headers)
def test_PUT_constraints_checking(self):
self._test_constraints_checking('PUT')
def test_POST_constraints_checking(self):
self._test_constraints_checking('POST')
def test_config_true_value_on_disable_encryption(self):
app = FakeSwift()
self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption)
for val in ('true', '1', 'yes', 'on', 't', 'y'):
app = encrypter.Encrypter(app, {'disable_encryption': val})
self.assertTrue(app.disable_encryption)
def test_PUT_app_exception(self):
app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {})
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
with self.assertRaises(HTTPException) as catcher:
req.get_response(app)
self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)
def test_encrypt_header_val(self):
# Prepare key and Crypto instance
object_key = fetch_crypto_keys()['object']
# - Normal string can be crypted
encrypted = encrypter.encrypt_header_val(Crypto(), 'aaa', object_key)
# sanity: return value is 2 item tuple
self.assertEqual(2, len(encrypted))
crypted_val, crypt_info = encrypted
expected_crypt_val = base64.b64encode(
encrypt('aaa', object_key, FAKE_IV))
expected_crypt_info = {
'cipher': 'AES_CTR_256',
'iv': 'This is an IV123'
}
self.assertEqual(expected_crypt_val, crypted_val)
self.assertEqual(expected_crypt_info, crypt_info)
# - Empty string raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), '', object_key)
self.assertEqual('empty value is not acceptable', cm.exception.message)
# - None also raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), None, object_key)
self.assertEqual('empty value is not acceptable', cm.exception.message)
class TestOioServerSideCopyMiddleware(TestServerSideCopyMiddleware):
def setUp(self):
self.app = FakeSwift()
self.ssc = copy.filter_factory({
'object_post_as_copy': 'yes',
})(self.app)
self.ssc.logger = self.app.logger
def tearDown(self):
# get_object_info() does not close response iterator,
# thus we have to disable the unclosed_requests test.
pass
def test_basic_put_with_x_copy_from(self):
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o2',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Content-Length': '0',
'X-Copy-From': 'c/o'
})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c/o') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertEqual('PUT', self.authorized[0].method)
self.assertEqual('/v1/a/c/o2', self.authorized[0].path)
self.assertEqual(self.app.swift_sources[0], 'SSC')
# For basic test cases, assert orig_req_method behavior
self.assertNotIn('swift.orig_req_method', req.environ)
def test_static_large_object_manifest(self):
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {
'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'
})
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {
'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'
}, 'passed')
self.app.register('PUT', '/v1/a/c/o2?multipart-manifest=put',
swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o2?multipart-manifest=get',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Content-Length': '0',
'X-Copy-From': 'c/o'
})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c/o') in headers)
self.assertEqual(3, len(self.app.calls))
self.assertEqual('HEAD', self.app.calls[0][0])
self.assertEqual('GET', self.app.calls[1][0])
get_path, qs = self.app.calls[1][1].split('?')
params = urllib.parse.parse_qs(qs)
self.assertDictEqual({
'format': ['raw'],
'multipart-manifest': ['get']
}, params)
self.assertEqual(get_path, '/v1/a/c/o')
self.assertEqual(self.app.calls[2],
('PUT', '/v1/a/c/o2?multipart-manifest=put'))
req_headers = self.app.headers[2]
self.assertNotIn('X-Static-Large-Object', req_headers)
self.assertNotIn('Etag', req_headers)
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/c/o', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/c/o2', self.authorized[1].path)
def test_static_large_object(self):
# Compared to the original copy middleware, we do an extra HEAD request
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {
'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'
}, 'passed')
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {
'X-Static-Large-Object': 'True',
'Etag': 'should not be sent'
}, 'passed')
self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o2',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Content-Length': '0',
'X-Copy-From': 'c/o'
})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c/o') in headers)
self.assertEqual(self.app.calls, [('HEAD', '/v1/a/c/o'),
('GET', '/v1/a/c/o'),
('PUT', '/v1/a/c/o2')])
req_headers = self.app.headers[1]
self.assertNotIn('X-Static-Large-Object', req_headers)
self.assertNotIn('Etag', req_headers)
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/c/o', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/c/o2', self.authorized[1].path)
def test_basic_put_with_x_copy_from_across_container(self):
self.app.register('HEAD', '/v1/a/c1/o1', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a/c2/o2', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Content-Length': '0',
'X-Copy-From': 'c1/o1'
})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c1/o1') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertEqual('PUT', self.authorized[0].method)
self.assertEqual('/v1/a/c2/o2', self.authorized[0].path)
def test_basic_put_with_x_copy_from_across_container_and_account(self):
self.app.register('HEAD', '/v1/a1/c1/o1', swob.HTTPOk, {})
self.app.register('PUT', '/v1/a2/c2/o2', swob.HTTPCreated, {},
'passed')
req = Request.blank('/v1/a2/c2/o2',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Content-Length': '0',
'X-Copy-From': 'c1/o1',
'X-Copy-From-Account': 'a1'
})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
self.assertTrue(('X-Copied-From', 'c1/o1') in headers)
self.assertTrue(('X-Copied-From-Account', 'a1') in headers)
self.assertEqual(len(self.authorized), 1)
self.assertEqual('PUT', self.authorized[0].method)
self.assertEqual('/v1/a2/c2/o2', self.authorized[0].path)
def test_copy_not_found_reading_source(self):
self.skipTest('To be fixed')
def test_copy_not_found_reading_source_and_account(self):
self.skipTest('To be fixed')
def test_copy_server_error_reading_source(self):
self.skipTest('To be fixed')
def test_copy_server_error_reading_source_and_account(self):
self.skipTest('To be fixed')
def test_copy_source_larger_than_max_file_size(self):
self.skipTest('To be fixed')
def test_COPY_source_metadata(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_leading_slash_in_x_copy_from_and_account(self):
self.skipTest('To be fixed')
def test_copy_with_object_metadata(self):
self.skipTest('To be fixed')
def test_copy_with_object_metadata_and_account(self):
self.skipTest('To be fixed')
def test_copy_with_slashes_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_slashes_in_x_copy_from_and_account(self):
self.skipTest('To be fixed')
def test_copy_with_spaces_in_x_copy_from(self):
self.skipTest('To be fixed')
def test_copy_with_spaces_in_x_copy_from_and_account(self):
self.skipTest('To be fixed')
class OioContainerHierarchy(unittest.TestCase):
def setUp(self):
conf = {'sds_default_account': 'OPENIO'}
self.filter_conf = {
'strip_v1': 'true',
'swift3_compat': 'true',
'account_first': 'true'
}
self.app = FakeSwift()
self.ch = container_hierarchy.filter_factory(
conf,
**self.filter_conf)(self.app)
def mock(self):
self.ch._create_placeholder = mock.MagicMock(return_value=None)
self.ch._remove_placeholder = mock.MagicMock(return_value=None)
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
def call_ch(self, req):
return self.call_app(req, app=self.ch)
def test_simple_put(self):
"""check number of request generated by Container Hierarchy"""
self.mock()
self.app.register(
'PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.ch._create_placeholder.assert_called_with(
mock.ANY, 'a', 'c', 'cnt', 'd1/d2/d3/')
def test_fake_directory_put(self):
self.mock()
req = Request.blank('/v1/a/c/d1/d2/d3/', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
self.ch._create_placeholder.assert_called_with(
mock.ANY, 'a', 'c', 'obj', 'd1/d2/d3/')
def test_get(self):
if self.ch.redis_keys_format == REDIS_KEYS_FORMAT_V1:
self.ch.conn.set("CS:a:c:cnt:d1/d2/d3/", 1)
self.app.register(
'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPOk, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_recursive_listing_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(return_value=['CS:a:cnt:d1/d2/d3/'])
self._test_recursive_listing()
def test_recursive_listing_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
# Will be called on CNT first (1 element), then on OBJ (empty)
self.ch.conn.hkeys = mock.MagicMock(side_effect=(['d1/d2/d3/'], []))
self._test_recursive_listing()
self.ch.conn.hkeys.assert_has_calls(
[mock.call('CS:a:c:cnt', match='d1/d2/*'),
mock.call('CS:a:c:obj', match='d1/d2/*')])
def _test_recursive_listing(self):
self.app.register(
'GET',
'/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/c?prefix=d1%2Fd2%2F', method='GET')
resp = self.call_ch(req)
data = json.loads(resp[2])
self.assertEqual(data[0]['name'], 'd1/d2/d3/o')
def test_listing_with_space_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(return_value=['CS:a:cnt:d 1/d2/'])
self._test_listing_with_space()
def test_listing_with_space_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
# Will be called on CNT first (1 element), then on OBJ (empty)
self.ch.conn.hkeys = mock.MagicMock(side_effect=(['d 1/d2/'], []))
self._test_listing_with_space()
self.ch.conn.hkeys.assert_has_calls(
[mock.call('CS:a:c:cnt', match='d 1/d2/*'),
mock.call('CS:a:c:obj', match='d 1/d2/*')])
def _test_listing_with_space(self):
self.app.register(
'GET',
'/v1/a/c%2Fd 1%2Fd2?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/c?prefix=d%201%2Fd2%2F', method='GET')
resp = self.call_ch(req)
data = json.loads(resp[2])
self.assertEqual(data[0]['name'], 'd 1/d2/o')
def test_global_listing(self):
self.app.register(
'GET', '/v1/a', swob.HTTPOk, {})
req = Request.blank('/v1/a', method='GET')
resp = self.call_ch(req)
self.assertEqual(resp[0], '200 OK')
def test_delete_object(self):
self.app.register(
'PUT', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='PUT')
resp = self.call_ch(req)
self.assertEqual(resp[0], '201 Created')
if self.ch.redis_keys_format == REDIS_KEYS_FORMAT_V1:
self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
else:
self.assertIn('d1/d2/d3/', self.ch.conn._keys['CS:a:c:cnt'])
self.app.register(
'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=1&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o",
"content_type": "application/octet-stream"}]))
self.app.register(
'DELETE', '/v1/a/c%2Fd1%2Fd2%2Fd3/o', swob.HTTPNoContent, {})
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
if self.ch.redis_keys_format == REDIS_KEYS_FORMAT_V1:
self.assertIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
else:
self.assertIn('d1/d2/d3/', self.ch.conn._keys['CS:a:c:cnt'])
self.app.register(
'GET', '/v1/a/c%2Fd1%2Fd2%2Fd3?prefix=&limit=1&format=json',
swob.HTTPOk, {}, json.dumps([]))
req = Request.blank('/v1/a/c/d1/d2/d3/o', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
self.assertNotIn('CS:a:c:cnt:d1/d2/d3/', self.ch.conn._keys)
def test_fake_directory(self):
req = Request.blank('/v1/a/container/d2/d3/', method='PUT')
resp = self.call_ch(req)
if self.ch.redis_keys_format == REDIS_KEYS_FORMAT_V1:
self.assertIn('CS:a:container:obj:d2/d3/', self.ch.conn._keys)
else:
self.assertIn('d2/d3/', self.ch.conn._keys['CS:a:container:obj'])
req = Request.blank('/v1/a/container/d2/d3/', method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], "204 No Content")
if self.ch.redis_keys_format == REDIS_KEYS_FORMAT_V1:
self.assertNotIn('CS:a:container:obj:d2/d3/', self.ch.conn._keys)
else:
self.assertNotIn('d2/d3/',
self.ch.conn._keys['CS:a:container:obj'])
def _listing(self, is_recursive):
self.ch.conn.exists = mock.MagicMock(return_value=True)
self.ch.conn.hexists = mock.MagicMock(return_value=True)
self.app.register(
'GET',
'/v1/a/bucket%2Fd1?prefix=d&limit=10&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o1",
"content_type": "application/octet-stream"}]))
if is_recursive:
self.app.register(
'GET',
'/v1/a/bucket%2Fd1%2Fd2?prefix=&limit=10&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "o2",
"content_type": "application/octet-stream"}]))
recursive = '' if is_recursive else '&delimiter=%2F'
req = Request.blank('/v1/a/bucket?prefix=d1/d&limit=10%s' % recursive,
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
return names
def test_listing_with_prefix_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/', 'CS:a:bucket:cnt:d1/d2/'])
names = self._listing(False)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/', names)
def test_listing_with_prefix_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt'])
self.ch.conn.hkeys = mock.MagicMock(
return_value=['d1/', 'd1/d2/'])
names = self._listing(False)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/', names)
def test_listing_with_prefix_recursive_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/', 'CS:a:bucket:cnt:d1/d2/'])
names = self._listing(True)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/o2', names)
def test_listing_with_prefix_recursive_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt'])
self.ch.conn.hkeys = mock.MagicMock(
return_value=['d1/', 'd1/d2/'])
names = self._listing(True)
self.assertIn('d1/o1', names)
self.assertIn('d1/d2/o2', names)
def test_listing_root_container_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/'])
self._test_listing_root_container()
def test_listing_root_container_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt'])
self.ch.conn.hkeys = mock.MagicMock(
return_value=['d1/'])
self._test_listing_root_container()
def _test_listing_root_container(self):
self.app.register(
'GET',
'/v1/a/bucket?prefix=d&limit=10&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "d0",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/bucket?prefix=d&limit=10&delimiter=%2F',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertIn("d0", names)
self.assertIn("d1/", names)
def test_listing_with_marker_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d0/',
'CS:a:bucket:cnt:d1/',
'CS:a:bucket:cnt:d2/'])
self._test_listing_with_marker()
def test_listing_with_marker_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt'])
self.ch.conn.hkeys = mock.MagicMock(
return_value=['d0/', 'd1/', 'd2/'])
self._test_listing_with_marker()
def _test_listing_with_marker(self):
# register root container
self.app.register(
'GET',
'/v1/a/bucket?marker=d1&prefix=&limit=10&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "zz",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertNotIn('d0/', names)
self.assertNotIn('d1/', names)
self.assertIn('d2/', names)
self.assertIn('zz', names)
def test_listing_with_marker_multi_container_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/',
'CS:a:bucket:cnt:d2/'])
self._test_listing_with_marker_multi_container()
def test_listing_with_marker_multi_container_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt'])
self.ch.conn.hkeys = mock.MagicMock(
return_value=['d1/', 'd2/'])
self._test_listing_with_marker_multi_container()
def _test_listing_with_marker_multi_container(self):
# register root container
self.app.register(
'GET',
'/v1/a/bucket?marker=d1&prefix=&limit=10&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "zz",
"content_type": "application/octet-stream"}]))
# with marker aa (as we inspect d1/)
self.app.register(
'GET',
'/v1/a/bucket%2Fd1?marker=aa&prefix=&limit=10&format=json', # noqa
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "d0",
"content_type": "application/octet-stream"}]))
# without marker on second container
self.app.register(
'GET',
'/v1/a/bucket%2Fd2?prefix=&limit=10&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": "d0",
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/bucket?limit=10&marker=d1/aa',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertIn('d1/d0', names)
self.assertIn('d2/d0', names)
self.assertIn('zz', names)
def test_duplicate_obj_cnt(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket:cnt:d1/cnt/',
'CS:a:bucket:obj:d1/obj/'])
self._test_duplicate_obj_cnt()
def test_duplicate_obj_cnt_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
self.ch.conn.hset('CS:a:bucket:cnt', 'd1/cnt', 1)
self.ch.conn.hset('CS:a:bucket:obj', 'd1/obj', 1)
self._test_duplicate_obj_cnt()
def _test_duplicate_obj_cnt(self):
self.app.register(
'GET',
'/v1/a/bucket?marker=d1&prefix=&limit=10&format=json',
swob.HTTPOk, {}, json.dumps([]))
req = Request.blank('/v1/a/bucket?limit=10&delimiter=%2F&marker=d1/',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertIn('d1/', names)
self.assertEqual(1, len(names))
def test_remove_bucket(self):
self.app.register(
'DELETE',
'/v1/a/bucket',
swob.HTTPNoContent, {},
"")
req = Request.blank('/v1/a/bucket',
method='DELETE')
resp = self.call_ch(req)
self.assertEqual(resp[0], "204 No Content")
def test_invalid_path(self):
req = Request.blank('/v1/a/',
method='GET')
with self.assertRaises(HTTPException) as cm:
self.call_ch(req)
self.assertEqual(cm.exception.status, "400 Bad Request")
def test_path(self):
cont = 'bucket'
path = 'dir1/dir2/object'
res = self.ch._fake_container_and_obj(cont, path.split('/'))
self.assertEqual(res, (cont + '%2Fdir1%2Fdir2', 'object'))
path = 'object'
res = self.ch._fake_container_and_obj(cont, path.split('/'))
self.assertEqual(res, (cont, 'object'))
def test_mpu_path(self):
cont = 'bucket+segments'
uploadid = 'MzNkYWZlNjItNjg3Yy00ZmIyLWIwOGYtOTA2OGVlZTA2MzA5'
path = ('dir1/dir2/object/%s/1' % uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont + '%2Fdir1%2Fdir2',
'object/%s/1' % uploadid))
path = ('dir1/dir2/object/' + uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont + '%2Fdir1%2Fdir2',
'object/' + uploadid))
path = ('object/%s/1' % uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont,
'object/%s/1' % uploadid))
path = ('object/' + uploadid).split('/')
res = self.ch._fake_container_and_obj(cont, path, is_mpu=True)
self.assertEqual(res, (cont,
'object/' + uploadid))
def test_upload_in_progress_v1(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V1
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket+segments:cnt:d1/d2/d3/'])
self._test_upload_in_progress()
def test_upload_in_progress_v2(self):
self.ch.redis_keys_format = REDIS_KEYS_FORMAT_V2
self.ch.conn.keys = mock.MagicMock(
return_value=['CS:a:bucket+segments:cnt'])
# Will be called on CNT first (1 element), then on OBJ (empty)
self.ch.conn.hkeys = mock.MagicMock(
side_effect=(['d1/d2/d3/'], []))
self._test_upload_in_progress()
def _test_upload_in_progress(self):
upload = ["obj/YmYwY2I1ZDYtNjMyYi00OGNiLWEzMzEtZDdhYTk0ODZkNWU2",
"root/MzNkYWZlNjItNjg3Yy00ZmIyLWIwOGYtOTA2OGVlZTA2MzA5"]
self.app.register(
'GET',
'/v1/a/bucket+segments%2Fd1%2Fd2%2Fd3?prefix=&limit=10000&format=json', # noqa
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": upload[0],
"content_type": "application/octet-stream"},
{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 400, "name": upload[0] + '/1',
"content_type": "application/octet-stream"}]))
self.app.register(
'GET',
'/v1/a/bucket+segments?prefix=&limit=10000&format=json',
swob.HTTPOk, {},
json.dumps([{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 0, "name": upload[1],
"content_type": "application/octet-stream"},
{"hash": "d41d8cd98f00b204e9800998ecf8427e",
"last_modified": "2018-04-20T09:40:59.000000",
"bytes": 400, "name": upload[1] + '/1',
"content_type": "application/octet-stream"}]))
req = Request.blank('/v1/a/bucket+segments',
method='GET')
resp = self.call_ch(req)
names = [item.get('name', item.get('subdir'))
for item in json.loads(resp[2])]
self.assertEqual(names,
['d1/d2/d3/' + upload[0],
'd1/d2/d3/' + upload[0] + '/1',
upload[1],
upload[1] + '/1'])
def test_copy_headers(self):
self.app.register(
'PUT', '/v1/a/bucket%2Fdir1/target',
swob.HTTPNoContent, {},
)
req = Request.blank(
'/v1/a/bucket/dir1/target',
method='PUT',
headers={'Oio-Copy-From': '/v1/a/bucket/sub1/source'})
resp = self.call_ch(req)
self.assertEqual(resp[0], '204 No Content')
self.assertEqual(self.app.headers[0]['Oio-Copy-From'],
"/v1%2Fa%2Fbucket%2Fsub1/source")