def _make_response_headers(self, content_length, plaintext_etag, keys,
body_key):
# helper method to make a typical set of response headers for a GET or
# HEAD request
cont_key = keys['container']
object_key = keys['object']
body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
'iv': FAKE_IV}
body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
return HeaderKeyDict({
'Etag': 'hashOfCiphertext',
'content-type': 'text/plain',
'content-length': content_length,
'X-Object-Sysmeta-Crypto-Etag': '%s; swift_meta=%s' % (
base64.b64encode(encrypt(plaintext_etag, object_key, FAKE_IV)),
get_crypto_meta_header()),
'X-Object-Sysmeta-Crypto-Body-Meta':
get_crypto_meta_header(body_crypto_meta),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', object_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-container-update-override-etag':
encrypt_and_append_meta('encrypt me, too', cont_key),
'x-object-sysmeta-test': 'do not encrypt me',
})
def test_GET_unencrypted_data(self):
# testing case of an unencrypted object with encrypted metadata from
# a later POST
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
obj_key = fetch_crypto_keys()['object']
hdrs = {'Etag': md5hex(body),
'content-type': 'text/plain',
'content-length': len(body),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', obj_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-test': 'do not encrypt me'}
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(md5hex(body), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
# POSTed user meta was encrypted
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
# PUT sysmeta was not encrypted
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
def test_headers_case(self):
body = 'fAkE ApP'
req = Request.blank('/v1/a/c/o', body='FaKe')
req.environ[CRYPTO_KEY_CALLBACK] = fetch_crypto_keys
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs.update({
'x-Object-mEta-ignoRes-caSe': 'thIs pArt WilL bE cOol',
})
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
status, headers, app_iter = req.call_application(self.decrypter)
self.assertEqual(status, '200 OK')
expected = {
'Etag': '7f7837924188f7b511a9e3881a9f77a8',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'encrypt me, too',
'X-Object-Meta-Test': 'encrypt me',
'Content-Length': '8',
'X-Object-Meta-Ignores-Case': 'thIs pArt WilL bE cOol',
'X-Object-Sysmeta-Test': 'do not encrypt me',
'Content-Type': 'text/plain',
}
self.assertEqual(dict(headers), expected)
self.assertEqual('fAkE ApP', ''.join(app_iter))
def test_GET_multipart_content_type(self):
# *just* having multipart content type shouldn't trigger the mime doc
# code path
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
# register request with fake swift
hdrs = self._make_response_headers(
len(ciphertext), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=ciphertext,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(plaintext), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
self.assertEqual(plaintext, resp.body)
def _test_request_success(self, method, body):
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)
# there shouldn't be any x-object-meta- headers, but if there are
# then the decrypted header will win where there is a name clash...
hdrs.update({
'x-object-meta-test': 'unexpected, overwritten by decrypted value',
'x-object-meta-distinct': 'unexpected but distinct from encrypted'
})
self.app.register(
method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
self.assertEqual('unexpected but distinct from encrypted',
resp.headers['x-object-meta-distinct'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
self.assertEqual(
'encrypt me, too',
resp.headers['X-Object-Sysmeta-Container-Update-Override-Etag'])
self.assertNotIn('X-Object-Sysmeta-Crypto-Body-Meta', resp.headers)
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', resp.headers)
return resp
def test_encrypt_header_val(self):
# Prepare key and Crypto instance
object_key = fetch_crypto_keys()['object']
# - Normal string can be crypted
encrypted = encrypter.encrypt_header_val(Crypto(), 'aaa', object_key)
# sanity: return value is 2 item tuple
self.assertEqual(2, len(encrypted))
crypted_val, crypt_info = encrypted
expected_crypt_val = base64.b64encode(
encrypt('aaa', object_key, FAKE_IV))
expected_crypt_info = {
'cipher': 'AES_CTR_256', 'iv': 'This is an IV123'}
self.assertEqual(expected_crypt_val, crypted_val)
self.assertEqual(expected_crypt_info, crypt_info)
# - Empty string raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), '', object_key)
self.assertEqual('empty value is not acceptable',
cm.exception.message)
# - None also raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), None, object_key)
self.assertEqual('empty value is not acceptable',
cm.exception.message)
def test_GET_multipart_ciphertext(self):
# build fake multipart response body
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET',
'/v1/a/c/o',
HTTPPartialContent,
body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart headers could be re-ordered, so parse response body to
# verify expected content
resp_lines = resp.body.split('\r\n')
resp_lines.reverse()
for start, end, ctype in parts:
self.assertEqual('--multipartboundary', resp_lines.pop())
expected_header_lines = {
'Content-Type: %s' % ctype,
'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
}
resp_header_lines = {resp_lines.pop(), resp_lines.pop()}
self.assertEqual(expected_header_lines, resp_header_lines)
self.assertEqual('', resp_lines.pop())
self.assertEqual(plaintext[start:end], resp_lines.pop())
self.assertEqual('--multipartboundary--', resp_lines.pop())
# we should have consumed the whole response body
self.assertFalse(resp_lines)
def test_GET_missing_etag_crypto_meta(self):
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted etag
hdrs['X-Object-Sysmeta-Crypto-Etag'] = \
base64.b64encode(encrypt(md5hex(body), key, FAKE_IV))
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header', resp.body)
self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multipart_ciphertext(self):
# build fake multipart response body
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
parts = ((0, 3, 'text/plain'),
(4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart headers could be re-ordered, so parse response body to
# verify expected content
resp_lines = resp.body.split('\r\n')
resp_lines.reverse()
for start, end, ctype in parts:
self.assertEqual('--multipartboundary', resp_lines.pop())
expected_header_lines = {
'Content-Type: %s' % ctype,
'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)}
resp_header_lines = {resp_lines.pop(), resp_lines.pop()}
self.assertEqual(expected_header_lines, resp_header_lines)
self.assertEqual('', resp_lines.pop())
self.assertEqual(plaintext[start:end], resp_lines.pop())
self.assertEqual('--multipartboundary--', resp_lines.pop())
# we should have consumed the whole response body
self.assertFalse(resp_lines)
def test_GET_missing_etag_crypto_meta(self):
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted etag
hdrs['X-Object-Sysmeta-Crypto-Etag'] = \
base64.b64encode(encrypt(md5hex(body), key, FAKE_IV))
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header', resp.body)
self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_bad_crypto_meta_for_user_metadata(self, method, bad_crypto_meta):
# use bad iv for metadata headers
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
enc_val = base64.b64encode(encrypt('encrypt me', key, FAKE_IV))
if bad_crypto_meta:
enc_val += ';swift_meta=' + get_crypto_meta_header(
crypto_meta=bad_crypto_meta)
hdrs['x-object-transient-sysmeta-crypto-meta-test'] = enc_val
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test', self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def _test_PUT_with_etag_override_in_headers(self, override_etag):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = b'FAKE APP'
plaintext_etag = md5hex(plaintext)
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
exp_etag = encrypt(override_etag.encode('ascii'), cont_key,
cont_etag_iv)
self.assertEqual(exp_etag, base64.b64decode(parts[0]))
def test_GET_cipher_mismatch_for_metadata(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(
len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs.update({'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', key, FAKE_IV)) +
';swift_meta=' +
get_crypto_meta_header(crypto_meta=bad_crypto_meta)})
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test', self.decrypter.logger.get_lines_for_level('error')[0])
def _test_PUT_with_etag_override_in_headers(self, override_etag):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = b'FAKE APP'
plaintext_etag = md5hex(plaintext)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag':
override_etag}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
exp_etag = encrypt(override_etag.encode('ascii'),
cont_key, cont_etag_iv)
self.assertEqual(exp_etag, base64.b64decode(parts[0]))
def test_GET_missing_key_callback(self):
# Do not provide keys, and do not set override flag
env = {'REQUEST_METHOD': 'GET'}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex('not the body'), fetch_crypto_keys(), 'not used')
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.',
resp.body)
self.assertIn('missing callback',
self.decrypter.logger.get_lines_for_level('error')[0])
def _verify_user_metadata(self, req_hdrs, name, value, key):
# verify encrypted version of user metadata
self.assertNotIn("X-Object-Meta-" + name, req_hdrs)
expected_hdr = "X-Object-Transient-Sysmeta-Crypto-Meta-" + name
self.assertIn(expected_hdr, req_hdrs)
enc_val, param = req_hdrs[expected_hdr].split(";")
param = param.strip()
self.assertTrue(param.startswith("swift_meta="))
actual_meta = json.loads(urllib.unquote_plus(param[len("swift_meta=") :]))
self.assertEqual(Crypto.cipher, actual_meta["cipher"])
meta_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(FAKE_IV, meta_iv)
self.assertEqual(base64.b64encode(encrypt(value, key, meta_iv)), enc_val)
# if there is any encrypted user metadata then this header should exist
self.assertIn("X-Object-Transient-Sysmeta-Crypto-Meta", req_hdrs)
common_meta = json.loads(urllib.unquote_plus(req_hdrs["X-Object-Transient-Sysmeta-Crypto-Meta"]))
self.assertDictEqual({"cipher": Crypto.cipher, "key_id": {"v": "fake", "path": "/a/c/fake"}}, common_meta)
def _test_bad_key(self, method):
# use bad key
def bad_fetch_crypto_keys():
keys = fetch_crypto_keys()
keys['object'] = 'bad key'
return keys
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
return req.get_response(self.decrypter)
def test_PUT_multiseg_good_client_etag(self):
body_key = os.urandom(32)
chunks = ["some", "chunks", "of data"]
body = "".join(chunks)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys, "wsgi.input": FileLikeIter(chunks)}
hdrs = {"content-type": "text/plain", "content-length": str(len(body)), "Etag": md5hex(body)}
req = Request.blank("/v1/a/c/o", environ=env, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
with mock.patch(
"swift.common.middleware.crypto.crypto_utils." "Crypto.create_random_key", lambda *args: body_key
):
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"})
self.assertEqual(encrypt(body, body_key, FAKE_IV), get_req.get_response(self.app).body)
def test_GET_missing_key_callback(self):
# Do not provide keys, and do not set override flag
env = {'REQUEST_METHOD': 'GET'}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex('not the body'),
fetch_crypto_keys(), 'not used')
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
self.assertIn('missing callback',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: raise_exc}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.',
resp.body)
self.assertIn('from callback: Testing',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta):
# use bad iv for object body
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_PUT_with_etag_override_in_headers(self):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = "FAKE APP"
plaintext_etag = md5hex(plaintext)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
"content-type": "text/plain",
"content-length": str(len(plaintext)),
"Etag": plaintext_etag,
"X-Object-Sysmeta-Container-Update-Override-Etag": "final etag",
}
req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
self.assertEqual(plaintext_etag, resp.headers["Etag"])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(("PUT", "/v1/a/c/o"), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()["container"]
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = "swift_meta="
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
self.assertEqual(Crypto().cipher, actual_meta["cipher"])
self.assertEqual(fetch_crypto_keys()["id"], actual_meta["key_id"])
cont_etag_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt("final etag", cont_key, cont_etag_iv), base64.b64decode(parts[0]))
def test_GET_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: raise_exc}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
self.assertIn('from callback: Testing',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multiseg(self):
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
hdrs = self._make_response_headers(
sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(),
body_key)
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta):
# use bad iv for object body
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_override_etag_bad_meta(self, method, bad_crypto_meta):
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted override etag
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
encrypt_and_append_meta(
md5hex(body), key, crypto_meta=bad_crypto_meta)
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header '
'X-Object-Sysmeta-Container-Update-Override-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def _test_request_success(self, method, body):
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(len(enc_body), plaintext_etag,
fetch_crypto_keys(), body_key)
# there shouldn't be any x-object-meta- headers, but if there are
# then the decrypted header will win where there is a name clash...
hdrs.update({
'x-object-meta-test':
'unexpected, overwritten by decrypted value',
'x-object-meta-distinct':
'unexpected but distinct from encrypted'
})
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
self.assertEqual('unexpected but distinct from encrypted',
resp.headers['x-object-meta-distinct'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
self.assertEqual(
'encrypt me, too',
resp.headers['X-Object-Sysmeta-Container-Update-Override-Etag'])
self.assertNotIn('X-Object-Sysmeta-Crypto-Body-Meta', resp.headers)
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', resp.headers)
return resp
def test_GET_multiseg(self):
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
hdrs = self._make_response_headers(sum(map(len,
enc_body)), plaintext_etag,
fetch_crypto_keys(), body_key)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
def _test_req_metadata_not_encrypted(self, method):
# check that metadata is not decrypted if it does not have crypto meta;
# testing for case of an unencrypted POST to an object.
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs.pop('x-object-transient-sysmeta-crypto-meta-test')
hdrs['x-object-meta-test'] = 'plaintext not encrypted'
self.app.register(
method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('plaintext not encrypted',
resp.headers['x-object-meta-test'])
def test_GET_cipher_mismatch_for_body(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(
len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
self.assertIn('Bad crypto meta: Cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta):
# build fake multipart response body
key = fetch_crypto_keys()['object']
ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, ctxt=ctxt)
parts = ((0, 3, 'text/plain'),
(4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), 'not used')
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_multiseg_with_range(self):
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
req.headers['Content-Range'] = 'bytes 3-10/17'
chunks = ['0123', '45678', '9abcdef']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]]
hdrs = self._make_response_headers(
sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(),
body_key)
hdrs['content-range'] = req.headers['Content-Range']
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('3456789a', resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
def test_PUT_multiseg_no_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def _test_bad_key(self, method):
# use bad key
def bad_fetch_crypto_keys():
keys = fetch_crypto_keys()
keys['object'] = 'bad key'
return keys
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
return req.get_response(self.decrypter)
def _verify_user_metadata(self, req_hdrs, name, value, key):
# verify encrypted version of user metadata
self.assertNotIn('X-Object-Meta-' + name, req_hdrs)
expected_hdr = 'X-Object-Transient-Sysmeta-Crypto-Meta-' + name
self.assertIn(expected_hdr, req_hdrs)
enc_val, param = req_hdrs[expected_hdr].split(';')
param = param.strip()
self.assertTrue(param.startswith('swift_meta='))
actual_meta = json.loads(
urlparse.unquote_plus(param[len('swift_meta='):]))
self.assertEqual(Crypto.cipher, actual_meta['cipher'])
meta_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, meta_iv)
self.assertEqual(
base64.b64encode(encrypt(value, key, meta_iv)),
enc_val)
# if there is any encrypted user metadata then this header should exist
self.assertIn('X-Object-Transient-Sysmeta-Crypto-Meta', req_hdrs)
common_meta = json.loads(urlparse.unquote_plus(
req_hdrs['X-Object-Transient-Sysmeta-Crypto-Meta']))
self.assertDictEqual({'cipher': Crypto.cipher,
'key_id': {'v': 'fake', 'path': '/a/c/fake'}},
common_meta)
def test_PUT_multiseg_no_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {
'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)
}
hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key', lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def _verify_user_metadata(self, req_hdrs, name, value, key):
# verify encrypted version of user metadata
self.assertNotIn('X-Object-Meta-' + name, req_hdrs)
expected_hdr = 'X-Object-Transient-Sysmeta-Crypto-Meta-' + name
self.assertIn(expected_hdr, req_hdrs)
enc_val, param = req_hdrs[expected_hdr].split(';')
param = param.strip()
self.assertTrue(param.startswith('swift_meta='))
actual_meta = json.loads(
urlparse.unquote_plus(param[len('swift_meta='):]))
self.assertEqual(Crypto.cipher, actual_meta['cipher'])
meta_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, meta_iv)
self.assertEqual(
base64.b64encode(encrypt(wsgi_to_bytes(value), key, meta_iv)),
wsgi_to_bytes(enc_val))
# if there is any encrypted user metadata then this header should exist
self.assertIn('X-Object-Transient-Sysmeta-Crypto-Meta', req_hdrs)
common_meta = json.loads(urlparse.unquote_plus(
req_hdrs['X-Object-Transient-Sysmeta-Crypto-Meta']))
self.assertDictEqual({'cipher': Crypto.cipher,
'key_id': {'v': 'fake', 'path': '/a/c/fake'}},
common_meta)
def test_GET_cipher_mismatch_for_body(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(len(enc_body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
self.assertIn('Bad crypto meta: Cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta):
# build fake multipart response body
key = fetch_crypto_keys()['object']
ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, ctxt=ctxt)
parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), 'not used')
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_PUT_with_other_footers(self, override_etag):
# verify handling of another middleware's footer callback
cont_key = fetch_crypto_keys()['container']
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
other_footers = {
'Etag': plaintext_etag,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Size':
'other override',
'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag
}
env = {
'REQUEST_METHOD':
'PUT',
CRYPTO_KEY_CALLBACK:
fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'correct etag is in footers'
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key', lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that other middleware's footers made it to app, including any
# container update overrides but nothing Etag-related
other_footers.pop('Etag')
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify encryption footers are ok
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)
# verify encrypted etag for container update
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
def test_PUT_req(self):
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'etag': plaintext_etag,
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
return_value=body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
# verify etag
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
# verify crypto_meta was appended to this etag
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
# verify encrypted version of plaintext etag
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
self.assertEqual(enc_etag, actual)
# verify etag MAC for conditional requests
actual_hmac = base64.b64decode(
req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac'])
self.assertEqual(
actual_hmac,
hmac.new(object_key, plaintext_etag, hashlib.sha256).digest())
# verify encrypted etag for container update
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# content-type is not encrypted
self.assertEqual('text/plain', req_hdrs['Content-Type'])
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(req_hdrs, 'Etag',
'not to be confused with the Etag!',
object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual(ciphertext, resp.body)
self.assertEqual(ciphertext_etag, resp.headers['Etag'])
def _test_ondisk_data_after_write_with_crypto(self, policy_name):
policy = storage_policy.POLICIES.get_by_name(policy_name)
self._create_container(self.proxy_app, policy_name=policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
# Verify container listing etag is encrypted by direct GET to container
# server. We can use any server for all nodes since they all share same
# devices dir.
cont_server = self._test_context['test_servers'][3]
cont_ring = Ring(self._test_context['testdir'], ring_name='container')
part, nodes = cont_ring.get_nodes('a', self.container_name)
for node in nodes:
req = Request.blank('/%s/%s/a/%s'
% (node['device'], part, self.container_name),
method='GET', query_string='format=json')
resp = req.get_response(cont_server)
listing = json.loads(resp.body)
# sanity checks...
self.assertEqual(1, len(listing))
self.assertEqual('o', listing[0]['name'])
self.assertEqual('application/test', listing[0]['content_type'])
# verify encrypted etag value
parts = listing[0]['hash'].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag,
self.km.create_key('/a/%s' % self.container_name),
listing_etag_iv))
self.assertEqual(exp_enc_listing_etag, parts[0])
# Verify diskfile data and metadata is encrypted
ring_object = self.proxy_app.get_object_ring(int(policy))
partition, nodes = ring_object.get_nodes('a', self.container_name, 'o')
conf = {'devices': self._test_context["testdir"],
'mount_check': 'false'}
df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy]
ondisk_data = []
exp_enc_body = None
for node_index, node in enumerate(nodes):
df = df_mgr.get_diskfile(node['device'], partition,
'a', self.container_name, 'o',
policy=policy)
with df.open():
meta = df.get_metadata()
contents = ''.join(df.reader())
metadata = dict((k.lower(), v) for k, v in meta.items())
# verify on disk data - body
body_iv = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['iv']
body_key_meta = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['body_key']
obj_key = self.km.create_key('/a/%s/o' % self.container_name)
body_key = Crypto().unwrap_key(obj_key, body_key_meta)
exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
ondisk_data.append((node, contents))
# verify on disk user metadata
enc_val, meta = metadata[
'x-object-transient-sysmeta-crypto-meta-fruit'].split(';')
meta = meta.strip()[len('swift_meta='):]
metadata_iv = load_crypto_meta(meta)['iv']
exp_enc_meta = base64.b64encode(encrypt('Kiwi', obj_key,
metadata_iv))
self.assertEqual(exp_enc_meta, enc_val)
self.assertNotIn('x-object-meta-fruit', metadata)
self.assertIn(
'x-object-transient-sysmeta-crypto-meta', metadata)
meta = load_crypto_meta(
metadata['x-object-transient-sysmeta-crypto-meta'])
self.assertIn('key_id', meta)
self.assertIn('path', meta['key_id'])
self.assertEqual(
'/a/%s/%s' % (self.container_name, self.object_name),
meta['key_id']['path'])
self.assertIn('v', meta['key_id'])
self.assertEqual('1', meta['key_id']['v'])
self.assertIn('cipher', meta)
self.assertEqual(Crypto.cipher, meta['cipher'])
# verify etag
actual_enc_etag, _junk, actual_etag_meta = metadata[
'x-object-sysmeta-crypto-etag'].partition('; swift_meta=')
etag_iv = load_crypto_meta(actual_etag_meta)['iv']
exp_enc_etag = base64.b64encode(encrypt(self.plaintext_etag,
obj_key, etag_iv))
self.assertEqual(exp_enc_etag, actual_enc_etag)
# verify etag hmac
exp_etag_mac = hmac.new(
obj_key, self.plaintext_etag, digestmod=hashlib.sha256)
exp_etag_mac = base64.b64encode(exp_etag_mac.digest())
self.assertEqual(exp_etag_mac,
metadata['x-object-sysmeta-crypto-etag-mac'])
# verify etag override for container updates
override = 'x-object-sysmeta-container-update-override-etag'
parts = metadata[override].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
cont_key = self.km.create_key('/a/%s' % self.container_name)
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag, cont_key,
listing_etag_iv))
self.assertEqual(exp_enc_listing_etag, parts[0])
self._check_GET_and_HEAD(self.crypto_app)
return exp_enc_body, ondisk_data
def _test_PUT_with_other_footers(self, override_etag):
# verify handling of another middleware's footer callback
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
other_footers = {
'Etag': plaintext_etag,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Size':
'other override',
'X-Object-Sysmeta-Container-Update-Override-Etag':
override_etag}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'correct etag is in footers'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that other middleware's footers made it to app, including any
# container update overrides but nothing Etag-related
other_footers.pop('Etag')
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify encryption footers are ok
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
self.assertTrue(etag_meta)
actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urlparse.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV,
base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
def test_PUT_nothing_read(self):
# simulate an artificial scenario of a downstream filter/app not
# actually reading the input stream from encrypter.
class NonReadingApp(object):
def __call__(self, env, start_response):
# note: no read from wsgi.input
req = Request(env)
env['swift.callback.update_footers'](req.headers)
call_headers.append(req.headers)
resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
return resp(env, start_response)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': 0,
'etag': 'etag from client'}
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
self.assertEqual('etag from client', call_headers[0]['etag'])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# check that an upstream footer callback gets called
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify encrypted override etag for container update.
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', call_headers[0])
parts = call_headers[0][
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt('other override', cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify that other middleware's footers made it to app
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# if upstream footer override etag is for an empty body then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# if upstream footer override etag is an empty string then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Container-Update-Override-Etag': ''}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
def encrypt_and_append_meta(value, key, crypto_meta=None):
return '%s; swift_meta=%s' % (base64.b64encode(encrypt(
value, key, FAKE_IV)), get_crypto_meta_header(crypto_meta))
def test_PUT_req(self):
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'etag': plaintext_etag,
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
return_value=body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urlparse.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV,
base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
# verify etag
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
# verify crypto_meta was appended to this etag
self.assertTrue(etag_meta)
actual_meta = json.loads(urlparse.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
# verify encrypted version of plaintext etag
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
self.assertEqual(enc_etag, actual)
# verify etag MAC for conditional requests
actual_hmac = base64.b64decode(
req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac'])
self.assertEqual(actual_hmac, hmac.new(
object_key, plaintext_etag, hashlib.sha256).digest())
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# content-type is not encrypted
self.assertEqual('text/plain', req_hdrs['Content-Type'])
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(
req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual(ciphertext, resp.body)
self.assertEqual(ciphertext_etag, resp.headers['Etag'])
def _test_ondisk_data_after_write_with_crypto(self, policy_name):
policy = storage_policy.POLICIES.get_by_name(policy_name)
self._create_container(self.proxy_app, policy_name=policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
# Verify container listing etag is encrypted by direct GET to container
# server. We can use any server for all nodes since they all share same
# devices dir.
cont_server = self._test_context['test_servers'][3]
cont_ring = Ring(self._test_context['testdir'], ring_name='container')
part, nodes = cont_ring.get_nodes('a', self.container_name)
for node in nodes:
req = Request.blank('/%s/%s/a/%s' %
(node['device'], part, self.container_name),
method='GET',
query_string='format=json')
resp = req.get_response(cont_server)
listing = json.loads(resp.body)
# sanity checks...
self.assertEqual(1, len(listing))
self.assertEqual('o', listing[0]['name'])
self.assertEqual('application/test', listing[0]['content_type'])
# verify encrypted etag value
parts = listing[0]['hash'].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'),
self.km.create_key('/a/%s' % self.container_name),
listing_etag_iv)).decode('ascii')
self.assertEqual(exp_enc_listing_etag, parts[0])
# Verify diskfile data and metadata is encrypted
ring_object = self.proxy_app.get_object_ring(int(policy))
partition, nodes = ring_object.get_nodes('a', self.container_name, 'o')
conf = {
'devices': self._test_context["testdir"],
'mount_check': 'false'
}
df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy]
ondisk_data = []
exp_enc_body = None
for node_index, node in enumerate(nodes):
df = df_mgr.get_diskfile(node['device'],
partition,
'a',
self.container_name,
'o',
policy=policy)
with df.open():
meta = df.get_metadata()
contents = b''.join(df.reader())
metadata = dict((k.lower(), v) for k, v in meta.items())
# verify on disk data - body
body_iv = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['iv']
body_key_meta = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['body_key']
obj_key = self.km.create_key('/a/%s/o' % self.container_name)
body_key = Crypto().unwrap_key(obj_key, body_key_meta)
exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
ondisk_data.append((node, contents))
# verify on disk user metadata
enc_val, meta = metadata[
'x-object-transient-sysmeta-crypto-meta-fruit'].split(';')
meta = meta.strip()[len('swift_meta='):]
metadata_iv = load_crypto_meta(meta)['iv']
exp_enc_meta = base64.b64encode(
encrypt(b'Kiwi', obj_key, metadata_iv)).decode('ascii')
self.assertEqual(exp_enc_meta, enc_val)
self.assertNotIn('x-object-meta-fruit', metadata)
self.assertIn('x-object-transient-sysmeta-crypto-meta',
metadata)
meta = load_crypto_meta(
metadata['x-object-transient-sysmeta-crypto-meta'])
self.assertIn('key_id', meta)
self.assertIn('path', meta['key_id'])
self.assertEqual(
'/a/%s/%s' % (self.container_name, self.object_name),
meta['key_id']['path'])
self.assertIn('v', meta['key_id'])
self.assertEqual('2', meta['key_id']['v'])
self.assertIn('cipher', meta)
self.assertEqual(Crypto.cipher, meta['cipher'])
# verify etag
actual_enc_etag, _junk, actual_etag_meta = metadata[
'x-object-sysmeta-crypto-etag'].partition('; swift_meta=')
etag_iv = load_crypto_meta(actual_etag_meta)['iv']
exp_enc_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'), obj_key,
etag_iv)).decode('ascii')
self.assertEqual(exp_enc_etag, actual_enc_etag)
# verify etag hmac
exp_etag_mac = hmac.new(obj_key,
self.plaintext_etag.encode('ascii'),
digestmod=hashlib.sha256).digest()
exp_etag_mac = base64.b64encode(exp_etag_mac).decode('ascii')
self.assertEqual(exp_etag_mac,
metadata['x-object-sysmeta-crypto-etag-mac'])
# verify etag override for container updates
override = 'x-object-sysmeta-container-update-override-etag'
parts = metadata[override].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
cont_key = self.km.create_key('/a/%s' % self.container_name)
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'), cont_key,
listing_etag_iv)).decode('ascii')
self.assertEqual(exp_enc_listing_etag, parts[0])
self._check_GET_and_HEAD(self.crypto_app)
return exp_enc_body, ondisk_data
def test_PUT_nothing_read(self):
# simulate an artificial scenario of a downstream filter/app not
# actually reading the input stream from encrypter.
class NonReadingApp(object):
def __call__(self, env, start_response):
# note: no read from wsgi.input
req = Request(env)
env['swift.callback.update_footers'](req.headers)
call_headers.append(req.headers)
resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
return resp(env, start_response)
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'content-type': 'text/plain',
'content-length': 0,
'etag': 'etag from client'
}
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
self.assertEqual('etag from client', call_headers[0]['etag'])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# check that an upstream footer callback gets called
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'
}
env.update({
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify encrypted override etag for container update.
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
call_headers[0])
parts = call_headers[0][
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt('other override', cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify that other middleware's footers made it to app
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# if upstream footer override etag is for an empty body then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG
}
env.update({
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)
})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))