def test_get_multiple_keys(self):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
mutliple_keys = self.crypto_context.get_multiple_keys(env)
self.assertEqual([
fetch_crypto_keys(),
fetch_crypto_keys(key_id={'secret_id': 'myid'})
], mutliple_keys)
def _test_override_etag_bad_meta(self, method, bad_crypto_meta):
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted override etag
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
encrypt_and_append_meta(
md5hex(body), key, crypto_meta=bad_crypto_meta)
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn(
'Error decrypting header '
'X-Object-Sysmeta-Container-Update-Override-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def test_get_keys(self):
# ok
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(fetch_crypto_keys(), keys)
# only default required keys are checked
subset_keys = {'object': fetch_crypto_keys()['object']}
env = {CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: subset_keys}
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(subset_keys, keys)
# only specified required keys are checked
subset_keys = {'container': fetch_crypto_keys()['container']}
env = {CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: subset_keys}
keys = self.crypto_context.get_keys(env, required=['container'])
self.assertDictEqual(subset_keys, keys)
subset_keys = {
'object': fetch_crypto_keys()['object'],
'container': fetch_crypto_keys()['container']
}
env = {CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: subset_keys}
keys = self.crypto_context.get_keys(env,
required=['object', 'container'])
self.assertDictEqual(subset_keys, keys)
def _test_bad_crypto_meta_for_user_metadata(self, method, bad_crypto_meta):
# use bad iv for metadata headers
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
enc_val = base64.b64encode(encrypt('encrypt me', key, FAKE_IV))
if bad_crypto_meta:
enc_val += ';swift_meta=' + get_crypto_meta_header(
crypto_meta=bad_crypto_meta)
hdrs['x-object-transient-sysmeta-crypto-meta-test'] = enc_val
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test',
self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def test_get_multiple_keys(self):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
mutliple_keys = self.crypto_context.get_multiple_keys(env)
self.assertEqual(
[fetch_crypto_keys(),
fetch_crypto_keys(key_id={'secret_id': 'myid'})],
mutliple_keys)
def test_GET_cipher_mismatch_for_metadata(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(len(enc_body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs.update({
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', key, FAKE_IV)) +
';swift_meta=' +
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
})
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting header', resp.body)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test',
self.decrypter.logger.get_lines_for_level('error')[0])
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ', '.join(plain_etags)
req = Request.blank(
'/v1/a/c/o',
environ=env,
method=method,
headers={match_header_name: match_header_value})
app = FakeSwift()
app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != '*':
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(', '))
# masked values for secret_id None
key = fetch_crypto_keys()['object']
masked_etags = [
'"%s"' % bytes_to_wsgi(
base64.b64encode(
hmac.new(key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')
]
# masked values for secret_id myid
key = fetch_crypto_keys(key_id={'secret_id': 'myid'})['object']
masked_etags_myid = [
'"%s"' % bytes_to_wsgi(
base64.b64encode(
hmac.new(key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')
]
expected_etags = set((expected_plain_etags or plain_etags) +
masked_etags + masked_etags_myid)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags),
set(req.headers[match_header_name].split(', ')))
def _test_PUT_with_etag_override_in_headers(self, override_etag):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = b'FAKE APP'
plaintext_etag = md5hex(plaintext)
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag
}
req = Request.blank('/v1/a/c/o',
environ=env,
body=plaintext,
headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
exp_etag = encrypt(override_etag.encode('ascii'), cont_key,
cont_etag_iv)
self.assertEqual(exp_etag, base64.b64decode(parts[0]))
def _test_PUT_with_etag_override_in_headers(self, override_etag):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = b'FAKE APP'
plaintext_etag = md5hex(plaintext)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag':
override_etag}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urlparse.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
exp_etag = encrypt(override_etag.encode('ascii'),
cont_key, cont_etag_iv)
self.assertEqual(exp_etag, base64.b64decode(parts[0]))
def test_encrypt_header_val(self):
# Prepare key and Crypto instance
object_key = fetch_crypto_keys()['object']
# - Normal string can be crypted
encrypted = encrypter.encrypt_header_val(Crypto(), 'aaa', object_key)
# sanity: return value is 2 item tuple
self.assertEqual(2, len(encrypted))
crypted_val, crypt_info = encrypted
expected_crypt_val = base64.b64encode(
encrypt('aaa', object_key, FAKE_IV))
expected_crypt_info = {
'cipher': 'AES_CTR_256', 'iv': 'This is an IV123'}
self.assertEqual(expected_crypt_val, crypted_val)
self.assertEqual(expected_crypt_info, crypt_info)
# - Empty string raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), '', object_key)
self.assertEqual('empty value is not acceptable',
cm.exception.message)
# - None also raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), None, object_key)
self.assertEqual('empty value is not acceptable',
cm.exception.message)
def _test_etag_is_at_not_duplicated(self, method):
# verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
# X-Backend-Etag-Is-At
key = fetch_crypto_keys()['object']
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={'If-Match': '"an etag"',
'If-None-Match': '"another etag"'})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'an etag', hashlib.sha256).digest()),
actual_headers['If-Match'])
self.assertIn('"another etag"', actual_headers['If-None-Match'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'another etag', hashlib.sha256).digest()),
actual_headers['If-None-Match'])
def test_GET_container_json(self):
content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
content_type_2 = 'text/plain; param=foo'
pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
key = fetch_crypto_keys()['container']
obj_dict_1 = {"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": encrypt_and_append_meta(
pt_etag1.encode('utf-8'), key),
"name": "testfile",
"content_type": content_type_1}
obj_dict_2 = {"bytes": 24,
"last_modified": "2015-04-14T23:33:06.519020",
"hash": encrypt_and_append_meta(
pt_etag2.encode('utf-8'), key),
"name": "testfile2",
"content_type": content_type_2}
listing = [obj_dict_1, obj_dict_2]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(2, len(body_json))
obj_dict_1['hash'] = pt_etag1
self.assertDictEqual(obj_dict_1, body_json[0])
obj_dict_2['hash'] = pt_etag2
self.assertDictEqual(obj_dict_2, body_json[1])
def test_cont_get_json_req_with_cipher_mismatch(self):
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
key = fetch_crypto_keys()['container']
pt_etag = 'c6e8196d7f0fff6444b90861fe8d609d'
ct_etag = encrypt_and_append_meta(pt_etag,
key,
crypto_meta=bad_crypto_meta)
obj_dict_1 = {
"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": ct_etag,
"name": "testfile",
"content_type": "image/jpeg"
}
listing = [obj_dict_1]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting container listing', resp.body)
self.assertIn("Cipher must be AES_CTR_256",
self.decrypter.logger.get_lines_for_level('error')[0])
def test_headers_case(self):
body = 'fAkE ApP'
req = Request.blank('/v1/a/c/o', body='FaKe')
req.environ[CRYPTO_KEY_CALLBACK] = fetch_crypto_keys
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(len(enc_body), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs.update({
'x-Object-mEta-ignoRes-caSe': 'thIs pArt WilL bE cOol',
})
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
status, headers, app_iter = req.call_application(self.decrypter)
self.assertEqual(status, '200 OK')
expected = {
'Etag': '7f7837924188f7b511a9e3881a9f77a8',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'encrypt me, too',
'X-Object-Meta-Test': 'encrypt me',
'Content-Length': '8',
'X-Object-Meta-Ignores-Case': 'thIs pArt WilL bE cOol',
'X-Object-Sysmeta-Test': 'do not encrypt me',
'Content-Type': 'text/plain',
}
self.assertEqual(dict(headers), expected)
self.assertEqual('fAkE ApP', ''.join(app_iter))
def test_POST_req(self):
body = 'FAKE APP'
env = {
'REQUEST_METHOD': 'POST',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
hdrs = {
'x-object-meta-test': 'encrypt me',
'x-object-meta-test2': '',
'x-object-sysmeta-test': 'do not encrypt me'
}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
key = fetch_crypto_keys()['object']
self.app.register('POST', '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertNotIn('Etag', resp.headers)
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('POST', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', key)
# unless it had no value
self.assertEqual('', req_hdrs['X-Object-Meta-Test2'])
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
def test_GET_missing_key_callback(self):
# Do not provide keys, and do not set override flag
env = {'REQUEST_METHOD': 'GET'}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex('not the body'), fetch_crypto_keys(), 'not used')
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.',
resp.body)
self.assertIn('missing callback',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_req_metadata_not_encrypted(self, method):
# check that metadata is not decrypted if it does not have crypto meta;
# testing for case of an unencrypted POST to an object.
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs.pop('x-object-transient-sysmeta-crypto-meta-test')
hdrs['x-object-meta-test'] = 'plaintext not encrypted'
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('plaintext not encrypted',
resp.headers['x-object-meta-test'])
def test_GET_multipart_content_type(self):
# *just* having multipart content type shouldn't trigger the mime doc
# code path
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
# register request with fake swift
hdrs = self._make_response_headers(
len(ciphertext), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=ciphertext,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(plaintext), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
self.assertEqual(plaintext, resp.body)
def test_GET_unencrypted_data(self):
# testing case of an unencrypted object with encrypted metadata from
# a later POST
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
obj_key = fetch_crypto_keys()['object']
hdrs = {
'Etag':
md5hex(body),
'content-type':
'text/plain',
'content-length':
len(body),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', obj_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-test':
'do not encrypt me'
}
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(md5hex(body), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
# POSTed user meta was encrypted
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
# PUT sysmeta was not encrypted
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
def test_GET_multiseg_with_range(self):
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
req.headers['Content-Range'] = 'bytes 3-10/17'
chunks = ['0123', '45678', '9abcdef']
body = ''.join(chunks)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]]
hdrs = self._make_response_headers(sum(map(len,
enc_body)), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs['content-range'] = req.headers['Content-Range']
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('3456789a', resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
def test_GET_unencrypted_data(self):
# testing case of an unencrypted object with encrypted metadata from
# a later POST
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
obj_key = fetch_crypto_keys()['object']
hdrs = {'Etag': md5hex(body),
'content-type': 'text/plain',
'content-length': len(body),
'x-object-transient-sysmeta-crypto-meta-test':
base64.b64encode(encrypt('encrypt me', obj_key, FAKE_IV)) +
';swift_meta=' + get_crypto_meta_header(),
'x-object-sysmeta-test': 'do not encrypt me'}
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual(body, resp.body)
self.assertEqual('200 OK', resp.status)
self.assertEqual(md5hex(body), resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
# POSTed user meta was encrypted
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
# PUT sysmeta was not encrypted
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
def test_GET_multipart_content_type(self):
# *just* having multipart content type shouldn't trigger the mime doc
# code path
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
# register request with fake swift
hdrs = self._make_response_headers(len(ciphertext), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=ciphertext,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(plaintext), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
self.assertEqual(plaintext, resp.body)
def test_POST_req(self):
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'POST',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'x-object-meta-test': 'encrypt me',
'x-object-meta-test2': '',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
key = fetch_crypto_keys()['object']
self.app.register('POST', '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertNotIn('Etag', resp.headers)
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('POST', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', key)
# unless it had no value
self.assertEqual('', req_hdrs['X-Object-Meta-Test2'])
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
def _test_request_success(self, method, body):
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)
# there shouldn't be any x-object-meta- headers, but if there are
# then the decrypted header will win where there is a name clash...
hdrs.update({
'x-object-meta-test': 'unexpected, overwritten by decrypted value',
'x-object-meta-distinct': 'unexpected but distinct from encrypted'
})
self.app.register(
method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual('text/plain', resp.headers['Content-Type'])
self.assertEqual('encrypt me', resp.headers['x-object-meta-test'])
self.assertEqual('unexpected but distinct from encrypted',
resp.headers['x-object-meta-distinct'])
self.assertEqual('do not encrypt me',
resp.headers['x-object-sysmeta-test'])
self.assertEqual(
'encrypt me, too',
resp.headers['X-Object-Sysmeta-Container-Update-Override-Etag'])
self.assertNotIn('X-Object-Sysmeta-Crypto-Body-Meta', resp.headers)
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', resp.headers)
return resp
def _test_etag_is_at_not_duplicated(self, method):
# verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
# X-Backend-Etag-Is-At
key = fetch_crypto_keys()['object']
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o',
environ=env,
method=method,
headers={
'If-Match': '"an etag"',
'If-None-Match': '"another etag"'
})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
self.assertIn(
'"%s"' % base64.b64encode(
hmac.new(key, 'an etag', hashlib.sha256).digest()),
actual_headers['If-Match'])
self.assertIn('"another etag"', actual_headers['If-None-Match'])
self.assertIn(
'"%s"' % base64.b64encode(
hmac.new(key, 'another etag', hashlib.sha256).digest()),
actual_headers['If-None-Match'])
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ", ".join(plain_etags)
req = Request.blank(
"/v1/a/c/o", environ=env, method=method, headers={match_header_name: match_header_value}
)
app = FakeSwift()
app.register(method, "/v1/a/c/o", HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual("200 OK", resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != "*":
self.assertIn("X-Backend-Etag-Is-At", actual_headers)
self.assertEqual("X-Object-Sysmeta-Crypto-Etag-Mac", actual_headers["X-Backend-Etag-Is-At"])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(", "))
key = fetch_crypto_keys()["object"]
masked_etags = [
'"%s"' % base64.b64encode(hmac.new(key, etag.strip('"'), hashlib.sha256).digest())
for etag in plain_etags
if etag not in ("*", "")
]
expected_etags = set((expected_plain_etags or plain_etags) + masked_etags)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags), set(req.headers[match_header_name].split(", ")))
def test_headers_case(self):
body = 'fAkE ApP'
req = Request.blank('/v1/a/c/o', body='FaKe')
req.environ[CRYPTO_KEY_CALLBACK] = fetch_crypto_keys
plaintext_etag = md5hex(body)
body_key = os.urandom(32)
enc_body = encrypt(body, body_key, FAKE_IV)
hdrs = self._make_response_headers(
len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs.update({
'x-Object-mEta-ignoRes-caSe': 'thIs pArt WilL bE cOol',
})
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
status, headers, app_iter = req.call_application(self.decrypter)
self.assertEqual(status, '200 OK')
expected = {
'Etag': '7f7837924188f7b511a9e3881a9f77a8',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'encrypt me, too',
'X-Object-Meta-Test': 'encrypt me',
'Content-Length': '8',
'X-Object-Meta-Ignores-Case': 'thIs pArt WilL bE cOol',
'X-Object-Sysmeta-Test': 'do not encrypt me',
'Content-Type': 'text/plain',
}
self.assertEqual(dict(headers), expected)
self.assertEqual('fAkE ApP', ''.join(app_iter))
def test_encrypt_header_val(self):
# Prepare key and Crypto instance
object_key = fetch_crypto_keys()['object']
# - Normal string can be crypted
encrypted = encrypter.encrypt_header_val(Crypto(), 'aaa', object_key)
# sanity: return value is 2 item tuple
self.assertEqual(2, len(encrypted))
crypted_val, crypt_info = encrypted
expected_crypt_val = base64.b64encode(
encrypt('aaa', object_key, FAKE_IV))
expected_crypt_info = {
'cipher': 'AES_CTR_256',
'iv': 'This is an IV123'
}
self.assertEqual(expected_crypt_val, crypted_val)
self.assertEqual(expected_crypt_info, crypt_info)
# - Empty string raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), '', object_key)
self.assertEqual('empty value is not acceptable', cm.exception.message)
# - None also raises a ValueError for safety
with self.assertRaises(ValueError) as cm:
encrypter.encrypt_header_val(Crypto(), None, object_key)
self.assertEqual('empty value is not acceptable', cm.exception.message)
def test_get_keys_with_crypto_meta(self):
# verify that key_id from crypto_meta is passed to fetch_crypto_keys
keys = fetch_crypto_keys()
mock_fetch_crypto_keys = mock.MagicMock(return_value=keys)
env = {CRYPTO_KEY_CALLBACK: mock_fetch_crypto_keys}
key_id = {'secret_id': '123'}
keys = self.crypto_context.get_keys(env, key_id=key_id)
self.assertDictEqual(fetch_crypto_keys(), keys)
mock_fetch_crypto_keys.assert_called_with(key_id={'secret_id': '123'})
# but it's ok for there to be no crypto_meta
keys = self.crypto_context.get_keys(env, key_id={})
self.assertDictEqual(fetch_crypto_keys(), keys)
mock_fetch_crypto_keys.assert_called_with(key_id={})
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(fetch_crypto_keys(), keys)
mock_fetch_crypto_keys.assert_called_with(key_id=None)
def test_get_keys_with_crypto_meta(self):
# verify that key_id from crypto_meta is passed to fetch_crypto_keys
keys = fetch_crypto_keys()
mock_fetch_crypto_keys = mock.MagicMock(return_value=keys)
env = {CRYPTO_KEY_CALLBACK: mock_fetch_crypto_keys}
key_id = {'secret_id': '123'}
keys = self.crypto_context.get_keys(env, key_id=key_id)
self.assertDictEqual(fetch_crypto_keys(), keys)
mock_fetch_crypto_keys.assert_called_with(key_id={'secret_id': '123'})
# but it's ok for there to be no crypto_meta
keys = self.crypto_context.get_keys(env, key_id={})
self.assertDictEqual(fetch_crypto_keys(), keys)
mock_fetch_crypto_keys.assert_called_with(key_id={})
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(fetch_crypto_keys(), keys)
mock_fetch_crypto_keys.assert_called_with(key_id=None)
def _test_bad_key(self, method):
# use bad key
def bad_fetch_crypto_keys():
keys = fetch_crypto_keys()
keys['object'] = 'bad key'
return keys
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
return req.get_response(self.decrypter)
def test_GET_multipart_ciphertext(self):
# build fake multipart response body
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET',
'/v1/a/c/o',
HTTPPartialContent,
body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart headers could be re-ordered, so parse response body to
# verify expected content
resp_lines = resp.body.split('\r\n')
resp_lines.reverse()
for start, end, ctype in parts:
self.assertEqual('--multipartboundary', resp_lines.pop())
expected_header_lines = {
'Content-Type: %s' % ctype,
'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
}
resp_header_lines = {resp_lines.pop(), resp_lines.pop()}
self.assertEqual(expected_header_lines, resp_header_lines)
self.assertEqual('', resp_lines.pop())
self.assertEqual(plaintext[start:end], resp_lines.pop())
self.assertEqual('--multipartboundary--', resp_lines.pop())
# we should have consumed the whole response body
self.assertFalse(resp_lines)
def test_GET_missing_key_callback(self):
# Do not provide keys, and do not set override flag
env = {'REQUEST_METHOD': 'GET'}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex('not the body'),
fetch_crypto_keys(), 'not used')
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
self.assertIn('missing callback',
self.decrypter.logger.get_lines_for_level('error')[0])
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ', '.join(plain_etags)
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={match_header_name: match_header_value})
app = FakeSwift()
app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != '*':
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(', '))
# masked values for secret_id None
key = fetch_crypto_keys()['object']
masked_etags = [
'"%s"' % bytes_to_wsgi(base64.b64encode(hmac.new(
key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')]
# masked values for secret_id myid
key = fetch_crypto_keys(key_id={'secret_id': 'myid'})['object']
masked_etags_myid = [
'"%s"' % bytes_to_wsgi(base64.b64encode(hmac.new(
key, wsgi_to_bytes(etag.strip('"')),
hashlib.sha256).digest()))
for etag in plain_etags if etag not in ('*', '')]
expected_etags = set((expected_plain_etags or plain_etags) +
masked_etags + masked_etags_myid)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags),
set(req.headers[match_header_name].split(', ')))
def test_GET_missing_etag_crypto_meta(self):
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted etag
hdrs['X-Object-Sysmeta-Crypto-Etag'] = \
base64.b64encode(encrypt(md5hex(body), key, FAKE_IV))
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header', resp.body)
self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_get_keys_missing_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys.pop('object')
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Missing key for 'object'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta):
# use bad iv for object body
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: raise_exc}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.',
resp.body)
self.assertIn('from callback: Testing',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_get_keys_missing_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys.pop('object')
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Missing key for 'object'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_bad_object_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys['object'] = 'the minor key'
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: bad_keys})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Bad key for 'object'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_GET_multipart_ciphertext(self):
# build fake multipart response body
body_key = os.urandom(32)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
parts = ((0, 3, 'text/plain'),
(4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), body_key)
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body,
headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('206 Partial Content', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(len(body), int(resp.headers['Content-Length']))
self.assertEqual('multipart/byteranges;boundary=multipartboundary',
resp.headers['Content-Type'])
# the multipart headers could be re-ordered, so parse response body to
# verify expected content
resp_lines = resp.body.split('\r\n')
resp_lines.reverse()
for start, end, ctype in parts:
self.assertEqual('--multipartboundary', resp_lines.pop())
expected_header_lines = {
'Content-Type: %s' % ctype,
'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)}
resp_header_lines = {resp_lines.pop(), resp_lines.pop()}
self.assertEqual(expected_header_lines, resp_header_lines)
self.assertEqual('', resp_lines.pop())
self.assertEqual(plaintext[start:end], resp_lines.pop())
self.assertEqual('--multipartboundary--', resp_lines.pop())
# we should have consumed the whole response body
self.assertFalse(resp_lines)
def test_PUT_with_etag_override_in_headers(self):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = "FAKE APP"
plaintext_etag = md5hex(plaintext)
env = {"REQUEST_METHOD": "PUT", CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
"content-type": "text/plain",
"content-length": str(len(plaintext)),
"Etag": plaintext_etag,
"X-Object-Sysmeta-Container-Update-Override-Etag": "final etag",
}
req = Request.blank("/v1/a/c/o", environ=env, body=plaintext, headers=hdrs)
self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual("201 Created", resp.status)
self.assertEqual(plaintext_etag, resp.headers["Etag"])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(("PUT", "/v1/a/c/o"), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn("X-Object-Sysmeta-Container-Update-Override-Etag", req_hdrs)
parts = req_hdrs["X-Object-Sysmeta-Container-Update-Override-Etag"].rsplit(";", 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()["container"]
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = "swift_meta="
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(urllib.unquote_plus(param[len(crypto_meta_tag) :]))
self.assertEqual(Crypto().cipher, actual_meta["cipher"])
self.assertEqual(fetch_crypto_keys()["id"], actual_meta["key_id"])
cont_etag_iv = base64.b64decode(actual_meta["iv"])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt("final etag", cont_key, cont_etag_iv), base64.b64decode(parts[0]))
def test_bad_object_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys['object'] = b'the minor key'
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: bad_keys})
self.assertIn('500 Internal Error', cm.exception.status)
self.assertIn("Bad key for 'object'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn(b'Unable to retrieve encryption keys.',
cm.exception.body)
def test_bad_container_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys['container'] = 'the major key'
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys},
required=['object', 'container'])
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Bad key for 'container'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_GET_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: raise_exc}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
self.assertIn('from callback: Testing',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_GET_missing_etag_crypto_meta(self):
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted etag
hdrs['X-Object-Sysmeta-Crypto-Etag'] = \
base64.b64encode(encrypt(md5hex(body), key, FAKE_IV))
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header', resp.body)
self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_get_keys_missing_container_key_for_specified_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys.pop('container')
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: bad_keys},
required=['object', 'container'])
self.assertIn('500 Internal Error', cm.exception.status)
self.assertIn("Missing key for 'container'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn(b'Unable to retrieve encryption keys.',
cm.exception.body)
def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta):
# use bad iv for object body
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_override_etag_bad_meta(self, method, bad_crypto_meta):
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
# simulate missing crypto meta from encrypted override etag
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
encrypt_and_append_meta(
md5hex(body), key, crypto_meta=bad_crypto_meta)
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('Error decrypting header '
'X-Object-Sysmeta-Container-Update-Override-Etag',
self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def test_GET_cipher_mismatch_for_body(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(
len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register(
'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
self.assertIn('Bad crypto meta: Cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta):
# build fake multipart response body
key = fetch_crypto_keys()['object']
ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, ctxt=ctxt)
parts = ((0, 3, 'text/plain'),
(4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(
len(body), plaintext_etag, fetch_crypto_keys(), 'not used')
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_PUT_zero_size_object(self):
# object body encryption should be skipped for zero sized object body
object_key = fetch_crypto_keys()['object']
plaintext_etag = EMPTY_ETAG
env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {
'etag': EMPTY_ETAG,
'content-type': 'text/plain',
'content-length': '0',
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'
}
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that there is no body crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs)
# verify etag is md5 of plaintext
self.assertEqual(EMPTY_ETAG, req_hdrs['Etag'])
# verify there is no etag crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs)
# verify there is no container update override for etag
self.assertNotIn('X-Object-Sysmeta-Container-Update-Override-Etag',
req_hdrs)
# user meta is still encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(req_hdrs, 'Etag',
'not to be confused with the Etag!',
object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is empty by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual('', resp.body)
self.assertEqual(EMPTY_ETAG, resp.headers['Etag'])
def _test_bad_crypto_meta_for_user_metadata(self, method, bad_crypto_meta):
# use bad iv for metadata headers
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(
len(body), md5hex(body), fetch_crypto_keys(), 'not used')
enc_val = base64.b64encode(encrypt('encrypt me', key, FAKE_IV))
if bad_crypto_meta:
enc_val += ';swift_meta=' + get_crypto_meta_header(
crypto_meta=bad_crypto_meta)
hdrs['x-object-transient-sysmeta-crypto-meta-test'] = enc_val
self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn(
'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-'
'Test', self.decrypter.logger.get_lines_for_level('error')[0])
return resp
def _test_bad_key(self, method):
# use bad key
def bad_fetch_crypto_keys():
keys = fetch_crypto_keys()
keys['object'] = 'bad key'
return keys
env = {
'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys
}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
key = fetch_crypto_keys()['object']
enc_body = encrypt(body, key, FAKE_IV)
hdrs = self._make_response_headers(len(body), md5hex(body),
fetch_crypto_keys(), 'not used')
self.app.register(method,
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
return req.get_response(self.decrypter)
def test_PUT_zero_size_object(self):
# object body encryption should be skipped for zero sized object body
object_key = fetch_crypto_keys()['object']
plaintext_etag = EMPTY_ETAG
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'etag': EMPTY_ETAG,
'content-type': 'text/plain',
'content-length': '0',
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank(
'/v1/a/c/o', environ=env, body='', headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that there is no body crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs)
# verify etag is md5 of plaintext
self.assertEqual(EMPTY_ETAG, req_hdrs['Etag'])
# verify there is no etag crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs)
# verify there is no container update override for etag
self.assertNotIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
# user meta is still encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(
req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is empty by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual('', resp.body)
self.assertEqual(EMPTY_ETAG, resp.headers['Etag'])
def test_GET_container_xml(self):
content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
content_type_2 = 'text/plain; param=foo'
pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d'
pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10'
key = fetch_crypto_keys()['container']
fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>\
''' + encrypt_and_append_meta(pt_etag1.encode('utf8'), key) + '''\
</hash><content_type>\
''' + content_type_1 + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
<object><hash>\
''' + encrypt_and_append_meta(pt_etag2.encode('utf8'), key) + '''\
</hash><content_type>\
''' + content_type_2 + '''\
</content_type><name>testfile2</name><bytes>24</bytes>\
<last_modified>2015-04-19T02:37:39.684740</last_modified></object>\
</container>'''
resp = self._make_cont_get_req(fake_body, 'xml')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
tree = minidom.parseString(body)
containers = tree.getElementsByTagName('container')
self.assertEqual(1, len(containers))
self.assertEqual('testc',
containers[0].attributes.getNamedItem("name").value)
objs = tree.getElementsByTagName('object')
self.assertEqual(2, len(objs))
obj_dict_1 = {"bytes": "16",
"last_modified": "2015-04-19T02:37:39.601660",
"hash": pt_etag1,
"name": "testfile",
"content_type": content_type_1}
self._assert_element_contains_dict(obj_dict_1, objs[0])
obj_dict_2 = {"bytes": "24",
"last_modified": "2015-04-19T02:37:39.684740",
"hash": pt_etag2,
"name": "testfile2",
"content_type": content_type_2}
self._assert_element_contains_dict(obj_dict_2, objs[1])
def test_GET_cipher_mismatch_for_body(self):
# Cipher does not match
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV)
bad_crypto_meta = fake_get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
hdrs = self._make_response_headers(len(enc_body), md5hex(body),
fetch_crypto_keys(), 'not used')
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(crypto_meta=bad_crypto_meta)
self.app.register('GET',
'/v1/a/c/o',
HTTPOk,
body=enc_body,
headers=hdrs)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
self.assertIn('Bad crypto meta: Cipher',
self.decrypter.logger.get_lines_for_level('error')[0])
def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta):
# build fake multipart response body
key = fetch_crypto_keys()['object']
ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV)
plaintext = 'Cwm fjord veg balks nth pyx quiz'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, ctxt=ctxt)
parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'),
(24, 32, 'text/plain'))
length = len(ciphertext)
body = ''
for start, end, ctype in parts:
body += '--multipartboundary\r\n'
body += 'Content-Type: %s\r\n' % ctype
body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)
body += '\r\n\r\n' + ciphertext[start:end] + '\r\n'
body += '--multipartboundary--'
# register request with fake swift
hdrs = self._make_response_headers(len(body), plaintext_etag,
fetch_crypto_keys(), 'not used')
hdrs['content-type'] = \
'multipart/byteranges;boundary=multipartboundary'
hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \
get_crypto_meta_header(bad_crypto_meta)
self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs)
# issue request
env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env)
resp = req.get_response(self.decrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting object', resp.body)
self.assertIn('Error decrypting object',
self.decrypter.logger.get_lines_for_level('error')[0])
def test_get_keys(self):
# ok
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(fetch_crypto_keys(), keys)
# only default required keys are checked
subset_keys = {'object': fetch_crypto_keys()['object']}
env = {CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: subset_keys}
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(subset_keys, keys)
# only specified required keys are checked
subset_keys = {'container': fetch_crypto_keys()['container']}
env = {CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: subset_keys}
keys = self.crypto_context.get_keys(env, required=['container'])
self.assertDictEqual(subset_keys, keys)
subset_keys = {'object': fetch_crypto_keys()['object'],
'container': fetch_crypto_keys()['container']}
env = {CRYPTO_KEY_CALLBACK: lambda *args, **kwargs: subset_keys}
keys = self.crypto_context.get_keys(
env, required=['object', 'container'])
self.assertDictEqual(subset_keys, keys)
