def test_encrypt_header_val(self): # Prepare key and Crypto instance object_key = fetch_crypto_keys()['object'] # - Normal string can be crypted encrypted = encrypter.encrypt_header_val(Crypto(), 'aaa', object_key) # sanity: return value is 2 item tuple self.assertEqual(2, len(encrypted)) crypted_val, crypt_info = encrypted expected_crypt_val = base64.b64encode( encrypt('aaa', object_key, FAKE_IV)) expected_crypt_info = { 'cipher': 'AES_CTR_256', 'iv': 'This is an IV123' } self.assertEqual(expected_crypt_val, crypted_val) self.assertEqual(expected_crypt_info, crypt_info) # - Empty string raises a ValueError for safety with self.assertRaises(ValueError) as cm: encrypter.encrypt_header_val(Crypto(), '', object_key) self.assertEqual('empty value is not acceptable', cm.exception.message) # - None also raises a ValueError for safety with self.assertRaises(ValueError) as cm: encrypter.encrypt_header_val(Crypto(), None, object_key) self.assertEqual('empty value is not acceptable', cm.exception.message)
def test_GET_multiseg_with_range(self): env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} req = Request.blank('/v1/a/c/o', environ=env) req.headers['Content-Range'] = 'bytes 3-10/17' chunks = ['0123', '45678', '9abcdef'] body = ''.join(chunks) plaintext_etag = md5hex(body) body_key = os.urandom(32) ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV) enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks] enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]] hdrs = self._make_response_headers(sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(), body_key) hdrs['content-range'] = req.headers['Content-Range'] self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) resp = req.get_response(self.decrypter) self.assertEqual('3456789a', resp.body) self.assertEqual('200 OK', resp.status) self.assertEqual(plaintext_etag, resp.headers['Etag']) self.assertEqual('text/plain', resp.headers['Content-Type'])
def _test_PUT_with_etag_override_in_headers(self, override_etag): # verify handling of another middleware's # container-update-override-etag in headers plaintext = b'FAKE APP' plaintext_etag = md5hex(plaintext) env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} hdrs = { 'content-type': 'text/plain', 'content-length': str(len(plaintext)), 'Etag': plaintext_etag, 'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag } req = Request.blank('/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) resp = req.get_response(self.encrypter) self.assertEqual('201 Created', resp.status) self.assertEqual(plaintext_etag, resp.headers['Etag']) # verify metadata items self.assertEqual(1, len(self.app.calls), self.app.calls) self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0]) req_hdrs = self.app.headers[0] # verify encrypted etag for container update self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) parts = req_hdrs[ 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) self.assertEqual(2, len(parts)) cont_key = fetch_crypto_keys()['container'] # extract crypto_meta from end of etag for container update param = parts[1].strip() crypto_meta_tag = 'swift_meta=' self.assertTrue(param.startswith(crypto_meta_tag), param) actual_meta = json.loads( urlparse.unquote_plus(param[len(crypto_meta_tag):])) self.assertEqual(Crypto().cipher, actual_meta['cipher']) self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id']) cont_etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(FAKE_IV, cont_etag_iv) exp_etag = encrypt(override_etag.encode('ascii'), cont_key, cont_etag_iv) self.assertEqual(exp_etag, base64.b64decode(parts[0]))
def test_GET_multiseg(self): env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} req = Request.blank('/v1/a/c/o', environ=env) chunks = ['some', 'chunks', 'of data'] body = ''.join(chunks) plaintext_etag = md5hex(body) body_key = os.urandom(32) ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV) enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks] hdrs = self._make_response_headers(sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(), body_key) self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) resp = req.get_response(self.decrypter) self.assertEqual(body, resp.body) self.assertEqual('200 OK', resp.status) self.assertEqual(plaintext_etag, resp.headers['Etag']) self.assertEqual('text/plain', resp.headers['Content-Type'])
def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta): # build fake multipart response body key = fetch_crypto_keys()['object'] ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV) plaintext = 'Cwm fjord veg balks nth pyx quiz' plaintext_etag = md5hex(plaintext) ciphertext = encrypt(plaintext, ctxt=ctxt) parts = ((0, 3, 'text/plain'), (4, 9, 'text/plain; charset=us-ascii'), (24, 32, 'text/plain')) length = len(ciphertext) body = '' for start, end, ctype in parts: body += '--multipartboundary\r\n' body += 'Content-Type: %s\r\n' % ctype body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length) body += '\r\n\r\n' + ciphertext[start:end] + '\r\n' body += '--multipartboundary--' # register request with fake swift hdrs = self._make_response_headers(len(body), plaintext_etag, fetch_crypto_keys(), 'not used') hdrs['content-type'] = \ 'multipart/byteranges;boundary=multipartboundary' hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \ get_crypto_meta_header(bad_crypto_meta) self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs) # issue request env = {'REQUEST_METHOD': 'GET', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} req = Request.blank('/v1/a/c/o', environ=env) resp = req.get_response(self.decrypter) self.assertEqual('500 Internal Error', resp.status) self.assertEqual('Error decrypting object', resp.body) self.assertIn('Error decrypting object', self.decrypter.logger.get_lines_for_level('error')[0])
def decrypt(key, iv, enc_val): dec_ctxt = Crypto({}).create_decryption_ctxt(key, iv, 0) dec_val = dec_ctxt.update(enc_val) return dec_val
def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route="encrypter") self.crypto = Crypto(conf) self.disable_encryption = config_true_value( conf.get('disable_encryption', 'false'))
def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route="decrypter") self.crypto = Crypto(conf)
class FakeFilter(object): app = None crypto = Crypto({})
def setUp(self): self.crypto = Crypto({})
def encrypt(val, key=None, iv=None, ctxt=None): if ctxt is None: ctxt = Crypto({}).create_encryption_ctxt(key, iv) enc_val = ctxt.update(val) return enc_val
def test_PUT_req(self): body_key = os.urandom(32) object_key = fetch_crypto_keys()['object'] plaintext = 'FAKE APP' plaintext_etag = md5hex(plaintext) ciphertext = encrypt(plaintext, body_key, FAKE_IV) ciphertext_etag = md5hex(ciphertext) env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} hdrs = { 'etag': plaintext_etag, 'content-type': 'text/plain', 'content-length': str(len(plaintext)), 'x-object-meta-etag': 'not to be confused with the Etag!', 'x-object-meta-test': 'encrypt me', 'x-object-sysmeta-test': 'do not encrypt me' } req = Request.blank('/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) with mock.patch( 'swift.common.middleware.crypto.crypto_utils.' 'Crypto.create_random_key', return_value=body_key): resp = req.get_response(self.encrypter) self.assertEqual('201 Created', resp.status) self.assertEqual(plaintext_etag, resp.headers['Etag']) # verify metadata items self.assertEqual(1, len(self.app.calls), self.app.calls) self.assertEqual('PUT', self.app.calls[0][0]) req_hdrs = self.app.headers[0] # verify body crypto meta actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] actual = json.loads(urllib.unquote_plus(actual)) self.assertEqual(Crypto().cipher, actual['cipher']) self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) # verify wrapped body key expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) self.assertEqual(expected_wrapped_key, base64.b64decode(actual['body_key']['key'])) self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv'])) self.assertEqual(fetch_crypto_keys()['id'], actual['key_id']) # verify etag self.assertEqual(ciphertext_etag, req_hdrs['Etag']) encrypted_etag, _junk, etag_meta = \ req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=') # verify crypto_meta was appended to this etag self.assertTrue(etag_meta) actual_meta = json.loads(urllib.unquote_plus(etag_meta)) self.assertEqual(Crypto().cipher, actual_meta['cipher']) # verify encrypted version of plaintext etag actual = base64.b64decode(encrypted_etag) etag_iv = base64.b64decode(actual_meta['iv']) enc_etag = encrypt(plaintext_etag, object_key, etag_iv) self.assertEqual(enc_etag, actual) # verify etag MAC for conditional requests actual_hmac = base64.b64decode( req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac']) self.assertEqual( actual_hmac, hmac.new(object_key, plaintext_etag, hashlib.sha256).digest()) # verify encrypted etag for container update self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) parts = req_hdrs[ 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) self.assertEqual(2, len(parts)) # extract crypto_meta from end of etag for container update param = parts[1].strip() crypto_meta_tag = 'swift_meta=' self.assertTrue(param.startswith(crypto_meta_tag), param) actual_meta = json.loads( urllib.unquote_plus(param[len(crypto_meta_tag):])) self.assertEqual(Crypto().cipher, actual_meta['cipher']) self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id']) cont_key = fetch_crypto_keys()['container'] cont_etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(FAKE_IV, cont_etag_iv) self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv), base64.b64decode(parts[0])) # content-type is not encrypted self.assertEqual('text/plain', req_hdrs['Content-Type']) # user meta is encrypted self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key) self._verify_user_metadata(req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key) # sysmeta is not encrypted self.assertEqual('do not encrypt me', req_hdrs['X-Object-Sysmeta-Test']) # verify object is encrypted by getting direct from the app get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = get_req.get_response(self.app) self.assertEqual(ciphertext, resp.body) self.assertEqual(ciphertext_etag, resp.headers['Etag'])
def test_PUT_nothing_read(self): # simulate an artificial scenario of a downstream filter/app not # actually reading the input stream from encrypter. class NonReadingApp(object): def __call__(self, env, start_response): # note: no read from wsgi.input req = Request(env) env['swift.callback.update_footers'](req.headers) call_headers.append(req.headers) resp = HTTPCreated(req=req, headers={'Etag': 'response etag'}) return resp(env, start_response) env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} hdrs = { 'content-type': 'text/plain', 'content-length': 0, 'etag': 'etag from client' } req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) call_headers = [] resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) self.assertEqual('201 Created', resp.status) self.assertEqual('response etag', resp.headers['Etag']) self.assertEqual(1, len(call_headers)) self.assertEqual('etag from client', call_headers[0]['etag']) # verify no encryption footers for k in call_headers[0]: self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-')) # check that an upstream footer callback gets called other_footers = { 'Etag': EMPTY_ETAG, 'X-Object-Sysmeta-Other': 'other sysmeta', 'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override' } env.update({ 'swift.callback.update_footers': lambda footers: footers.update(other_footers) }) req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) call_headers = [] resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) self.assertEqual('201 Created', resp.status) self.assertEqual('response etag', resp.headers['Etag']) self.assertEqual(1, len(call_headers)) # verify encrypted override etag for container update. self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', call_headers[0]) parts = call_headers[0][ 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) self.assertEqual(2, len(parts)) cont_key = fetch_crypto_keys()['container'] param = parts[1].strip() crypto_meta_tag = 'swift_meta=' self.assertTrue(param.startswith(crypto_meta_tag), param) actual_meta = json.loads( urllib.unquote_plus(param[len(crypto_meta_tag):])) self.assertEqual(Crypto().cipher, actual_meta['cipher']) self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id']) cont_etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(FAKE_IV, cont_etag_iv) self.assertEqual(encrypt('other override', cont_key, cont_etag_iv), base64.b64decode(parts[0])) # verify that other middleware's footers made it to app other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag') for k, v in other_footers.items(): self.assertEqual(v, call_headers[0][k]) # verify no encryption footers for k in call_headers[0]: self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-')) # if upstream footer override etag is for an empty body then check that # it is not encrypted other_footers = { 'Etag': EMPTY_ETAG, 'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG } env.update({ 'swift.callback.update_footers': lambda footers: footers.update(other_footers) }) req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) call_headers = [] resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) self.assertEqual('201 Created', resp.status) self.assertEqual('response etag', resp.headers['Etag']) self.assertEqual(1, len(call_headers)) # verify that other middleware's footers made it to app for k, v in other_footers.items(): self.assertEqual(v, call_headers[0][k]) # verify no encryption footers for k in call_headers[0]: self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
def _test_PUT_with_other_footers(self, override_etag): # verify handling of another middleware's footer callback cont_key = fetch_crypto_keys()['container'] body_key = os.urandom(32) object_key = fetch_crypto_keys()['object'] plaintext = 'FAKE APP' plaintext_etag = md5hex(plaintext) ciphertext = encrypt(plaintext, body_key, FAKE_IV) ciphertext_etag = md5hex(ciphertext) other_footers = { 'Etag': plaintext_etag, 'X-Object-Sysmeta-Other': 'other sysmeta', 'X-Object-Sysmeta-Container-Update-Override-Size': 'other override', 'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag } env = { 'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys, 'swift.callback.update_footers': lambda footers: footers.update(other_footers) } hdrs = { 'content-type': 'text/plain', 'content-length': str(len(plaintext)), 'Etag': 'correct etag is in footers' } req = Request.blank('/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) with mock.patch( 'swift.common.middleware.crypto.crypto_utils.' 'Crypto.create_random_key', lambda *args: body_key): resp = req.get_response(self.encrypter) self.assertEqual('201 Created', resp.status) self.assertEqual(plaintext_etag, resp.headers['Etag']) # verify metadata items self.assertEqual(1, len(self.app.calls), self.app.calls) self.assertEqual('PUT', self.app.calls[0][0]) req_hdrs = self.app.headers[0] # verify that other middleware's footers made it to app, including any # container update overrides but nothing Etag-related other_footers.pop('Etag') other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag') for k, v in other_footers.items(): self.assertEqual(v, req_hdrs[k]) # verify encryption footers are ok encrypted_etag, _junk, etag_meta = \ req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=') self.assertTrue(etag_meta) actual_meta = json.loads(urllib.unquote_plus(etag_meta)) self.assertEqual(Crypto().cipher, actual_meta['cipher']) self.assertEqual(ciphertext_etag, req_hdrs['Etag']) actual = base64.b64decode(encrypted_etag) etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual) # verify encrypted etag for container update self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) parts = req_hdrs[ 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) self.assertEqual(2, len(parts)) # extract crypto_meta from end of etag for container update param = parts[1].strip() crypto_meta_tag = 'swift_meta=' self.assertTrue(param.startswith(crypto_meta_tag), param) actual_meta = json.loads( urllib.unquote_plus(param[len(crypto_meta_tag):])) self.assertEqual(Crypto().cipher, actual_meta['cipher']) cont_key = fetch_crypto_keys()['container'] cont_etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(FAKE_IV, cont_etag_iv) self.assertEqual(encrypt(override_etag, cont_key, cont_etag_iv), base64.b64decode(parts[0])) # verify body crypto meta actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] actual = json.loads(urllib.unquote_plus(actual)) self.assertEqual(Crypto().cipher, actual['cipher']) self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) # verify wrapped body key expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) self.assertEqual(expected_wrapped_key, base64.b64decode(actual['body_key']['key'])) self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv'])) self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
class TestCrypto(unittest.TestCase): def setUp(self): self.crypto = Crypto({}) def test_create_encryption_context(self): value = 'encrypt me' * 100 # more than one cipher block key = os.urandom(32) iv = os.urandom(16) ctxt = self.crypto.create_encryption_ctxt(key, iv) expected = Cipher(algorithms.AES(key), modes.CTR(iv), backend=default_backend()).encryptor().update(value) self.assertEqual(expected, ctxt.update(value)) for bad_iv in ('a little too long', 'too short'): self.assertRaises(ValueError, self.crypto.create_encryption_ctxt, key, bad_iv) for bad_key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24): self.assertRaises(ValueError, self.crypto.create_encryption_ctxt, bad_key, iv) def test_create_decryption_context(self): value = 'decrypt me' * 100 # more than one cipher block key = os.urandom(32) iv = os.urandom(16) ctxt = self.crypto.create_decryption_ctxt(key, iv, 0) expected = Cipher(algorithms.AES(key), modes.CTR(iv), backend=default_backend()).decryptor().update(value) self.assertEqual(expected, ctxt.update(value)) for bad_iv in ('a little too long', 'too short'): self.assertRaises(ValueError, self.crypto.create_decryption_ctxt, key, bad_iv, 0) for bad_key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24): self.assertRaises(ValueError, self.crypto.create_decryption_ctxt, bad_key, iv, 0) with self.assertRaises(ValueError) as cm: self.crypto.create_decryption_ctxt(key, iv, -1) self.assertEqual("Offset must not be negative", cm.exception.message) def test_enc_dec_small_chunks(self): self.enc_dec_chunks(['encrypt me', 'because I', 'am sensitive']) def test_enc_dec_large_chunks(self): self.enc_dec_chunks([os.urandom(65536), os.urandom(65536)]) def enc_dec_chunks(self, chunks): key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki' iv = self.crypto.create_iv() enc_ctxt = self.crypto.create_encryption_ctxt(key, iv) enc_val = [enc_ctxt.update(chunk) for chunk in chunks] self.assertTrue(''.join(enc_val) != chunks) dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 0) dec_val = [dec_ctxt.update(chunk) for chunk in enc_val] self.assertEqual( ''.join(chunks), ''.join(dec_val), 'Expected value {%s} but got {%s}' % (''.join(chunks), ''.join(dec_val))) def test_decrypt_range(self): chunks = ['0123456789abcdef', 'ghijklmnopqrstuv'] key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki' iv = self.crypto.create_iv() enc_ctxt = self.crypto.create_encryption_ctxt(key, iv) enc_val = [enc_ctxt.update(chunk) for chunk in chunks] self.assertTrue(''.join(enc_val) != chunks) # Simulate a ranged GET from byte 19 to 32 : 'jklmnopqrstuv' dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 19) ranged_chunks = [enc_val[1][3:]] dec_val = [dec_ctxt.update(chunk) for chunk in ranged_chunks] self.assertEqual( 'jklmnopqrstuv', ''.join(dec_val), 'Expected value {%s} but got {%s}' % ('jklmnopqrstuv', ''.join(dec_val))) def test_create_decryption_context_non_zero_offset(self): # Verify that iv increments for each 16 bytes of offset. # For a ranged GET we pass a non-zero offset so that the decrypter # counter is incremented to the correct value to start decrypting at # that offset into the object body. The counter should increment by one # from the starting IV value for every 16 bytes offset into the object # body, until it reaches 2^128 -1 when it should wrap to zero. We check # that is happening by verifying a decrypted value using various # offsets. key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki' def do_test(): for offset, exp_iv in mappings.items(): dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, offset) offset_in_block = offset % 16 cipher = Cipher(algorithms.AES(key), modes.CTR(exp_iv), backend=default_backend()) expected = cipher.decryptor().update('p' * offset_in_block + 'ciphertext') actual = dec_ctxt.update('ciphertext') expected = expected[offset % 16:] self.assertEqual( expected, actual, 'Expected %r but got %r, iv=%s and offset=%s' % (expected, actual, iv, offset)) iv = '0000000010000000' mappings = { 2: '0000000010000000', 16: '0000000010000001', 19: '0000000010000001', 48: '0000000010000003', 1024: '000000001000000p', 5119: '000000001000001o' } do_test() # choose max iv value and test that it wraps to zero iv = chr(0xff) * 16 mappings = { 2: iv, 16: str(bytearray.fromhex('00' * 16)), # iv wraps to 0 19: str(bytearray.fromhex('00' * 16)), 48: str(bytearray.fromhex('00' * 15 + '02')), 1024: str(bytearray.fromhex('00' * 15 + '3f')), 5119: str(bytearray.fromhex('00' * 14 + '013E')) } do_test() iv = chr(0x0) * 16 mappings = { 2: iv, 16: str(bytearray.fromhex('00' * 15 + '01')), 19: str(bytearray.fromhex('00' * 15 + '01')), 48: str(bytearray.fromhex('00' * 15 + '03')), 1024: str(bytearray.fromhex('00' * 15 + '40')), 5119: str(bytearray.fromhex('00' * 14 + '013F')) } do_test() iv = chr(0x0) * 8 + chr(0xff) * 8 mappings = { 2: iv, 16: str(bytearray.fromhex('00' * 7 + '01' + '00' * 8)), 19: str(bytearray.fromhex('00' * 7 + '01' + '00' * 8)), 48: str(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '02')), 1024: str(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '3F')), 5119: str(bytearray.fromhex('00' * 7 + '01' + '00' * 6 + '013E')) } do_test() def test_check_key(self): for key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24): with self.assertRaises(ValueError) as cm: self.crypto.check_key(key) self.assertEqual("Key must be length 32 bytes", cm.exception.message) def test_check_crypto_meta(self): meta = {'cipher': 'AES_CTR_256'} with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: Missing 'iv'", cm.exception.message) for bad_iv in ('a little too long', 'too short'): meta['iv'] = bad_iv with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: IV must be length 16 bytes", cm.exception.message) meta = {'iv': os.urandom(16)} with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: Missing 'cipher'", cm.exception.message) meta['cipher'] = 'Mystery cipher' with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: Cipher must be AES_CTR_256", cm.exception.message) def test_create_iv(self): self.assertEqual(16, len(self.crypto.create_iv())) # crude check that we get back different values on each call self.assertNotEqual(self.crypto.create_iv(), self.crypto.create_iv()) def test_get_crypto_meta(self): meta = self.crypto.create_crypto_meta() self.assertIsInstance(meta, dict) # this is deliberately brittle so that if new items are added then the # test will need to be updated self.assertEqual(2, len(meta)) self.assertIn('iv', meta) self.assertEqual(16, len(meta['iv'])) self.assertIn('cipher', meta) self.assertEqual('AES_CTR_256', meta['cipher']) self.crypto.check_crypto_meta(meta) # sanity check meta2 = self.crypto.create_crypto_meta() self.assertNotEqual(meta['iv'], meta2['iv']) # crude sanity check def test_create_random_key(self): # crude check that we get unique keys on each call keys = set() for i in range(10): key = self.crypto.create_random_key() self.assertEqual(32, len(key)) keys.add(key) self.assertEqual(10, len(keys)) def test_wrap_unwrap_key(self): wrapping_key = os.urandom(32) key_to_wrap = os.urandom(32) iv = os.urandom(16) with mock.patch( 'swift.common.middleware.crypto.crypto_utils.Crypto.create_iv', return_value=iv): wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) cipher = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv), backend=default_backend()) expected = {'key': cipher.encryptor().update(key_to_wrap), 'iv': iv} self.assertEqual(expected, wrapped) unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped) self.assertEqual(key_to_wrap, unwrapped) def test_unwrap_bad_key(self): # verify that ValueError is raised if unwrapped key is invalid wrapping_key = os.urandom(32) for length in (0, 16, 24, 31, 33): key_to_wrap = os.urandom(length) wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) with self.assertRaises(ValueError) as cm: self.crypto.unwrap_key(wrapping_key, wrapped) self.assertEqual(cm.exception.message, 'Key must be length 32 bytes')
def _test_ondisk_data_after_write_with_crypto(self, policy_name): policy = storage_policy.POLICIES.get_by_name(policy_name) self._create_container(self.proxy_app, policy_name=policy_name) self._put_object(self.crypto_app, self.plaintext) self._post_object(self.crypto_app) # Verify container listing etag is encrypted by direct GET to container # server. We can use any server for all nodes since they all share same # devices dir. cont_server = self._test_context['test_servers'][3] cont_ring = Ring(self._test_context['testdir'], ring_name='container') part, nodes = cont_ring.get_nodes('a', self.container_name) for node in nodes: req = Request.blank('/%s/%s/a/%s' % (node['device'], part, self.container_name), method='GET', query_string='format=json') resp = req.get_response(cont_server) listing = json.loads(resp.body) # sanity checks... self.assertEqual(1, len(listing)) self.assertEqual('o', listing[0]['name']) self.assertEqual('application/test', listing[0]['content_type']) # verify encrypted etag value parts = listing[0]['hash'].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), self.km.create_key('/a/%s' % self.container_name), listing_etag_iv)).decode('ascii') self.assertEqual(exp_enc_listing_etag, parts[0]) # Verify diskfile data and metadata is encrypted ring_object = self.proxy_app.get_object_ring(int(policy)) partition, nodes = ring_object.get_nodes('a', self.container_name, 'o') conf = { 'devices': self._test_context["testdir"], 'mount_check': 'false' } df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy] ondisk_data = [] exp_enc_body = None for node_index, node in enumerate(nodes): df = df_mgr.get_diskfile(node['device'], partition, 'a', self.container_name, 'o', policy=policy) with df.open(): meta = df.get_metadata() contents = b''.join(df.reader()) metadata = dict((k.lower(), v) for k, v in meta.items()) # verify on disk data - body body_iv = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['iv'] body_key_meta = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['body_key'] obj_key = self.km.create_key('/a/%s/o' % self.container_name) body_key = Crypto().unwrap_key(obj_key, body_key_meta) exp_enc_body = encrypt(self.plaintext, body_key, body_iv) ondisk_data.append((node, contents)) # verify on disk user metadata enc_val, meta = metadata[ 'x-object-transient-sysmeta-crypto-meta-fruit'].split(';') meta = meta.strip()[len('swift_meta='):] metadata_iv = load_crypto_meta(meta)['iv'] exp_enc_meta = base64.b64encode( encrypt(b'Kiwi', obj_key, metadata_iv)).decode('ascii') self.assertEqual(exp_enc_meta, enc_val) self.assertNotIn('x-object-meta-fruit', metadata) self.assertIn('x-object-transient-sysmeta-crypto-meta', metadata) meta = load_crypto_meta( metadata['x-object-transient-sysmeta-crypto-meta']) self.assertIn('key_id', meta) self.assertIn('path', meta['key_id']) self.assertEqual( '/a/%s/%s' % (self.container_name, self.object_name), meta['key_id']['path']) self.assertIn('v', meta['key_id']) self.assertEqual('2', meta['key_id']['v']) self.assertIn('cipher', meta) self.assertEqual(Crypto.cipher, meta['cipher']) # verify etag actual_enc_etag, _junk, actual_etag_meta = metadata[ 'x-object-sysmeta-crypto-etag'].partition('; swift_meta=') etag_iv = load_crypto_meta(actual_etag_meta)['iv'] exp_enc_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), obj_key, etag_iv)).decode('ascii') self.assertEqual(exp_enc_etag, actual_enc_etag) # verify etag hmac exp_etag_mac = hmac.new(obj_key, self.plaintext_etag.encode('ascii'), digestmod=hashlib.sha256).digest() exp_etag_mac = base64.b64encode(exp_etag_mac).decode('ascii') self.assertEqual(exp_etag_mac, metadata['x-object-sysmeta-crypto-etag-mac']) # verify etag override for container updates override = 'x-object-sysmeta-container-update-override-etag' parts = metadata[override].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] cont_key = self.km.create_key('/a/%s' % self.container_name) exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), cont_key, listing_etag_iv)).decode('ascii') self.assertEqual(exp_enc_listing_etag, parts[0]) self._check_GET_and_HEAD(self.crypto_app) return exp_enc_body, ondisk_data
class TestCrypto(unittest.TestCase): def setUp(self): self.crypto = Crypto({}) def test_create_encryption_context(self): value = b'encrypt me' * 100 # more than one cipher block key = os.urandom(32) iv = os.urandom(16) ctxt = self.crypto.create_encryption_ctxt(key, iv) expected = Cipher( algorithms.AES(key), modes.CTR(iv), backend=default_backend()).encryptor().update(value) self.assertEqual(expected, ctxt.update(value)) for bad_iv in (b'a little too long', b'too short'): self.assertRaises( ValueError, self.crypto.create_encryption_ctxt, key, bad_iv) for bad_key in (b'objKey', b'a' * 31, b'a' * 33, b'a' * 16, b'a' * 24): self.assertRaises( ValueError, self.crypto.create_encryption_ctxt, bad_key, iv) def test_create_decryption_context(self): value = b'decrypt me' * 100 # more than one cipher block key = os.urandom(32) iv = os.urandom(16) ctxt = self.crypto.create_decryption_ctxt(key, iv, 0) expected = Cipher( algorithms.AES(key), modes.CTR(iv), backend=default_backend()).decryptor().update(value) self.assertEqual(expected, ctxt.update(value)) for bad_iv in (b'a little too long', b'too short'): self.assertRaises( ValueError, self.crypto.create_decryption_ctxt, key, bad_iv, 0) for bad_key in (b'objKey', b'a' * 31, b'a' * 33, b'a' * 16, b'a' * 24): self.assertRaises( ValueError, self.crypto.create_decryption_ctxt, bad_key, iv, 0) with self.assertRaises(ValueError) as cm: self.crypto.create_decryption_ctxt(key, iv, -1) self.assertEqual("Offset must not be negative", cm.exception.args[0]) def test_enc_dec_small_chunks(self): self.enc_dec_chunks([b'encrypt me', b'because I', b'am sensitive']) def test_enc_dec_large_chunks(self): self.enc_dec_chunks([os.urandom(65536), os.urandom(65536)]) def enc_dec_chunks(self, chunks): key = b'objL7wjV6L79Sfs4y7dy41273l0k6Wki' iv = self.crypto.create_iv() enc_ctxt = self.crypto.create_encryption_ctxt(key, iv) enc_val = [enc_ctxt.update(chunk) for chunk in chunks] self.assertTrue(b''.join(enc_val) != chunks) dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 0) dec_val = [dec_ctxt.update(chunk) for chunk in enc_val] self.assertEqual(b''.join(chunks), b''.join(dec_val), 'Expected value {%s} but got {%s}' % (b''.join(chunks), b''.join(dec_val))) def test_decrypt_range(self): chunks = [b'0123456789abcdef', b'ghijklmnopqrstuv'] key = b'objL7wjV6L79Sfs4y7dy41273l0k6Wki' iv = self.crypto.create_iv() enc_ctxt = self.crypto.create_encryption_ctxt(key, iv) enc_val = [enc_ctxt.update(chunk) for chunk in chunks] # Simulate a ranged GET from byte 19 to 32 : 'jklmnopqrstuv' dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 19) ranged_chunks = [enc_val[1][3:]] dec_val = [dec_ctxt.update(chunk) for chunk in ranged_chunks] self.assertEqual(b'jklmnopqrstuv', b''.join(dec_val), 'Expected value {%s} but got {%s}' % (b'jklmnopqrstuv', b''.join(dec_val))) def test_create_decryption_context_non_zero_offset(self): # Verify that iv increments for each 16 bytes of offset. # For a ranged GET we pass a non-zero offset so that the decrypter # counter is incremented to the correct value to start decrypting at # that offset into the object body. The counter should increment by one # from the starting IV value for every 16 bytes offset into the object # body, until it reaches 2^128 -1 when it should wrap to zero. We check # that is happening by verifying a decrypted value using various # offsets. key = b'objL7wjV6L79Sfs4y7dy41273l0k6Wki' def do_test(): for offset, exp_iv in mappings.items(): dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, offset) offset_in_block = offset % 16 cipher = Cipher(algorithms.AES(key), modes.CTR(exp_iv), backend=default_backend()) expected = cipher.decryptor().update( b'p' * offset_in_block + b'ciphertext') actual = dec_ctxt.update(b'ciphertext') expected = expected[offset % 16:] self.assertEqual(expected, actual, 'Expected %r but got %r, iv=%s and offset=%s' % (expected, actual, iv, offset)) iv = b'0000000010000000' mappings = { 2: b'0000000010000000', 16: b'0000000010000001', 19: b'0000000010000001', 48: b'0000000010000003', 1024: b'000000001000000p', 5119: b'000000001000001o' } do_test() # choose max iv value and test that it wraps to zero iv = b'\xff' * 16 mappings = { 2: iv, 16: bytes(bytearray.fromhex('00' * 16)), # iv wraps to 0 19: bytes(bytearray.fromhex('00' * 16)), 48: bytes(bytearray.fromhex('00' * 15 + '02')), 1024: bytes(bytearray.fromhex('00' * 15 + '3f')), 5119: bytes(bytearray.fromhex('00' * 14 + '013E')) } do_test() iv = b'\x00' * 16 mappings = { 2: iv, 16: bytes(bytearray.fromhex('00' * 15 + '01')), 19: bytes(bytearray.fromhex('00' * 15 + '01')), 48: bytes(bytearray.fromhex('00' * 15 + '03')), 1024: bytes(bytearray.fromhex('00' * 15 + '40')), 5119: bytes(bytearray.fromhex('00' * 14 + '013F')) } do_test() iv = b'\x00' * 8 + b'\xff' * 8 mappings = { 2: iv, 16: bytes(bytearray.fromhex('00' * 7 + '01' + '00' * 8)), 19: bytes(bytearray.fromhex('00' * 7 + '01' + '00' * 8)), 48: bytes(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '02')), 1024: bytes(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '3F')), 5119: bytes(bytearray.fromhex('00' * 7 + '01' + '00' * 6 + '013E')) } do_test() def test_check_key(self): for key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24): with self.assertRaises(ValueError) as cm: self.crypto.check_key(key) self.assertEqual("Key must be length 32 bytes", cm.exception.args[0]) def test_check_crypto_meta(self): meta = {'cipher': 'AES_CTR_256'} with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: Missing 'iv'", cm.exception.args[0]) for bad_iv in ('a little too long', 'too short'): meta['iv'] = bad_iv with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: IV must be length 16 bytes", cm.exception.args[0]) meta = {'iv': os.urandom(16)} with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: Missing 'cipher'", cm.exception.args[0]) meta['cipher'] = 'Mystery cipher' with self.assertRaises(EncryptionException) as cm: self.crypto.check_crypto_meta(meta) self.assertEqual("Bad crypto meta: Cipher must be AES_CTR_256", cm.exception.args[0]) def test_create_iv(self): self.assertEqual(16, len(self.crypto.create_iv())) # crude check that we get back different values on each call self.assertNotEqual(self.crypto.create_iv(), self.crypto.create_iv()) def test_get_crypto_meta(self): meta = self.crypto.create_crypto_meta() self.assertIsInstance(meta, dict) # this is deliberately brittle so that if new items are added then the # test will need to be updated self.assertEqual(2, len(meta)) self.assertIn('iv', meta) self.assertEqual(16, len(meta['iv'])) self.assertIn('cipher', meta) self.assertEqual('AES_CTR_256', meta['cipher']) self.crypto.check_crypto_meta(meta) # sanity check meta2 = self.crypto.create_crypto_meta() self.assertNotEqual(meta['iv'], meta2['iv']) # crude sanity check def test_create_random_key(self): # crude check that we get unique keys on each call keys = set() for i in range(10): key = self.crypto.create_random_key() self.assertEqual(32, len(key)) keys.add(key) self.assertEqual(10, len(keys)) def test_wrap_unwrap_key(self): wrapping_key = os.urandom(32) key_to_wrap = os.urandom(32) iv = os.urandom(16) with mock.patch( 'swift.common.middleware.crypto.crypto_utils.Crypto.create_iv', return_value=iv): wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) cipher = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv), backend=default_backend()) expected = {'key': cipher.encryptor().update(key_to_wrap), 'iv': iv} self.assertEqual(expected, wrapped) unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped) self.assertEqual(key_to_wrap, unwrapped) def test_unwrap_bad_key(self): # verify that ValueError is raised if unwrapped key is invalid wrapping_key = os.urandom(32) for length in (0, 16, 24, 31, 33): key_to_wrap = os.urandom(length) wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) with self.assertRaises(ValueError) as cm: self.crypto.unwrap_key(wrapping_key, wrapped) self.assertEqual( cm.exception.args[0], 'Key must be length 32 bytes')