def test_load_crypto_meta(self): actual = crypto_utils.load_crypto_meta(self.serialized_meta) self.assertEqual(self.meta, actual) actual = crypto_utils.load_crypto_meta(self.serialized_meta_with_key) self.assertEqual(self.meta_with_key, actual) def assert_raises(value, message): with self.assertRaises(EncryptionException) as cm: crypto_utils.load_crypto_meta(value) self.assertIn('Bad crypto meta %r' % value, cm.exception.message) self.assertIn(message, cm.exception.message) assert_raises(None, 'crypto meta not a string') assert_raises(99, 'crypto meta not a string') assert_raises('', 'No JSON object could be decoded') assert_raises('abc', 'No JSON object could be decoded') assert_raises('[]', 'crypto meta not a Mapping') assert_raises('{"iv": "abcdef"}', 'Incorrect padding') assert_raises('{"iv": []}', 'must be string or buffer') assert_raises('{"iv": {}}', 'must be string or buffer') assert_raises('{"iv": 99}', 'must be string or buffer') assert_raises('{"key": "abcdef"}', 'Incorrect padding') assert_raises('{"key": []}', 'must be string or buffer') assert_raises('{"key": {}}', 'must be string or buffer') assert_raises('{"key": 99}', 'must be string or buffer') assert_raises('{"body_key": {"iv": "abcdef"}}', 'Incorrect padding') assert_raises('{"body_key": {"iv": []}}', 'must be string or buffer') assert_raises('{"body_key": {"iv": {}}}', 'must be string or buffer') assert_raises('{"body_key": {"iv": 99}}', 'must be string or buffer') assert_raises('{"body_key": {"key": "abcdef"}}', 'Incorrect padding') assert_raises('{"body_key": {"key": []}}', 'must be string or buffer') assert_raises('{"body_key": {"key": {}}}', 'must be string or buffer') assert_raises('{"body_key": {"key": 99}}', 'must be string or buffer')
def test_dump_then_load_crypto_meta(self): actual = crypto_utils.load_crypto_meta( crypto_utils.dump_crypto_meta(self.meta)) self.assertEqual(self.meta, actual) actual = crypto_utils.load_crypto_meta( crypto_utils.dump_crypto_meta(self.meta_with_key)) self.assertEqual(self.meta_with_key, actual)
def assert_raises(value, message): with self.assertRaises(EncryptionException) as cm: crypto_utils.load_crypto_meta(value) self.assertIn('Bad crypto meta %r' % value, cm.exception.args[0]) if isinstance(message, (tuple, list)): for opt in message: if opt in cm.exception.args[0]: break else: self.fail('Expected to find one of %r in %r' % ( message, cm.exception.args[0])) else: self.assertIn(message, cm.exception.args[0])
def test_load_crypto_meta(self): actual = crypto_utils.load_crypto_meta(self.serialized_meta) self.assertEqual(self.meta, actual) actual = crypto_utils.load_crypto_meta(self.serialized_meta_with_key) self.assertEqual(self.meta_with_key, actual) def assert_raises(value, message): with self.assertRaises(EncryptionException) as cm: crypto_utils.load_crypto_meta(value) self.assertIn('Bad crypto meta %r' % value, cm.exception.args[0]) if isinstance(message, (tuple, list)): for opt in message: if opt in cm.exception.args[0]: break else: self.fail('Expected to find one of %r in %r' % ( message, cm.exception.args[0])) else: self.assertIn(message, cm.exception.args[0]) assert_raises(None, 'crypto meta not a string') assert_raises(99, 'crypto meta not a string') assert_raises('', ('No JSON object could be decoded', 'Expecting value: line 1 column 1')) assert_raises('abc', ('No JSON object could be decoded', 'Expecting value: line 1 column 1')) assert_raises('[]', 'crypto meta not a Mapping') bad_type_messages = [ 'must be string or buffer', 'argument should be a bytes-like object or ASCII string', ] assert_raises('{"iv": "abcdef"}', 'Incorrect padding') assert_raises('{"iv": []}', bad_type_messages) assert_raises('{"iv": {}}', bad_type_messages) assert_raises('{"iv": 99}', bad_type_messages) assert_raises('{"key": "abcdef"}', 'Incorrect padding') assert_raises('{"key": []}', bad_type_messages) assert_raises('{"key": {}}', bad_type_messages) assert_raises('{"key": 99}', bad_type_messages) assert_raises('{"body_key": {"iv": "abcdef"}}', 'Incorrect padding') assert_raises('{"body_key": {"iv": []}}', bad_type_messages) assert_raises('{"body_key": {"iv": {}}}', bad_type_messages) assert_raises('{"body_key": {"iv": 99}}', bad_type_messages) assert_raises('{"body_key": {"key": "abcdef"}}', 'Incorrect padding') assert_raises('{"body_key": {"key": []}}', bad_type_messages) assert_raises('{"body_key": {"key": {}}}', bad_type_messages) assert_raises('{"body_key": {"key": 99}}', bad_type_messages)
def get_crypto_meta(self, header_name): """ Extract a crypto_meta dict from a header. :param header_name: name of header that may have crypto_meta :return: A dict containing crypto_meta items :raises EncryptionException: if an error occurs while parsing the crypto meta """ crypto_meta_json = self._response_header_value(header_name) if crypto_meta_json is None: return None crypto_meta = load_crypto_meta(crypto_meta_json) self.crypto.check_crypto_meta(crypto_meta) return crypto_meta
def _test_ondisk_data_after_write_with_crypto(self, policy_name): policy = storage_policy.POLICIES.get_by_name(policy_name) self._create_container(self.proxy_app, policy_name=policy_name) self._put_object(self.crypto_app, self.plaintext) self._post_object(self.crypto_app) # Verify container listing etag is encrypted by direct GET to container # server. We can use any server for all nodes since they all share same # devices dir. cont_server = self._test_context['test_servers'][3] cont_ring = Ring(self._test_context['testdir'], ring_name='container') part, nodes = cont_ring.get_nodes('a', self.container_name) for node in nodes: req = Request.blank('/%s/%s/a/%s' % (node['device'], part, self.container_name), method='GET', query_string='format=json') resp = req.get_response(cont_server) listing = json.loads(resp.body) # sanity checks... self.assertEqual(1, len(listing)) self.assertEqual('o', listing[0]['name']) self.assertEqual('application/test', listing[0]['content_type']) # verify encrypted etag value parts = listing[0]['hash'].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), self.km.create_key('/a/%s' % self.container_name), listing_etag_iv)).decode('ascii') self.assertEqual(exp_enc_listing_etag, parts[0]) # Verify diskfile data and metadata is encrypted ring_object = self.proxy_app.get_object_ring(int(policy)) partition, nodes = ring_object.get_nodes('a', self.container_name, 'o') conf = { 'devices': self._test_context["testdir"], 'mount_check': 'false' } df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy] ondisk_data = [] exp_enc_body = None for node_index, node in enumerate(nodes): df = df_mgr.get_diskfile(node['device'], partition, 'a', self.container_name, 'o', policy=policy) with df.open(): meta = df.get_metadata() contents = b''.join(df.reader()) metadata = dict((k.lower(), v) for k, v in meta.items()) # verify on disk data - body body_iv = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['iv'] body_key_meta = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['body_key'] obj_key = self.km.create_key('/a/%s/o' % self.container_name) body_key = Crypto().unwrap_key(obj_key, body_key_meta) exp_enc_body = encrypt(self.plaintext, body_key, body_iv) ondisk_data.append((node, contents)) # verify on disk user metadata enc_val, meta = metadata[ 'x-object-transient-sysmeta-crypto-meta-fruit'].split(';') meta = meta.strip()[len('swift_meta='):] metadata_iv = load_crypto_meta(meta)['iv'] exp_enc_meta = base64.b64encode( encrypt(b'Kiwi', obj_key, metadata_iv)).decode('ascii') self.assertEqual(exp_enc_meta, enc_val) self.assertNotIn('x-object-meta-fruit', metadata) self.assertIn('x-object-transient-sysmeta-crypto-meta', metadata) meta = load_crypto_meta( metadata['x-object-transient-sysmeta-crypto-meta']) self.assertIn('key_id', meta) self.assertIn('path', meta['key_id']) self.assertEqual( '/a/%s/%s' % (self.container_name, self.object_name), meta['key_id']['path']) self.assertIn('v', meta['key_id']) self.assertEqual('2', meta['key_id']['v']) self.assertIn('cipher', meta) self.assertEqual(Crypto.cipher, meta['cipher']) # verify etag actual_enc_etag, _junk, actual_etag_meta = metadata[ 'x-object-sysmeta-crypto-etag'].partition('; swift_meta=') etag_iv = load_crypto_meta(actual_etag_meta)['iv'] exp_enc_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), obj_key, etag_iv)).decode('ascii') self.assertEqual(exp_enc_etag, actual_enc_etag) # verify etag hmac exp_etag_mac = hmac.new(obj_key, self.plaintext_etag.encode('ascii'), digestmod=hashlib.sha256).digest() exp_etag_mac = base64.b64encode(exp_etag_mac).decode('ascii') self.assertEqual(exp_etag_mac, metadata['x-object-sysmeta-crypto-etag-mac']) # verify etag override for container updates override = 'x-object-sysmeta-container-update-override-etag' parts = metadata[override].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] cont_key = self.km.create_key('/a/%s' % self.container_name) exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), cont_key, listing_etag_iv)).decode('ascii') self.assertEqual(exp_enc_listing_etag, parts[0]) self._check_GET_and_HEAD(self.crypto_app) return exp_enc_body, ondisk_data
def _test_ondisk_data_after_write_with_crypto(self, policy_name): policy = storage_policy.POLICIES.get_by_name(policy_name) self._create_container(self.proxy_app, policy_name=policy_name) self._put_object(self.crypto_app, self.plaintext) self._post_object(self.crypto_app) # Verify container listing etag is encrypted by direct GET to container # server. We can use any server for all nodes since they all share same # devices dir. cont_server = self._test_context['test_servers'][3] cont_ring = Ring(self._test_context['testdir'], ring_name='container') part, nodes = cont_ring.get_nodes('a', self.container_name) for node in nodes: req = Request.blank('/%s/%s/a/%s' % (node['device'], part, self.container_name), method='GET', query_string='format=json') resp = req.get_response(cont_server) listing = json.loads(resp.body) # sanity checks... self.assertEqual(1, len(listing)) self.assertEqual('o', listing[0]['name']) self.assertEqual('application/test', listing[0]['content_type']) # verify encrypted etag value parts = listing[0]['hash'].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag, self.km.create_key('/a/%s' % self.container_name), listing_etag_iv)) self.assertEqual(exp_enc_listing_etag, parts[0]) # Verify diskfile data and metadata is encrypted ring_object = self.proxy_app.get_object_ring(int(policy)) partition, nodes = ring_object.get_nodes('a', self.container_name, 'o') conf = {'devices': self._test_context["testdir"], 'mount_check': 'false'} df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy] ondisk_data = [] exp_enc_body = None for node_index, node in enumerate(nodes): df = df_mgr.get_diskfile(node['device'], partition, 'a', self.container_name, 'o', policy=policy) with df.open(): meta = df.get_metadata() contents = ''.join(df.reader()) metadata = dict((k.lower(), v) for k, v in meta.items()) # verify on disk data - body body_iv = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['iv'] body_key_meta = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['body_key'] obj_key = self.km.create_key('/a/%s/o' % self.container_name) body_key = Crypto().unwrap_key(obj_key, body_key_meta) exp_enc_body = encrypt(self.plaintext, body_key, body_iv) ondisk_data.append((node, contents)) # verify on disk user metadata enc_val, meta = metadata[ 'x-object-transient-sysmeta-crypto-meta-fruit'].split(';') meta = meta.strip()[len('swift_meta='):] metadata_iv = load_crypto_meta(meta)['iv'] exp_enc_meta = base64.b64encode(encrypt('Kiwi', obj_key, metadata_iv)) self.assertEqual(exp_enc_meta, enc_val) self.assertNotIn('x-object-meta-fruit', metadata) self.assertIn( 'x-object-transient-sysmeta-crypto-meta', metadata) meta = load_crypto_meta( metadata['x-object-transient-sysmeta-crypto-meta']) self.assertIn('key_id', meta) self.assertIn('path', meta['key_id']) self.assertEqual( '/a/%s/%s' % (self.container_name, self.object_name), meta['key_id']['path']) self.assertIn('v', meta['key_id']) self.assertEqual('1', meta['key_id']['v']) self.assertIn('cipher', meta) self.assertEqual(Crypto.cipher, meta['cipher']) # verify etag actual_enc_etag, _junk, actual_etag_meta = metadata[ 'x-object-sysmeta-crypto-etag'].partition('; swift_meta=') etag_iv = load_crypto_meta(actual_etag_meta)['iv'] exp_enc_etag = base64.b64encode(encrypt(self.plaintext_etag, obj_key, etag_iv)) self.assertEqual(exp_enc_etag, actual_enc_etag) # verify etag hmac exp_etag_mac = hmac.new( obj_key, self.plaintext_etag, digestmod=hashlib.sha256) exp_etag_mac = base64.b64encode(exp_etag_mac.digest()) self.assertEqual(exp_etag_mac, metadata['x-object-sysmeta-crypto-etag-mac']) # verify etag override for container updates override = 'x-object-sysmeta-container-update-override-etag' parts = metadata[override].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] cont_key = self.km.create_key('/a/%s' % self.container_name) exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag, cont_key, listing_etag_iv)) self.assertEqual(exp_enc_listing_etag, parts[0]) self._check_GET_and_HEAD(self.crypto_app) return exp_enc_body, ondisk_data
def assert_raises(value, message): with self.assertRaises(EncryptionException) as cm: crypto_utils.load_crypto_meta(value) self.assertIn('Bad crypto meta %r' % value, cm.exception.message) self.assertIn(message, cm.exception.message)
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata) for label, meta in [ ('Data crypto details', sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')), ('Metadata crypto details', transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')), ]: if meta is None: continue print('%s: %s' % ( label, json.dumps(load_crypto_meta(meta, b64decode=False), indent=2, sort_keys=True, separators=(',', ': '))))