def test_load_crypto_meta(self):
actual = crypto_utils.load_crypto_meta(self.serialized_meta)
self.assertEqual(self.meta, actual)
actual = crypto_utils.load_crypto_meta(self.serialized_meta_with_key)
self.assertEqual(self.meta_with_key, actual)
def assert_raises(value, message):
with self.assertRaises(EncryptionException) as cm:
crypto_utils.load_crypto_meta(value)
self.assertIn('Bad crypto meta %r' % value, cm.exception.message)
self.assertIn(message, cm.exception.message)
assert_raises(None, 'crypto meta not a string')
assert_raises(99, 'crypto meta not a string')
assert_raises('', 'No JSON object could be decoded')
assert_raises('abc', 'No JSON object could be decoded')
assert_raises('[]', 'crypto meta not a Mapping')
assert_raises('{"iv": "abcdef"}', 'Incorrect padding')
assert_raises('{"iv": []}', 'must be string or buffer')
assert_raises('{"iv": {}}', 'must be string or buffer')
assert_raises('{"iv": 99}', 'must be string or buffer')
assert_raises('{"key": "abcdef"}', 'Incorrect padding')
assert_raises('{"key": []}', 'must be string or buffer')
assert_raises('{"key": {}}', 'must be string or buffer')
assert_raises('{"key": 99}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"iv": []}}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": {}}}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": 99}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"key": []}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": {}}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": 99}}', 'must be string or buffer')
def test_load_crypto_meta(self):
actual = crypto_utils.load_crypto_meta(self.serialized_meta)
self.assertEqual(self.meta, actual)
actual = crypto_utils.load_crypto_meta(self.serialized_meta_with_key)
self.assertEqual(self.meta_with_key, actual)
def assert_raises(value, message):
with self.assertRaises(EncryptionException) as cm:
crypto_utils.load_crypto_meta(value)
self.assertIn('Bad crypto meta %r' % value, cm.exception.message)
self.assertIn(message, cm.exception.message)
assert_raises(None, 'crypto meta not a string')
assert_raises(99, 'crypto meta not a string')
assert_raises('', 'No JSON object could be decoded')
assert_raises('abc', 'No JSON object could be decoded')
assert_raises('[]', 'crypto meta not a Mapping')
assert_raises('{"iv": "abcdef"}', 'Incorrect padding')
assert_raises('{"iv": []}', 'must be string or buffer')
assert_raises('{"iv": {}}', 'must be string or buffer')
assert_raises('{"iv": 99}', 'must be string or buffer')
assert_raises('{"key": "abcdef"}', 'Incorrect padding')
assert_raises('{"key": []}', 'must be string or buffer')
assert_raises('{"key": {}}', 'must be string or buffer')
assert_raises('{"key": 99}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"iv": []}}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": {}}}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": 99}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"key": []}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": {}}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": 99}}', 'must be string or buffer')
def test_dump_then_load_crypto_meta(self):
actual = crypto_utils.load_crypto_meta(
crypto_utils.dump_crypto_meta(self.meta))
self.assertEqual(self.meta, actual)
actual = crypto_utils.load_crypto_meta(
crypto_utils.dump_crypto_meta(self.meta_with_key))
self.assertEqual(self.meta_with_key, actual)
def test_dump_then_load_crypto_meta(self):
actual = crypto_utils.load_crypto_meta(
crypto_utils.dump_crypto_meta(self.meta))
self.assertEqual(self.meta, actual)
actual = crypto_utils.load_crypto_meta(
crypto_utils.dump_crypto_meta(self.meta_with_key))
self.assertEqual(self.meta_with_key, actual)
def assert_raises(value, message):
with self.assertRaises(EncryptionException) as cm:
crypto_utils.load_crypto_meta(value)
self.assertIn('Bad crypto meta %r' % value, cm.exception.args[0])
if isinstance(message, (tuple, list)):
for opt in message:
if opt in cm.exception.args[0]:
break
else:
self.fail('Expected to find one of %r in %r' % (
message, cm.exception.args[0]))
else:
self.assertIn(message, cm.exception.args[0])
def test_load_crypto_meta(self):
actual = crypto_utils.load_crypto_meta(self.serialized_meta)
self.assertEqual(self.meta, actual)
actual = crypto_utils.load_crypto_meta(self.serialized_meta_with_key)
self.assertEqual(self.meta_with_key, actual)
def assert_raises(value, message):
with self.assertRaises(EncryptionException) as cm:
crypto_utils.load_crypto_meta(value)
self.assertIn('Bad crypto meta %r' % value, cm.exception.args[0])
if isinstance(message, (tuple, list)):
for opt in message:
if opt in cm.exception.args[0]:
break
else:
self.fail('Expected to find one of %r in %r' % (
message, cm.exception.args[0]))
else:
self.assertIn(message, cm.exception.args[0])
assert_raises(None, 'crypto meta not a string')
assert_raises(99, 'crypto meta not a string')
assert_raises('', ('No JSON object could be decoded',
'Expecting value: line 1 column 1'))
assert_raises('abc', ('No JSON object could be decoded',
'Expecting value: line 1 column 1'))
assert_raises('[]', 'crypto meta not a Mapping')
bad_type_messages = [
'must be string or buffer',
'argument should be a bytes-like object or ASCII string',
]
assert_raises('{"iv": "abcdef"}', 'Incorrect padding')
assert_raises('{"iv": []}', bad_type_messages)
assert_raises('{"iv": {}}', bad_type_messages)
assert_raises('{"iv": 99}', bad_type_messages)
assert_raises('{"key": "abcdef"}', 'Incorrect padding')
assert_raises('{"key": []}', bad_type_messages)
assert_raises('{"key": {}}', bad_type_messages)
assert_raises('{"key": 99}', bad_type_messages)
assert_raises('{"body_key": {"iv": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"iv": []}}', bad_type_messages)
assert_raises('{"body_key": {"iv": {}}}', bad_type_messages)
assert_raises('{"body_key": {"iv": 99}}', bad_type_messages)
assert_raises('{"body_key": {"key": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"key": []}}', bad_type_messages)
assert_raises('{"body_key": {"key": {}}}', bad_type_messages)
assert_raises('{"body_key": {"key": 99}}', bad_type_messages)
def get_crypto_meta(self, header_name):
"""
Extract a crypto_meta dict from a header.
:param header_name: name of header that may have crypto_meta
:return: A dict containing crypto_meta items
:raises EncryptionException: if an error occurs while parsing the
crypto meta
"""
crypto_meta_json = self._response_header_value(header_name)
if crypto_meta_json is None:
return None
crypto_meta = load_crypto_meta(crypto_meta_json)
self.crypto.check_crypto_meta(crypto_meta)
return crypto_meta
def get_crypto_meta(self, header_name):
"""
Extract a crypto_meta dict from a header.
:param header_name: name of header that may have crypto_meta
:return: A dict containing crypto_meta items
:raises EncryptionException: if an error occurs while parsing the
crypto meta
"""
crypto_meta_json = self._response_header_value(header_name)
if crypto_meta_json is None:
return None
crypto_meta = load_crypto_meta(crypto_meta_json)
self.crypto.check_crypto_meta(crypto_meta)
return crypto_meta
def _test_ondisk_data_after_write_with_crypto(self, policy_name):
policy = storage_policy.POLICIES.get_by_name(policy_name)
self._create_container(self.proxy_app, policy_name=policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
# Verify container listing etag is encrypted by direct GET to container
# server. We can use any server for all nodes since they all share same
# devices dir.
cont_server = self._test_context['test_servers'][3]
cont_ring = Ring(self._test_context['testdir'], ring_name='container')
part, nodes = cont_ring.get_nodes('a', self.container_name)
for node in nodes:
req = Request.blank('/%s/%s/a/%s' %
(node['device'], part, self.container_name),
method='GET',
query_string='format=json')
resp = req.get_response(cont_server)
listing = json.loads(resp.body)
# sanity checks...
self.assertEqual(1, len(listing))
self.assertEqual('o', listing[0]['name'])
self.assertEqual('application/test', listing[0]['content_type'])
# verify encrypted etag value
parts = listing[0]['hash'].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'),
self.km.create_key('/a/%s' % self.container_name),
listing_etag_iv)).decode('ascii')
self.assertEqual(exp_enc_listing_etag, parts[0])
# Verify diskfile data and metadata is encrypted
ring_object = self.proxy_app.get_object_ring(int(policy))
partition, nodes = ring_object.get_nodes('a', self.container_name, 'o')
conf = {
'devices': self._test_context["testdir"],
'mount_check': 'false'
}
df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy]
ondisk_data = []
exp_enc_body = None
for node_index, node in enumerate(nodes):
df = df_mgr.get_diskfile(node['device'],
partition,
'a',
self.container_name,
'o',
policy=policy)
with df.open():
meta = df.get_metadata()
contents = b''.join(df.reader())
metadata = dict((k.lower(), v) for k, v in meta.items())
# verify on disk data - body
body_iv = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['iv']
body_key_meta = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['body_key']
obj_key = self.km.create_key('/a/%s/o' % self.container_name)
body_key = Crypto().unwrap_key(obj_key, body_key_meta)
exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
ondisk_data.append((node, contents))
# verify on disk user metadata
enc_val, meta = metadata[
'x-object-transient-sysmeta-crypto-meta-fruit'].split(';')
meta = meta.strip()[len('swift_meta='):]
metadata_iv = load_crypto_meta(meta)['iv']
exp_enc_meta = base64.b64encode(
encrypt(b'Kiwi', obj_key, metadata_iv)).decode('ascii')
self.assertEqual(exp_enc_meta, enc_val)
self.assertNotIn('x-object-meta-fruit', metadata)
self.assertIn('x-object-transient-sysmeta-crypto-meta',
metadata)
meta = load_crypto_meta(
metadata['x-object-transient-sysmeta-crypto-meta'])
self.assertIn('key_id', meta)
self.assertIn('path', meta['key_id'])
self.assertEqual(
'/a/%s/%s' % (self.container_name, self.object_name),
meta['key_id']['path'])
self.assertIn('v', meta['key_id'])
self.assertEqual('2', meta['key_id']['v'])
self.assertIn('cipher', meta)
self.assertEqual(Crypto.cipher, meta['cipher'])
# verify etag
actual_enc_etag, _junk, actual_etag_meta = metadata[
'x-object-sysmeta-crypto-etag'].partition('; swift_meta=')
etag_iv = load_crypto_meta(actual_etag_meta)['iv']
exp_enc_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'), obj_key,
etag_iv)).decode('ascii')
self.assertEqual(exp_enc_etag, actual_enc_etag)
# verify etag hmac
exp_etag_mac = hmac.new(obj_key,
self.plaintext_etag.encode('ascii'),
digestmod=hashlib.sha256).digest()
exp_etag_mac = base64.b64encode(exp_etag_mac).decode('ascii')
self.assertEqual(exp_etag_mac,
metadata['x-object-sysmeta-crypto-etag-mac'])
# verify etag override for container updates
override = 'x-object-sysmeta-container-update-override-etag'
parts = metadata[override].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
cont_key = self.km.create_key('/a/%s' % self.container_name)
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'), cont_key,
listing_etag_iv)).decode('ascii')
self.assertEqual(exp_enc_listing_etag, parts[0])
self._check_GET_and_HEAD(self.crypto_app)
return exp_enc_body, ondisk_data
def _test_ondisk_data_after_write_with_crypto(self, policy_name):
policy = storage_policy.POLICIES.get_by_name(policy_name)
self._create_container(self.proxy_app, policy_name=policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
# Verify container listing etag is encrypted by direct GET to container
# server. We can use any server for all nodes since they all share same
# devices dir.
cont_server = self._test_context['test_servers'][3]
cont_ring = Ring(self._test_context['testdir'], ring_name='container')
part, nodes = cont_ring.get_nodes('a', self.container_name)
for node in nodes:
req = Request.blank('/%s/%s/a/%s'
% (node['device'], part, self.container_name),
method='GET', query_string='format=json')
resp = req.get_response(cont_server)
listing = json.loads(resp.body)
# sanity checks...
self.assertEqual(1, len(listing))
self.assertEqual('o', listing[0]['name'])
self.assertEqual('application/test', listing[0]['content_type'])
# verify encrypted etag value
parts = listing[0]['hash'].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag,
self.km.create_key('/a/%s' % self.container_name),
listing_etag_iv))
self.assertEqual(exp_enc_listing_etag, parts[0])
# Verify diskfile data and metadata is encrypted
ring_object = self.proxy_app.get_object_ring(int(policy))
partition, nodes = ring_object.get_nodes('a', self.container_name, 'o')
conf = {'devices': self._test_context["testdir"],
'mount_check': 'false'}
df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy]
ondisk_data = []
exp_enc_body = None
for node_index, node in enumerate(nodes):
df = df_mgr.get_diskfile(node['device'], partition,
'a', self.container_name, 'o',
policy=policy)
with df.open():
meta = df.get_metadata()
contents = ''.join(df.reader())
metadata = dict((k.lower(), v) for k, v in meta.items())
# verify on disk data - body
body_iv = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['iv']
body_key_meta = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['body_key']
obj_key = self.km.create_key('/a/%s/o' % self.container_name)
body_key = Crypto().unwrap_key(obj_key, body_key_meta)
exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
ondisk_data.append((node, contents))
# verify on disk user metadata
enc_val, meta = metadata[
'x-object-transient-sysmeta-crypto-meta-fruit'].split(';')
meta = meta.strip()[len('swift_meta='):]
metadata_iv = load_crypto_meta(meta)['iv']
exp_enc_meta = base64.b64encode(encrypt('Kiwi', obj_key,
metadata_iv))
self.assertEqual(exp_enc_meta, enc_val)
self.assertNotIn('x-object-meta-fruit', metadata)
self.assertIn(
'x-object-transient-sysmeta-crypto-meta', metadata)
meta = load_crypto_meta(
metadata['x-object-transient-sysmeta-crypto-meta'])
self.assertIn('key_id', meta)
self.assertIn('path', meta['key_id'])
self.assertEqual(
'/a/%s/%s' % (self.container_name, self.object_name),
meta['key_id']['path'])
self.assertIn('v', meta['key_id'])
self.assertEqual('1', meta['key_id']['v'])
self.assertIn('cipher', meta)
self.assertEqual(Crypto.cipher, meta['cipher'])
# verify etag
actual_enc_etag, _junk, actual_etag_meta = metadata[
'x-object-sysmeta-crypto-etag'].partition('; swift_meta=')
etag_iv = load_crypto_meta(actual_etag_meta)['iv']
exp_enc_etag = base64.b64encode(encrypt(self.plaintext_etag,
obj_key, etag_iv))
self.assertEqual(exp_enc_etag, actual_enc_etag)
# verify etag hmac
exp_etag_mac = hmac.new(
obj_key, self.plaintext_etag, digestmod=hashlib.sha256)
exp_etag_mac = base64.b64encode(exp_etag_mac.digest())
self.assertEqual(exp_etag_mac,
metadata['x-object-sysmeta-crypto-etag-mac'])
# verify etag override for container updates
override = 'x-object-sysmeta-container-update-override-etag'
parts = metadata[override].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
cont_key = self.km.create_key('/a/%s' % self.container_name)
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag, cont_key,
listing_etag_iv))
self.assertEqual(exp_enc_listing_etag, parts[0])
self._check_GET_and_HEAD(self.crypto_app)
return exp_enc_body, ondisk_data
def assert_raises(value, message):
with self.assertRaises(EncryptionException) as cm:
crypto_utils.load_crypto_meta(value)
self.assertIn('Bad crypto meta %r' % value, cm.exception.message)
self.assertIn(message, cm.exception.message)
def print_obj_metadata(metadata, drop_prefixes=False):
"""
Print out basic info and metadata from object, as returned from
:func:`swift.obj.diskfile.read_metadata`.
Metadata should include the keys: name, Content-Type, and
X-Timestamp.
Additional metadata is displayed unmodified.
:param metadata: dict of object metadata
:param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-",
and "X-Object-Transient-Sysmeta-" when displaying
User Metadata, System Metadata, and Transient
System Metadata entries
:raises ValueError:
"""
user_metadata = {}
sys_metadata = {}
transient_sys_metadata = {}
other_metadata = {}
if not metadata:
raise ValueError('Metadata is None')
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = Timestamp(metadata.pop('X-Timestamp', 0))
account = container = obj = obj_hash = None
if path:
try:
account, container, obj = path.split('/', 3)[1:]
except ValueError:
raise ValueError('Path is invalid for object %r' % path)
else:
obj_hash = hash_path(account, container, obj)
print('Path: %s' % path)
print(' Account: %s' % account)
print(' Container: %s' % container)
print(' Object: %s' % obj)
print(' Object hash: %s' % obj_hash)
else:
print('Path: Not found in metadata')
if content_type:
print('Content-Type: %s' % content_type)
else:
print('Content-Type: Not found in metadata')
if ts:
print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
else:
print('Timestamp: Not found in metadata')
for key, value in metadata.items():
if is_user_meta('Object', key):
if drop_prefixes:
key = strip_user_meta_prefix('Object', key)
user_metadata[key] = value
elif is_sys_meta('Object', key):
if drop_prefixes:
key = strip_sys_meta_prefix('Object', key)
sys_metadata[key] = value
elif is_object_transient_sysmeta(key):
if drop_prefixes:
key = strip_object_transient_sysmeta_prefix(key)
transient_sys_metadata[key] = value
else:
other_metadata[key] = value
def print_metadata(title, items):
print(title)
if items:
for key, value in sorted(items.items()):
print(' %s: %s' % (key, value))
else:
print(' No metadata found')
print_metadata('System Metadata:', sys_metadata)
print_metadata('Transient System Metadata:', transient_sys_metadata)
print_metadata('User Metadata:', user_metadata)
print_metadata('Other Metadata:', other_metadata)
for label, meta in [
('Data crypto details',
sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')),
('Metadata crypto details',
transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')),
]:
if meta is None:
continue
print('%s: %s' % (
label,
json.dumps(load_crypto_meta(meta, b64decode=False), indent=2,
sort_keys=True, separators=(',', ': '))))
def print_obj_metadata(metadata, drop_prefixes=False):
"""
Print out basic info and metadata from object, as returned from
:func:`swift.obj.diskfile.read_metadata`.
Metadata should include the keys: name, Content-Type, and
X-Timestamp.
Additional metadata is displayed unmodified.
:param metadata: dict of object metadata
:param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-",
and "X-Object-Transient-Sysmeta-" when displaying
User Metadata, System Metadata, and Transient
System Metadata entries
:raises ValueError:
"""
user_metadata = {}
sys_metadata = {}
transient_sys_metadata = {}
other_metadata = {}
if not metadata:
raise ValueError('Metadata is None')
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = Timestamp(metadata.pop('X-Timestamp', 0))
account = container = obj = obj_hash = None
if path:
try:
account, container, obj = path.split('/', 3)[1:]
except ValueError:
raise ValueError('Path is invalid for object %r' % path)
else:
obj_hash = hash_path(account, container, obj)
print('Path: %s' % path)
print(' Account: %s' % account)
print(' Container: %s' % container)
print(' Object: %s' % obj)
print(' Object hash: %s' % obj_hash)
else:
print('Path: Not found in metadata')
if content_type:
print('Content-Type: %s' % content_type)
else:
print('Content-Type: Not found in metadata')
if ts:
print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
else:
print('Timestamp: Not found in metadata')
for key, value in metadata.items():
if is_user_meta('Object', key):
if drop_prefixes:
key = strip_user_meta_prefix('Object', key)
user_metadata[key] = value
elif is_sys_meta('Object', key):
if drop_prefixes:
key = strip_sys_meta_prefix('Object', key)
sys_metadata[key] = value
elif is_object_transient_sysmeta(key):
if drop_prefixes:
key = strip_object_transient_sysmeta_prefix(key)
transient_sys_metadata[key] = value
else:
other_metadata[key] = value
def print_metadata(title, items):
print(title)
if items:
for key, value in sorted(items.items()):
print(' %s: %s' % (key, value))
else:
print(' No metadata found')
print_metadata('System Metadata:', sys_metadata)
print_metadata('Transient System Metadata:', transient_sys_metadata)
print_metadata('User Metadata:', user_metadata)
print_metadata('Other Metadata:', other_metadata)
for label, meta in [
('Data crypto details',
sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')),
('Metadata crypto details',
transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')),
]:
if meta is None:
continue
print('%s: %s' % (
label,
json.dumps(load_crypto_meta(meta, b64decode=False), indent=2,
sort_keys=True, separators=(',', ': '))))
def assert_raises(value, message):
with self.assertRaises(EncryptionException) as cm:
crypto_utils.load_crypto_meta(value)
self.assertIn('Bad crypto meta %r' % value, cm.exception.message)
self.assertIn(message, cm.exception.message)
