def _crypt(self, offset, data):
offset_big = offset // 16
offset_small = offset % 16
iv = binascii.unhexlify("%032x" % offset_big)
cipher = AES(self.key, iv=iv)
cipher.process("\x00" * offset_small)
return cipher.process(data)
def _decrypt_rwcapdata(self, encwrcap):
salt = encwrcap[:16]
crypttext = encwrcap[16:-32]
key = hashutil.mutable_rwcap_key_hash(salt, self._node.get_writekey())
cryptor = AES(key)
plaintext = cryptor.process(crypttext)
return plaintext
def _crypt(self, offset, data):
offset_big = offset // 16
offset_small = offset % 16
iv = binascii.unhexlify("%032x" % offset_big)
cipher = AES(self.key, iv=iv)
cipher.process("\x00"*offset_small)
return cipher.process(data)
def _decrypt_rwcapdata(self, encwrcap):
salt = encwrcap[:16]
crypttext = encwrcap[16:-32]
key = hashutil.mutable_rwcap_key_hash(salt, self._node.get_writekey())
cryptor = AES(key)
plaintext = cryptor.process(crypttext)
return plaintext
def _decrypt(self, crypttext, IV, readkey):
self._status.set_status("decrypting")
started = time.time()
key = hashutil.ssk_readkey_data_hash(IV, readkey)
decryptor = AES(key)
plaintext = decryptor.process(crypttext)
self._status.timings["decrypt"] = time.time() - started
return plaintext
def _decrypt(self, crypttext, IV, readkey):
self._status.set_status("decrypting")
started = time.time()
key = hashutil.ssk_readkey_data_hash(IV, readkey)
decryptor = AES(key)
plaintext = decryptor.process(crypttext)
self._status.timings["decrypt"] = time.time() - started
return plaintext
def test_previous_upload_failed(self):
self.basedir = "helper/AssistedUpload/test_previous_upload_failed"
self.setUpHelper(self.basedir)
# we want to make sure that an upload which fails (leaving the
# ciphertext in the CHK_encoding/ directory) does not prevent a later
# attempt to upload that file from working. We simulate this by
# populating the directory manually. The hardest part is guessing the
# storage index.
k = FakeClient.DEFAULT_ENCODING_PARAMETERS["k"]
n = FakeClient.DEFAULT_ENCODING_PARAMETERS["n"]
max_segsize = FakeClient.DEFAULT_ENCODING_PARAMETERS[
"max_segment_size"]
segsize = min(max_segsize, len(DATA))
# this must be a multiple of 'required_shares'==k
segsize = mathutil.next_multiple(segsize, k)
key = hashutil.convergence_hash(k, n, segsize, DATA,
"test convergence string")
assert len(key) == 16
encryptor = AES(key)
SI = hashutil.storage_index_hash(key)
SI_s = si_b2a(SI)
encfile = os.path.join(self.basedir, "CHK_encoding", SI_s)
f = open(encfile, "wb")
f.write(encryptor.process(DATA))
f.close()
u = upload.Uploader(self.helper_furl)
u.setServiceParent(self.s)
d = wait_a_few_turns()
def _ready(res):
assert u._helper
return upload_data(u, DATA, convergence="test convergence string")
d.addCallback(_ready)
def _uploaded(results):
the_uri = results.get_uri()
assert "CHK" in the_uri
d.addCallback(_uploaded)
def _check_empty(res):
files = os.listdir(os.path.join(self.basedir, "CHK_encoding"))
self.failUnlessEqual(files, [])
files = os.listdir(os.path.join(self.basedir, "CHK_incoming"))
self.failUnlessEqual(files, [])
d.addCallback(_check_empty)
return d
def _encrypt_rw_uri(writekey, rw_uri):
precondition(isinstance(rw_uri, str), rw_uri)
precondition(isinstance(writekey, str), writekey)
salt = hashutil.mutable_rwcap_salt_hash(rw_uri)
key = hashutil.mutable_rwcap_key_hash(salt, writekey)
cryptor = AES(key)
crypttext = cryptor.process(rw_uri)
mac = hashutil.hmac(key, salt + crypttext)
assert len(mac) == 32
return salt + crypttext + mac
def _encrypt_rw_uri(writekey, rw_uri):
precondition(isinstance(rw_uri, str), rw_uri)
precondition(isinstance(writekey, str), writekey)
salt = hashutil.mutable_rwcap_salt_hash(rw_uri)
key = hashutil.mutable_rwcap_key_hash(salt, writekey)
cryptor = AES(key)
crypttext = cryptor.process(rw_uri)
mac = hashutil.hmac(key, salt + crypttext)
assert len(mac) == 32
return salt + crypttext + mac
def test_previous_upload_failed(self):
self.basedir = "helper/AssistedUpload/test_previous_upload_failed"
self.setUpHelper(self.basedir)
# we want to make sure that an upload which fails (leaving the
# ciphertext in the CHK_encoding/ directory) does not prevent a later
# attempt to upload that file from working. We simulate this by
# populating the directory manually. The hardest part is guessing the
# storage index.
k = FakeClient.DEFAULT_ENCODING_PARAMETERS["k"]
n = FakeClient.DEFAULT_ENCODING_PARAMETERS["n"]
max_segsize = FakeClient.DEFAULT_ENCODING_PARAMETERS["max_segment_size"]
segsize = min(max_segsize, len(DATA))
# this must be a multiple of 'required_shares'==k
segsize = mathutil.next_multiple(segsize, k)
key = hashutil.convergence_hash(k, n, segsize, DATA, "test convergence string")
assert len(key) == 16
encryptor = AES(key)
SI = hashutil.storage_index_hash(key)
SI_s = si_b2a(SI)
encfile = os.path.join(self.basedir, "CHK_encoding", SI_s)
f = open(encfile, "wb")
f.write(encryptor.process(DATA))
f.close()
u = upload.Uploader(self.helper_furl)
u.setServiceParent(self.s)
d = wait_a_few_turns()
def _ready(res):
assert u._helper
return upload_data(u, DATA, convergence="test convergence string")
d.addCallback(_ready)
def _uploaded(results):
the_uri = results.get_uri()
assert "CHK" in the_uri
d.addCallback(_uploaded)
def _check_empty(res):
files = os.listdir(os.path.join(self.basedir, "CHK_encoding"))
self.failUnlessEqual(files, [])
files = os.listdir(os.path.join(self.basedir, "CHK_incoming"))
self.failUnlessEqual(files, [])
d.addCallback(_check_empty)
return d
def __init__(self, consumer, readkey, offset):
self._consumer = consumer
self._read_event = None
# TODO: pycryptopp CTR-mode needs random-access operations: I want
# either a=AES(readkey, offset) or better yet both of:
# a=AES(readkey, offset=0)
# a.process(ciphertext, offset=xyz)
# For now, we fake it with the existing iv= argument.
offset_big = offset // 16
offset_small = offset % 16
iv = binascii.unhexlify("%032x" % offset_big)
self._decryptor = AES(readkey, iv=iv)
self._decryptor.process("\x00" * offset_small)
def _decrypt_segment(self, segment_and_salt):
"""
I take a single segment and its salt, and decrypt it. I return
the plaintext of the segment that is in my argument.
"""
segment, salt = segment_and_salt
self._set_current_status("decrypting")
self.log("decrypting segment %d" % self._current_segment)
started = time.time()
key = hashutil.ssk_readkey_data_hash(salt, self._node.get_readkey())
decryptor = AES(key)
plaintext = decryptor.process(segment)
self._status.accumulate_decrypt_time(time.time() - started)
return plaintext
def _decrypt_segment(self, segment_and_salt):
"""
I take a single segment and its salt, and decrypt it. I return
the plaintext of the segment that is in my argument.
"""
segment, salt = segment_and_salt
self._set_current_status("decrypting")
self.log("decrypting segment %d" % self._current_segment)
started = time.time()
key = hashutil.ssk_readkey_data_hash(salt, self._node.get_readkey())
decryptor = AES(key)
plaintext = decryptor.process(segment)
self._status.accumulate_decrypt_time(time.time() - started)
return plaintext
def encrypt_sym(data, key, method):
"""
Encrypt C{data} using a {password}.
Currently, the only encryption methods supported are AES-256 in CTR
mode and XSalsa20.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt C{data} (must be 256 bits long).
:type key: str
:param method: The encryption method to use.
:type method: str
:return: A tuple with the initial value and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' % (len(key) * 8))
iv = None
# AES-256 in CTR mode
if method == EncryptionMethods.AES_256_CTR:
iv = os.urandom(16)
ciphertext = AES(key=key, iv=iv).process(data)
# XSalsa20
elif method == EncryptionMethods.XSALSA20:
iv = os.urandom(24)
ciphertext = XSalsa20(key=key, iv=iv).process(data)
else:
# raise if method is unknown
raise UnknownEncryptionMethod('Unkwnown method: %s' % method)
return binascii.b2a_base64(iv), ciphertext
def decrypt_sym(data, key, method, **kwargs):
"""
Decrypt data using symmetric secret.
Currently, the only encryption method supported is AES-256 CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The key used to decrypt C{data} (must be 256 bits long).
:type key: str
:param method: The encryption method to use.
:type method: str
:param kwargs: Other parameters specific to each encryption method.
:type kwargs: dict
:return: The decrypted data.
:rtype: str
:raise UnknownEncryptionMethodError: Raised when C{method} is unknown.
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
soledad_assert('iv' in kwargs, '%s needs an initial value.' % method)
_assert_known_encryption_method(method)
# AES-256 in CTR mode
if method == crypto.EncryptionMethods.AES_256_CTR:
return AES(key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data)
elif method == crypto.EncryptionMethods.XSALSA20:
return XSalsa20(key=key,
iv=binascii.a2b_base64(kwargs['iv'])).process(data)
class DecryptingConsumer:
"""I sit between a CiphertextDownloader (which acts as a Producer) and
the real Consumer, decrypting everything that passes by. The real
Consumer sees the real Producer, but the Producer sees us instead of the
real consumer."""
implements(IConsumer, IDownloadStatusHandlingConsumer)
def __init__(self, consumer, readkey, offset):
self._consumer = consumer
self._read_ev = None
self._download_status = None
# TODO: pycryptopp CTR-mode needs random-access operations: I want
# either a=AES(readkey, offset) or better yet both of:
# a=AES(readkey, offset=0)
# a.process(ciphertext, offset=xyz)
# For now, we fake it with the existing iv= argument.
offset_big = offset // 16
offset_small = offset % 16
iv = binascii.unhexlify("%032x" % offset_big)
self._decryptor = AES(readkey, iv=iv)
self._decryptor.process("\x00" * offset_small)
def set_download_status_read_event(self, read_ev):
self._read_ev = read_ev
def set_download_status(self, ds):
self._download_status = ds
def registerProducer(self, producer, streaming):
# this passes through, so the real consumer can flow-control the real
# producer. Therefore we don't need to provide any IPushProducer
# methods. We implement all the IConsumer methods as pass-throughs,
# and only intercept write() to perform decryption.
self._consumer.registerProducer(producer, streaming)
def unregisterProducer(self):
self._consumer.unregisterProducer()
def write(self, ciphertext):
started = now()
plaintext = self._decryptor.process(ciphertext)
if self._read_ev:
elapsed = now() - started
self._read_ev.update(0, elapsed, 0)
if self._download_status:
self._download_status.add_misc_event("AES", started, now())
self._consumer.write(plaintext)
class DecryptingConsumer:
"""I sit between a CiphertextDownloader (which acts as a Producer) and
the real Consumer, decrypting everything that passes by. The real
Consumer sees the real Producer, but the Producer sees us instead of the
real consumer."""
implements(IConsumer, IDownloadStatusHandlingConsumer)
def __init__(self, consumer, readkey, offset):
self._consumer = consumer
self._read_ev = None
self._download_status = None
# TODO: pycryptopp CTR-mode needs random-access operations: I want
# either a=AES(readkey, offset) or better yet both of:
# a=AES(readkey, offset=0)
# a.process(ciphertext, offset=xyz)
# For now, we fake it with the existing iv= argument.
offset_big = offset // 16
offset_small = offset % 16
iv = binascii.unhexlify("%032x" % offset_big)
self._decryptor = AES(readkey, iv=iv)
self._decryptor.process("\x00"*offset_small)
def set_download_status_read_event(self, read_ev):
self._read_ev = read_ev
def set_download_status(self, ds):
self._download_status = ds
def registerProducer(self, producer, streaming):
# this passes through, so the real consumer can flow-control the real
# producer. Therefore we don't need to provide any IPushProducer
# methods. We implement all the IConsumer methods as pass-throughs,
# and only intercept write() to perform decryption.
self._consumer.registerProducer(producer, streaming)
def unregisterProducer(self):
self._consumer.unregisterProducer()
def write(self, ciphertext):
started = now()
plaintext = self._decryptor.process(ciphertext)
if self._read_ev:
elapsed = now() - started
self._read_ev.update(0, elapsed, 0)
if self._download_status:
self._download_status.add_misc_event("AES", started, now())
self._consumer.write(plaintext)
def _encrypt_and_encode(self):
# this returns a Deferred that fires with a list of (sharedata,
# sharenum) tuples. TODO: cache the ciphertext, only produce the
# shares that we care about.
self.log("_encrypt_and_encode")
self._status.set_status("Encrypting")
started = time.time()
key = hashutil.ssk_readkey_data_hash(self.salt, self.readkey)
enc = AES(key)
crypttext = enc.process(self.newdata)
assert len(crypttext) == len(self.newdata)
now = time.time()
self._status.timings["encrypt"] = now - started
started = now
# now apply FEC
self._status.set_status("Encoding")
fec = codec.CRSEncoder()
fec.set_params(self.segment_size, self.required_shares,
self.total_shares)
piece_size = fec.get_block_size()
crypttext_pieces = [None] * self.required_shares
for i in range(len(crypttext_pieces)):
offset = i * piece_size
piece = crypttext[offset:offset + piece_size]
piece = piece + "\x00" * (piece_size - len(piece)) # padding
crypttext_pieces[i] = piece
assert len(piece) == piece_size
d = fec.encode(crypttext_pieces)
def _done_encoding(res):
elapsed = time.time() - started
self._status.timings["encode"] = elapsed
return res
d.addCallback(_done_encoding)
return d
def _encrypt_and_encode(self):
# this returns a Deferred that fires with a list of (sharedata,
# sharenum) tuples. TODO: cache the ciphertext, only produce the
# shares that we care about.
self.log("_encrypt_and_encode")
self._status.set_status("Encrypting")
started = time.time()
key = hashutil.ssk_readkey_data_hash(self.salt, self.readkey)
enc = AES(key)
crypttext = enc.process(self.newdata)
assert len(crypttext) == len(self.newdata)
now = time.time()
self._status.timings["encrypt"] = now - started
started = now
# now apply FEC
self._status.set_status("Encoding")
fec = codec.CRSEncoder()
fec.set_params(self.segment_size,
self.required_shares, self.total_shares)
piece_size = fec.get_block_size()
crypttext_pieces = [None] * self.required_shares
for i in range(len(crypttext_pieces)):
offset = i * piece_size
piece = crypttext[offset:offset+piece_size]
piece = piece + "\x00"*(piece_size - len(piece)) # padding
crypttext_pieces[i] = piece
assert len(piece) == piece_size
d = fec.encode(crypttext_pieces)
def _done_encoding(res):
elapsed = time.time() - started
self._status.timings["encode"] = elapsed
return res
d.addCallback(_done_encoding)
return d
def aes(key, data, counter=False):
""" encrypt data with aes, using either pycryptopp or PyCrypto.
Args
key: The encryption key
data: plain text data
counter: a callable, usually not needed
"""
# using either pycryptopp...
if hasattr(AES, "process"):
a = AES(key)
return a.process(data)
# ... or PyCrypto
counter = counter or Counter()
a = AES.new(key, AES.MODE_CTR, counter=counter)
rest = len(data) % 16
if not rest:
return a.encrypt(data)
# Data length must be a multiple of 16
# Pad with bytes all of the same value as the number of padding bytes
pad = (16 - rest)
data += chr(pad) * pad
return a.encrypt(data)[:-pad]
def __init__(self, consumer, readkey, offset):
self._consumer = consumer
self._read_event = None
# TODO: pycryptopp CTR-mode needs random-access operations: I want
# either a=AES(readkey, offset) or better yet both of:
# a=AES(readkey, offset=0)
# a.process(ciphertext, offset=xyz)
# For now, we fake it with the existing iv= argument.
offset_big = offset // 16
offset_small = offset % 16
iv = binascii.unhexlify("%032x" % offset_big)
self._decryptor = AES(readkey, iv=iv)
self._decryptor.process("\x00"*offset_small)
def encrypt_sym(data, key):
"""
Encrypt data using AES-256 cipher in CTR mode.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt data (must be 256 bits long).
:type key: str
:return: A tuple with the initialization vector and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' % (len(key) * 8))
iv = os.urandom(16)
ciphertext = AES(key=key, iv=iv).process(data)
return binascii.b2a_base64(iv), ciphertext
def decrypt_sym(data, key, iv):
"""
Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The symmetric key used to decrypt data (must be 256 bits
long).
:type key: str
:param iv: The initialization vector.
:type iv: long
:return: The decrypted data.
:rtype: str
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
return AES(key=key, iv=binascii.a2b_base64(iv)).process(data)
def aes(key, data, counter=False):
""" encrypt data with aes, using either pycryptopp or PyCrypto.
Args
key: The encryption key
data: plain text data
counter: a callable, usually not needed
"""
# using either pycryptopp...
if hasattr(AES, "process"):
a = AES(key)
return a.process(data)
# ... or PyCrypto
counter = counter or Counter()
a = AES.new(key, AES.MODE_CTR, counter=counter)
rest = len(data) % 16
if not rest:
return a.encrypt(data)
# Data length must be a multiple of 16
# Pad with bytes all of the same value as the number of padding bytes
pad = (16 - rest)
data += chr(pad) * pad
return a.encrypt(data)[:-pad]
def _decrypt_privkey(self, enc_privkey):
enc = AES(self._writekey)
privkey = enc.process(enc_privkey)
return privkey
def _encrypt_privkey(self, writekey, privkey):
enc = AES(writekey)
crypttext = enc.process(privkey)
return crypttext
def _decrypt_privkey(self, enc_privkey):
enc = AES(self._writekey)
privkey = enc.process(enc_privkey)
return privkey
def _encrypt_privkey(self, writekey, privkey):
enc = AES(writekey)
crypttext = enc.process(privkey)
return crypttext
