def _import_recovery_document(self, data):
"""
Import storage secrets for symmetric encryption and uuid (if present)
from a recovery document.
Note that this method does not store the imported data on disk. For
that, use C{self._store_secrets()}.
:param data: The recovery document.
:type data: dict
:return: A tuple containing the number of imported secrets and the
secret_id of the last active secret.
:rtype: (int, str)
"""
soledad_assert(self.STORAGE_SECRETS_KEY in data)
# include secrets in the secret pool.
secret_count = 0
secrets = data[self.STORAGE_SECRETS_KEY].items()
active_secret = None
# XXX remove check for existence of key (included for backwards
# compatibility)
if self.ACTIVE_SECRET_KEY in data:
active_secret = data[self.ACTIVE_SECRET_KEY]
for secret_id, encrypted_secret in secrets:
if secret_id not in self._secrets:
try:
self._secrets[secret_id] = \
self._decrypt_storage_secret(encrypted_secret)
secret_count += 1
except SecretsException as e:
logger.error("Failed to decrypt storage secret: %s"
% str(e))
return secret_count, active_secret
def _put_secrets_in_shared_db(self):
"""
Assert local keys are the same as shared db's ones.
Try to fetch keys from shared recovery database. If they already exist
in the remote db, assert that that data is the same as local data.
Otherwise, upload keys to shared recovery database.
"""
soledad_assert(
self._has_secret(),
'Tried to send keys to server but they don\'t exist in local '
'storage.')
# try to get secrets doc from server, otherwise create it
doc = self._get_secrets_from_shared_db()
if doc is None:
doc = document.SoledadDocument(
doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
doc.content = self._export_recovery_document()
# upload secrets to server
user_data = self._get_user_data()
events.emit_async(events.SOLEDAD_UPLOADING_KEYS, user_data)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
db.put_doc(doc)
events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, user_data)
def insert_encrypted_received_doc(self, doc_id, doc_rev, content, gen, trans_id, idx):
"""
Decrypt and insert a received document into local staging area to be
processed later on.
:param doc_id: The document ID.
:type doc_id: str
:param doc_rev: The document Revision
:param doc_rev: str
:param content: The content of the document
:type content: dict
:param gen: The document Generation
:type gen: int
:param trans_id: Transaction ID
:type trans_id: str
:param idx: The index of this document in the current sync process.
:type idx: int
:return: A deferred that will fire after the decrypted document has
been inserted in the sync db.
:rtype: twisted.internet.defer.Deferred
"""
soledad_assert(self._crypto is not None, "need a crypto object")
key = self._crypto.doc_passphrase(doc_id)
secret = self._crypto.secret
args = doc_id, doc_rev, content, gen, trans_id, key, secret, idx
# decrypt asynchronously
doc = decrypt_doc_task(*args)
# callback will insert it for later processing
return self._decrypt_doc_cb(doc)
def decrypt_sym(data, key, iv):
"""
Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The symmetric key used to decrypt data (must be 256 bits
long).
:type key: str
:param iv: The initialization vector.
:type iv: long
:return: The decrypted data.
:rtype: str
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
backend = MultiBackend([OpenSSLBackend()])
iv = binascii.a2b_base64(iv)
cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
decryptor = cipher.decryptor()
return decryptor.update(data) + decryptor.finalize()
def decrypt_sym(data, key, method, **kwargs):
"""
Decrypt data using symmetric secret.
Currently, the only encryption method supported is AES-256 CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The key used to decrypt C{data} (must be 256 bits long).
:type key: str
:param method: The encryption method to use.
:type method: str
:param kwargs: Other parameters specific to each encryption method.
:type kwargs: dict
:return: The decrypted data.
:rtype: str
:raise UnknownEncryptionMethodError: Raised when C{method} is unknown.
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
soledad_assert('iv' in kwargs, '%s needs an initial value.' % method)
_assert_known_encryption_method(method)
# AES-256 in CTR mode
if method == crypto.EncryptionMethods.AES_256_CTR:
return AES(key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data)
elif method == crypto.EncryptionMethods.XSALSA20:
return XSalsa20(key=key,
iv=binascii.a2b_base64(kwargs['iv'])).process(data)
def _async_decrypt_doc(self, doc_id, rev, content, gen, trans_id, idx):
"""
Dispatch an asynchronous document decrypting routine and save the
result object.
:param doc_id: The ID for the document with contents to be encrypted.
:type doc: str
:param rev: The revision of the document.
:type rev: str
:param content: The serialized content of the document.
:type content: str
:param gen: The generation corresponding to the modification of that
document.
:type gen: int
:param trans_id: The transaction id corresponding to the modification
of that document.
:type trans_id: str
:param idx: The index of this document in the current sync process.
:type idx: int
"""
soledad_assert(self._crypto is not None, "need a crypto object")
content = json.loads(content)
key = self._crypto.doc_passphrase(doc_id)
secret = self._crypto.secret
args = doc_id, rev, content, gen, trans_id, key, secret, idx
# decrypt asynchronously
self._async_results.append(
self._pool.apply_async(
decrypt_doc_task, args))
def insert_encrypted_received_doc(self, doc_id, doc_rev, content, gen,
trans_id, idx):
"""
Decrypt and insert a received document into local staging area to be
processed later on.
:param doc_id: The document ID.
:type doc_id: str
:param doc_rev: The document Revision
:param doc_rev: str
:param content: The content of the document
:type content: dict
:param gen: The document Generation
:type gen: int
:param trans_id: Transaction ID
:type trans_id: str
:param idx: The index of this document in the current sync process.
:type idx: int
:return: A deferred that will fire after the decrypted document has
been inserted in the sync db.
:rtype: twisted.internet.defer.Deferred
"""
soledad_assert(self._crypto is not None, "need a crypto object")
key = self._crypto.doc_passphrase(doc_id)
secret = self._crypto.secret
args = doc_id, doc_rev, content, gen, trans_id, key, secret, idx
# decrypt asynchronously
doc = decrypt_doc_task(*args)
# callback will insert it for later processing
return self._decrypt_doc_cb(doc)
def _import_recovery_document(self, data):
"""
Import storage secrets for symmetric encryption and uuid (if present)
from a recovery document.
Note that this method does not store the imported data on disk. For
that, use C{self._store_secrets()}.
:param data: The recovery document.
:type data: dict
:return: A tuple containing the number of imported secrets and the
secret_id of the last active secret.
:rtype: (int, str)
"""
soledad_assert(self.STORAGE_SECRETS_KEY in data)
# include secrets in the secret pool.
secret_count = 0
secrets = data[self.STORAGE_SECRETS_KEY].items()
active_secret = None
# XXX remove check for existence of key (included for backwards
# compatibility)
if self.ACTIVE_SECRET_KEY in data:
active_secret = data[self.ACTIVE_SECRET_KEY]
for secret_id, encrypted_secret in secrets:
if secret_id not in self._secrets:
try:
self._secrets[secret_id] = \
self._decrypt_storage_secret(encrypted_secret)
secret_count += 1
except SecretsException as e:
logger.error("Failed to decrypt storage secret: %s" %
str(e))
return secret_count, active_secret
def _put_secrets_in_shared_db(self):
"""
Assert local keys are the same as shared db's ones.
Try to fetch keys from shared recovery database. If they already exist
in the remote db, assert that that data is the same as local data.
Otherwise, upload keys to shared recovery database.
"""
soledad_assert(
self._has_secret(),
'Tried to send keys to server but they don\'t exist in local '
'storage.')
# try to get secrets doc from server, otherwise create it
doc = self._get_secrets_from_shared_db()
if doc is None:
doc = document.SoledadDocument(doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
doc.content = self._export_recovery_document()
# upload secrets to server
user_data = self._get_user_data()
events.emit_async(events.SOLEDAD_UPLOADING_KEYS, user_data)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
db.put_doc(doc)
events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, user_data)
def encrypt_doc(self, doc, workers=True):
"""
Symmetrically encrypt a document.
:param doc: The document with contents to be encrypted.
:type doc: SoledadDocument
:param workers: Whether to defer the decryption to the multiprocess
pool of workers. Useful for debugging purposes.
:type workers: bool
"""
soledad_assert(self._crypto is not None, "need a crypto object")
docstr = doc.get_json()
key = self._crypto.doc_passphrase(doc.doc_id)
secret = self._crypto.secret
args = doc.doc_id, doc.rev, docstr, key, secret
try:
if workers:
res = self._pool.apply_async(
encrypt_doc_task, args,
callback=self.encrypt_doc_cb)
else:
# encrypt inline
res = encrypt_doc_task(*args)
self.encrypt_doc_cb(res)
except Exception as exc:
logger.exception(exc)
def _async_decrypt_doc(self, doc_id, rev, content, gen, trans_id, idx):
"""
Dispatch an asynchronous document decrypting routine and save the
result object.
:param doc_id: The ID for the document with contents to be encrypted.
:type doc: str
:param rev: The revision of the document.
:type rev: str
:param content: The serialized content of the document.
:type content: str
:param gen: The generation corresponding to the modification of that
document.
:type gen: int
:param trans_id: The transaction id corresponding to the modification
of that document.
:type trans_id: str
:param idx: The index of this document in the current sync process.
:type idx: int
"""
soledad_assert(self._crypto is not None, "need a crypto object")
content = json.loads(content)
key = self._crypto.doc_passphrase(doc_id)
secret = self._crypto.secret
args = doc_id, rev, content, gen, trans_id, key, secret, idx
# decrypt asynchronously
self._async_results.append(
self._pool.apply_async(decrypt_doc_task, args))
def decrypt_sym(data, key, iv):
"""
Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The symmetric key used to decrypt data (must be 256 bits
long).
:type key: str
:param iv: The initialization vector.
:type iv: long
:return: The decrypted data.
:rtype: str
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
backend = MultiBackend([OpenSSLBackend()])
iv = binascii.a2b_base64(iv)
cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
decryptor = cipher.decryptor()
return decryptor.update(data) + decryptor.finalize()
def _decrypt_v1(self, data):
# get encrypted secret from dictionary: the old format allowed for
# storage of more than one secret, but this feature was never used and
# soledad has been using only one secret so far. As there is a corner
# case where the old 'active_secret' key might not be set, we just
# ignore it and pop the only secret found in the 'storage_secrets' key.
secret_id = data['storage_secrets'].keys().pop()
encrypted = data['storage_secrets'][secret_id]
# assert that we know how to decrypt the secret
soledad_assert('cipher' in encrypted)
cipher = encrypted['cipher']
if cipher == 'aes256':
cipher = ENC_METHOD.aes_256_ctr
soledad_assert(cipher in ENC_METHOD)
# decrypt
salt = binascii.a2b_base64(encrypted['kdf_salt'])
key = self._get_key(salt)
separator = ':'
iv, ciphertext = encrypted['secret'].split(separator, 1)
ciphertext = binascii.a2b_base64(ciphertext)
plaintext = self._decrypt(key, iv, ciphertext, encrypted, cipher)
# create secrets dictionary
secrets = {
'remote_secret': plaintext[0:512],
'local_salt': plaintext[512:576],
'local_secret': plaintext[576:1024],
}
return secrets
def decrypt_sym(data, key, method, **kwargs):
"""
Decrypt data using symmetric secret.
Currently, the only encryption method supported is AES-256 CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The key used to decrypt C{data} (must be 256 bits long).
:type key: str
:param method: The encryption method to use.
:type method: str
:param kwargs: Other parameters specific to each encryption method.
:type kwargs: dict
:return: The decrypted data.
:rtype: str
:raise UnknownEncryptionMethodError: Raised when C{method} is unknown.
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(len(key) == 32, "Wrong key size: %s (must be 256 bits long)." % len(key)) # 32 x 8 = 256 bits.
soledad_assert("iv" in kwargs, "%s needs an initial value." % method)
_assert_known_encryption_method(method)
# AES-256 in CTR mode
if method == crypto.EncryptionMethods.AES_256_CTR:
return AES(key=key, iv=binascii.a2b_base64(kwargs["iv"])).process(data)
elif method == crypto.EncryptionMethods.XSALSA20:
return XSalsa20(key=key, iv=binascii.a2b_base64(kwargs["iv"])).process(data)
def encrypt_doc(self, doc, workers=True):
"""
Symmetrically encrypt a document.
:param doc: The document with contents to be encrypted.
:type doc: SoledadDocument
:param workers: Whether to defer the decryption to the multiprocess
pool of workers. Useful for debugging purposes.
:type workers: bool
"""
soledad_assert(self._crypto is not None, "need a crypto object")
docstr = doc.get_json()
key = self._crypto.doc_passphrase(doc.doc_id)
secret = self._crypto.secret
args = doc.doc_id, doc.rev, docstr, key, secret
try:
if workers:
res = self._pool.apply_async(encrypt_doc_task,
args,
callback=self.encrypt_doc_cb)
else:
# encrypt inline
res = encrypt_doc_task(*args)
self.encrypt_doc_cb(res)
except Exception as exc:
logger.exception(exc)
def _initialize_sync_db(self, opts):
"""
Initialize the Symmetrically-Encrypted document to be synced database,
and the queue to communicate with subprocess workers.
:param opts:
:type opts: SQLCipherOptions
"""
soledad_assert(opts.sync_db_key is not None)
sync_db_path = None
if opts.path != ":memory:":
sync_db_path = "%s-sync" % opts.path
else:
sync_db_path = ":memory:"
# we copy incoming options because the opts object might be used
# somewhere else
sync_opts = sqlcipher.SQLCipherOptions.copy(opts,
path=sync_db_path,
create=True)
self._sync_db = sqlcipher.getConnectionPool(
sync_opts, extra_queries=self._sync_db_extra_init)
if self._defer_encryption:
# initialize syncing queue encryption pool
self._sync_enc_pool = encdecpool.SyncEncrypterPool(
self._crypto, self._sync_db)
self._sync_enc_pool.start()
def encrypt_sym(data, key, method):
"""
Encrypt C{data} using a {password}.
Currently, the only encryption methods supported are AES-256 in CTR
mode and XSalsa20.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt C{data} (must be 256 bits long).
:type key: str
:param method: The encryption method to use.
:type method: str
:return: A tuple with the initial value and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' % (len(key) * 8))
iv = None
# AES-256 in CTR mode
if method == EncryptionMethods.AES_256_CTR:
iv = os.urandom(16)
ciphertext = AES(key=key, iv=iv).process(data)
# XSalsa20
elif method == EncryptionMethods.XSALSA20:
iv = os.urandom(24)
ciphertext = XSalsa20(key=key, iv=iv).process(data)
else:
# raise if method is unknown
raise UnknownEncryptionMethod('Unkwnown method: %s' % method)
return binascii.b2a_base64(iv), ciphertext
def _decrypt_v1(self, data):
# get encrypted secret from dictionary: the old format allowed for
# storage of more than one secret, but this feature was never used and
# soledad has been using only one secret so far. As there is a corner
# case where the old 'active_secret' key might not be set, we just
# ignore it and pop the only secret found in the 'storage_secrets' key.
secret_id = data['storage_secrets'].keys().pop()
encrypted = data['storage_secrets'][secret_id]
# assert that we know how to decrypt the secret
soledad_assert('cipher' in encrypted)
cipher = encrypted['cipher']
if cipher == 'aes256':
cipher = ENC_METHOD.aes_256_ctr
soledad_assert(cipher in ENC_METHOD)
# decrypt
salt = binascii.a2b_base64(encrypted['kdf_salt'])
key = self._get_key(salt)
separator = ':'
iv, ciphertext = encrypted['secret'].split(separator, 1)
ciphertext = binascii.a2b_base64(ciphertext)
plaintext = self._decrypt(key, iv, ciphertext, encrypted, cipher)
# create secrets dictionary
secrets = {
'remote_secret': plaintext[0:512],
'local_salt': plaintext[512:576],
'local_secret': plaintext[576:1024],
}
return secrets
def _initialize_sync_db(self, opts):
"""
Initialize the Symmetrically-Encrypted document to be synced database,
and the queue to communicate with subprocess workers.
:param opts:
:type opts: SQLCipherOptions
"""
soledad_assert(opts.sync_db_key is not None)
sync_db_path = None
if opts.path != ":memory:":
sync_db_path = "%s-sync" % opts.path
else:
sync_db_path = ":memory:"
# we copy incoming options because the opts object might be used
# somewhere else
sync_opts = sqlcipher.SQLCipherOptions.copy(
opts, path=sync_db_path, create=True)
self._sync_db = sqlcipher.getConnectionPool(
sync_opts, extra_queries=self._sync_db_extra_init)
if self._defer_encryption:
# initialize syncing queue encryption pool
self._sync_enc_pool = encdecpool.SyncEncrypterPool(
self._crypto, self._sync_db)
self._sync_enc_pool.start()
def __init__(self,
doc_info,
ciphertext_fd,
result=None,
secret=None,
armor=True,
start_stream=True,
tag=None):
if not secret:
raise EncryptionDecryptionError('no secret given')
self.doc_id = doc_info.doc_id
self.rev = doc_info.rev
self.fd = ciphertext_fd
self.armor = armor
self._producer = None
self.result = result or BytesIO()
sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
self.size = None
self.tag = None
preamble, iv = self._consume_preamble()
soledad_assert(preamble)
soledad_assert(iv)
self._aes = AESWriter(sym_key, iv, self.result, tag=tag or self.tag)
self._aes.authenticate(preamble)
if start_stream:
self._start_stream()
def encrypt_sym(data, key):
"""
Encrypt data using AES-256 cipher in CTR mode.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt data (must be 256 bits long).
:type key: str
:return: A tuple with the initialization vector and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' %
(len(key) * 8))
iv = os.urandom(16)
backend = MultiBackend([OpenSSLBackend()])
cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
encryptor = cipher.encryptor()
ciphertext = encryptor.update(data) + encryptor.finalize()
return binascii.b2a_base64(iv), ciphertext
def _init_config_with_defaults(self):
"""
Initialize configuration using default values for missing params.
"""
soledad_assert_type(self._passphrase, unicode)
initialize = lambda attr, val: getattr(self, attr, None) is None and setattr(self, attr, val)
initialize("_secrets_path", os.path.join(self.default_prefix, self.secrets_file_name))
initialize("_local_db_path", os.path.join(self.default_prefix, self.local_db_file_name))
# initialize server_url
soledad_assert(self._server_url is not None, "Missing URL for Soledad server.")
def tearDownClass(cls):
"""
Stop CouchDB instance for test.
"""
# from BaseLeapTest
soledad_assert(
cls.tempdir.startswith('/tmp/leap_tests-'),
"beware! tried to remove a dir which does not "
"live in temporal folder!")
shutil.rmtree(cls.tempdir)
# from CouchDBTestCase
cls.wrapper.stop()
def set_init_pragmas(conn, opts=None, extra_queries=None):
"""
Set the initialization pragmas.
This includes the crypto pragmas, and any other options that must
be passed early to sqlcipher db.
"""
soledad_assert(opts is not None)
extra_queries = [] if extra_queries is None else extra_queries
with _db_init_lock:
# only one execution path should initialize the db
_set_init_pragmas(conn, opts, extra_queries)
def tearDownClass(cls):
"""
Stop CouchDB instance for test.
"""
# from BaseLeapTest
soledad_assert(
cls.tempdir.startswith('/tmp/leap_tests-'),
"beware! tried to remove a dir which does not "
"live in temporal folder!")
shutil.rmtree(cls.tempdir)
# from CouchDBTestCase
cls.wrapper.stop()
def set_init_pragmas(conn, opts=None, extra_queries=None):
"""
Set the initialization pragmas.
This includes the crypto pragmas, and any other options that must
be passed early to sqlcipher db.
"""
soledad_assert(opts is not None)
extra_queries = [] if extra_queries is None else extra_queries
with _db_init_lock:
# only one execution path should initialize the db
_set_init_pragmas(conn, opts, extra_queries)
def _assert_known_encryption_method(method):
"""
Assert that we can encrypt/decrypt the given C{method}
:param method: The encryption method to assert.
:type method: str
:raise UnknownEncryptionMethodError: Raised when C{method} is unknown.
"""
valid_methods = [crypto.EncryptionMethods.AES_256_CTR, crypto.EncryptionMethods.XSALSA20]
try:
soledad_assert(method in valid_methods)
except AssertionError:
raise crypto.UnknownEncryptionMethodError
def _delete_temporary_dirs():
# XXX should not access "private" attrs
for f in [self._soledad.local_db_path,
self._soledad.secrets.secrets_path]:
if os.path.isfile(f):
os.unlink(f)
# The following snippet comes from BaseLeapTest.setUpClass, but we
# repeat it here because twisted.trial does not work with
# setUpClass/tearDownClass.
soledad_assert(
self.tempdir.startswith('/tmp/leap_tests-'),
"beware! tried to remove a dir which does not "
"live in temporal folder!")
shutil.rmtree(self.tempdir)
def delete_doc(self, doc):
"""
Mark a document as deleted.
Will abort if the current revision doesn't match doc.rev.
This will also set doc.content to None.
:param doc: A document to be deleted.
:type doc: leap.soledad.common.document.Document
:return: A deferred.
:rtype: twisted.internet.defer.Deferred
"""
soledad_assert(doc is not None, "delete_doc doesn't accept None.")
return self._defer("delete_doc", doc)
def tearDown(self):
self._delete_dbfiles()
# the following come from BaseLeapTest.tearDownClass, because
# twisted.trial doesn't support such class methods for tearing down
# test classes.
os.environ["PATH"] = self.old_path
os.environ["HOME"] = self.old_home
# safety check! please do not wipe my home...
# XXX needs to adapt to non-linuces
soledad_assert(
self.tempdir.startswith("/tmp/leap_tests-") or self.tempdir.startswith("/var/folder"),
"beware! tried to remove a dir which does not " "live in temporal folder!",
)
shutil.rmtree(self.tempdir)
def _decrypt_v2(self, encrypted):
cipher = encrypted['cipher']
soledad_assert(cipher in ENC_METHOD)
salt = binascii.a2b_base64(encrypted['kdf_salt'])
key = self._get_key(salt)
iv = encrypted['iv']
ciphertext = binascii.a2b_base64(encrypted['secrets'])
plaintext = self._decrypt(key, iv, ciphertext, encrypted, cipher)
encoded = json.loads(plaintext)
secrets = {}
for name, value in encoded.iteritems():
secrets[name] = binascii.a2b_base64(value)
return secrets
def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
mac_method, secret):
"""
Calculate a MAC for C{doc} using C{ciphertext}.
Current MAC method used is HMAC, with the following parameters:
* key: sha256(storage_secret, doc_id)
* msg: doc_id + doc_rev + ciphertext
* digestmod: sha256
:param doc_id: The id of the document.
:type doc_id: str
:param doc_rev: The revision of the document.
:type doc_rev: str
:param ciphertext: The content of the document.
:type ciphertext: str
:param enc_scheme: The encryption scheme.
:type enc_scheme: str
:param enc_method: The encryption method.
:type enc_method: str
:param enc_iv: The encryption initialization vector.
:type enc_iv: str
:param mac_method: The MAC method to use.
:type mac_method: str
:param secret: The Soledad storage secret
:type secret: str
:return: The calculated MAC.
:rtype: str
:raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
"""
try:
soledad_assert(mac_method == crypto.MacMethods.HMAC)
except AssertionError:
raise crypto.UnknownMacMethodError
template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}"
content = template.format(
doc_id=doc_id,
doc_rev=doc_rev,
ciphertext=ciphertext,
enc_scheme=enc_scheme,
enc_method=enc_method,
enc_iv=enc_iv)
return hmac.new(
doc_mac_key(doc_id, secret),
content,
hashlib.sha256).digest()
def _decrypt_v2(self, encrypted):
cipher = encrypted['cipher']
soledad_assert(cipher in ENC_METHOD)
salt = binascii.a2b_base64(encrypted['kdf_salt'])
key = self._get_key(salt)
iv = encrypted['iv']
ciphertext = binascii.a2b_base64(encrypted['secrets'])
plaintext = self._decrypt(
key, iv, ciphertext, encrypted, cipher)
encoded = json.loads(plaintext)
secrets = {}
for name, value in encoded.iteritems():
secrets[name] = binascii.a2b_base64(value)
return secrets
def _init_config_with_defaults(self):
"""
Initialize configuration using default values for missing params.
"""
soledad_assert_type(self._passphrase, unicode)
initialize = lambda attr, val: getattr(
self, attr, None) is None and setattr(self, attr, val)
initialize("_secrets_path",
os.path.join(self.default_prefix, self.secrets_file_name))
initialize("_local_db_path",
os.path.join(self.default_prefix, self.local_db_file_name))
# initialize server_url
soledad_assert(self._server_url is not None,
'Missing URL for Soledad server.')
def tearDown(self):
self._delete_dbfiles()
# the following come from BaseLeapTest.tearDownClass, because
# twisted.trial doesn't support such class methods for tearing down
# test classes.
os.environ["PATH"] = self.old_path
os.environ["HOME"] = self.old_home
# safety check! please do not wipe my home...
# XXX needs to adapt to non-linuces
soledad_assert(
self.tempdir.startswith('/tmp/leap_tests-')
or self.tempdir.startswith('/var/folder'),
"beware! tried to remove a dir which does not "
"live in temporal folder!")
shutil.rmtree(self.tempdir)
def encrypt_doc(self, doc):
"""
Encrypt document asynchronously then insert it on
local staging database.
:param doc: The document to be encrypted.
:type doc: SoledadDocument
"""
soledad_assert(self._crypto is not None, "need a crypto object")
docstr = doc.get_json()
key = self._crypto.doc_passphrase(doc.doc_id)
secret = self._crypto.secret
args = doc.doc_id, doc.rev, docstr, key, secret
# encrypt asynchronously
d = threads.deferToThread(encrypt_doc_task, *args)
d.addCallback(self._encrypt_doc_cb)
def _delete_temporary_dirs():
# XXX should not access "private" attrs
for f in [
self._soledad.local_db_path,
self._soledad.secrets.secrets_path
]:
if os.path.isfile(f):
os.unlink(f)
# The following snippet comes from BaseLeapTest.setUpClass, but we
# repeat it here because twisted.trial does not work with
# setUpClass/tearDownClass.
soledad_assert(
self.tempdir.startswith('/tmp/leap_tests-'),
"beware! tried to remove a dir which does not "
"live in temporal folder!")
shutil.rmtree(self.tempdir)
def encrypt_doc(self, doc):
"""
Encrypt document asynchronously then insert it on
local staging database.
:param doc: The document to be encrypted.
:type doc: SoledadDocument
"""
soledad_assert(self._crypto is not None, "need a crypto object")
docstr = doc.get_json()
key = self._crypto.doc_passphrase(doc.doc_id)
secret = self._crypto.secret
args = doc.doc_id, doc.rev, docstr, key, secret
# encrypt asynchronously
d = threads.deferToThread(encrypt_doc_task, *args)
d.addCallback(self._encrypt_doc_cb)
def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
mac_method, secret):
"""
Calculate a MAC for C{doc} using C{ciphertext}.
Current MAC method used is HMAC, with the following parameters:
* key: sha256(storage_secret, doc_id)
* msg: doc_id + doc_rev + ciphertext
* digestmod: sha256
:param doc_id: The id of the document.
:type doc_id: str
:param doc_rev: The revision of the document.
:type doc_rev: str
:param ciphertext: The content of the document.
:type ciphertext: str
:param enc_scheme: The encryption scheme.
:type enc_scheme: str
:param enc_method: The encryption method.
:type enc_method: str
:param enc_iv: The encryption initialization vector.
:type enc_iv: str
:param mac_method: The MAC method to use.
:type mac_method: str
:param secret: The Soledad storage secret
:type secret: str
:return: The calculated MAC.
:rtype: str
:raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
"""
try:
soledad_assert(mac_method == crypto.MacMethods.HMAC)
except AssertionError:
raise crypto.UnknownMacMethodError
template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}"
content = template.format(doc_id=doc_id,
doc_rev=doc_rev,
ciphertext=ciphertext,
enc_scheme=enc_scheme,
enc_method=enc_method,
enc_iv=enc_iv)
return hmac.new(doc_mac_key(doc_id, secret), content,
hashlib.sha256).digest()
def encrypt_doc(crypto, doc):
"""
Encrypt C{doc}'s content.
Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
string representing the following:
{
ENC_JSON_KEY: '<encrypted doc JSON string>',
ENC_SCHEME_KEY: 'symkey',
ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
ENC_IV_KEY: '<the initial value used to encrypt>',
MAC_KEY: '<mac>'
MAC_METHOD_KEY: 'hmac'
}
:param crypto: A SoledadCryto instance used to perform the encryption.
:type crypto: leap.soledad.crypto.SoledadCrypto
:param doc: The document with contents to be encrypted.
:type doc: SoledadDocument
:return: The JSON serialization of the dict representing the encrypted
content.
:rtype: str
"""
soledad_assert(doc.is_tombstone() is False)
# encrypt content using AES-256 CTR mode
iv, ciphertext = crypto.encrypt_sym(
str(doc.get_json()), # encryption/decryption routines expect str
crypto.doc_passphrase(doc.doc_id),
method=EncryptionMethods.AES_256_CTR)
# Return a representation for the encrypted content. In the following, we
# convert binary data to hexadecimal representation so the JSON
# serialization does not complain about what it tries to serialize.
hex_ciphertext = binascii.b2a_hex(ciphertext)
return json.dumps({
ENC_JSON_KEY: hex_ciphertext,
ENC_SCHEME_KEY: EncryptionSchemes.SYMKEY,
ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
ENC_IV_KEY: iv,
MAC_KEY: binascii.b2a_hex(mac_doc( # store the mac as hex.
crypto, doc.doc_id, doc.rev,
ciphertext,
MacMethods.HMAC)),
MAC_METHOD_KEY: MacMethods.HMAC,
})
def _assert_known_encryption_method(method):
"""
Assert that we can encrypt/decrypt the given C{method}
:param method: The encryption method to assert.
:type method: str
:raise UnknownEncryptionMethodError: Raised when C{method} is unknown.
"""
valid_methods = [
crypto.EncryptionMethods.AES_256_CTR,
crypto.EncryptionMethods.XSALSA20,
]
try:
soledad_assert(method in valid_methods)
except AssertionError:
raise crypto.UnknownEncryptionMethodError
def _init_config(self, secrets_path, local_db_path, server_url):
"""
Initialize configuration using default values for missing params.
"""
# initialize secrets_path
self._secrets_path = secrets_path
if self._secrets_path is None:
self._secrets_path = os.path.join(self.DEFAULT_PREFIX,
self.STORAGE_SECRETS_FILE_NAME)
# initialize local_db_path
self._local_db_path = local_db_path
if self._local_db_path is None:
self._local_db_path = os.path.join(self.DEFAULT_PREFIX,
self.LOCAL_DATABASE_FILE_NAME)
# initialize server_url
self._server_url = server_url
soledad_assert(self._server_url is not None,
'Missing URL for Soledad server.')
def _encrypt_doc(self, doc):
"""
Symmetrically encrypt a document.
:param doc: The document with contents to be encrypted.
:type doc: SoledadDocument
:param workers: Whether to defer the decryption to the multiprocess
pool of workers. Useful for debugging purposes.
:type workers: bool
"""
soledad_assert(self._crypto is not None, "need a crypto object")
docstr = doc.get_json()
key = self._crypto.doc_passphrase(doc.doc_id)
secret = self._crypto.secret
args = doc.doc_id, doc.rev, docstr, key, secret
# encrypt asynchronously
self._pool.apply_async(encrypt_doc_task, args, callback=self._encrypt_doc_cb)
def _init_config(self, secrets_path, local_db_path, server_url):
"""
Initialize configuration using default values for missing params.
"""
# initialize secrets_path
self._secrets_path = secrets_path
if self._secrets_path is None:
self._secrets_path = os.path.join(
self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME)
# initialize local_db_path
self._local_db_path = local_db_path
if self._local_db_path is None:
self._local_db_path = os.path.join(
self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME)
# initialize server_url
self._server_url = server_url
soledad_assert(
self._server_url is not None,
'Missing URL for Soledad server.')
def doc_passphrase(self, doc_id):
"""
Generate a passphrase for symmetric encryption of document's contents.
The password is derived using HMAC having sha256 as underlying hash
function. The key used for HMAC are the first
C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage
secret stripped from the first MAC_KEY_LENGTH characters. The HMAC
message is C{doc_id}.
:param doc_id: The id of the document that will be encrypted using
this passphrase.
:type doc_id: str
:return: The passphrase.
:rtype: str
"""
soledad_assert(self.secret is not None)
return hmac.new(self.secret[MAC_KEY_LENGTH:], doc_id, hashlib.sha256).digest()
def _encrypt_doc(self, doc):
"""
Symmetrically encrypt a document.
:param doc: The document with contents to be encrypted.
:type doc: SoledadDocument
:param workers: Whether to defer the decryption to the multiprocess
pool of workers. Useful for debugging purposes.
:type workers: bool
"""
soledad_assert(self._crypto is not None, "need a crypto object")
docstr = doc.get_json()
key = self._crypto.doc_passphrase(doc.doc_id)
secret = self._crypto.secret
args = doc.doc_id, doc.rev, docstr, key, secret
# encrypt asynchronously
self._pool.apply_async(encrypt_doc_task,
args,
callback=self._encrypt_doc_cb)
def doc_mac_key(doc_id, secret):
"""
Generate a key for calculating a MAC for a document whose id is
C{doc_id}.
The key is derived using HMAC having sha256 as underlying hash
function. The key used for HMAC is the first MAC_KEY_LENGTH characters
of Soledad's storage secret. The HMAC message is C{doc_id}.
:param doc_id: The id of the document.
:type doc_id: str
:param secret: The Soledad storage secret
:type secret: str
:return: The key.
:rtype: str
"""
soledad_assert(secret is not None)
return hmac.new(secret[:MAC_KEY_LENGTH], doc_id, hashlib.sha256).digest()
def doc_passphrase(self, doc_id):
"""
Generate a passphrase for symmetric encryption of document's contents.
The password is derived using HMAC having sha256 as underlying hash
function. The key used for HMAC are the first
C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage
secret stripped from the first MAC_KEY_LENGTH characters. The HMAC
message is C{doc_id}.
:param doc_id: The id of the document that will be encrypted using
this passphrase.
:type doc_id: str
:return: The passphrase.
:rtype: str
"""
soledad_assert(self.secret is not None)
return hmac.new(self.secret[MAC_KEY_LENGTH:], doc_id,
hashlib.sha256).digest()
def doc_mac_key(doc_id, secret):
"""
Generate a key for calculating a MAC for a document whose id is
C{doc_id}.
The key is derived using HMAC having sha256 as underlying hash
function. The key used for HMAC is the first MAC_KEY_LENGTH characters
of Soledad's storage secret. The HMAC message is C{doc_id}.
:param doc_id: The id of the document.
:type doc_id: str
:param secret: The Soledad storage secret
:type secret: str
:return: The key.
:rtype: str
"""
soledad_assert(secret is not None)
return hmac.new(secret[:MAC_KEY_LENGTH], doc_id, hashlib.sha256).digest()
def encrypt_sym(data, key):
"""
Encrypt data using AES-256 cipher in CTR mode.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt data (must be 256 bits long).
:type key: str
:return: A tuple with the initialization vector and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' % (len(key) * 8))
iv = os.urandom(16)
ciphertext = AES(key=key, iv=iv).process(data)
return binascii.b2a_base64(iv), ciphertext
def _import_recovery_document(self, data):
"""
Import storage secrets for symmetric encryption from a recovery
document.
Note that this method does not store the imported data on disk. For
that, use C{self._store_secrets()}.
:param data: The recovery document.
:type data: dict
:return: A tuple containing the number of imported secrets, the
secret_id of the last active secret, and the recovery
document format version.
:rtype: (int, str, int)
"""
soledad_assert(self.STORAGE_SECRETS_KEY in data)
version = data.get(self.RECOVERY_DOC_VERSION_KEY, 1)
meth = getattr(self, '_import_recovery_document_version_%d' % version)
secret_count, active_secret = meth(data)
return secret_count, active_secret, version
def decrypt_sym(data, key, iv):
"""
Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The symmetric key used to decrypt data (must be 256 bits
long).
:type key: str
:param iv: The initialization vector.
:type iv: long
:return: The decrypted data.
:rtype: str
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
return AES(key=key, iv=binascii.a2b_base64(iv)).process(data)
def encrypt_sym(data, key):
"""
Encrypt data using AES-256 cipher in CTR mode.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt data (must be 256 bits long).
:type key: str
:return: A tuple with the initialization vector and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' %
(len(key) * 8))
iv = os.urandom(16)
ciphertext = AES(key=key, iv=iv).process(data)
return binascii.b2a_base64(iv), ciphertext
def decrypt_sym(data, key, iv):
"""
Decrypt some data previously encrypted using AES-256 cipher in CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The symmetric key used to decrypt data (must be 256 bits
long).
:type key: str
:param iv: The initialization vector.
:type iv: long
:return: The decrypted data.
:rtype: str
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
return AES(
key=key, iv=binascii.a2b_base64(iv)).process(data)
def _decrypt(self, key, iv, ciphertext, encrypted, method):
# assert some properties of the stored secret
soledad_assert(encrypted['kdf'] == 'scrypt')
soledad_assert(encrypted['kdf_length'] == len(key))
# decrypt
plaintext = decrypt_sym(ciphertext, key, iv, method)
soledad_assert(encrypted['length'] == len(plaintext))
return plaintext
def _decrypt(self, key, iv, ciphertext, encrypted, method):
# assert some properties of the stored secret
soledad_assert(encrypted['kdf'] == 'scrypt')
soledad_assert(encrypted['kdf_length'] == len(key))
# decrypt
plaintext = decrypt_sym(ciphertext, key, iv, method)
soledad_assert(encrypted['length'] == len(plaintext))
return plaintext
def encrypt_sym(data, key):
"""
Encrypt data using AES-256 cipher in CTR mode.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt data (must be 256 bits long).
:type key: str
:return: A tuple with the initialization vector and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' % (len(key) * 8))
iv = os.urandom(16)
cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend)
encryptor = cipher.encryptor()
ciphertext = encryptor.update(data) + encryptor.finalize()
return binascii.b2a_base64(iv), ciphertext
def __init__(self, doc_info, ciphertext_fd, result=None,
secret=None, armor=True, start_stream=True, tag=None):
if not secret:
raise EncryptionDecryptionError('no secret given')
self.doc_id = doc_info.doc_id
self.rev = doc_info.rev
self.fd = ciphertext_fd
self.armor = armor
self._producer = None
self.result = result or BytesIO()
sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
self.size = None
self.tag = None
preamble, iv = self._consume_preamble()
soledad_assert(preamble)
soledad_assert(iv)
self._aes = AESWriter(sym_key, iv, self.result, tag=tag or self.tag)
self._aes.authenticate(preamble)
if start_stream:
self._start_stream()
def decrypt_sym(self, data, key,
method=EncryptionMethods.AES_256_CTR, **kwargs):
"""
Decrypt data using symmetric secret.
Currently, the only encryption method supported is AES-256 CTR mode.
:param data: The data to be decrypted.
:type data: str
:param key: The key used to decrypt C{data} (must be 256 bits long).
:type key: str
:param method: The encryption method to use.
:type method: str
:param kwargs: Other parameters specific to each encryption method.
:type kwargs: dict
:return: The decrypted data.
:rtype: str
"""
soledad_assert_type(key, str)
# assert params
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s (must be 256 bits long).' % len(key))
soledad_assert(
'iv' in kwargs,
'%s needs an initial value.' % method)
# AES-256 in CTR mode
if method == EncryptionMethods.AES_256_CTR:
return AES(
key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data)
elif method == EncryptionMethods.XSALSA20:
return XSalsa20(
key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data)
# raise if method is unknown
raise UnknownEncryptionMethod('Unkwnown method: %s' % method)
def encrypt_sym(data, key, method):
"""
Encrypt C{data} using a {password}.
Currently, the only encryption methods supported are AES-256 in CTR
mode and XSalsa20.
:param data: The data to be encrypted.
:type data: str
:param key: The key used to encrypt C{data} (must be 256 bits long).
:type key: str
:param method: The encryption method to use.
:type method: str
:return: A tuple with the initial value and the encrypted data.
:rtype: (long, str)
"""
soledad_assert_type(key, str)
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' %
(len(key) * 8))
iv = None
# AES-256 in CTR mode
if method == EncryptionMethods.AES_256_CTR:
iv = os.urandom(16)
ciphertext = AES(key=key, iv=iv).process(data)
# XSalsa20
elif method == EncryptionMethods.XSALSA20:
iv = os.urandom(24)
ciphertext = XSalsa20(key=key, iv=iv).process(data)
else:
# raise if method is unknown
raise UnknownEncryptionMethod('Unkwnown method: %s' % method)
return binascii.b2a_base64(iv), ciphertext
