def test_doc_decryption(soledad_client, benchmark, payload):
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
benchmark(crypto.decrypt_doc, doc)
def _insert_received_doc(self, response, idx, total):
"""
Insert a received document into the local replica.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if self._sync_decr_pool and not self._sync_decr_pool.running:
self._sync_decr_pool.start(number_of_changes)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(self._crypto.decrypt_doc(doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
_emit_receive_status(user_data, self._received_docs, total)
return number_of_changes, new_generation, new_transaction_id
def _insert_received_doc(self, response, idx, total):
"""
Insert a received document into the local replica.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if self._sync_decr_pool and not self._sync_decr_pool.running:
self._sync_decr_pool.start(number_of_changes)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(self._crypto.decrypt_doc(doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
_emit_receive_status(user_data, self._received_docs, total)
return number_of_changes, new_generation, new_transaction_id
def _insert_received_doc(self, idx, total, response):
"""
Insert a received document into the local replica.
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
:param response: The body and headers of the response.
:type response: tuple(str, dict)
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(decrypt_doc(self._crypto, doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
msg = "%d/%d" % (idx, total)
emit(SOLEDAD_SYNC_RECEIVE_STATUS, msg)
logger.debug("Soledad sync receive status: %s" % msg)
return number_of_changes, new_generation, new_transaction_id
def test_doc_decryption(soledad_client, txbenchmark, payload):
"""
Decrypt a document of a given size.
"""
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(
doc_id=uuid4().hex, rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = yield crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
yield txbenchmark(crypto.decrypt_doc, doc)
def test_doc_decryption(soledad_client, txbenchmark, payload):
"""
Decrypt a document of a given size.
"""
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = yield crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
yield txbenchmark(crypto.decrypt_doc, doc)
def test_decrypt_with_unknown_mac_method_raises(self):
"""
Trying to decrypt a document with unknown MAC method should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(self._soledad._crypto.encrypt_doc(doc))
self.assertTrue(MAC_KEY in doc.content)
self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC method
doc.content[MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(UnknownMacMethodError,
self._soledad._crypto.decrypt_doc, doc)
def test_decrypt_with_wrong_mac_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(self._soledad._crypto.encrypt_doc(doc))
self.assertTrue(MAC_KEY in doc.content)
self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC
doc.content[MAC_KEY] = '1234567890ABCDEF'
# try to decrypt doc
self.assertRaises(WrongMacError, self._soledad._crypto.decrypt_doc,
doc)
def _insert_received_doc(self, idx, total, response):
"""
Insert a received document into the local replica.
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
:param response: The body and headers of the response.
:type response: tuple(str, dict)
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(decrypt_doc(self._crypto, doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
msg = "%d/%d" % (idx, total)
emit(SOLEDAD_SYNC_RECEIVE_STATUS, msg)
logger.debug("Soledad sync receive status: %s" % msg)
return number_of_changes, new_generation, new_transaction_id
def test_decrypt_with_wrong_mac_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
self.assertTrue(MAC_KEY in doc.content)
self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC
doc.content[MAC_KEY] = '1234567890ABCDEF'
# try to decrypt doc
self.assertRaises(
WrongMacError,
crypto.decrypt_doc, self._soledad._crypto, doc)
def test_encrypt_and_decrypt(self):
"""
Check that encrypting and decrypting gives same doc.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
assert encrypted != payload
assert 'raw' in encrypted
doc2 = SoledadDocument('id1', '1')
doc2.set_json(encrypted)
assert _crypto.is_symmetrically_encrypted(encrypted)
decrypted = (yield crypto.decrypt_doc(doc2)).getvalue()
assert len(decrypted) != 0
assert json.loads(decrypted) == payload
def test_encrypt_and_decrypt(self):
"""
Check that encrypting and decrypting gives same doc.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
assert encrypted != payload
assert 'raw' in encrypted
doc2 = SoledadDocument('id1', '1')
doc2.set_json(encrypted)
assert _crypto.is_symmetrically_encrypted(encrypted)
decrypted = (yield crypto.decrypt_doc(doc2)).getvalue()
assert len(decrypted) != 0
assert json.loads(decrypted) == payload
def test_decrypt_with_unknown_mac_method_raises(self):
"""
Trying to decrypt a document with unknown MAC method should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
self.assertTrue(MAC_KEY in doc.content)
self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC method
doc.content[MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(
UnknownMacMethodError,
crypto.decrypt_doc, self._soledad._crypto, doc)
def __atomic_doc_parse(self, doc_info, content, total):
doc = SoledadDocument(doc_info['id'], doc_info['rev'], content)
if is_symmetrically_encrypted(content):
content = yield self._crypto.decrypt_doc(doc)
elif old_crypto.is_symmetrically_encrypted(doc):
content = self._deprecated_crypto.decrypt_doc(doc)
doc.set_json(content)
# TODO insert blobs here on the blob backend
# FIXME: This is wrong. Using the very same SQLite connection object
# from multiple threads is dangerous. We should bring the dbpool here
# or find an alternative. Deferring to a thread only helps releasing
# the reactor for other tasks as this is an IO intensive call.
yield threads.deferToThread(self._insert_doc_cb, doc, doc_info['gen'],
doc_info['trans_id'])
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
_emit_receive_status(user_data, self._received_docs, total=total)
def test_encrypt_decrypt_json(self):
"""
Test encrypting and decrypting documents.
"""
simpledoc = {'key': 'val'}
doc1 = SoledadDocument(doc_id='id')
doc1.content = simpledoc
# encrypt doc
doc1.set_json(self._soledad._crypto.encrypt_doc(doc1))
# assert content is different and includes keys
self.assertNotEqual(simpledoc, doc1.content,
'incorrect document encryption')
self.assertTrue(ENC_JSON_KEY in doc1.content)
self.assertTrue(ENC_SCHEME_KEY in doc1.content)
# decrypt doc
doc1.set_json(self._soledad._crypto.decrypt_doc(doc1))
self.assertEqual(simpledoc, doc1.content,
'incorrect document encryption')
def test_encrypt_decrypt_json(self):
"""
Test encrypting and decrypting documents.
"""
simpledoc = {'key': 'val'}
doc1 = SoledadDocument(doc_id='id')
doc1.content = simpledoc
# encrypt doc
doc1.set_json(target.encrypt_doc(self._soledad._crypto, doc1))
# assert content is different and includes keys
self.assertNotEqual(
simpledoc, doc1.content,
'incorrect document encryption')
self.assertTrue(target.ENC_JSON_KEY in doc1.content)
self.assertTrue(target.ENC_SCHEME_KEY in doc1.content)
# decrypt doc
doc1.set_json(target.decrypt_doc(self._soledad._crypto, doc1))
self.assertEqual(
simpledoc, doc1.content, 'incorrect document encryption')
def test_decrypt_with_wrong_tag_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
encdict = json.loads(encrypted)
preamble, raw = _crypto._split(str(encdict['raw']))
# mess with tag
messed = raw[:-16] + '0' * 16
preamble = base64.urlsafe_b64encode(preamble)
newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed))
doc2 = SoledadDocument('id1', '1')
doc2.set_json(json.dumps({"raw": str(newraw)}))
with pytest.raises(_crypto.InvalidBlob):
yield crypto.decrypt_doc(doc2)
def test_decrypt_with_wrong_tag_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
encdict = json.loads(encrypted)
preamble, raw = str(encdict['raw']).split()
preamble = base64.urlsafe_b64decode(preamble)
raw = base64.urlsafe_b64decode(raw)
# mess with tag
messed = raw[:-16] + '0' * 16
preamble = base64.urlsafe_b64encode(preamble)
newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed))
doc2 = SoledadDocument('id1', '1')
doc2.set_json(json.dumps({"raw": str(newraw)}))
with pytest.raises(_crypto.InvalidBlob):
yield crypto.decrypt_doc(doc2)
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
"""
Parse incoming synchronization stream and insert documents in the
local database.
If an incoming document's encryption scheme is equal to
EncryptionSchemes.SYMKEY, then this method will decrypt it with
Soledad's symmetric key.
:param data: The body of the HTTP response.
:type data: str
:param return_doc_cb: A callback to insert docs from target.
:type return_doc_cb: function
:param ensure_callback: A callback to ensure we have the correct
target_replica_uid, if it was just created.
:type ensure_callback: function
:raise BrokenSyncStream: If C{data} is malformed.
:return: A dictionary representing the first line of the response got
from remote replica.
:rtype: list of str
"""
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
#-------------------------------------------------------------
# symmetric decryption of document's contents
#-------------------------------------------------------------
# if arriving content was symmetrically encrypted, we decrypt
# it.
doc = SoledadDocument(
entry['id'], entry['rev'], entry['content'])
if doc.content and ENC_SCHEME_KEY in doc.content:
if doc.content[ENC_SCHEME_KEY] == \
EncryptionSchemes.SYMKEY:
doc.set_json(decrypt_doc(self._crypto, doc))
#-------------------------------------------------------------
# end of symmetric decryption
#-------------------------------------------------------------
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res