def test_stage2_bootstrap_signals(self):
"""
Test that if there are keys in server, soledad will download them and
emit corresponding signals.
"""
# get existing instance so we have access to keys
sol = self._soledad_instance()
# create a document with secrets
doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id())
doc.content = sol.secrets._export_recovery_document()
sol.close()
# reset mock
soledad.client.secrets.events.emit.reset_mock()
# get a fresh instance so it emits all bootstrap signals
shared_db = self.get_default_shared_mock(get_doc_return_value=doc)
sol = self._soledad_instance(secrets_path='alternative_stage2.json',
local_db_path='alternative_stage2.u1db',
shared_db_class=shared_db)
# reverse call order so we can verify in the order the signals were
# expected
soledad.client.secrets.events.emit.mock_calls.reverse()
soledad.client.secrets.events.emit.call_args = \
soledad.client.secrets.events.emit.call_args_list[0]
soledad.client.secrets.events.emit.call_args_list.reverse()
# assert download keys signals
soledad.client.secrets.events.emit.assert_called_with(
catalog.SOLEDAD_DOWNLOADING_KEYS,
ADDRESS,
)
self._pop_mock_call(soledad.client.secrets.events.emit)
soledad.client.secrets.events.emit.assert_called_with(
catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
ADDRESS,
)
sol.close()
def test_stage2_bootstrap_signals(self):
"""
Test that if there are keys in server, soledad will download them and
emit corresponding signals.
"""
# get existing instance so we have access to keys
sol = self._soledad_instance()
# create a document with secrets
doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id())
doc.content = sol.secrets._export_recovery_document()
sol.close()
# reset mock
soledad.client.secrets.events.emit_async.reset_mock()
# get a fresh instance so it emits all bootstrap signals
shared_db = self.get_default_shared_mock(get_doc_return_value=doc)
sol = self._soledad_instance(
secrets_path="alternative_stage2.json", local_db_path="alternative_stage2.u1db", shared_db_class=shared_db
)
# reverse call order so we can verify in the order the signals were
# expected
soledad.client.secrets.events.emit_async.mock_calls.reverse()
soledad.client.secrets.events.emit_async.call_args = soledad.client.secrets.events.emit_async.call_args_list[0]
soledad.client.secrets.events.emit_async.call_args_list.reverse()
# assert download keys signals
soledad.client.secrets.events.emit_async.assert_called_with(
catalog.SOLEDAD_DOWNLOADING_KEYS, {"userid": ADDRESS, "uuid": ADDRESS}
)
self._pop_mock_call(soledad.client.secrets.events.emit_async)
soledad.client.secrets.events.emit_async.assert_called_with(
catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, {"userid": ADDRESS, "uuid": ADDRESS}
)
sol.close()
def _put_secrets_in_shared_db(self):
"""
Assert local keys are the same as shared db's ones.
Try to fetch keys from shared recovery database. If they already exist
in the remote db, assert that that data is the same as local data.
Otherwise, upload keys to shared recovery database.
"""
soledad_assert(
self._has_secret(),
'Tried to send keys to server but they don\'t exist in local '
'storage.')
# try to get secrets doc from server, otherwise create it
doc = self._get_secrets_from_shared_db()
if doc is None:
doc = SoledadDocument(doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
doc.content = self.export_recovery_document()
# upload secrets to server
signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
db.put_doc(doc)
signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
def _put_secrets_in_shared_db(self):
"""
Assert local keys are the same as shared db's ones.
Try to fetch keys from shared recovery database. If they already exist
in the remote db, assert that that data is the same as local data.
Otherwise, upload keys to shared recovery database.
"""
soledad_assert(
self._has_secret(),
'Tried to send keys to server but they don\'t exist in local '
'storage.')
# try to get secrets doc from server, otherwise create it
doc = self._get_secrets_from_shared_db()
if doc is None:
doc = SoledadDocument(
doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
doc.content = self.export_recovery_document()
# upload secrets to server
signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
db.put_doc(doc)
signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
def test_doc_update_fails_with_wrong_rev(self):
# create a document in shared db
doc_id = 'some-random-doc'
self.assertIsNone(self._db.get_doc(doc_id))
# create a document in shared db
doc = SoledadDocument(doc_id=doc_id)
self._db.put_doc(doc)
# try to update document without including revision of old version
doc.rev = 'wrong-rev'
self.assertRaises(RevisionConflict, self._db.put_doc, doc)
def test_doc_update_fails_with_wrong_rev(self):
# create a document in shared db
doc_id = 'some-random-doc'
self.assertIsNone(self._db.get_doc(doc_id))
# create a document in shared db
doc = SoledadDocument(doc_id=doc_id)
self._db.put_doc(doc)
# try to update document without including revision of old version
doc.rev = 'wrong-rev'
self.assertRaises(RevisionConflict, self._db.put_doc, doc)
def test_repair_only_deletes_key_docs(self):
soledad = mock()
key = self._public_key(SOME_EMAIL_ADDRESS, SOME_KEY_ID)
key_doc = SoledadDocument(doc_id='some_doc', json=key.get_active_json(SOME_EMAIL_ADDRESS))
other_doc = SoledadDocument(doc_id='something', json='{}')
when(soledad).get_all_docs().thenReturn(defer.succeed((1, [key_doc, other_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad, never).delete_doc(other_doc)
def save_remote(self, encrypted):
doc = self._remote_doc
if not doc:
doc = SoledadDocument(doc_id=self._remote_doc_id())
doc.content = encrypted
db = self._shared_db
if not db:
logger.warn('no shared db found')
return
db.put_doc(doc)
def test_doc_decryption(soledad_client, benchmark, payload):
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
benchmark(crypto.decrypt_doc, doc)
def _insert_received_doc(self, response, idx, total):
"""
Insert a received document into the local replica.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if self._sync_decr_pool and not self._sync_decr_pool.running:
self._sync_decr_pool.start(number_of_changes)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(self._crypto.decrypt_doc(doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
_emit_receive_status(user_data, self._received_docs, total)
return number_of_changes, new_generation, new_transaction_id
def test_doc_update_succeeds(self):
doc_id = 'some-random-doc'
self.assertIsNone(self._db.get_doc(doc_id))
# create a document in shared db
doc = SoledadDocument(doc_id=doc_id)
self._db.put_doc(doc)
# update that document
expected = {'new': 'content'}
doc.content = expected
self._db.put_doc(doc)
# ensure expected content was saved
doc = self._db.get_doc(doc_id)
self.assertEqual(expected, doc.content)
def test_doc_update_succeeds(self):
doc_id = 'some-random-doc'
self.assertIsNone(self._db.get_doc(doc_id))
# create a document in shared db
doc = SoledadDocument(doc_id=doc_id)
self._db.put_doc(doc)
# update that document
expected = {'new': 'content'}
doc.content = expected
self._db.put_doc(doc)
# ensure expected content was saved
doc = self._db.get_doc(doc_id)
self.assertEqual(expected, doc.content)
def test_repair_delete_public_key_docs(self):
soledad = mock()
key = self._public_key(SOME_EMAIL_ADDRESS, SOME_FINGERPRINT)
active_doc = SoledadDocument(doc_id='some_doc',
json=key.get_active_json())
key_doc = SoledadDocument(doc_id='some_doc', json=key.get_json())
when(soledad).get_all_docs().thenReturn(
defer.succeed((1, [key_doc, active_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad).delete_doc(active_doc)
verify(soledad).delete_doc(key_doc)
def test_extra_comma(self):
doc = SoledadDocument('i', rev='r')
doc.content = {'a': 'b'}
encrypted_docstr = _crypto.SoledadCrypto('safe').encrypt_doc(doc)
with self.assertRaises(l2db.errors.BrokenSyncStream):
self.parse("[\r\n{},\r\n]")
with self.assertRaises(l2db.errors.BrokenSyncStream):
self.parse(('[\r\n{},\r\n{"id": "i", "rev": "r", ' +
'"gen": 3, "trans_id": "T-sid"},\r\n' + '%s,\r\n]') %
encrypted_docstr)
def _insert_received_doc(self, response, idx, total):
"""
Insert a received document into the local replica.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if self._sync_decr_pool and not self._sync_decr_pool.running:
self._sync_decr_pool.start(number_of_changes)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(self._crypto.decrypt_doc(doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
_emit_receive_status(user_data, self._received_docs, total)
return number_of_changes, new_generation, new_transaction_id
def __init__(self,
doc_id=None,
rev=None,
json='{}',
has_conflicts=False,
syncable=True,
store=None):
SoledadDocument.__init__(self,
doc_id=doc_id,
rev=rev,
json=json,
has_conflicts=has_conflicts,
syncable=syncable)
self.set_store(store)
def test_extra_comma(self):
doc = SoledadDocument('i', rev='r')
doc.content = {'a': 'b'}
encrypted_docstr = _crypto.SoledadCrypto('safe').encrypt_doc(doc)
with self.assertRaises(l2db.errors.BrokenSyncStream):
self.parse("[\r\n{},\r\n]")
with self.assertRaises(l2db.errors.BrokenSyncStream):
self.parse(
('[\r\n{},\r\n{"id": "i", "rev": "r", ' +
'"gen": 3, "trans_id": "T-sid"},\r\n' +
'%s,\r\n]') % encrypted_docstr)
def test_repair_keeps_active_and_key_doc_if_private_key_exists(self):
soledad = mock()
key = self._public_key(SOME_EMAIL_ADDRESS, SOME_KEY_ID)
private_key = self._private_key(SOME_EMAIL_ADDRESS, SOME_KEY_ID)
active_doc = SoledadDocument(doc_id='some_doc', json=key.get_active_json(SOME_EMAIL_ADDRESS))
key_doc = SoledadDocument(doc_id='some_doc', json=key.get_json())
private_key_doc = SoledadDocument(doc_id='some_doc', json=private_key.get_json())
when(soledad).get_all_docs().thenReturn(defer.succeed((1, [key_doc, active_doc, private_key_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad, never).delete_doc(key_doc)
verify(soledad, never).delete_doc(active_doc)
verify(soledad, never).delete_doc(private_key_doc)
def _encrypt_message(self, uuid_pubkey, address, message):
"""
Given a UUID, a public key, address and a message, it encrypts
the message to that public key.
The address is needed in order to build the OpenPGPKey object.
@param uuid_pubkey: tuple that holds the uuid and the public
key as it is returned by the previous call in the chain
@type uuid_pubkey: tuple (str, str)
@param address: mail address for this message
@type address: str
@param message: message contents
@type message: str
@return: uuid, doc to sync with Soledad
@rtype: tuple(str, SoledadDocument)
"""
uuid, pubkey = uuid_pubkey
log.msg("Encrypting message to %s's pubkey" % (uuid,))
log.msg("Pubkey: %s" % (pubkey,))
doc = SoledadDocument(doc_id=str(pyuuid.uuid4()))
data = {'incoming': True, 'content': message}
if pubkey is None or len(pubkey) == 0:
doc.content = {
self.INCOMING_KEY: True,
ENC_SCHEME_KEY: EncryptionSchemes.NONE,
ENC_JSON_KEY: json.dumps(data)
}
return uuid, doc
openpgp_key = None
with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg:
gpg.import_keys(pubkey)
key = gpg.list_keys().pop()
openpgp_key = openpgp._build_key_from_gpg(address, key, pubkey)
doc.content = {
self.INCOMING_KEY: True,
ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
ENC_JSON_KEY: str(gpg.encrypt(
json.dumps(data),
openpgp_key.fingerprint,
symmetric=False))
}
return uuid, doc
def _insert_received_doc(self, idx, total, response):
"""
Insert a received document into the local replica.
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
:param response: The body and headers of the response.
:type response: tuple(str, dict)
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(decrypt_doc(self._crypto, doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
msg = "%d/%d" % (idx, total)
emit(SOLEDAD_SYNC_RECEIVE_STATUS, msg)
logger.debug("Soledad sync receive status: %s" % msg)
return number_of_changes, new_generation, new_transaction_id
def test_doc_decryption(soledad_client, txbenchmark, payload):
"""
Decrypt a document of a given size.
"""
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = yield crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
yield txbenchmark(crypto.decrypt_doc, doc)
def test_doc_decryption(soledad_client, txbenchmark, payload):
"""
Decrypt a document of a given size.
"""
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(
doc_id=uuid4().hex, rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = yield crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
yield txbenchmark(crypto.decrypt_doc, doc)
def test_decrypt_with_unknown_mac_method_raises(self):
"""
Trying to decrypt a document with unknown MAC method should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
self.assertTrue(crypto.MAC_KEY in doc.content)
self.assertTrue(crypto.MAC_METHOD_KEY in doc.content)
# mess with MAC method
doc.content[crypto.MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(UnknownMacMethod, crypto.decrypt_doc,
self._soledad._crypto, doc)
def _mock_create_soledad_doc(self, doc_id, doc):
soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
if doc.future_doc_id:
when(self.soledad).create_doc(doc.serialize(), doc_id=doc_id).thenReturn(defer.succeed(soledad_doc))
else:
when(self.soledad).create_doc(doc.serialize()).thenReturn(defer.succeed(soledad_doc))
self.doc_by_id[doc_id] = soledad_doc
def test_decrypt_with_wrong_mac_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
self.assertTrue(crypto.MAC_KEY in doc.content)
self.assertTrue(crypto.MAC_METHOD_KEY in doc.content)
# mess with MAC
doc.content[crypto.MAC_KEY] = '1234567890ABCDEF'
# try to decrypt doc
self.assertRaises(WrongMac, crypto.decrypt_doc, self._soledad._crypto,
doc)
def _mock_get_soledad_doc(self, doc_id, doc):
soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
# when(self.soledad).get_doc(doc_id).thenReturn(defer.succeed(soledad_doc))
when(self.soledad).get_doc(doc_id).thenAnswer(lambda: defer.succeed(soledad_doc))
self.doc_by_id[doc_id] = soledad_doc
def test_processing_order(self):
"""
This test ensures that processing of documents only occur if there is
a sequence in place.
"""
crypto = self._soledad._crypto
docs = []
for i in xrange(1, 10):
i = str(i)
doc = SoledadDocument(doc_id=DOC_ID + i,
rev=DOC_REV + i,
json=json.dumps(DOC_CONTENT))
encrypted_content = json.loads(crypto.encrypt_doc(doc))
docs.append((doc, encrypted_content))
# insert the encrypted document in the pool
yield self._pool.start(10) # pool is expecting to process 10 docs
self._pool._loop.stop() # we are processing manually
# first three arrives, forming a sequence
for i, (doc, encrypted_content) in enumerate(docs[:3]):
gen = idx = i + 1
yield self._pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx)
# last one arrives alone, so it can't be processed
doc, encrypted_content = docs[-1]
yield self._pool.insert_encrypted_received_doc(doc.doc_id, doc.rev,
encrypted_content, 10,
"trans_id", 10)
yield self._pool._decrypt_and_recurse()
self.assertEqual(3, self._pool._processed_docs)
def _insert_decrypted_local_doc(self, doc_id, doc_rev, content, gen,
trans_id, idx):
"""
Insert the decrypted document into the local replica.
Make use of the passed callback `insert_doc_cb` passed to the caller
by u1db sync.
:param doc_id: The document id.
:type doc_id: str
:param doc_rev: The document revision.
:type doc_rev: str
:param content: The serialized content of the document.
:type content: str
:param gen: The generation corresponding to the modification of that
document.
:type gen: int
:param trans_id: The transaction id corresponding to the modification
of that document.
:type trans_id: str
"""
# could pass source_replica in params for callback chain
logger.debug("Sync decrypter pool: inserting doc in local db: "
"%s:%s %s" % (doc_id, doc_rev, gen))
# convert deleted documents to avoid error on document creation
if content == 'null':
content = None
doc = SoledadDocument(doc_id, doc_rev, content)
gen = int(gen)
self._insert_doc_cb(doc, gen, trans_id)
# store info about processed docs
self._last_inserted_idx = idx
self._processed_docs += 1
def test_store_attachment_twice_does_not_cause_exception(self):
attachment_id = '9863729729D2E2EE8E52F0A7115CE33AD18DDA4B58E49AE08DD092D1C8E699B0'
content = 'this is some attachment content'
content_type = 'text/plain'
cdoc_serialized = {
'content_transfer_encoding': 'base64',
'lkf': [],
'content_disposition': 'attachment',
'ctype': '',
'raw': 'dGhpcyBpcyBzb21lIGF0dGFjaG1lbnQgY29udGVudA==',
'phash':
'9863729729D2E2EE8E52F0A7115CE33AD18DDA4B58E49AE08DD092D1C8E699B0',
'content_type': 'text/plain',
'type': 'cnt'
}
doc = SoledadDocument(json=json.dumps({
'content_type': content_type,
'raw': content
}))
when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt',
attachment_id).thenReturn(
defer.succeed([doc]))
store = LeapAttachmentStore(self.soledad)
when(self.soledad).create_doc(cdoc_serialized,
doc_id=attachment_id).thenRaise(
u1db.errors.RevisionConflict())
actual_attachment_id = yield store.add_attachment(
content, content_type)
self.assertEqual(attachment_id, actual_attachment_id)
def _insert_received_doc(self, idx, total, response):
"""
Insert a received document into the local replica.
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
:param response: The body and headers of the response.
:type response: tuple(str, dict)
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(decrypt_doc(self._crypto, doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
msg = "%d/%d" % (idx, total)
emit(SOLEDAD_SYNC_RECEIVE_STATUS, msg)
logger.debug("Soledad sync receive status: %s" % msg)
return number_of_changes, new_generation, new_transaction_id
def test_decrypt_with_unknown_mac_method_raises(self):
"""
Trying to decrypt a document with unknown MAC method should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
self.assertTrue(MAC_KEY in doc.content)
self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC method
doc.content[MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(
UnknownMacMethodError,
crypto.decrypt_doc, self._soledad._crypto, doc)
def testLogErrorIfDecryptFails(self):
def assert_failure(_):
mock_logger_error.assert_any_call('_decrypt_doc: '
'Error decrypting document with '
'ID 1')
with patch.object(Logger, 'error') as mock_logger_error:
doc = SoledadDocument()
doc.doc_id = '1'
doc.content = {'_enc_json': ''}
self.fetcher._process_decrypted_doc = Mock()
self.km.decrypt = Mock(return_value=defer.fail(Exception()))
d = self.fetcher._decrypt_doc(doc)
d.addCallback(assert_failure)
return d
def test_stage2_bootstrap_signals(self):
"""
Test that if there are keys in server, soledad will download them and
emit corresponding signals.
"""
# get existing instance so we have access to keys
sol = self._soledad_instance()
# create a document with secrets
doc = SoledadDocument(doc_id=sol._shared_db_doc_id())
doc.content = sol.export_recovery_document()
class Stage2MockSharedDB(object):
get_doc = Mock(return_value=doc)
put_doc = Mock()
lock = Mock(return_value=('atoken', 300))
unlock = Mock()
def __call__(self):
return self
sol.close()
# reset mock
soledad.client.signal.reset_mock()
# get a fresh instance so it emits all bootstrap signals
sol = self._soledad_instance(
secrets_path='alternative_stage2.json',
local_db_path='alternative_stage2.u1db',
shared_db_class=Stage2MockSharedDB)
# reverse call order so we can verify in the order the signals were
# expected
soledad.client.signal.mock_calls.reverse()
soledad.client.signal.call_args = \
soledad.client.signal.call_args_list[0]
soledad.client.signal.call_args_list.reverse()
# assert download keys signals
soledad.client.signal.assert_called_with(
proto.SOLEDAD_DOWNLOADING_KEYS,
ADDRESS,
)
self._pop_mock_call(soledad.client.signal)
soledad.client.signal.assert_called_with(
proto.SOLEDAD_DONE_DOWNLOADING_KEYS,
ADDRESS,
)
sol.close()
def test_encrypt_and_decrypt(self):
"""
Check that encrypting and decrypting gives same doc.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
assert encrypted != payload
assert 'raw' in encrypted
doc2 = SoledadDocument('id1', '1')
doc2.set_json(encrypted)
assert _crypto.is_symmetrically_encrypted(encrypted)
decrypted = (yield crypto.decrypt_doc(doc2)).getvalue()
assert len(decrypted) != 0
assert json.loads(decrypted) == payload
def test_extra_comma(self):
"""
Test adapted to use encrypted content.
"""
doc = SoledadDocument('i', rev='r')
doc.content = {}
enc_json = target.encrypt_doc(self._soledad._crypto, doc)
tgt = target.SoledadSyncTarget("http://foo/foo",
crypto=self._soledad._crypto)
self.assertRaises(u1db.errors.BrokenSyncStream, tgt._parse_sync_stream,
"[\r\n{},\r\n]", None)
self.assertRaises(
u1db.errors.BrokenSyncStream, tgt._parse_sync_stream,
'[\r\n{},\r\n{"id": "i", "rev": "r", '
'"content": %s, "gen": 3, "trans_id": "T-sid"}'
',\r\n]' % json.dumps(enc_json), lambda doc, gen, trans_id: None)
def test_encrypt_and_decrypt(self):
"""
Check that encrypting and decrypting gives same doc.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
assert encrypted != payload
assert 'raw' in encrypted
doc2 = SoledadDocument('id1', '1')
doc2.set_json(encrypted)
assert _crypto.is_symmetrically_encrypted(encrypted)
decrypted = (yield crypto.decrypt_doc(doc2)).getvalue()
assert len(decrypted) != 0
assert json.loads(decrypted) == payload
def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
syncable=True):
"""
Return a default Soledad Document.
Used in the initialization for SQLCipherDatabase
"""
return SoledadDocument(doc_id=doc_id, rev=rev, json=json,
has_conflicts=has_conflicts, syncable=syncable)
def test_decrypt_with_wrong_mac_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
self.assertTrue(MAC_KEY in doc.content)
self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC
doc.content[MAC_KEY] = '1234567890ABCDEF'
# try to decrypt doc
self.assertRaises(
WrongMacError,
crypto.decrypt_doc, self._soledad._crypto, doc)
def test_stage2_bootstrap_signals(self):
"""
Test that if there are keys in server, soledad will download them and
emit corresponding signals.
"""
# get existing instance so we have access to keys
sol = self._soledad_instance()
# create a document with secrets
doc = SoledadDocument(doc_id=sol._shared_db_doc_id())
doc.content = sol.export_recovery_document()
class Stage2MockSharedDB(object):
get_doc = Mock(return_value=doc)
put_doc = Mock()
lock = Mock(return_value=('atoken', 300))
unlock = Mock()
def __call__(self):
return self
sol.close()
# reset mock
soledad.client.signal.reset_mock()
# get a fresh instance so it emits all bootstrap signals
sol = self._soledad_instance(secrets_path='alternative_stage2.json',
local_db_path='alternative_stage2.u1db',
shared_db_class=Stage2MockSharedDB)
# reverse call order so we can verify in the order the signals were
# expected
soledad.client.signal.mock_calls.reverse()
soledad.client.signal.call_args = \
soledad.client.signal.call_args_list[0]
soledad.client.signal.call_args_list.reverse()
# assert download keys signals
soledad.client.signal.assert_called_with(
proto.SOLEDAD_DOWNLOADING_KEYS,
ADDRESS,
)
self._pop_mock_call(soledad.client.signal)
soledad.client.signal.assert_called_with(
proto.SOLEDAD_DONE_DOWNLOADING_KEYS,
ADDRESS,
)
sol.close()
def test_extra_comma(self):
"""
Test adapted to use encrypted content.
"""
doc = SoledadDocument('i', rev='r')
doc.content = {}
enc_json = target.encrypt_doc(self._soledad._crypto, doc)
tgt = target.SoledadSyncTarget(
"http://foo/foo", crypto=self._soledad._crypto)
self.assertRaises(u1db.errors.BrokenSyncStream,
tgt._parse_sync_stream, "[\r\n{},\r\n]", None)
self.assertRaises(u1db.errors.BrokenSyncStream,
tgt._parse_sync_stream,
'[\r\n{},\r\n{"id": "i", "rev": "r", '
'"content": %s, "gen": 3, "trans_id": "T-sid"}'
',\r\n]' % json.dumps(enc_json),
lambda doc, gen, trans_id: None)
def test_get_or_create_key_returns_key(self):
soledad = mock()
when(soledad).get_from_index('by-type', 'index_key').thenReturn(
[SoledadDocument(json='{"value": "somekey"}')])
key = yield SearchIndexStorageKey(soledad).get_or_create_key()
self.assertEqual('somekey', key)
def __atomic_doc_parse(self, doc_info, content, total):
doc = SoledadDocument(doc_info['id'], doc_info['rev'], content)
if is_symmetrically_encrypted(content):
content = yield self._crypto.decrypt_doc(doc)
elif old_crypto.is_symmetrically_encrypted(doc):
content = self._deprecated_crypto.decrypt_doc(doc)
doc.set_json(content)
# TODO insert blobs here on the blob backend
# FIXME: This is wrong. Using the very same SQLite connection object
# from multiple threads is dangerous. We should bring the dbpool here
# or find an alternative. Deferring to a thread only helps releasing
# the reactor for other tasks as this is an IO intensive call.
yield threads.deferToThread(self._insert_doc_cb, doc, doc_info['gen'],
doc_info['trans_id'])
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
_emit_receive_status(user_data, self._received_docs, total=total)
def test_doc_encryption(soledad_client, txbenchmark, payload):
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
yield txbenchmark(crypto.encrypt_doc, doc)
def factory(doc_id=None,
rev=None,
json='{}',
has_conflicts=False,
syncable=True):
return SoledadDocument(doc_id=doc_id,
rev=rev,
json=json,
has_conflicts=has_conflicts,
syncable=syncable)
def test_extra_comma(self):
"""
Test adapted to use encrypted content.
"""
doc = SoledadDocument('i', rev='r')
doc.content = {}
_crypto = self._soledad._crypto
key = _crypto.doc_passphrase(doc.doc_id)
secret = _crypto.secret
enc_json = crypto.encrypt_docstr(
doc.get_json(), doc.doc_id, doc.rev,
key, secret)
with self.assertRaises(u1db.errors.BrokenSyncStream):
self.target._parse_received_doc_response("[\r\n{},\r\n]")
with self.assertRaises(u1db.errors.BrokenSyncStream):
self.target._parse_received_doc_response(
('[\r\n{},\r\n{"id": "i", "rev": "r", ' +
'"content": %s, "gen": 3, "trans_id": "T-sid"}' +
',\r\n]') % json.dumps(enc_json))
def test_decrypt_with_wrong_tag_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
encdict = json.loads(encrypted)
preamble, raw = str(encdict['raw']).split()
preamble = base64.urlsafe_b64decode(preamble)
raw = base64.urlsafe_b64decode(raw)
# mess with tag
messed = raw[:-16] + '0' * 16
preamble = base64.urlsafe_b64encode(preamble)
newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed))
doc2 = SoledadDocument('id1', '1')
doc2.set_json(json.dumps({"raw": str(newraw)}))
with pytest.raises(_crypto.InvalidBlob):
yield crypto.decrypt_doc(doc2)
def test_encrypt_decrypt_json(self):
"""
Test encrypting and decrypting documents.
"""
simpledoc = {'key': 'val'}
doc1 = SoledadDocument(doc_id='id')
doc1.content = simpledoc
# encrypt doc
doc1.set_json(target.encrypt_doc(self._soledad._crypto, doc1))
# assert content is different and includes keys
self.assertNotEqual(
simpledoc, doc1.content,
'incorrect document encryption')
self.assertTrue(target.ENC_JSON_KEY in doc1.content)
self.assertTrue(target.ENC_SCHEME_KEY in doc1.content)
# decrypt doc
doc1.set_json(target.decrypt_doc(self._soledad._crypto, doc1))
self.assertEqual(
simpledoc, doc1.content, 'incorrect document encryption')
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
"""
Parse incoming synchronization stream and insert documents in the
local database.
If an incoming document's encryption scheme is equal to
EncryptionSchemes.SYMKEY, then this method will decrypt it with
Soledad's symmetric key.
:param data: The body of the HTTP response.
:type data: str
:param return_doc_cb: A callback to insert docs from target.
:type return_doc_cb: function
:param ensure_callback: A callback to ensure we have the correct
target_replica_uid, if it was just created.
:type ensure_callback: function
:raise BrokenSyncStream: If C{data} is malformed.
:return: A dictionary representing the first line of the response got
from remote replica.
:rtype: list of str
"""
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
#-------------------------------------------------------------
# symmetric decryption of document's contents
#-------------------------------------------------------------
# if arriving content was symmetrically encrypted, we decrypt
# it.
doc = SoledadDocument(
entry['id'], entry['rev'], entry['content'])
if doc.content and ENC_SCHEME_KEY in doc.content:
if doc.content[ENC_SCHEME_KEY] == \
EncryptionSchemes.SYMKEY:
doc.set_json(decrypt_doc(self._crypto, doc))
#-------------------------------------------------------------
# end of symmetric decryption
#-------------------------------------------------------------
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res
