def test_repair_only_deletes_key_docs(self):
soledad = mock()
key = self._public_key(SOME_EMAIL_ADDRESS, SOME_KEY_ID)
key_doc = SoledadDocument(doc_id='some_doc', json=key.get_active_json(SOME_EMAIL_ADDRESS))
other_doc = SoledadDocument(doc_id='something', json='{}')
when(soledad).get_all_docs().thenReturn(defer.succeed((1, [key_doc, other_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad, never).delete_doc(other_doc)
def test_repair_delete_public_key_docs(self):
soledad = mock()
key = self._public_key(SOME_EMAIL_ADDRESS, SOME_FINGERPRINT)
active_doc = SoledadDocument(doc_id='some_doc',
json=key.get_active_json())
key_doc = SoledadDocument(doc_id='some_doc', json=key.get_json())
when(soledad).get_all_docs().thenReturn(
defer.succeed((1, [key_doc, active_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad).delete_doc(active_doc)
verify(soledad).delete_doc(key_doc)
def test_repair_keeps_active_and_key_doc_if_private_key_exists(self):
soledad = mock()
key = self._public_key(SOME_EMAIL_ADDRESS, SOME_KEY_ID)
private_key = self._private_key(SOME_EMAIL_ADDRESS, SOME_KEY_ID)
active_doc = SoledadDocument(doc_id='some_doc', json=key.get_active_json(SOME_EMAIL_ADDRESS))
key_doc = SoledadDocument(doc_id='some_doc', json=key.get_json())
private_key_doc = SoledadDocument(doc_id='some_doc', json=private_key.get_json())
when(soledad).get_all_docs().thenReturn(defer.succeed((1, [key_doc, active_doc, private_key_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad, never).delete_doc(key_doc)
verify(soledad, never).delete_doc(active_doc)
verify(soledad, never).delete_doc(private_key_doc)
def _insert_decrypted_local_doc(self, doc_id, doc_rev, content, gen,
trans_id, idx):
"""
Insert the decrypted document into the local replica.
Make use of the passed callback `insert_doc_cb` passed to the caller
by u1db sync.
:param doc_id: The document id.
:type doc_id: str
:param doc_rev: The document revision.
:type doc_rev: str
:param content: The serialized content of the document.
:type content: str
:param gen: The generation corresponding to the modification of that
document.
:type gen: int
:param trans_id: The transaction id corresponding to the modification
of that document.
:type trans_id: str
"""
# could pass source_replica in params for callback chain
logger.debug("Sync decrypter pool: inserting doc in local db: "
"%s:%s %s" % (doc_id, doc_rev, gen))
# convert deleted documents to avoid error on document creation
if content == 'null':
content = None
doc = SoledadDocument(doc_id, doc_rev, content)
gen = int(gen)
self._insert_doc_cb(doc, gen, trans_id)
# store info about processed docs
self._last_inserted_idx = idx
self._processed_docs += 1
def test_store_attachment_twice_does_not_cause_exception(self):
attachment_id = '9863729729D2E2EE8E52F0A7115CE33AD18DDA4B58E49AE08DD092D1C8E699B0'
content = 'this is some attachment content'
content_type = 'text/plain'
cdoc_serialized = {
'content_transfer_encoding': 'base64',
'lkf': [],
'content_disposition': 'attachment',
'ctype': '',
'raw': 'dGhpcyBpcyBzb21lIGF0dGFjaG1lbnQgY29udGVudA==',
'phash':
'9863729729D2E2EE8E52F0A7115CE33AD18DDA4B58E49AE08DD092D1C8E699B0',
'content_type': 'text/plain',
'type': 'cnt'
}
doc = SoledadDocument(json=json.dumps({
'content_type': content_type,
'raw': content
}))
when(self.soledad).get_from_index('by-type-and-payloadhash', 'cnt',
attachment_id).thenReturn(
defer.succeed([doc]))
store = LeapAttachmentStore(self.soledad)
when(self.soledad).create_doc(cdoc_serialized,
doc_id=attachment_id).thenRaise(
u1db.errors.RevisionConflict())
actual_attachment_id = yield store.add_attachment(
content, content_type)
self.assertEqual(attachment_id, actual_attachment_id)
def _mock_create_soledad_doc(self, doc_id, doc):
soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
if doc.future_doc_id:
when(self.soledad).create_doc(doc.serialize(), doc_id=doc_id).thenReturn(defer.succeed(soledad_doc))
else:
when(self.soledad).create_doc(doc.serialize()).thenReturn(defer.succeed(soledad_doc))
self.doc_by_id[doc_id] = soledad_doc
def _mock_get_soledad_doc(self, doc_id, doc):
soledad_doc = SoledadDocument(doc_id, json=json.dumps(doc.serialize()))
# when(self.soledad).get_doc(doc_id).thenReturn(defer.succeed(soledad_doc))
when(self.soledad).get_doc(doc_id).thenAnswer(lambda: defer.succeed(soledad_doc))
self.doc_by_id[doc_id] = soledad_doc
def _put_secrets_in_shared_db(self):
"""
Assert local keys are the same as shared db's ones.
Try to fetch keys from shared recovery database. If they already exist
in the remote db, assert that that data is the same as local data.
Otherwise, upload keys to shared recovery database.
"""
soledad_assert(
self._has_secret(),
'Tried to send keys to server but they don\'t exist in local '
'storage.')
# try to get secrets doc from server, otherwise create it
doc = self._get_secrets_from_shared_db()
if doc is None:
doc = SoledadDocument(doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
doc.content = self.export_recovery_document()
# upload secrets to server
signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
db.put_doc(doc)
signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
def test_processing_order(self):
"""
This test ensures that processing of documents only occur if there is
a sequence in place.
"""
crypto = self._soledad._crypto
docs = []
for i in xrange(1, 10):
i = str(i)
doc = SoledadDocument(doc_id=DOC_ID + i,
rev=DOC_REV + i,
json=json.dumps(DOC_CONTENT))
encrypted_content = json.loads(crypto.encrypt_doc(doc))
docs.append((doc, encrypted_content))
# insert the encrypted document in the pool
yield self._pool.start(10) # pool is expecting to process 10 docs
self._pool._loop.stop() # we are processing manually
# first three arrives, forming a sequence
for i, (doc, encrypted_content) in enumerate(docs[:3]):
gen = idx = i + 1
yield self._pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx)
# last one arrives alone, so it can't be processed
doc, encrypted_content = docs[-1]
yield self._pool.insert_encrypted_received_doc(doc.doc_id, doc.rev,
encrypted_content, 10,
"trans_id", 10)
yield self._pool._decrypt_and_recurse()
self.assertEqual(3, self._pool._processed_docs)
def test_stage2_bootstrap_signals(self):
"""
Test that if there are keys in server, soledad will download them and
emit corresponding signals.
"""
# get existing instance so we have access to keys
sol = self._soledad_instance()
# create a document with secrets
doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id())
doc.content = sol.secrets._export_recovery_document()
sol.close()
# reset mock
soledad.client.secrets.events.emit.reset_mock()
# get a fresh instance so it emits all bootstrap signals
shared_db = self.get_default_shared_mock(get_doc_return_value=doc)
sol = self._soledad_instance(secrets_path='alternative_stage2.json',
local_db_path='alternative_stage2.u1db',
shared_db_class=shared_db)
# reverse call order so we can verify in the order the signals were
# expected
soledad.client.secrets.events.emit.mock_calls.reverse()
soledad.client.secrets.events.emit.call_args = \
soledad.client.secrets.events.emit.call_args_list[0]
soledad.client.secrets.events.emit.call_args_list.reverse()
# assert download keys signals
soledad.client.secrets.events.emit.assert_called_with(
catalog.SOLEDAD_DOWNLOADING_KEYS,
ADDRESS,
)
self._pop_mock_call(soledad.client.secrets.events.emit)
soledad.client.secrets.events.emit.assert_called_with(
catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
ADDRESS,
)
sol.close()
def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
syncable=True):
"""
Return a default Soledad Document.
Used in the initialization for SQLCipherDatabase
"""
return SoledadDocument(doc_id=doc_id, rev=rev, json=json,
has_conflicts=has_conflicts, syncable=syncable)
def test_encrypt_and_decrypt(self):
"""
Check that encrypting and decrypting gives same doc.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
assert encrypted != payload
assert 'raw' in encrypted
doc2 = SoledadDocument('id1', '1')
doc2.set_json(encrypted)
assert _crypto.is_symmetrically_encrypted(encrypted)
decrypted = (yield crypto.decrypt_doc(doc2)).getvalue()
assert len(decrypted) != 0
assert json.loads(decrypted) == payload
def test_get_or_create_key_returns_key(self):
soledad = mock()
when(soledad).get_from_index('by-type', 'index_key').thenReturn(
[SoledadDocument(json='{"value": "somekey"}')])
key = yield SearchIndexStorageKey(soledad).get_or_create_key()
self.assertEqual('somekey', key)
def test_doc_encryption(soledad_client, txbenchmark, payload):
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
yield txbenchmark(crypto.encrypt_doc, doc)
def factory(doc_id=None,
rev=None,
json='{}',
has_conflicts=False,
syncable=True):
return SoledadDocument(doc_id=doc_id,
rev=rev,
json=json,
has_conflicts=has_conflicts,
syncable=syncable)
def test_doc_update_fails_with_wrong_rev(self):
# create a document in shared db
doc_id = 'some-random-doc'
self.assertIsNone(self._db.get_doc(doc_id))
# create a document in shared db
doc = SoledadDocument(doc_id=doc_id)
self._db.put_doc(doc)
# try to update document without including revision of old version
doc.rev = 'wrong-rev'
self.assertRaises(RevisionConflict, self._db.put_doc, doc)
def save_remote(self, encrypted):
doc = self._remote_doc
if not doc:
doc = SoledadDocument(doc_id=self._remote_doc_id())
doc.content = encrypted
db = self._shared_db
if not db:
logger.warn('no shared db found')
return
db.put_doc(doc)
def test_repair_recreates_public_key_active_doc_if_necessary(self):
soledad = mock()
private_key = self._private_key(SOME_EMAIL_ADDRESS, SOME_KEY_ID)
private_key_doc = SoledadDocument(doc_id='some_doc', json=private_key.get_json())
when(soledad).get_all_docs().thenReturn(defer.succeed((1, [private_key_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad).create_doc_from_json('{"key_id": "4914254E384E264C", "tags": ["keymanager-active"], "type": "OpenPGPKey-active", "private": false, "address": "*****@*****.**"}')
def test_doc_decryption(soledad_client, benchmark, payload):
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
benchmark(crypto.decrypt_doc, doc)
def test_decrypt_with_wrong_tag_raises(self):
"""
Trying to decrypt a document with wrong MAC should raise.
"""
crypto = _crypto.SoledadCrypto('A' * 96)
payload = {'key': 'someval'}
doc1 = SoledadDocument('id1', '1', json.dumps(payload))
encrypted = yield crypto.encrypt_doc(doc1)
encdict = json.loads(encrypted)
preamble, raw = _crypto._split(str(encdict['raw']))
# mess with tag
messed = raw[:-16] + '0' * 16
preamble = base64.urlsafe_b64encode(preamble)
newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed))
doc2 = SoledadDocument('id1', '1')
doc2.set_json(json.dumps({"raw": str(newraw)}))
with pytest.raises(_crypto.InvalidBlob):
yield crypto.decrypt_doc(doc2)
def test_encrypt_doc_and_get_it_back(self):
"""
Test that the pool actually encrypts a document added to the queue.
"""
doc = SoledadDocument(doc_id=DOC_ID,
rev=DOC_REV,
json=json.dumps(DOC_CONTENT))
yield self._pool.encrypt_doc(doc)
encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV)
self.assertIsNotNone(encrypted)
def setup():
client = soledad_client()
pool = SyncEncrypterPool(client._crypto, client._sync_db)
pool.start()
request.addfinalizer(pool.stop)
docs = [
SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
for _ in xrange(amount)
]
return pool, docs
def test_extra_comma(self):
doc = SoledadDocument('i', rev='r')
doc.content = {'a': 'b'}
encrypted_docstr = _crypto.SoledadCrypto('safe').encrypt_doc(doc)
with self.assertRaises(l2db.errors.BrokenSyncStream):
self.parse("[\r\n{},\r\n]")
with self.assertRaises(l2db.errors.BrokenSyncStream):
self.parse(('[\r\n{},\r\n{"id": "i", "rev": "r", ' +
'"gen": 3, "trans_id": "T-sid"},\r\n' + '%s,\r\n]') %
encrypted_docstr)
def test_doc_update_succeeds(self):
doc_id = 'some-random-doc'
self.assertIsNone(self._db.get_doc(doc_id))
# create a document in shared db
doc = SoledadDocument(doc_id=doc_id)
self._db.put_doc(doc)
# update that document
expected = {'new': 'content'}
doc.content = expected
self._db.put_doc(doc)
# ensure expected content was saved
doc = self._db.get_doc(doc_id)
self.assertEqual(expected, doc.content)
def _insert_received_doc(self, response, idx, total):
"""
Insert a received document into the local replica.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if self._sync_decr_pool and not self._sync_decr_pool.running:
self._sync_decr_pool.start(number_of_changes)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(self._crypto.decrypt_doc(doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
self._received_docs += 1
user_data = {'uuid': self.uuid, 'userid': self.userid}
_emit_receive_status(user_data, self._received_docs, total)
return number_of_changes, new_generation, new_transaction_id
def test_repair_recreates_public_key_active_doc_if_necessary(self):
soledad = mock()
private_key = self._private_key(SOME_EMAIL_ADDRESS, SOME_FINGERPRINT)
private_key_doc = SoledadDocument(doc_id='some_doc',
json=private_key.get_active_json())
when(soledad).get_all_docs().thenReturn(
defer.succeed((1, [private_key_doc])))
yield SoledadMaintenance(soledad).repair()
verify(soledad).create_doc_from_json(
'{"encr_used": false, "sign_used": false, "validation": "Weak_Chain", "version": 1, "address": "*****@*****.**", "last_audited_at": 0, "fingerprint": "4914254E384E264C", "type": "OpenPGPKey-active", "private": false, "tags": ["keymanager-active"]}'
)
def _mock_get_mailbox(self, mailbox_name, create_new_uuid=False):
mbox_uuid = self.mbox_uuid if not create_new_uuid else str(uuid4())
when(self.soledad).list_indexes().thenReturn(defer.succeed(MAIL_INDEXES)).thenReturn(
defer.succeed(MAIL_INDEXES))
doc_id = str(uuid4())
mbox = MailboxWrapper(doc_id=doc_id, mbox=mailbox_name, uuid=mbox_uuid)
soledad_doc = SoledadDocument(doc_id, json=json.dumps(mbox.serialize()))
when(self.soledad).get_from_index('by-type-and-mbox', 'mbox', mailbox_name).thenReturn(defer.succeed([soledad_doc]))
self._mock_get_soledad_doc(doc_id, mbox)
self.mbox_uuid_by_name[mailbox_name] = mbox_uuid
self.mbox_soledad_docs.append(soledad_doc)
return mbox, soledad_doc
def test_insert_encrypted_received_doc_many(self, many=100):
"""
Test that many encrypted documents added to the pool are decrypted and
inserted using the callback.
"""
crypto = self._soledad._crypto
self._pool.start(many)
docs = []
# insert many encrypted docs in the pool
for i in xrange(many):
gen = idx = i + 1
doc_id = "doc_id: %d" % idx
rev = "rev: %d" % idx
content = {'idx': idx}
trans_id = "trans_id: %d" % idx
doc = SoledadDocument(doc_id=doc_id,
rev=rev,
json=json.dumps(content))
encrypted_content = json.loads(crypto.encrypt_doc(doc))
docs.append((doc_id, rev, encrypted_content, gen, trans_id, idx))
shuffle(docs)
for doc in docs:
self._pool.insert_encrypted_received_doc(*doc)
def _assert_docs_were_decrypted_and_inserted(_):
self.assertEqual(many, len(self._inserted_docs))
idx = 1
for doc, gen, trans_id in self._inserted_docs:
expected_gen = idx
expected_doc_id = "doc_id: %d" % idx
expected_rev = "rev: %d" % idx
expected_content = json.dumps({'idx': idx})
expected_trans_id = "trans_id: %d" % idx
self.assertEqual(expected_doc_id, doc.doc_id)
self.assertEqual(expected_rev, doc.rev)
self.assertEqual(expected_content, json.dumps(doc.content))
self.assertEqual(expected_gen, gen)
self.assertEqual(expected_trans_id, trans_id)
idx += 1
self._pool.deferred.addCallback(
_assert_docs_were_decrypted_and_inserted)
return self._pool.deferred
def test_doc_decryption(soledad_client, txbenchmark, payload):
"""
Decrypt a document of a given size.
"""
crypto = soledad_client()._crypto
DOC_CONTENT = {'payload': payload(size)}
doc = SoledadDocument(doc_id=uuid4().hex,
rev='rev',
json=json.dumps(DOC_CONTENT))
encrypted_doc = yield crypto.encrypt_doc(doc)
doc.set_json(encrypted_doc)
yield txbenchmark(crypto.decrypt_doc, doc)
def test_decrypt_with_unknown_mac_method_raises(self):
"""
Trying to decrypt a document with unknown MAC method should raise.
"""
simpledoc = {'key': 'val'}
doc = SoledadDocument(doc_id='id')
doc.content = simpledoc
# encrypt doc
doc.set_json(self._soledad._crypto.encrypt_doc(doc))
self.assertTrue(MAC_KEY in doc.content)
self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC method
doc.content[MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(UnknownMacMethodError,
self._soledad._crypto.decrypt_doc, doc)
