def _put_secrets_in_shared_db(self):
"""
Assert local keys are the same as shared db's ones.
Try to fetch keys from shared recovery database. If they already exist
in the remote db, assert that that data is the same as local data.
Otherwise, upload keys to shared recovery database.
"""
soledad_assert(
self._has_secret(),
'Tried to send keys to server but they don\'t exist in local '
'storage.')
# try to get secrets doc from server, otherwise create it
doc = self._get_secrets_from_shared_db()
if doc is None:
doc = document.SoledadDocument(
doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
doc.content = self._export_recovery_document()
# upload secrets to server
events.emit(events.SOLEDAD_UPLOADING_KEYS, self._uuid)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
db.put_doc(doc)
events.emit(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
def _send_docs(self, docs_by_generation, last_known_generation,
last_known_trans_id, sync_id):
if not docs_by_generation:
defer.returnValue([None, None])
headers = self._auth_header.copy()
headers.update({'content-type': ['application/x-soledad-sync-put']})
# add remote replica metadata to the request
first_entries = ['[']
self._prepare('',
first_entries,
last_known_generation=last_known_generation,
last_known_trans_id=last_known_trans_id,
sync_id=sync_id,
ensure=self._ensure_callback is not None)
idx = 0
total = len(docs_by_generation)
for doc, gen, trans_id in docs_by_generation:
idx += 1
result = yield self._send_one_doc(headers, first_entries, doc, gen,
trans_id, total, idx)
if self._defer_encryption:
self._sync_enc_pool.delete_encrypted_doc(doc.doc_id, doc.rev)
msg = "%d/%d" % (idx, total)
content = {'sent': idx, 'total': total}
emit(SOLEDAD_SYNC_SEND_STATUS, content)
logger.debug("Sync send status: %s" % msg)
response_dict = json.loads(result)[0]
gen_after_send = response_dict['new_generation']
trans_id_after_send = response_dict['new_transaction_id']
defer.returnValue([gen_after_send, trans_id_after_send])
def _put_secrets_in_shared_db(self):
"""
Assert local keys are the same as shared db's ones.
Try to fetch keys from shared recovery database. If they already exist
in the remote db, assert that that data is the same as local data.
Otherwise, upload keys to shared recovery database.
"""
soledad_assert(
self._has_secret(),
'Tried to send keys to server but they don\'t exist in local '
'storage.')
# try to get secrets doc from server, otherwise create it
doc = self._get_secrets_from_shared_db()
if doc is None:
doc = document.SoledadDocument(
doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
doc.content = self._export_recovery_document()
# upload secrets to server
events.emit(events.SOLEDAD_UPLOADING_KEYS, self._uuid)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
db.put_doc(doc)
events.emit(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
def _send_docs(self, docs_by_generation, last_known_generation,
last_known_trans_id, sync_id):
if not docs_by_generation:
defer.returnValue([None, None])
headers = self._auth_header.copy()
headers.update({'content-type': ['application/x-soledad-sync-put']})
# add remote replica metadata to the request
first_entries = ['[']
self._prepare(
'', first_entries,
last_known_generation=last_known_generation,
last_known_trans_id=last_known_trans_id,
sync_id=sync_id,
ensure=self._ensure_callback is not None)
idx = 0
total = len(docs_by_generation)
for doc, gen, trans_id in docs_by_generation:
idx += 1
result = yield self._send_one_doc(
headers, first_entries, doc,
gen, trans_id, total, idx)
if self._defer_encryption:
self._sync_enc_pool.delete_encrypted_doc(
doc.doc_id, doc.rev)
emit(SOLEDAD_SYNC_SEND_STATUS,
"Soledad sync send status: %d/%d"
% (idx, total))
response_dict = json.loads(result)[0]
gen_after_send = response_dict['new_generation']
trans_id_after_send = response_dict['new_transaction_id']
defer.returnValue([gen_after_send, trans_id_after_send])
def _insert_received_doc(self, response, idx, total):
"""
Insert a received document into the local replica.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(self._crypto.decrypt_doc(doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id, idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
self._received_docs += 1
msg = "%d/%d" % (self._received_docs, total)
content = {'received': self._received_docs, 'total': total}
emit(SOLEDAD_SYNC_RECEIVE_STATUS, content)
logger.debug("Sync receive status: %s" % msg)
return number_of_changes, new_generation, new_transaction_id
def _insert_received_doc(self, idx, total, response):
"""
Insert a received document into the local replica.
:param idx: The index count of the current operation.
:type idx: int
:param total: The total number of operations.
:type total: int
:param response: The body and headers of the response.
:type response: tuple(str, dict)
"""
new_generation, new_transaction_id, number_of_changes, doc_id, \
rev, content, gen, trans_id = \
self._parse_received_doc_response(response)
if doc_id is not None:
# decrypt incoming document and insert into local database
# -------------------------------------------------------------
# symmetric decryption of document's contents
# -------------------------------------------------------------
# If arriving content was symmetrically encrypted, we decrypt it.
# We do it inline if defer_decryption flag is False or no sync_db
# was defined, otherwise we defer it writing it to the received
# docs table.
doc = SoledadDocument(doc_id, rev, content)
if is_symmetrically_encrypted(doc):
if self._queue_for_decrypt:
self._sync_decr_pool.insert_encrypted_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
# defer_decryption is False or no-sync-db fallback
doc.set_json(decrypt_doc(self._crypto, doc))
self._insert_doc_cb(doc, gen, trans_id)
else:
# not symmetrically encrypted doc, insert it directly
# or save it in the decrypted stage.
if self._queue_for_decrypt:
self._sync_decr_pool.insert_received_doc(
doc.doc_id, doc.rev, doc.content, gen, trans_id,
idx)
else:
self._insert_doc_cb(doc, gen, trans_id)
# -------------------------------------------------------------
# end of symmetric decryption
# -------------------------------------------------------------
msg = "%d/%d" % (idx, total)
emit(SOLEDAD_SYNC_RECEIVE_STATUS, msg)
logger.debug("Soledad sync receive status: %s" % msg)
return number_of_changes, new_generation, new_transaction_id
def _sync_callback(local_gen):
self._last_received_docs = docs = self._dbsyncer.received_docs
# Post-Sync Hooks
if docs:
iface = soledad_interfaces.ISoledadPostSyncPlugin
suitable_plugins = collect_plugins(iface)
for plugin in suitable_plugins:
watched = plugin.watched_doc_types
r = [filter(lambda s: s.startswith(preffix), docs) for preffix in watched]
filtered = list(chain(*r))
plugin.process_received_docs(filtered)
soledad_events.emit(soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid)
return local_gen
def _get_secrets_from_shared_db(self):
"""
Retrieve the document with encrypted key material from the shared
database.
:return: a document with encrypted key material in its contents
:rtype: document.SoledadDocument
"""
events.emit(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
doc = db.get_doc(self._shared_db_doc_id())
events.emit(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid)
return doc
def _get_secrets_from_shared_db(self):
"""
Retrieve the document with encrypted key material from the shared
database.
:return: a document with encrypted key material in its contents
:rtype: document.SoledadDocument
"""
events.emit(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid)
db = self._shared_db
if not db:
logger.warning('No shared db found')
return
doc = db.get_doc(self._shared_db_doc_id())
events.emit(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid)
return doc
def _sync_callback(local_gen):
self._last_received_docs = docs = self._dbsyncer.received_docs
# Post-Sync Hooks
if docs:
iface = soledad_interfaces.ISoledadPostSyncPlugin
suitable_plugins = collect_plugins(iface)
for plugin in suitable_plugins:
watched = plugin.watched_doc_types
r = [
filter(lambda s: s.startswith(preffix), docs)
for preffix in watched
]
filtered = list(chain(*r))
plugin.process_received_docs(filtered)
soledad_events.emit(soledad_events.SOLEDAD_DONE_DATA_SYNC,
self.uuid)
return local_gen
def _gen_secret(self):
"""
Generate a secret for symmetric encryption and store in a local
encrypted file.
This method emits the following events.signals:
* SOLEDAD_CREATING_KEYS
* SOLEDAD_DONE_CREATING_KEYS
:return: The id of the generated secret.
:rtype: str
"""
events.emit(events.SOLEDAD_CREATING_KEYS, self._uuid)
# generate random secret
secret = os.urandom(self.GEN_SECRET_LENGTH)
secret_id = sha256(secret).hexdigest()
self._secrets[secret_id] = secret
self._store_secrets()
events.emit(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid)
return secret_id
def _gen_secret(self):
"""
Generate a secret for symmetric encryption and store in a local
encrypted file.
This method emits the following events.signals:
* SOLEDAD_CREATING_KEYS
* SOLEDAD_DONE_CREATING_KEYS
:return: The id of the generated secret.
:rtype: str
"""
events.emit(events.SOLEDAD_CREATING_KEYS, self._uuid)
# generate random secret
secret = os.urandom(self.GEN_SECRET_LENGTH)
secret_id = sha256(secret).hexdigest()
self._secrets[secret_id] = secret
self._store_secrets()
events.emit(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid)
return secret_id
def _emit_done_data_sync(passthrough):
soledad_events.emit(
soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid)
return passthrough
def _emit_done_data_sync(passthrough):
soledad_events.emit(soledad_events.SOLEDAD_DONE_DATA_SYNC,
self.uuid)
return passthrough