async def _vlob_create(self, encryption_revision: int, entry_id: EntryID,
ciphered: bytes, now: Pendulum):
"""
Raises:
FSError
FSRemoteSyncError
FSBackendOfflineError
FSWorkspaceInMaintenance
FSBadEncryptionRevision
FSWorkspaceNoAccess
"""
# Vlob upload
rep = await self._backend_cmds("vlob_create", self.workspace_id,
encryption_revision, entry_id, now,
ciphered)
if rep["status"] == "already_exists":
raise FSRemoteSyncError(entry_id)
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoWriteAccess(
"Cannot upload manifest: no write access")
elif rep["status"] == "bad_encryption_revision":
raise FSBadEncryptionRevision(
f"Cannot create vlob {entry_id}: Bad encryption revision provided"
)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot create vlob while the workspace is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot create vlob {entry_id}: `{rep['status']}`")
async def load_block(self, access: BlockAccess) -> None:
"""
Raises:
FSError
FSRemoteBlockNotFound
FSBackendOfflineError
FSWorkspaceInMaintenance
FSWorkspaceNoAccess
"""
# Download
rep = await self._backend_cmds("block_read", access.id)
if rep["status"] == "not_found":
raise FSRemoteBlockNotFound(access)
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoReadAccess("Cannot load block: no read access")
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot download block while the workspace in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot download block: `{rep['status']}`")
# Decryption
try:
block = access.key.decrypt(rep["block"])
# Decryption error
except CryptoError as exc:
raise FSError(f"Cannot decrypt block: {exc}") from exc
# TODO: let encryption manager do the digest check ?
assert HashDigest.from_data(block) == access.digest, access
await self.local_storage.set_clean_block(access.id, block)
async def list_versions(
self, entry_id: EntryID) -> Dict[int, Tuple[DateTime, DeviceID]]:
"""
Raises:
FSError
FSRemoteOperationError
FSBackendOfflineError
FSWorkspaceInMaintenance
FSRemoteManifestNotFound
"""
with translate_backend_cmds_errors():
rep = await self.backend_cmds.vlob_list_versions(
VlobID(entry_id.uuid))
if rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoReadAccess(
"Cannot load manifest: no read access")
elif rep["status"] == "not_found":
raise FSRemoteManifestNotFound(entry_id)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot download vlob while the workspace is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot fetch vlob {entry_id}: `{rep['status']}`")
return rep["versions"]
async def _fetch_remote_user_manifest(self,
version: int = None) -> UserManifest:
"""
Raises:
FSError
FSWorkspaceInMaintenance
FSBackendOfflineError
"""
try:
# Note encryption_revision is always 1 given we never reencrypt
# the user manifest's realm
rep = await self.backend_cmds.vlob_read(1, self.user_manifest_id,
version)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
if rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot access workspace data while it is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot fetch user manifest from backend: {rep}")
expected_author = rep["author"]
expected_timestamp = rep["timestamp"]
expected_version = rep["version"]
blob = rep["blob"]
try:
author = await self.remote_devices_manager.get_device(
expected_author)
except RemoteDevicesManagerBackendOfflineError as exc:
raise FSBackendOfflineError(str(exc)) from exc
except RemoteDevicesManagerError as exc:
raise FSError(f"Cannot retrieve author public key: {exc}") from exc
try:
manifest = UserManifest.decrypt_verify_and_load(
blob,
key=self.device.user_manifest_key,
author_verify_key=author.verify_key,
expected_id=self.device.user_manifest_id,
expected_author=expected_author,
expected_timestamp=expected_timestamp,
expected_version=version
if version is not None else expected_version,
)
except DataError as exc:
raise FSError(f"Invalid user manifest: {exc}") from exc
return manifest
async def _vlob_update(
self,
encryption_revision: int,
entry_id: EntryID,
ciphered: bytes,
now: DateTime,
version: int,
) -> None:
"""
Raises:
FSError
FSRemoteSyncError
FSBackendOfflineError
FSWorkspaceInMaintenance
FSBadEncryptionRevision
FSWorkspaceNoAccess
"""
# Vlob upload
with translate_backend_cmds_errors():
rep = await self.backend_cmds.vlob_update(encryption_revision,
entry_id, version, now,
ciphered)
if rep["status"] == "not_found":
raise FSRemoteSyncError(entry_id)
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoWriteAccess(
"Cannot upload manifest: no write access")
elif rep["status"] == "bad_version":
raise FSRemoteSyncError(entry_id)
elif rep["status"] == "bad_timestamp":
# Quick and dirty fix before a better version with a retry loop : go offline so we
# don't have to deal with another client updating manifest with a later timestamp
raise FSBackendOfflineError(rep)
elif rep["status"] == "bad_encryption_revision":
raise FSBadEncryptionRevision(
f"Cannot update vlob {entry_id}: Bad encryption revision provided"
)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot create vlob while the workspace is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot update vlob {entry_id}: `{rep['status']}`")
async def _vlob_update(
self,
encryption_revision: int,
entry_id: EntryID,
ciphered: bytes,
now: DateTime,
version: int,
) -> None:
"""
Raises:
FSError
FSRemoteSyncError
FSBackendOfflineError
FSRemoteOperationError
FSWorkspaceInMaintenance
FSBadEncryptionRevision
FSWorkspaceNoAccess
"""
# Vlob upload
with translate_backend_cmds_errors():
rep = await self.backend_cmds.vlob_update(encryption_revision,
VlobID(entry_id.uuid),
version, now, ciphered)
if rep["status"] == "not_found":
raise FSRemoteSyncError(entry_id)
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoWriteAccess(
"Cannot upload manifest: no write access")
elif rep["status"] == "require_greater_timestamp":
raise VlobRequireGreaterTimestampError(
rep["strictly_greater_than"])
elif rep["status"] == "bad_version":
raise FSRemoteSyncError(entry_id)
elif rep["status"] == "bad_encryption_revision":
raise FSBadEncryptionRevision(
f"Cannot update vlob {entry_id}: Bad encryption revision provided"
)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot create vlob while the workspace is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot update vlob {entry_id}: `{rep['status']}`")
async def _send_start_reencryption_cmd(
self,
workspace_id: EntryID,
encryption_revision: int,
timestamp: DateTime,
per_user_ciphered_msgs: Dict[UserID, bytes],
) -> bool:
"""
Raises:
FSError
FSBackendOfflineError
FSWorkspaceNoAccess
BackendCmdsParticipantsMismatchError
"""
# Finally send command to the backend
try:
rep = await self.backend_cmds.realm_start_reencryption_maintenance(
RealmID(workspace_id.uuid), encryption_revision, timestamp,
per_user_ciphered_msgs)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(
f"Cannot start maintenance on workspace {workspace_id}: {exc}"
) from exc
if rep["status"] == "participants_mismatch":
# Catched by caller
return False
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Workspace {workspace_id} already in maintenance: {rep}")
elif rep["status"] == "not_allowed":
raise FSWorkspaceNoAccess(
f"Not allowed to start maintenance on workspace {workspace_id}: {rep}"
)
elif rep["status"] != "ok":
raise FSError(
f"Cannot start maintenance on workspace {workspace_id}: {rep}")
return True
async def upload_block(self, access: BlockAccess, data: bytes) -> None:
"""
Raises:
FSError
FSBackendOfflineError
FSRemoteOperationError
FSWorkspaceInMaintenance
FSWorkspaceNoAccess
"""
# Encryption
try:
ciphered = access.key.encrypt(data)
# Encryption error
except CryptoError as exc:
raise FSError(f"Cannot encrypt block: {exc}") from exc
# Upload block
with translate_backend_cmds_errors():
rep = await self.backend_cmds.block_create(
access.id, RealmID(self.workspace_id.uuid), ciphered)
if rep["status"] == "already_exists":
# Ignore exception if the block has already been uploaded
# This might happen when a failure occurs before the local storage is updated
pass
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoWriteAccess(
"Cannot upload block: no write access")
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot upload block while the workspace in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot upload block: {rep}")
# Update local storage
await self.local_storage.set_clean_block(access.id, data)
await self.local_storage.clear_chunk(ChunkID(access.id.uuid),
miss_ok=True)
async def load_manifest(
self,
entry_id: EntryID,
version: int = None,
timestamp: Pendulum = None,
expected_backend_timestamp: Pendulum = None,
) -> RemoteManifest:
"""
Download a manifest.
Only one from version or timestamp parameters can be specified at the same time.
expected_backend_timestamp enables to check a timestamp against the one returned by the
backend.
Raises:
FSError
FSBackendOfflineError
FSWorkspaceInMaintenance
FSRemoteManifestNotFound
FSBadEncryptionRevision
FSWorkspaceNoAccess
"""
if timestamp is not None and version is not None:
raise FSError(
f"Supplied both version {version} and timestamp `{timestamp}` for manifest "
f"`{entry_id}`")
# Download the vlob
workspace_entry = self.get_workspace_entry()
rep = await self._backend_cmds(
"vlob_read",
workspace_entry.encryption_revision,
entry_id,
version=version,
timestamp=timestamp if version is None else None,
)
if rep["status"] == "not_found":
raise FSRemoteManifestNotFound(entry_id)
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoReadAccess(
"Cannot load manifest: no read access")
elif rep["status"] == "bad_version":
raise FSRemoteManifestNotFoundBadVersion(entry_id)
elif rep["status"] == "bad_timestamp":
raise FSRemoteManifestNotFoundBadTimestamp(entry_id)
elif rep["status"] == "bad_encryption_revision":
raise FSBadEncryptionRevision(
f"Cannot fetch vlob {entry_id}: Bad encryption revision provided"
)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot download vlob while the workspace is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot fetch vlob {entry_id}: `{rep['status']}`")
expected_version = rep["version"]
expected_author = rep["author"]
expected_timestamp = rep["timestamp"]
if version not in (None, expected_version):
raise FSError(
f"Backend returned invalid version for vlob {entry_id} (expecting {version}, "
f"got {expected_version})")
if expected_backend_timestamp and expected_backend_timestamp != expected_timestamp:
raise FSError(
f"Backend returned invalid expected timestamp for vlob {entry_id} at version "
f"{version} (expecting {expected_backend_timestamp}, got {expected_timestamp})"
)
author = await self.remote_device_manager.get_device(expected_author)
try:
remote_manifest = RemoteManifest.decrypt_verify_and_load(
rep["blob"],
key=workspace_entry.key,
author_verify_key=author.verify_key,
expected_author=expected_author,
expected_timestamp=expected_timestamp,
expected_version=expected_version,
expected_id=entry_id,
)
except DataError as exc:
raise FSError(f"Cannot decrypt vlob: {exc}") from exc
# Finally make sure author was allowed to create this manifest
role_at_timestamp = await self._get_user_realm_role_at(
expected_author.user_id, expected_timestamp)
if role_at_timestamp is None:
raise FSError(
f"Manifest was created at {expected_timestamp} by `{expected_author}` "
"which had no right to access the workspace at that time")
elif role_at_timestamp == RealmRole.READER:
raise FSError(
f"Manifest was created at {expected_timestamp} by `{expected_author}` "
"which had write right on the workspace at that time")
return remote_manifest
async def workspace_share(
self, workspace_id: EntryID, recipient: UserID, role: Optional[WorkspaceRole]
) -> None:
"""
Raises:
FSError
FSWorkspaceNotFoundError
FSBackendOfflineError
FSSharingNotAllowedError
"""
if self.device.user_id == recipient:
raise FSError("Cannot share to oneself")
user_manifest = self.get_user_manifest()
workspace_entry = user_manifest.get_workspace_entry(workspace_id)
if not workspace_entry:
raise FSWorkspaceNotFoundError(f"Unknown workspace `{workspace_id}`")
# Make sure the workspace is not a placeholder
await self._workspace_minimal_sync(workspace_entry)
# Retrieve the user
try:
recipient_user, revoked_recipient_user = await self.remote_devices_manager.get_user(
recipient
)
except RemoteDevicesManagerBackendOfflineError as exc:
raise FSBackendOfflineError(str(exc)) from exc
except RemoteDevicesManagerError as exc:
raise FSError(f"Cannot retreive recipient: {exc}") from exc
if revoked_recipient_user:
raise FSError(f"User {recipient} revoked")
# Note we don't bother to check workspace's access roles given they
# could be outdated (and backend will do the check anyway)
now = pendulum_now()
# Build the sharing message
try:
if role is not None:
recipient_message = SharingGrantedMessageContent(
author=self.device.device_id,
timestamp=now,
name=workspace_entry.name,
id=workspace_entry.id,
encryption_revision=workspace_entry.encryption_revision,
encrypted_on=workspace_entry.encrypted_on,
key=workspace_entry.key,
)
else:
recipient_message = SharingRevokedMessageContent(
author=self.device.device_id, timestamp=now, id=workspace_entry.id
)
ciphered_recipient_message = recipient_message.dump_sign_and_encrypt_for(
author_signkey=self.device.signing_key, recipient_pubkey=recipient_user.public_key
)
except DataError as exc:
raise FSError(f"Cannot create sharing message for `{recipient}`: {exc}") from exc
# Build role certificate
role_certificate = RealmRoleCertificateContent(
author=self.device.device_id,
timestamp=now,
realm_id=workspace_id,
user_id=recipient,
role=role,
).dump_and_sign(self.device.signing_key)
# Actually send the command to the backend
try:
rep = await self.backend_cmds.realm_update_roles(
role_certificate, ciphered_recipient_message
)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(f"Error while trying to set vlob group roles in backend: {exc}") from exc
if rep["status"] == "not_allowed":
raise FSSharingNotAllowedError(
f"Must be Owner or Manager on the workspace is mandatory to share it: {rep}"
)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot share workspace while it is in maintenance: {rep}"
)
elif rep["status"] == "already_granted":
# Stay idempotent
return
elif rep["status"] != "ok":
raise FSError(f"Error while trying to set vlob group roles in backend: {rep}")
async def _outbound_sync_inner(self) -> bool:
base_um = self.get_user_manifest()
if not base_um.need_sync:
return True
# Make sure the corresponding realm has been created in the backend
if base_um.is_placeholder:
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=self.device.device_id,
timestamp=pendulum_now(),
realm_id=self.device.user_manifest_id,
).dump_and_sign(self.device.signing_key)
try:
rep = await self.backend_cmds.realm_create(certif)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(f"Cannot create user manifest's realm in backend: {exc}") from exc
if rep["status"] == "already_exists":
# It's possible a previous attempt to create this realm
# succeeded but we didn't receive the confirmation, hence
# we play idempotent here.
pass
elif rep["status"] != "ok":
raise FSError(f"Cannot create user manifest's realm in backend: {rep}")
# Sync placeholders
for w in base_um.workspaces:
await self._workspace_minimal_sync(w)
# Build vlob
now = pendulum_now()
to_sync_um = base_um.to_remote(author=self.device.device_id, timestamp=now)
ciphered = to_sync_um.dump_sign_and_encrypt(
author_signkey=self.device.signing_key, key=self.device.user_manifest_key
)
# Sync the vlob with backend
try:
# Note encryption_revision is always 1 given we never reencrypt
# the user manifest's realm
if to_sync_um.version == 1:
rep = await self.backend_cmds.vlob_create(
self.user_manifest_id, 1, self.user_manifest_id, now, ciphered
)
else:
rep = await self.backend_cmds.vlob_update(
1, self.user_manifest_id, to_sync_um.version, now, ciphered
)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(f"Cannot sync user manifest: {exc}") from exc
if rep["status"] in ("already_exists", "bad_version"):
# Concurrency error (handled by the caller)
return False
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot modify workspace data while it is in maintenance: {rep}"
)
elif rep["status"] != "ok":
raise FSError(f"Cannot sync user manifest: {rep}")
# Merge back the manifest in local
async with self._update_user_manifest_lock:
diverged_um = self.get_user_manifest()
# Final merge could have been achieved by a concurrent operation
if to_sync_um.version > diverged_um.base_version:
merged_um = merge_local_user_manifests(diverged_um, to_sync_um)
await self.set_user_manifest(merged_um)
self.event_bus.send("fs.entry.synced", id=self.user_manifest_id)
return True
async def load_manifest(
self,
entry_id: EntryID,
version: Optional[int] = None,
timestamp: Optional[DateTime] = None,
expected_backend_timestamp: Optional[DateTime] = None,
workspace_entry: Optional[WorkspaceEntry] = None,
) -> BaseRemoteManifest:
"""
Download a manifest.
Only one from version or timestamp parameters can be specified at the same time.
expected_backend_timestamp enables to check a timestamp against the one returned by the
backend.
Raises:
FSError
FSBackendOfflineError
FSRemoteOperationError
FSWorkspaceInMaintenance
FSRemoteManifestNotFound
FSBadEncryptionRevision
FSWorkspaceNoAccess
FSUserNotFoundError
FSDeviceNotFoundError
FSInvalidTrustchainError
"""
assert (timestamp is None or version is None
), "Either timestamp or version argument should be provided"
# Get the current and requested workspace entry
# They're usually the same, except when loading from a workspace while it's in maintenance
current_workspace_entry = self.get_workspace_entry()
workspace_entry = current_workspace_entry if workspace_entry is None else workspace_entry
# Download the vlob
with translate_backend_cmds_errors():
rep = await self.backend_cmds.vlob_read(
workspace_entry.encryption_revision,
VlobID(entry_id.uuid),
version=version,
timestamp=timestamp if version is None else None,
)
# Special case for loading manifest while in maintenance.
# This is done to allow users to fetch data from a workspace while it's being reencrypted.
# If the workspace is in maintenance for another reason (such as garbage collection),
# the recursive call to load manifest will simply also fail with an FSWorkspaceInMaintenance.
if (rep["status"] == "in_maintenance"
and workspace_entry.encryption_revision
== current_workspace_entry.encryption_revision):
# Getting the last workspace entry with the previous encryption revision
# requires one or several calls to the backend, meaning the following exceptions might get raised:
# - FSError
# - FSBackendOfflineError
# - FSWorkspaceInMaintenance
# It is fine to let those exceptions bubble up as there all valid reasons for failing to load a manifest.
previous_workspace_entry = await self.get_previous_workspace_entry(
)
if previous_workspace_entry is not None:
# Make sure we don't fall into an infinite loop because of some other bug
assert (previous_workspace_entry.encryption_revision <
self.get_workspace_entry().encryption_revision)
# Recursive call to `load_manifest`, requiring an older encryption revision than the current one
return await self.load_manifest(
entry_id,
version=version,
timestamp=timestamp,
expected_backend_timestamp=expected_backend_timestamp,
workspace_entry=previous_workspace_entry,
)
if rep["status"] == "not_found":
raise FSRemoteManifestNotFound(entry_id)
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoReadAccess(
"Cannot load manifest: no read access")
elif rep["status"] == "bad_version":
raise FSRemoteManifestNotFoundBadVersion(entry_id)
elif rep["status"] == "bad_encryption_revision":
raise FSBadEncryptionRevision(
f"Cannot fetch vlob {entry_id}: Bad encryption revision provided"
)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot download vlob while the workspace is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot fetch vlob {entry_id}: `{rep['status']}`")
expected_version = rep["version"]
expected_author = rep["author"]
expected_timestamp = rep["timestamp"]
if version not in (None, expected_version):
raise FSError(
f"Backend returned invalid version for vlob {entry_id} (expecting {version}, "
f"got {expected_version})")
if expected_backend_timestamp and expected_backend_timestamp != expected_timestamp:
raise FSError(
f"Backend returned invalid expected timestamp for vlob {entry_id} at version "
f"{version} (expecting {expected_backend_timestamp}, got {expected_timestamp})"
)
with translate_remote_devices_manager_errors():
author = await self.remote_devices_manager.get_device(
expected_author)
try:
remote_manifest = BaseRemoteManifest.decrypt_verify_and_load(
rep["blob"],
key=workspace_entry.key,
author_verify_key=author.verify_key,
expected_author=expected_author,
expected_timestamp=expected_timestamp,
expected_version=expected_version,
expected_id=entry_id,
)
except DataError as exc:
raise FSError(f"Cannot decrypt vlob: {exc}") from exc
# Get the timestamp of the last role for this particular user
author_last_role_granted_on = rep["author_last_role_granted_on"]
# Compatibility with older backends (best effort strategy)
if author_last_role_granted_on is None:
author_last_role_granted_on = self.device.timestamp()
# Finally make sure author was allowed to create this manifest
role_at_timestamp = await self._get_user_realm_role_at(
expected_author.user_id, expected_timestamp,
author_last_role_granted_on)
if role_at_timestamp is None:
raise FSError(
f"Manifest was created at {expected_timestamp} by `{expected_author}` "
"which had no right to access the workspace at that time")
elif role_at_timestamp == RealmRole.READER:
raise FSError(
f"Manifest was created at {expected_timestamp} by `{expected_author}` "
"which had no right to write on the workspace at that time")
return remote_manifest
