async def run(
cls: Type[UserFSTypeVar],
device: LocalDevice,
path: Path,
backend_cmds: BackendAuthenticatedCmds,
remote_devices_manager: RemoteDevicesManager,
event_bus: EventBus,
prevent_sync_pattern: Pattern[str],
) -> AsyncIterator[UserFSTypeVar]:
self = cls(device, path, backend_cmds, remote_devices_manager,
event_bus, prevent_sync_pattern)
# Run user storage
async with UserStorage.run(self.device, self.path) as self.storage:
# Nursery for workspace storages
async with open_service_nursery(
) as self._workspace_storage_nursery:
# Make sure all the workspaces are loaded
# In particular, we want to make sure that any workspace available through
# `userfs.get_user_manifest().workspaces` is also available through
# `userfs.get_workspace(workspace_id)`.
for workspace_entry in self.get_user_manifest().workspaces:
await self._load_workspace(workspace_entry.id)
yield self
# Stop the workspace storages
self._workspace_storage_nursery.cancel_scope.cancel()
async def read(self, organization_id: OrganizationID,
id: BlockID) -> bytes:
value = None
async def _single_blockstore_read(
nursery, blockstore: BaseBlockStoreComponent) -> None:
nonlocal value
try:
value = await blockstore.read(organization_id, id)
nursery.cancel_scope.cancel()
except BlockStoreError:
pass
async with open_service_nursery() as nursery:
for blockstore in self.blockstores:
nursery.start_soon(_single_blockstore_read, nursery,
blockstore)
if not value:
self._logger.warning(
"Block read error: All nodes have failed",
organization_id=organization_id,
block_id=id,
)
raise BlockStoreError("All RAID1 nodes have failed")
return value
async def populate_paths(self, manifest_cache, entry_id: EntryID,
early: DateTime, late: DateTime):
# TODO : Use future manifest source field to follow files and directories
async with open_service_nursery() as child_nursery:
child_nursery.start_soon(self.populate_source_path, manifest_cache,
entry_id, early.add(microseconds=-1))
child_nursery.start_soon(self.populate_destination_path,
manifest_cache, entry_id, late)
child_nursery.start_soon(self.populate_current_path,
manifest_cache, entry_id, early)
async def components_factory(
config: BackendConfig, event_bus: EventBus
) -> AsyncGenerator[dict, None]:
dbh = PGHandler(config.db_url, config.db_min_connections, config.db_max_connections, event_bus)
async def _send_event(
event: BackendEvent, conn: Optional[triopg._triopg.TrioConnectionProxy] = None, **kwargs
) -> None:
if conn is None:
async with dbh.pool.acquire() as conn:
await send_signal(conn, event, **kwargs)
else:
await send_signal(conn, event, **kwargs)
webhooks = WebhooksComponent(config)
http = HTTPComponent(config)
organization = PGOrganizationComponent(dbh, webhooks, config)
user = PGUserComponent(dbh, event_bus)
invite = PGInviteComponent(dbh, event_bus, config)
message = PGMessageComponent(dbh)
realm = PGRealmComponent(dbh)
vlob = PGVlobComponent(dbh)
ping = PGPingComponent(dbh)
blockstore = blockstore_factory(config.blockstore_config, postgresql_dbh=dbh)
block = PGBlockComponent(dbh, blockstore, vlob)
pki = PGPkiEnrollmentComponent(dbh)
events = EventsComponent(realm, send_event=_send_event)
components = {
"events": events,
"webhooks": webhooks,
"http": http,
"organization": organization,
"user": user,
"invite": invite,
"message": message,
"realm": realm,
"vlob": vlob,
"ping": ping,
"block": block,
"blockstore": blockstore,
"pki": pki,
}
for component in components.values():
method = getattr(component, "register_components", None)
if method is not None:
method(**components)
async with open_service_nursery() as nursery:
await dbh.init(nursery)
try:
yield components
finally:
await dbh.teardown()
async def components_factory(config: BackendConfig, event_bus: EventBus):
send_events_channel, receive_events_channel = trio.open_memory_channel[
Tuple[Enum, Dict[str, object]]](math.inf)
async def _send_event(event: Enum, **kwargs):
await send_events_channel.send((event, kwargs))
async def _dispatch_event():
async for event, kwargs in receive_events_channel:
await trio.sleep(0)
event_bus.send(event, **kwargs)
webhooks = WebhooksComponent(config)
http = HTTPComponent(config)
organization = MemoryOrganizationComponent(_send_event, webhooks, config)
user = MemoryUserComponent(_send_event, event_bus)
invite = MemoryInviteComponent(_send_event, event_bus, config)
message = MemoryMessageComponent(_send_event)
realm = MemoryRealmComponent(_send_event)
vlob = MemoryVlobComponent(_send_event)
ping = MemoryPingComponent(_send_event)
pki = MemoryPkiEnrollmentComponent(_send_event)
block = MemoryBlockComponent()
blockstore = blockstore_factory(config.blockstore_config)
events = EventsComponent(realm, send_event=_send_event)
components = {
"events": events,
"webhooks": webhooks,
"http": http,
"organization": organization,
"user": user,
"invite": invite,
"message": message,
"realm": realm,
"vlob": vlob,
"ping": ping,
"pki": pki,
"block": block,
"blockstore": blockstore,
}
for component in components.values():
method = getattr(component, "register_components", None)
if method is not None:
method(**components)
async with open_service_nursery() as nursery:
nursery.start_soon(_dispatch_event)
try:
yield components
finally:
nursery.cancel_scope.cancel()
async def create(self, organization_id: OrganizationID, id: UUID, block: bytes) -> None:
async def _single_blockstore_create(blockstore):
try:
await blockstore.create(organization_id, id, block)
except BlockAlreadyExistsError:
# It's possible a previous tentative to upload this block has
# failed due to another blockstore not available. In such case
# a retrial will raise AlreadyExistsError on all the blockstores
# that sucessfully uploaded the block during last attempt.
# Only solution to solve this is to ignore AlreadyExistsError.
pass
async with open_service_nursery() as nursery:
for blockstore in self.blockstores:
nursery.start_soon(_single_blockstore_create, blockstore)
async def upload_blocks(self, blocks: List[BlockAccess]) -> None:
blocks_iter = iter(blocks)
async def _uploader() -> None:
while True:
access = next(blocks_iter, None)
if not access:
break
try:
data = await self.local_storage.get_dirty_block(access.id)
except FSLocalMissError:
continue
await self.upload_block(access, data)
async with open_service_nursery() as nursery:
for _ in range(4):
nursery.start_soon(_uploader)
async def create(self, organization_id: OrganizationID, id: UUID,
block: bytes) -> None:
nb_chunks = len(self.blockstores) - 1
chunks = split_block_in_chunks(block, nb_chunks)
assert len(chunks) == nb_chunks
checksum_chunk = generate_checksum_chunk(chunks)
# Actually do the upload
error_count = 0
async def _subblockstore_create(nursery, blockstore_index,
chunk_or_checksum):
nonlocal error_count
try:
await self.blockstores[blockstore_index].create(
organization_id, id, chunk_or_checksum)
except BlockAlreadyExistsError:
# It's possible a previous tentative to upload this block has
# failed due to another blockstore not available. In such case
# a retrial will raise AlreadyExistsError on all the blockstores
# that sucessfully uploaded the block during last attempt.
# Only solution to solve this is to ignore AlreadyExistsError.
pass
except BlockTimeoutError as exc:
error_count += 1
logger.warning(
f"Cannot reach RAID5 blockstore #{blockstore_index} to create block {id}",
exc_info=exc,
)
if error_count > 1:
# Early exit
nursery.cancel_scope.cancel()
async with open_service_nursery() as nursery:
for i, chunk_or_checksum in enumerate([*chunks, checksum_chunk]):
nursery.start_soon(_subblockstore_create, nursery, i,
chunk_or_checksum)
if error_count > 1:
# Only a single blockstore is allowed to fail
logger.error(
f"Block {id} cannot be created: Too many failing blockstores in the RAID5 cluster"
)
raise BlockTimeoutError(
"More than 1 blockstores has failed in the RAID5 cluster")
async def read(self, organization_id: OrganizationID, id: UUID) -> bytes:
async def _single_blockstore_read(nursery, blockstore):
nonlocal value
try:
value = await blockstore.read(organization_id, id)
nursery.cancel_scope.cancel()
except (BlockNotFoundError, BlockTimeoutError):
pass
value = None
async with open_service_nursery() as nursery:
for blockstore in self.blockstores:
nursery.start_soon(_single_blockstore_read, nursery, blockstore)
if not value:
raise BlockNotFoundError()
return value
async def run(
cls: Type[UserFSTypeVar],
data_base_dir: Path,
device: LocalDevice,
backend_cmds: BackendAuthenticatedCmds,
remote_devices_manager: RemoteDevicesManager,
event_bus: EventBus,
prevent_sync_pattern: Pattern[str],
preferred_language: str = "en",
workspace_storage_cache_size:
int = DEFAULT_WORKSPACE_STORAGE_CACHE_SIZE,
) -> AsyncIterator[UserFSTypeVar]:
self = cls(
data_base_dir,
device,
backend_cmds,
remote_devices_manager,
event_bus,
prevent_sync_pattern,
preferred_language,
workspace_storage_cache_size,
)
# Run user storage
async with UserStorage.run(self.data_base_dir,
self.device) as self.storage:
# Nursery for workspace storages
async with open_service_nursery(
) as self._workspace_storage_nursery:
# Make sure all the workspaces are loaded
# In particular, we want to make sure that any workspace available through
# `userfs.get_user_manifest().workspaces` is also available through
# `userfs.get_workspace(workspace_id)`.
for workspace_entry in self.get_user_manifest().workspaces:
await self._load_workspace(workspace_entry.id)
yield self
# Stop the workspace storages
self._workspace_storage_nursery.cancel_scope.cancel()
async def create(self, organization_id: OrganizationID, id: BlockID,
block: bytes) -> None:
at_least_one_success = False
at_least_one_error = False
async def _single_blockstore_create(
cancel_scope, blockstore: BaseBlockStoreComponent) -> None:
nonlocal at_least_one_success
nonlocal at_least_one_error
try:
await blockstore.create(organization_id, id, block)
at_least_one_success = True
except BlockStoreError:
at_least_one_error = True
if not self._partial_create_ok:
# Early exit given the create cannot succeed
cancel_scope.cancel()
async with open_service_nursery() as nursery:
for blockstore in self.blockstores:
nursery.start_soon(_single_blockstore_create,
nursery.cancel_scope, blockstore)
if self._partial_create_ok:
if not at_least_one_success:
self._logger.warning(
"Block create error: All nodes have failed",
organization_id=str(organization_id),
block_id=str(id),
)
raise BlockStoreError("All RAID1 nodes have failed")
else:
if at_least_one_error:
self._logger.warning(
"Block create error: A node have failed",
organization_id=str(organization_id),
block_id=str(id),
)
raise BlockStoreError("A RAID1 node have failed")
async def mountpoint_manager_factory(
user_fs,
event_bus,
base_mountpoint_path,
*,
debug: bool = False,
mount_all: bool = False,
mount_on_workspace_created: bool = False,
mount_on_workspace_shared: bool = False,
unmount_on_workspace_revoked: bool = False,
exclude_from_mount_all: list = (),
):
config = {"debug": debug}
runner = get_mountpoint_runner()
# Now is a good time to perform some cleanup in the registry
if sys.platform == "win32":
cleanup_parsec_drive_icons()
elif sys.platform == "darwin":
await cleanup_macos_mountpoint_folder(base_mountpoint_path)
def on_event(event, new_entry, previous_entry=None):
# Workspace created
if event == CoreEvent.FS_WORKSPACE_CREATED:
if mount_on_workspace_created:
mount_nursery.start_soon(mountpoint_manager.safe_mount,
new_entry.id)
return
# Workspace revoked
if event == CoreEvent.SHARING_UPDATED and new_entry.role is None:
if unmount_on_workspace_revoked:
mount_nursery.start_soon(mountpoint_manager.safe_unmount,
new_entry.id)
return
# Workspace shared
if event == CoreEvent.SHARING_UPDATED and previous_entry is None:
if mount_on_workspace_shared:
mount_nursery.start_soon(mountpoint_manager.safe_mount,
new_entry.id)
return
# Instantiate the mountpoint manager with its own nursery
async with open_service_nursery() as nursery:
mountpoint_manager = MountpointManager(user_fs, event_bus,
base_mountpoint_path, config,
runner, nursery)
# Exit this context by unmounting all mountpoints
try:
# A nursery dedicated to new workspace events
async with open_service_nursery() as mount_nursery:
# Setup new workspace events
with event_bus.connect_in_context(
(CoreEvent.FS_WORKSPACE_CREATED, on_event),
(CoreEvent.SHARING_UPDATED, on_event),
):
# Mount required workspaces
if mount_all:
await mountpoint_manager.safe_mount_all(
exclude=exclude_from_mount_all)
# Yield point
yield mountpoint_manager
# Cancel current mount_workspace tasks
mount_nursery.cancel_scope.cancel()
# Unmount all the workspaces (should this be shielded?)
finally:
await mountpoint_manager.safe_unmount_all()
# Cancel the mountpoint tasks (although they should all be finised by now)
nursery.cancel_scope.cancel()
async def load_blocks(self, accesses: List[BlockAccess]) -> None:
async with open_service_nursery() as nursery:
async with await self.receive_load_blocks(
accesses, nursery) as receive_channel:
async for value in receive_channel:
pass
async def read(self, organization_id: OrganizationID, id: UUID) -> bytes:
timeout_count = 0
fetch_results: List[Optional[bytes]] = [None] * len(self.blockstores)
async def _partial_blockstore_read(nursery, blockstore_index):
nonlocal timeout_count
nonlocal fetch_results
try:
fetch_results[blockstore_index] = await self.blockstores[
blockstore_index].read(organization_id, id)
except BlockNotFoundError as exc:
# We don't know yet if this id doesn't exists globally or only in this blockstore...
fetch_results[blockstore_index] = exc
except BlockTimeoutError as exc:
fetch_results[blockstore_index] = exc
timeout_count += 1
logger.warning(
f"Cannot reach RAID5 blockstore #{blockstore_index} to read block {id}",
exc_info=exc,
)
if timeout_count > 1:
nursery.cancel_scope.cancel()
else:
# Try to fetch the checksum to rebuild the current missing chunk...
nursery.start_soon(_partial_blockstore_read, nursery,
len(self.blockstores) - 1)
async with open_service_nursery() as nursery:
# Don't fetch the checksum by default
for blockstore_index in range(len(self.blockstores) - 1):
nursery.start_soon(_partial_blockstore_read, nursery,
blockstore_index)
if timeout_count == 0:
# Sanity check: no errors and we didn't fetch the checksum
assert len([res for res in fetch_results if res is None]) == 1
assert fetch_results[-1] is None
assert not len(
[res for res in fetch_results if isinstance(res, Exception)])
return rebuild_block_from_chunks(fetch_results[:-1], None)
elif timeout_count == 1:
checksum = fetch_results[-1]
# Sanity check: one error and we have fetched the checksum
assert len([res for res in fetch_results if res is None]) == 0
assert isinstance(checksum, (bytes, bytearray))
assert len([
res for res in fetch_results if isinstance(res, Exception)
]) == 1
return rebuild_block_from_chunks(
[
res if isinstance(res, (bytes, bytearray)) else None
for res in fetch_results[:-1]
],
checksum,
)
else:
logger.error(
f"Block {id} cannot be read: Too many failing blockstores in the RAID5 cluster"
)
raise BlockTimeoutError(
"More than 1 blockstores has failed in the RAID5 cluster")