async def _transactions_factory(device, backend_cmds, local_storage, cls=SyncTransactions): def _get_workspace_entry(): return workspace_entry workspace_entry = WorkspaceEntry.new("test") workspace_manifest = LocalWorkspaceManifest.new_placeholder( id=workspace_entry.id, now=Pendulum(2000, 1, 1) ) async with local_storage.lock_entry_id(workspace_entry.id): await local_storage.set_manifest(workspace_entry.id, workspace_manifest) remote_devices_manager = remote_devices_manager_factory(device) remote_loader = RemoteLoader( device, workspace_entry.id, _get_workspace_entry, backend_cmds, remote_devices_manager, local_storage, ) return cls( workspace_entry.id, _get_workspace_entry, device, local_storage, remote_loader, event_bus, )
async def test_create_workspace(initial_user_manifest_state, alice_user_fs, alice): with freeze_time("2000-01-02"): wid = await alice_user_fs.workspace_create("w1") um = alice_user_fs.get_user_manifest() expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend( alice) expected_um = LocalUserManifest( base=expected_base_um, need_sync=True, updated=datetime(2000, 1, 2), last_processed_message=expected_base_um.last_processed_message, workspaces=(WorkspaceEntry( name="w1", id=wid, key=ANY, encryption_revision=1, encrypted_on=datetime(2000, 1, 2), role_cached_on=datetime(2000, 1, 2), role=WorkspaceRole.OWNER, ), ), ) assert um == expected_um w_manifest = await alice_user_fs.get_workspace( wid).local_storage.get_manifest(wid) expected_w_manifest = LocalWorkspaceManifest.new_placeholder( alice.device_id, id=w_manifest.id, now=datetime(2000, 1, 2)) assert w_manifest == expected_w_manifest
async def _load_workspace_manifest(self) -> None: try: await self.manifest_storage.get_manifest(self.workspace_id) except FSLocalMissError: # It is possible to lack the workspace manifest in local if our # device hasn't tried to access it yet (and we are not the creator # of the workspace, in which case the workspacefs local db is # initialized with a non-speculative local manifest placeholder). # In such case it is easy to fall back on an empty manifest # which is a good enough aproximation of the very first version # of the manifest (field `created` is invalid, but it will be # correction by the merge during sync). # This approach also guarantees the workspace root folder is always # consistent (ls/touch/mkdir always works on it), which is not the # case for the others files and folders (as their access may # require communication with the backend). # This is especially important when the workspace is accessed from # file system mountpoint given having a weird error popup when clicking # on the mountpoint from the file explorer really feel like a bug :/ timestamp = self.device.timestamp() manifest = LocalWorkspaceManifest.new_placeholder( author=self.device.device_id, id=self.workspace_id, timestamp=timestamp, speculative=True, ) await self.manifest_storage.set_manifest(self.workspace_id, manifest)
def _recursive_process_copy_map(copy_map): manifest = copy_map["manifest"] cpy_access = ManifestAccess() if is_file_manifest(manifest): cpy_manifest = LocalFileManifest( author=self.local_author, size=manifest.size, blocks=manifest.blocks, dirty_blocks=manifest.dirty_blocks, ) else: cpy_children = {} for child_name in manifest.children.keys(): child_copy_map = copy_map["children"][child_name] new_child_access = _recursive_process_copy_map( child_copy_map) cpy_children[child_name] = new_child_access if is_folder_manifest(manifest): cpy_manifest = LocalFolderManifest( author=self.local_author, children=cpy_children) else: assert is_workspace_manifest(manifest) cpy_manifest = LocalWorkspaceManifest( self.local_author, children=cpy_children) self.set_manifest(cpy_access, cpy_manifest) return cpy_access
async def workspace_create(self, name: AnyEntryName) -> EntryID: """ Raises: Nothing ! """ name = EntryName(name) workspace_entry = WorkspaceEntry.new(name) workspace_manifest = LocalWorkspaceManifest.new_placeholder(id=workspace_entry.id) async with self._update_user_manifest_lock: user_manifest = self.get_user_manifest() user_manifest = user_manifest.evolve_workspaces_and_mark_updated(workspace_entry) await self._create_workspace(workspace_entry.id, workspace_manifest) await self.set_user_manifest(user_manifest) self.event_bus.send("fs.entry.updated", id=self.user_manifest_id) self.event_bus.send("fs.workspace.created", new_entry=workspace_entry) return workspace_entry.id
async def workspace_create(self, name: AnyEntryName) -> EntryID: """ Raises: Nothing ! """ name = EntryName(name) workspace_entry = WorkspaceEntry.new(name) workspace_manifest = LocalWorkspaceManifest.new_placeholder( self.device.device_id, id=workspace_entry.id ) async with self._update_user_manifest_lock: user_manifest = self.get_user_manifest() user_manifest = user_manifest.evolve_workspaces_and_mark_updated(workspace_entry) await self._create_workspace(workspace_entry.id, workspace_manifest) await self.set_user_manifest(user_manifest) self.event_bus.send(CoreEvent.FS_ENTRY_UPDATED, id=self.user_manifest_id) self.event_bus.send(CoreEvent.FS_WORKSPACE_CREATED, new_entry=workspace_entry) return workspace_entry.id
async def _transactions_factory(device, local_storage, cls=SyncTransactions): def _get_workspace_entry(): return workspace_entry async def _get_previous_workspace_entry(): # The tests shouldn't need this yet assert False workspace_entry = WorkspaceEntry.new(EntryName("test"), device.timestamp()) workspace_manifest = LocalWorkspaceManifest.new_placeholder( device.device_id, id=workspace_entry.id, timestamp=datetime(2000, 1, 1)) async with local_storage.lock_entry_id(workspace_entry.id): await local_storage.set_manifest(workspace_entry.id, workspace_manifest) async with backend_authenticated_cmds_factory( device.organization_addr, device.device_id, device.signing_key) as cmds: remote_devices_manager = remote_devices_manager_factory(device) remote_loader = RemoteLoader( device, workspace_entry.id, _get_workspace_entry, _get_previous_workspace_entry, cmds, remote_devices_manager, local_storage, ) yield cls( workspace_entry.id, _get_workspace_entry, device, local_storage, remote_loader, event_bus, core_config, )
async def workspace_storage_non_speculative_init( data_base_dir: Path, device: LocalDevice, workspace_id: EntryID) -> None: db_path = get_workspace_data_storage_db_path(data_base_dir, device, workspace_id) # Local data storage service async with LocalDatabase.run(db_path) as data_localdb: # Manifest storage service async with ManifestStorage.run(device, data_localdb, workspace_id) as manifest_storage: timestamp = device.timestamp() manifest = LocalWorkspaceManifest.new_placeholder( author=device.device_id, id=workspace_id, timestamp=timestamp, speculative=False) await manifest_storage.set_manifest(workspace_id, manifest)
def workspace_create(self, path: FsPath) -> None: if not path.parent.is_root(): raise PermissionError( 13, "Permission denied (workspace only allowed at root level)", str(path)) root_manifest = self.get_user_manifest() if path.name in root_manifest.children: raise FileExistsError(17, "File exists", str(path)) child_access = ManifestAccess() child_manifest = LocalWorkspaceManifest(self.local_author) root_manifest = root_manifest.evolve_children_and_mark_updated( {path.name: child_access}) self.set_manifest(self.root_access, root_manifest) self.set_manifest(child_access, child_manifest) self.event_bus.send("fs.entry.updated", id=self.root_access.id) self.event_bus.send("fs.entry.updated", id=child_access.id) self.event_bus.send("fs.workspace.loaded", path=str(path), id=child_access.id)
async def _transactions_factory(device, backend_cmds, local_storage, cls=SyncTransactions): def _get_workspace_entry(): return workspace_entry async def _get_previous_workspace_entry(): # The tests shouldn't need this yet assert False workspace_entry = WorkspaceEntry.new("test") workspace_manifest = LocalWorkspaceManifest.new_placeholder( device.device_id, id=workspace_entry.id, now=datetime(2000, 1, 1)) async with local_storage.lock_entry_id(workspace_entry.id): await local_storage.set_manifest(workspace_entry.id, workspace_manifest) remote_devices_manager = remote_devices_manager_factory(device) remote_loader = RemoteLoader( device, workspace_entry.id, _get_workspace_entry, _get_previous_workspace_entry, backend_cmds, remote_devices_manager, local_storage, ) return cls( workspace_entry.id, _get_workspace_entry, device, local_storage, remote_loader, event_bus, )
def test_merge_speculative_with_it_unsuspected_former_self( local_changes, core_config): d1 = datetime(2000, 1, 1) d2 = datetime(2000, 1, 2) d3 = datetime(2000, 1, 3) d4 = datetime(2000, 1, 4) d5 = datetime(2000, 1, 5) my_device = DeviceID("a@a") # 1) Workspace manifest is originally created by our device local = LocalWorkspaceManifest.new_placeholder(author=my_device, timestamp=d1) foo_id = EntryID.new() local = local.evolve(updated=d2, children=FrozenDict({EntryName("foo"): foo_id})) # 2) We sync the workspace manifest v1 = local.to_remote(author=my_device, timestamp=d3) # 3) Now let's pretend we lost local storage, hence creating a new speculative manifest new_local = LocalWorkspaceManifest.new_placeholder(author=my_device, id=local.id, timestamp=d3, speculative=True) if local_changes: bar_id = EntryID.new() new_local = new_local.evolve(updated=d4, children=FrozenDict( {EntryName("bar"): bar_id})) # 4) When syncing the manifest, we shouldn't remove any data from the remote merged = merge_manifests( local_author=my_device, timestamp=d5, prevent_sync_pattern=empty_pattern, local_manifest=new_local, remote_manifest=v1, ) if local_changes: assert merged == LocalWorkspaceManifest( base=v1, need_sync=True, updated=d5, children=FrozenDict({ **v1.children, **new_local.children }), local_confinement_points=frozenset(), remote_confinement_points=frozenset(), speculative=False, ) else: assert merged == LocalWorkspaceManifest( base=v1, need_sync=False, updated=v1.updated, children=v1.children, local_confinement_points=frozenset(), remote_confinement_points=frozenset(), speculative=False, )