def download( remote_history: RemoteNodeHistory, stored_history: Optional[StoredNodeHistory], session: Session, ) -> SyncActionResult: """ 1. Without local history - Find latest base - Download latest base - Store history in local DB 2. With local history - Diff remote and local history and find shortest path - Fetch deltas one by one patch - Store history in local DB """ history = cast(NodeHistory, remote_history.history) if stored_history is not None: entries, is_absolute = history.diff(stored_history.history) if is_absolute: local_path = file_transfer.download_to_root( session, history.path, entries[0].base_version) entries = entries[1:] else: local_path = session.root_folder.path / history.path if entries: patch_file(session, os.fspath(local_path), [e.key for e in entries]) local_node = LocalNode.create(local_path, session) stored_history.data = history.dict() # type: ignore stored_history.local_modified_time = local_node.created_time stored_history.local_created_time = local_node.modified_time stored_history.remote_history_etag = remote_history.etag else: entries, is_absolute = history.diff(None) local_path = file_transfer.download_to_root(session, history.path, entries[0].base_version) if entries[1:]: patch_file(session, os.fspath(local_path), [e.key for e in entries[1:]]) local_node = LocalNode.create(local_path, session) stored_history = StoredNodeHistory( key=remote_history.key, root_folder=RootFolder.for_session(session), data=history.dict(), local_modified_time=local_node.created_time, local_created_time=local_node.modified_time, remote_history_etag=remote_history.etag) last_entry = entries[-1] file_transfer.download_metadata( session, last_entry.key, "signature", os.fspath(session.signature_folder / last_entry.key)) stored_history.save() return SyncActionResult()
def create_full_version(session: Session, local_path: str): node = LocalNode.create(Path(local_path).resolve(), session) remote_history = RemoteNodeHistory(history=None, key=node.key, etag=None) remote_history.load(session) history = remote_history.history last = history.last prev = history.entries[-2] assert last.base_version is None assert last.has_delta assert prev.base_version is not None with create_temp_file() as base_path: s3_path = f"{session.s3_prefix}/{node.path}" download_file(session.s3_client, session.storage_bucket, s3_path, base_path, version=prev.base_version) patch_file(session, base_path, [last.key]) upload_file( session.s3_client, base_path, session.storage_bucket, s3_path, ) obj = get_file_metadata(session.s3_client, session.storage_bucket, s3_path) last.base_version = obj["VersionId"] last.base_size = int(obj.get("Size", 0)) remote_history.save(session)
def upload_again(session: Session, path: str): path = Path(path).resolve() with open(path, "ab") as f: f.write(b"1") node = LocalNode.create(path, session) remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) upload(remote_history, node)(session) show_history(session, node.key)
def download_new_version(session: Session, path: str): path = Path(path).resolve() node = LocalNode.create(path, session) remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) stored_history = models.StoredNodeHistory\ .get(models.StoredNodeHistory.key == node.key) download(remote_history, stored_history)(session) show_history(session, node.key)
def clear_remote(session: Session): # upload path = create_file(session.root_folder.path, MB(1)) node = LocalNode.create(path, session) upload(None, node)(session) remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) stored_history = models.StoredNodeHistory\ .get(models.StoredNodeHistory.key == node.key) delete_remote(remote_history, stored_history)(session) node.local_path.unlink() show_history(session, node.key)
def download_first_time(session: Session): # upload path = create_file(session.root_folder.path, MB(1)) node = LocalNode.create(path, session) upload(None, node)(session) # clear local delete_local( node, models.StoredNodeHistory.get(models.StoredNodeHistory.key == node.key) )(session) # download remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) download(remote_history, None)(session) show_history(session, node.key)
def scan_local_files(session: Session) -> Iterable[LocalNode]: return (LocalNode.create(p, session) for p in iter_folder(session.root_folder.path))
def upload_new(session: Session, filename=None): path = create_file(session.root_folder.path, MB(1), filename=filename) node = LocalNode.create(path, session) action = upload(None, node) action(session) show_history(session, node.key)