def download( remote_history: RemoteNodeHistory, stored_history: Optional[StoredNodeHistory], session: Session, ) -> SyncActionResult: """ 1. Without local history - Find latest base - Download latest base - Store history in local DB 2. With local history - Diff remote and local history and find shortest path - Fetch deltas one by one patch - Store history in local DB """ history = cast(NodeHistory, remote_history.history) if stored_history is not None: entries, is_absolute = history.diff(stored_history.history) if is_absolute: local_path = file_transfer.download_to_root( session, history.path, entries[0].base_version) entries = entries[1:] else: local_path = session.root_folder.path / history.path if entries: patch_file(session, os.fspath(local_path), [e.key for e in entries]) local_node = LocalNode.create(local_path, session) stored_history.data = history.dict() # type: ignore stored_history.local_modified_time = local_node.created_time stored_history.local_created_time = local_node.modified_time stored_history.remote_history_etag = remote_history.etag else: entries, is_absolute = history.diff(None) local_path = file_transfer.download_to_root(session, history.path, entries[0].base_version) if entries[1:]: patch_file(session, os.fspath(local_path), [e.key for e in entries[1:]]) local_node = LocalNode.create(local_path, session) stored_history = StoredNodeHistory( key=remote_history.key, root_folder=RootFolder.for_session(session), data=history.dict(), local_modified_time=local_node.created_time, local_created_time=local_node.modified_time, remote_history_etag=remote_history.etag) last_entry = entries[-1] file_transfer.download_metadata( session, last_entry.key, "signature", os.fspath(session.signature_folder / last_entry.key)) stored_history.save() return SyncActionResult()
def create_full_version(session: Session, local_path: str): node = LocalNode.create(Path(local_path).resolve(), session) remote_history = RemoteNodeHistory(history=None, key=node.key, etag=None) remote_history.load(session) history = remote_history.history last = history.last prev = history.entries[-2] assert last.base_version is None assert last.has_delta assert prev.base_version is not None with create_temp_file() as base_path: s3_path = f"{session.s3_prefix}/{node.path}" download_file(session.s3_client, session.storage_bucket, s3_path, base_path, version=prev.base_version) patch_file(session, base_path, [last.key]) upload_file( session.s3_client, base_path, session.storage_bucket, s3_path, ) obj = get_file_metadata(session.s3_client, session.storage_bucket, s3_path) last.base_version = obj["VersionId"] last.base_size = int(obj.get("Size", 0)) remote_history.save(session)
def upload_again(session: Session, path: str): path = Path(path).resolve() with open(path, "ab") as f: f.write(b"1") node = LocalNode.create(path, session) remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) upload(remote_history, node)(session) show_history(session, node.key)
def download_new_version(session: Session, path: str): path = Path(path).resolve() node = LocalNode.create(path, session) remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) stored_history = models.StoredNodeHistory\ .get(models.StoredNodeHistory.key == node.key) download(remote_history, stored_history)(session) show_history(session, node.key)
def local(self, **extra_attrs): attrs = { "root_folder": Path(self.root_folder.path), "path": self.path, "modified_time": self.modified_time, "created_time": self.created_time, "size": self.size, "etag": self.etag, **extra_attrs, } return LocalNode(**attrs)
def clear_remote(session: Session): # upload path = create_file(session.root_folder.path, MB(1)) node = LocalNode.create(path, session) upload(None, node)(session) remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) stored_history = models.StoredNodeHistory\ .get(models.StoredNodeHistory.key == node.key) delete_remote(remote_history, stored_history)(session) node.local_path.unlink() show_history(session, node.key)
def download_first_time(session: Session): # upload path = create_file(session.root_folder.path, MB(1)) node = LocalNode.create(path, session) upload(None, node)(session) # clear local delete_local( node, models.StoredNodeHistory.get(models.StoredNodeHistory.key == node.key) )(session) # download remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None) remote_history.load(session) download(remote_history, None)(session) show_history(session, node.key)
def handle_node(remote: RemoteNodeHistory, local: LocalNode, stored: StoredNodeHistory) -> SyncAction: if not remote and not local and not stored: return nop() elif not remote and not local and stored: return delete_history(stored) elif not remote and local and not stored: return upload(None, local) elif not remote and local and stored: return delete_local(local, stored) elif remote.exists and not local and not stored: return download(remote, None) elif remote.exists and not local and stored: return delete_remote(remote, stored) elif remote.deleted and not local and stored: return delete_history(stored) elif remote and local and not stored: if remote.deleted: return delete_local(local, stored) elif cast(NodeHistory, remote.history).etag == local.etag: return save_history(remote, local) else: return conflict(remote, local, stored) elif remote and local and stored: local_updated = local.updated(stored) remote_updated = remote.updated(stored) if remote.deleted: if local_updated: return conflict(remote, local, stored) else: return delete_local(local, stored) elif local_updated and remote_updated: if cast(NodeHistory, remote.history).etag == local.etag: return nop() else: return conflict(remote, local, stored) return nop() elif local_updated: return upload(remote, local) elif remote_updated: return download(remote, stored) else: return nop() return nop()
def scan_local_files(session: Session) -> Iterable[LocalNode]: return (LocalNode.create(p, session) for p in iter_folder(session.root_folder.path))
def upload(remote_history: Optional[RemoteNodeHistory], node: LocalNode, session: Session) -> SyncActionResult: """ 1. Without remote history: - Calc signature - Generate id - Create new history - Upload base - Upload history - Store history in local DB 2. With remote history: - Generate key - Calc delta - Calc signature - Upload delta - Upload signature - Add history record - Upload history - Store history in local DB """ new_key = NodeHistoryEntry.generate_key() if remote_history is not None: history = cast(NodeHistory, remote_history.history) with create_temp_file() as delta_path: calc_delta(session, node.local_fspath, history.last.key, delta_path) file_transfer.upload_metadata(session, delta_path, new_key, "delta") delta_size = Path(delta_path).stat().st_size with create_temp_file() as signature_path: calc_signature(session, node.local_fspath, new_key, signature_path) file_transfer.upload_metadata(session, signature_path, new_key, "signature") history.add_entry( NodeHistoryEntry.create_delta_only(new_key, node.calc_etag(), delta_size)) else: with create_temp_file() as signature_path: calc_signature(session, node.local_fspath, new_key, signature_path) file_transfer.upload_metadata(session, signature_path, new_key, "signature") version = file_transfer.upload_to_root(session, node) history = NodeHistory(key=node.key, path=node.path, entries=[]) history.add_entry( NodeHistoryEntry.create_base_only(new_key, node.calc_etag(), version, node.size)) remote_history = RemoteNodeHistory(history=history, key=node.key, etag=None) remote_history.save(session) stored_history = StoredNodeHistory.get_or_none( StoredNodeHistory.key == history.key) if stored_history is not None: stored_history.data = history.dict() stored_history.remote_history_etag = remote_history.etag stored_history.local_modified_time = node.created_time stored_history.local_created_time = node.modified_time stored_history.save() else: StoredNodeHistory.create(key=remote_history.key, root_folder=RootFolder.for_session(session), data=cast(NodeHistory, remote_history.history).dict(), local_modified_time=node.created_time, local_created_time=node.modified_time, remote_history_etag=remote_history.etag) return SyncActionResult()
def upload_new(session: Session, filename=None): path = create_file(session.root_folder.path, MB(1), filename=filename) node = LocalNode.create(path, session) action = upload(None, node) action(session) show_history(session, node.key)