def download(
remote_history: RemoteNodeHistory,
stored_history: Optional[StoredNodeHistory],
session: Session,
) -> SyncActionResult:
"""
1. Without local history
- Find latest base
- Download latest base
- Store history in local DB
2. With local history
- Diff remote and local history and find shortest path
- Fetch deltas one by one patch
- Store history in local DB
"""
history = cast(NodeHistory, remote_history.history)
if stored_history is not None:
entries, is_absolute = history.diff(stored_history.history)
if is_absolute:
local_path = file_transfer.download_to_root(
session, history.path, entries[0].base_version)
entries = entries[1:]
else:
local_path = session.root_folder.path / history.path
if entries:
patch_file(session, os.fspath(local_path),
[e.key for e in entries])
local_node = LocalNode.create(local_path, session)
stored_history.data = history.dict() # type: ignore
stored_history.local_modified_time = local_node.created_time
stored_history.local_created_time = local_node.modified_time
stored_history.remote_history_etag = remote_history.etag
else:
entries, is_absolute = history.diff(None)
local_path = file_transfer.download_to_root(session, history.path,
entries[0].base_version)
if entries[1:]:
patch_file(session, os.fspath(local_path),
[e.key for e in entries[1:]])
local_node = LocalNode.create(local_path, session)
stored_history = StoredNodeHistory(
key=remote_history.key,
root_folder=RootFolder.for_session(session),
data=history.dict(),
local_modified_time=local_node.created_time,
local_created_time=local_node.modified_time,
remote_history_etag=remote_history.etag)
last_entry = entries[-1]
file_transfer.download_metadata(
session, last_entry.key, "signature",
os.fspath(session.signature_folder / last_entry.key))
stored_history.save()
return SyncActionResult()
def create_full_version(session: Session, local_path: str):
node = LocalNode.create(Path(local_path).resolve(), session)
remote_history = RemoteNodeHistory(history=None, key=node.key, etag=None)
remote_history.load(session)
history = remote_history.history
last = history.last
prev = history.entries[-2]
assert last.base_version is None
assert last.has_delta
assert prev.base_version is not None
with create_temp_file() as base_path:
s3_path = f"{session.s3_prefix}/{node.path}"
download_file(session.s3_client,
session.storage_bucket,
s3_path,
base_path,
version=prev.base_version)
patch_file(session, base_path, [last.key])
upload_file(
session.s3_client,
base_path,
session.storage_bucket,
s3_path,
)
obj = get_file_metadata(session.s3_client, session.storage_bucket,
s3_path)
last.base_version = obj["VersionId"]
last.base_size = int(obj.get("Size", 0))
remote_history.save(session)
def upload_again(session: Session, path: str):
path = Path(path).resolve()
with open(path, "ab") as f:
f.write(b"1")
node = LocalNode.create(path, session)
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
upload(remote_history, node)(session)
show_history(session, node.key)
def download_new_version(session: Session, path: str):
path = Path(path).resolve()
node = LocalNode.create(path, session)
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
stored_history = models.StoredNodeHistory\
.get(models.StoredNodeHistory.key == node.key)
download(remote_history, stored_history)(session)
show_history(session, node.key)
def clear_remote(session: Session):
# upload
path = create_file(session.root_folder.path, MB(1))
node = LocalNode.create(path, session)
upload(None, node)(session)
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
stored_history = models.StoredNodeHistory\
.get(models.StoredNodeHistory.key == node.key)
delete_remote(remote_history, stored_history)(session)
node.local_path.unlink()
show_history(session, node.key)
def download_first_time(session: Session):
# upload
path = create_file(session.root_folder.path, MB(1))
node = LocalNode.create(path, session)
upload(None, node)(session)
# clear local
delete_local(
node,
models.StoredNodeHistory.get(models.StoredNodeHistory.key == node.key)
)(session)
# download
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
download(remote_history, None)(session)
show_history(session, node.key)
def scan_local_files(session: Session) -> Iterable[LocalNode]:
return (LocalNode.create(p, session)
for p in iter_folder(session.root_folder.path))
def upload_new(session: Session, filename=None):
path = create_file(session.root_folder.path, MB(1), filename=filename)
node = LocalNode.create(path, session)
action = upload(None, node)
action(session)
show_history(session, node.key)