def download(
remote_history: RemoteNodeHistory,
stored_history: Optional[StoredNodeHistory],
session: Session,
) -> SyncActionResult:
"""
1. Without local history
- Find latest base
- Download latest base
- Store history in local DB
2. With local history
- Diff remote and local history and find shortest path
- Fetch deltas one by one patch
- Store history in local DB
"""
history = cast(NodeHistory, remote_history.history)
if stored_history is not None:
entries, is_absolute = history.diff(stored_history.history)
if is_absolute:
local_path = file_transfer.download_to_root(
session, history.path, entries[0].base_version)
entries = entries[1:]
else:
local_path = session.root_folder.path / history.path
if entries:
patch_file(session, os.fspath(local_path),
[e.key for e in entries])
local_node = LocalNode.create(local_path, session)
stored_history.data = history.dict() # type: ignore
stored_history.local_modified_time = local_node.created_time
stored_history.local_created_time = local_node.modified_time
stored_history.remote_history_etag = remote_history.etag
else:
entries, is_absolute = history.diff(None)
local_path = file_transfer.download_to_root(session, history.path,
entries[0].base_version)
if entries[1:]:
patch_file(session, os.fspath(local_path),
[e.key for e in entries[1:]])
local_node = LocalNode.create(local_path, session)
stored_history = StoredNodeHistory(
key=remote_history.key,
root_folder=RootFolder.for_session(session),
data=history.dict(),
local_modified_time=local_node.created_time,
local_created_time=local_node.modified_time,
remote_history_etag=remote_history.etag)
last_entry = entries[-1]
file_transfer.download_metadata(
session, last_entry.key, "signature",
os.fspath(session.signature_folder / last_entry.key))
stored_history.save()
return SyncActionResult()
def create_full_version(session: Session, local_path: str):
node = LocalNode.create(Path(local_path).resolve(), session)
remote_history = RemoteNodeHistory(history=None, key=node.key, etag=None)
remote_history.load(session)
history = remote_history.history
last = history.last
prev = history.entries[-2]
assert last.base_version is None
assert last.has_delta
assert prev.base_version is not None
with create_temp_file() as base_path:
s3_path = f"{session.s3_prefix}/{node.path}"
download_file(session.s3_client,
session.storage_bucket,
s3_path,
base_path,
version=prev.base_version)
patch_file(session, base_path, [last.key])
upload_file(
session.s3_client,
base_path,
session.storage_bucket,
s3_path,
)
obj = get_file_metadata(session.s3_client, session.storage_bucket,
s3_path)
last.base_version = obj["VersionId"]
last.base_size = int(obj.get("Size", 0))
remote_history.save(session)
def upload_again(session: Session, path: str):
path = Path(path).resolve()
with open(path, "ab") as f:
f.write(b"1")
node = LocalNode.create(path, session)
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
upload(remote_history, node)(session)
show_history(session, node.key)
def download_new_version(session: Session, path: str):
path = Path(path).resolve()
node = LocalNode.create(path, session)
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
stored_history = models.StoredNodeHistory\
.get(models.StoredNodeHistory.key == node.key)
download(remote_history, stored_history)(session)
show_history(session, node.key)
def local(self, **extra_attrs):
attrs = {
"root_folder": Path(self.root_folder.path),
"path": self.path,
"modified_time": self.modified_time,
"created_time": self.created_time,
"size": self.size,
"etag": self.etag,
**extra_attrs,
}
return LocalNode(**attrs)
def clear_remote(session: Session):
# upload
path = create_file(session.root_folder.path, MB(1))
node = LocalNode.create(path, session)
upload(None, node)(session)
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
stored_history = models.StoredNodeHistory\
.get(models.StoredNodeHistory.key == node.key)
delete_remote(remote_history, stored_history)(session)
node.local_path.unlink()
show_history(session, node.key)
def download_first_time(session: Session):
# upload
path = create_file(session.root_folder.path, MB(1))
node = LocalNode.create(path, session)
upload(None, node)(session)
# clear local
delete_local(
node,
models.StoredNodeHistory.get(models.StoredNodeHistory.key == node.key)
)(session)
# download
remote_history = RemoteNodeHistory(key=node.key, etag=None, history=None)
remote_history.load(session)
download(remote_history, None)(session)
show_history(session, node.key)
def handle_node(remote: RemoteNodeHistory, local: LocalNode,
stored: StoredNodeHistory) -> SyncAction:
if not remote and not local and not stored:
return nop()
elif not remote and not local and stored:
return delete_history(stored)
elif not remote and local and not stored:
return upload(None, local)
elif not remote and local and stored:
return delete_local(local, stored)
elif remote.exists and not local and not stored:
return download(remote, None)
elif remote.exists and not local and stored:
return delete_remote(remote, stored)
elif remote.deleted and not local and stored:
return delete_history(stored)
elif remote and local and not stored:
if remote.deleted:
return delete_local(local, stored)
elif cast(NodeHistory, remote.history).etag == local.etag:
return save_history(remote, local)
else:
return conflict(remote, local, stored)
elif remote and local and stored:
local_updated = local.updated(stored)
remote_updated = remote.updated(stored)
if remote.deleted:
if local_updated:
return conflict(remote, local, stored)
else:
return delete_local(local, stored)
elif local_updated and remote_updated:
if cast(NodeHistory, remote.history).etag == local.etag:
return nop()
else:
return conflict(remote, local, stored)
return nop()
elif local_updated:
return upload(remote, local)
elif remote_updated:
return download(remote, stored)
else:
return nop()
return nop()
def scan_local_files(session: Session) -> Iterable[LocalNode]:
return (LocalNode.create(p, session)
for p in iter_folder(session.root_folder.path))
def upload(remote_history: Optional[RemoteNodeHistory], node: LocalNode,
session: Session) -> SyncActionResult:
"""
1. Without remote history:
- Calc signature
- Generate id
- Create new history
- Upload base
- Upload history
- Store history in local DB
2. With remote history:
- Generate key
- Calc delta
- Calc signature
- Upload delta
- Upload signature
- Add history record
- Upload history
- Store history in local DB
"""
new_key = NodeHistoryEntry.generate_key()
if remote_history is not None:
history = cast(NodeHistory, remote_history.history)
with create_temp_file() as delta_path:
calc_delta(session, node.local_fspath, history.last.key,
delta_path)
file_transfer.upload_metadata(session, delta_path, new_key,
"delta")
delta_size = Path(delta_path).stat().st_size
with create_temp_file() as signature_path:
calc_signature(session, node.local_fspath, new_key, signature_path)
file_transfer.upload_metadata(session, signature_path, new_key,
"signature")
history.add_entry(
NodeHistoryEntry.create_delta_only(new_key, node.calc_etag(),
delta_size))
else:
with create_temp_file() as signature_path:
calc_signature(session, node.local_fspath, new_key, signature_path)
file_transfer.upload_metadata(session, signature_path, new_key,
"signature")
version = file_transfer.upload_to_root(session, node)
history = NodeHistory(key=node.key, path=node.path, entries=[])
history.add_entry(
NodeHistoryEntry.create_base_only(new_key, node.calc_etag(),
version, node.size))
remote_history = RemoteNodeHistory(history=history,
key=node.key,
etag=None)
remote_history.save(session)
stored_history = StoredNodeHistory.get_or_none(
StoredNodeHistory.key == history.key)
if stored_history is not None:
stored_history.data = history.dict()
stored_history.remote_history_etag = remote_history.etag
stored_history.local_modified_time = node.created_time
stored_history.local_created_time = node.modified_time
stored_history.save()
else:
StoredNodeHistory.create(key=remote_history.key,
root_folder=RootFolder.for_session(session),
data=cast(NodeHistory,
remote_history.history).dict(),
local_modified_time=node.created_time,
local_created_time=node.modified_time,
remote_history_etag=remote_history.etag)
return SyncActionResult()
def upload_new(session: Session, filename=None):
path = create_file(session.root_folder.path, MB(1), filename=filename)
node = LocalNode.create(path, session)
action = upload(None, node)
action(session)
show_history(session, node.key)