def is_equal_digests(
self,
local_digest: Optional[str],
remote_digest: Optional[str],
local_path: Path,
remote_digest_algorithm: str = None,
) -> bool:
"""
Compare 2 document's digests.
:param str local_digest: Digest of the local document.
Set to None to force digest computation.
:param str remote_digest: Digest of the remote document.
:param str local_path: Local path of the document.
:param str remote_digest_algorithm: Remote document digest algorithm
:return bool: Digest are equals.
"""
if local_digest == remote_digest:
return True
if remote_digest_algorithm is None:
if not remote_digest:
return False
remote_digest_algorithm = get_digest_algorithm(remote_digest)
if not remote_digest_algorithm:
raise UnknownDigest(str(remote_digest))
file_info = self.try_get_info(local_path)
if not file_info:
return False
digest = file_info.get_digest(digest_func=remote_digest_algorithm)
return digest == remote_digest
def check_integrity(self, digest: str,
download_action: DownloadAction) -> None:
"""
Check the integrity of a downloaded chunked file.
Update the progress of the verification during the computation of the digest.
"""
digester = get_digest_algorithm(digest)
filepath = download_action.tmppath or download_action.filepath
# Terminate the download action to be able to start the verification one as we are allowing
# only 1 action per thread.
# Note that this is not really needed as the verification action would replace the download
# one, but let's do things right.
DownloadAction.finish_action()
verif_action = VerificationAction(filepath,
reporter=QApplication.instance())
def callback(_):
verif_action.progress += FILE_BUFFER_SIZE
try:
computed_digest = compute_digest(filepath,
digester,
callback=callback)
if digest != computed_digest:
# TMP file and Downloads table entry will be deleted
# by the calling method
raise CorruptedFile(filepath, digest, computed_digest)
finally:
VerificationAction.finish_action()
def _guess_digest_and_algo(item: Dict[str, Any]) -> Tuple[str, str]:
"""Guess the digest and the algorithm used, if not already specified."""
digest = item.get("digest") or ""
digest_algorithm = item.get("digestAlgorithm") or ""
if digest_algorithm:
digest_algorithm = digest_algorithm.lower().replace("-", "")
elif digest:
digest_algorithm = get_digest_algorithm(digest) or ""
return digest, digest_algorithm
def from_dict(blob: Dict[str, Any]) -> "Blob":
""" Convert Dict to Blob object. """
name = blob["name"]
digest = blob.get("digest") or ""
digest_algorithm = get_digest_algorithm(blob.get("digestAlgorithm", "")) or ""
size = int(blob.get("length", 0))
mimetype = blob.get("mime-type", "application/octet-stream")
data = blob.get("data", "")
if digest_algorithm:
digest_algorithm = digest_algorithm.lower().replace("-", "")
return Blob(name, digest, digest_algorithm, size, mimetype, data)
def digest_status(digest: str) -> DigestStatus:
"""Determine the given *digest* status. It will be use to know when a document can be synced."""
if not digest:
return DigestStatus.REMOTE_HASH_EMPTY
# Likely a crafted digest to tell us it will be computed async (NXDRIVE-2140)
if "-" in digest:
return DigestStatus.REMOTE_HASH_ASYNC
# The digest seems good
if get_digest_algorithm(digest):
return DigestStatus.OK
# The digest will not be locally computable (likely a Live Connect document)
return DigestStatus.REMOTE_HASH_EXOTIC
def check_integrity(self, digest: str,
download_action: DownloadAction) -> None:
"""
Check the integrity of a downloaded chunked file.
Update the progress of the verification during the computation of the digest.
"""
if Options.disabled_file_integrity_check:
log.debug(
"disabled_file_integrity_check is True, skipping file integrity check then"
)
return
digester = get_digest_algorithm(digest)
if not digester:
log.warning(
f"Empty or non-standard digest {digest!r}, skipping the file integrity check"
)
return
size = download_action.size
filepath = download_action.tmppath or download_action.filepath
# Terminate the download action to be able to start the verification one as we are allowing
# only 1 action per thread.
# Note that this is not really needed as the verification action would replace the download
# one, but let's do things right.
DownloadAction.finish_action()
verif_action = VerificationAction(filepath,
size,
reporter=QApplication.instance())
def callback(_: Path) -> None:
verif_action.progress += FILE_BUFFER_SIZE
try:
computed_digest = compute_digest(filepath,
digester,
callback=callback)
if digest != computed_digest:
# TMP file and Downloads table entry will be deleted
# by the calling method
raise CorruptedFile(filepath, digest, computed_digest)
finally:
VerificationAction.finish_action()
def check_integrity_simple(self, digest: str, file: Path) -> None:
"""Check the integrity of a relatively small downloaded file."""
if Options.disabled_file_integrity_check:
log.debug(
"disabled_file_integrity_check is True, skipping file integrity check then"
)
return
digester = get_digest_algorithm(digest)
if not digester:
log.warning(
f"Empty or non-standard digest {digest!r}, skipping the file integrity check"
)
return
computed_digest = compute_digest(file, digester)
if digest != computed_digest:
raise CorruptedFile(file, digest, computed_digest)
def _prepare_edit(self,
server_url: str,
doc_id: str,
user: str = None,
download_url: str = None) -> Optional[Path]:
start_time = current_milli_time()
engine = self._get_engine(server_url, doc_id=doc_id, user=user)
if not engine:
return None
# Avoid any link with the engine, remote_doc are not cached so we
# can do that
info = self._get_info(engine, doc_id)
if not info:
return None
url = None
url_info: Dict[str, str] = {}
if download_url:
import re
urlmatch = re.match(
r"([^\/]+\/){3}(?P<xpath>.+)\/(?P<filename>[^\?]*).*",
download_url,
re.I,
)
if urlmatch:
url_info = urlmatch.groupdict()
url = server_url
if not url.endswith("/"):
url += "/"
url += download_url
xpath = url_info.get("xpath")
if not xpath:
if info.doc_type == "Note":
xpath = "note:note"
else:
xpath = "file:content"
elif xpath == "blobholder:0":
xpath = "file:content"
blob = info.get_blob(xpath)
if not blob:
log.warning(
f"No blob associated with xpath {xpath} for file {info.path}")
return None
filename = blob.name
self.directEditStarting.emit(engine.hostname, filename)
# Create local structure
folder_name = safe_filename(f"{doc_id}_{xpath}")
dir_path = self._folder / folder_name
dir_path.mkdir(exist_ok=True)
log.info(f"Editing {filename!r}")
file_path = dir_path / filename
tmp_folder = self._folder / f"{doc_id}.dl"
tmp_folder.mkdir(parents=True, exist_ok=True)
file_out = tmp_folder / filename
try:
# Download the file
tmp_file = self._download(engine,
info,
file_path,
file_out,
blob,
xpath,
url=url)
if tmp_file is None:
log.warning("Download failed")
return None
except CONNECTION_ERROR:
log.warning("Unable to perform DirectEdit", exc_info=True)
return None
# Set the remote_id
dir_path = self.local.get_path(dir_path)
self.local.set_remote_id(dir_path, doc_id)
self.local.set_remote_id(dir_path, server_url, name="nxdirectedit")
if user:
self.local.set_remote_id(dir_path, user, name="nxdirectedituser")
if xpath:
self.local.set_remote_id(dir_path, xpath, name="nxdirecteditxpath")
if blob.digest:
self.local.set_remote_id(dir_path,
blob.digest,
name="nxdirecteditdigest")
# Set digest algorithm if not sent by the server
digest_algorithm = blob.digest_algorithm
if not digest_algorithm:
digest_algorithm = get_digest_algorithm(blob.digest)
if not digest_algorithm:
raise UnknownDigest(blob.digest)
self.local.set_remote_id(
dir_path,
digest_algorithm.encode("utf-8"),
name="nxdirecteditdigestalgorithm",
)
self.local.set_remote_id(dir_path, filename, name="nxdirecteditname")
safe_rename(tmp_file, file_path)
timing = current_milli_time() - start_time
self.openDocument.emit(filename, timing)
return file_path
def from_dict(fs_item: Dict[str, Any]) -> "RemoteFileInfo":
"""Convert Automation file system item description to RemoteFileInfo"""
try:
uid = fs_item["id"]
parent_uid = fs_item["parentId"]
path = fs_item["path"]
name = unicodedata.normalize("NFC", fs_item["name"])
except KeyError:
raise DriveError(
f"This item is missing mandatory information: {fs_item}")
folderish = fs_item.get("folder", False)
def to_date(timestamp: Optional[int]) -> Optional[datetime]:
if not isinstance(timestamp, int):
return None
try:
return datetime.fromtimestamp(timestamp // 1000)
except (OSError, OverflowError):
# OSError: [Errno 22] Invalid argument (Windows 7, see NXDRIVE-1600)
# OverflowError: timestamp out of range for platform time_t
return None
last_update = to_date(fs_item.get("lastModificationDate"))
creation = to_date(fs_item.get("creationDate"))
if folderish:
digest = None
digest_algorithm = None
download_url = None
can_update = False
can_create_child = fs_item.get("canCreateChild", False)
# Scroll API availability
can_scroll = fs_item.get("canScrollDescendants", False)
can_scroll_descendants = can_scroll
else:
digest = fs_item["digest"]
digest_algorithm = fs_item.get("digestAlgorithm")
if digest_algorithm:
digest_algorithm = digest_algorithm.lower().replace("-", "")
else:
digest_algorithm = get_digest_algorithm(digest)
if not digest_algorithm:
raise UnknownDigest(digest)
download_url = fs_item.get("downloadURL")
can_update = fs_item.get("canUpdate", False)
can_create_child = False
can_scroll_descendants = False
# Lock info
lock_info = fs_item.get("lockInfo")
lock_owner = lock_created = None
if lock_info:
lock_owner = lock_info.get("owner")
lock_created = lock_info.get("created")
if lock_created:
lock_created = datetime.fromtimestamp(lock_created // 1000)
return RemoteFileInfo(
name,
uid,
parent_uid,
path,
folderish,
last_update,
creation,
fs_item.get("lastContributor"),
digest,
digest_algorithm,
download_url,
fs_item.get("canRename", False),
fs_item.get("canDelete", False),
can_update,
can_create_child,
lock_owner,
lock_created,
can_scroll_descendants,
)
def check_integrity_simple(self, digest: str, file: Path) -> None:
"""Check the integrity of a relatively small downloaded file."""
digester = get_digest_algorithm(digest)
computed_digest = compute_digest(file, digester)
if digest != computed_digest:
raise CorruptedFile(file, digest, computed_digest)
def _prepare_edit(
self, server_url: str, doc_id: str, user: str = None, download_url: str = None
) -> Optional[Path]:
start_time = current_milli_time()
engine = self._get_engine(server_url, doc_id=doc_id, user=user)
if not engine:
return None
# Avoid any link with the engine, remote_doc are not cached so we
# can do that
info = self._get_info(engine, doc_id)
if not info:
return None
url = None
url_info: Dict[str, str] = {}
if download_url:
import re
urlmatch = re.match(
r"([^\/]+\/){3}(?P<xpath>.+)\/(?P<filename>[^\?]*).*",
download_url,
re.I,
)
if urlmatch:
url_info = urlmatch.groupdict()
url = server_url
if not url.endswith("/"):
url += "/"
url += download_url
xpath = url_info.get("xpath")
if not xpath:
if info.doc_type == "Note":
xpath = "note:note"
else:
xpath = "file:content"
elif xpath == "blobholder:0":
xpath = "file:content"
blob = info.get_blob(xpath)
if not blob:
log.warning(f"No blob associated with xpath {xpath} for file {info.path}")
return None
filename = blob.name
self.directEditStarting.emit(engine.hostname, filename)
# Create local structure
folder_name = safe_os_filename(f"{doc_id}_{xpath}")
dir_path = self._folder / folder_name
dir_path.mkdir(exist_ok=True)
log.info(f"Editing {filename!r}")
file_path = dir_path / filename
# Download the file
FileAction("Download", file_path, size=0, reporter=QApplication.instance())
try:
tmp_file = self._download(engine, info, file_path, blob, xpath, url=url)
if tmp_file is None:
log.warning("Download failed")
return None
finally:
FileAction.finish_action()
# Set the remote_id
dir_path = self.local.get_path(dir_path)
self.local.set_remote_id(dir_path, doc_id)
self.local.set_remote_id(dir_path, server_url, name="nxdirectedit")
if user:
self.local.set_remote_id(dir_path, user, name="nxdirectedituser")
if xpath:
self.local.set_remote_id(dir_path, xpath, name="nxdirecteditxpath")
if blob.digest:
self.local.set_remote_id(dir_path, blob.digest, name="nxdirecteditdigest")
# Set digest algorithm if not sent by the server
digest_algorithm = blob.digest_algorithm
if not digest_algorithm:
digest_algorithm = get_digest_algorithm(blob.digest)
if not digest_algorithm:
raise UnknownDigest(blob.digest)
self.local.set_remote_id(
dir_path,
digest_algorithm.encode("utf-8"),
name="nxdirecteditdigestalgorithm",
)
self.local.set_remote_id(dir_path, filename, name="nxdirecteditname")
# Rename to final filename
# Under Windows first need to delete target file if exists,
# otherwise will get a 183 WindowsError
if WINDOWS and file_path.exists():
file_path.unlink()
tmp_file.rename(file_path)
self._last_action_timing = current_milli_time() - start_time
self.openDocument.emit(filename)
return file_path
:param str local_digest: Digest of the local document.
Set to None to force digest computation.
:param str remote_digest: Digest of the remote document.
:param str local_path: Local path of the document.
:param str remote_digest_algorithm: Remote document digest algorithm
:return bool: Digest are equals.
"""
if local_digest == remote_digest:
return True
if remote_digest_algorithm is None:
if not remote_digest:
return False
remote_digest_algorithm = get_digest_algorithm(remote_digest)
if not remote_digest_algorithm:
raise UnknownDigest(str(remote_digest))
file_info = self.try_get_info(local_path)
if not file_info:
return False
digest = file_info.get_digest(digest_func=remote_digest_algorithm)
return digest == remote_digest
def is_ignored(self, parent_ref: Path, file_name: str, /) -> bool:
""" Note: added parent_ref to be able to filter on size if needed. """
file_name = safe_filename(force_decode(file_name.lower()))
if file_name.endswith(
