def is_equal_digests( self, local_digest: Optional[str], remote_digest: Optional[str], local_path: Path, remote_digest_algorithm: str = None, ) -> bool: """ Compare 2 document's digests. :param str local_digest: Digest of the local document. Set to None to force digest computation. :param str remote_digest: Digest of the remote document. :param str local_path: Local path of the document. :param str remote_digest_algorithm: Remote document digest algorithm :return bool: Digest are equals. """ if local_digest == remote_digest: return True if remote_digest_algorithm is None: if not remote_digest: return False remote_digest_algorithm = get_digest_algorithm(remote_digest) if not remote_digest_algorithm: raise UnknownDigest(str(remote_digest)) file_info = self.try_get_info(local_path) if not file_info: return False digest = file_info.get_digest(digest_func=remote_digest_algorithm) return digest == remote_digest
def check_integrity(self, digest: str, download_action: DownloadAction) -> None: """ Check the integrity of a downloaded chunked file. Update the progress of the verification during the computation of the digest. """ digester = get_digest_algorithm(digest) filepath = download_action.tmppath or download_action.filepath # Terminate the download action to be able to start the verification one as we are allowing # only 1 action per thread. # Note that this is not really needed as the verification action would replace the download # one, but let's do things right. DownloadAction.finish_action() verif_action = VerificationAction(filepath, reporter=QApplication.instance()) def callback(_): verif_action.progress += FILE_BUFFER_SIZE try: computed_digest = compute_digest(filepath, digester, callback=callback) if digest != computed_digest: # TMP file and Downloads table entry will be deleted # by the calling method raise CorruptedFile(filepath, digest, computed_digest) finally: VerificationAction.finish_action()
def _guess_digest_and_algo(item: Dict[str, Any]) -> Tuple[str, str]: """Guess the digest and the algorithm used, if not already specified.""" digest = item.get("digest") or "" digest_algorithm = item.get("digestAlgorithm") or "" if digest_algorithm: digest_algorithm = digest_algorithm.lower().replace("-", "") elif digest: digest_algorithm = get_digest_algorithm(digest) or "" return digest, digest_algorithm
def from_dict(blob: Dict[str, Any]) -> "Blob": """ Convert Dict to Blob object. """ name = blob["name"] digest = blob.get("digest") or "" digest_algorithm = get_digest_algorithm(blob.get("digestAlgorithm", "")) or "" size = int(blob.get("length", 0)) mimetype = blob.get("mime-type", "application/octet-stream") data = blob.get("data", "") if digest_algorithm: digest_algorithm = digest_algorithm.lower().replace("-", "") return Blob(name, digest, digest_algorithm, size, mimetype, data)
def digest_status(digest: str) -> DigestStatus: """Determine the given *digest* status. It will be use to know when a document can be synced.""" if not digest: return DigestStatus.REMOTE_HASH_EMPTY # Likely a crafted digest to tell us it will be computed async (NXDRIVE-2140) if "-" in digest: return DigestStatus.REMOTE_HASH_ASYNC # The digest seems good if get_digest_algorithm(digest): return DigestStatus.OK # The digest will not be locally computable (likely a Live Connect document) return DigestStatus.REMOTE_HASH_EXOTIC
def check_integrity(self, digest: str, download_action: DownloadAction) -> None: """ Check the integrity of a downloaded chunked file. Update the progress of the verification during the computation of the digest. """ if Options.disabled_file_integrity_check: log.debug( "disabled_file_integrity_check is True, skipping file integrity check then" ) return digester = get_digest_algorithm(digest) if not digester: log.warning( f"Empty or non-standard digest {digest!r}, skipping the file integrity check" ) return size = download_action.size filepath = download_action.tmppath or download_action.filepath # Terminate the download action to be able to start the verification one as we are allowing # only 1 action per thread. # Note that this is not really needed as the verification action would replace the download # one, but let's do things right. DownloadAction.finish_action() verif_action = VerificationAction(filepath, size, reporter=QApplication.instance()) def callback(_: Path) -> None: verif_action.progress += FILE_BUFFER_SIZE try: computed_digest = compute_digest(filepath, digester, callback=callback) if digest != computed_digest: # TMP file and Downloads table entry will be deleted # by the calling method raise CorruptedFile(filepath, digest, computed_digest) finally: VerificationAction.finish_action()
def check_integrity_simple(self, digest: str, file: Path) -> None: """Check the integrity of a relatively small downloaded file.""" if Options.disabled_file_integrity_check: log.debug( "disabled_file_integrity_check is True, skipping file integrity check then" ) return digester = get_digest_algorithm(digest) if not digester: log.warning( f"Empty or non-standard digest {digest!r}, skipping the file integrity check" ) return computed_digest = compute_digest(file, digester) if digest != computed_digest: raise CorruptedFile(file, digest, computed_digest)
def _prepare_edit(self, server_url: str, doc_id: str, user: str = None, download_url: str = None) -> Optional[Path]: start_time = current_milli_time() engine = self._get_engine(server_url, doc_id=doc_id, user=user) if not engine: return None # Avoid any link with the engine, remote_doc are not cached so we # can do that info = self._get_info(engine, doc_id) if not info: return None url = None url_info: Dict[str, str] = {} if download_url: import re urlmatch = re.match( r"([^\/]+\/){3}(?P<xpath>.+)\/(?P<filename>[^\?]*).*", download_url, re.I, ) if urlmatch: url_info = urlmatch.groupdict() url = server_url if not url.endswith("/"): url += "/" url += download_url xpath = url_info.get("xpath") if not xpath: if info.doc_type == "Note": xpath = "note:note" else: xpath = "file:content" elif xpath == "blobholder:0": xpath = "file:content" blob = info.get_blob(xpath) if not blob: log.warning( f"No blob associated with xpath {xpath} for file {info.path}") return None filename = blob.name self.directEditStarting.emit(engine.hostname, filename) # Create local structure folder_name = safe_filename(f"{doc_id}_{xpath}") dir_path = self._folder / folder_name dir_path.mkdir(exist_ok=True) log.info(f"Editing {filename!r}") file_path = dir_path / filename tmp_folder = self._folder / f"{doc_id}.dl" tmp_folder.mkdir(parents=True, exist_ok=True) file_out = tmp_folder / filename try: # Download the file tmp_file = self._download(engine, info, file_path, file_out, blob, xpath, url=url) if tmp_file is None: log.warning("Download failed") return None except CONNECTION_ERROR: log.warning("Unable to perform DirectEdit", exc_info=True) return None # Set the remote_id dir_path = self.local.get_path(dir_path) self.local.set_remote_id(dir_path, doc_id) self.local.set_remote_id(dir_path, server_url, name="nxdirectedit") if user: self.local.set_remote_id(dir_path, user, name="nxdirectedituser") if xpath: self.local.set_remote_id(dir_path, xpath, name="nxdirecteditxpath") if blob.digest: self.local.set_remote_id(dir_path, blob.digest, name="nxdirecteditdigest") # Set digest algorithm if not sent by the server digest_algorithm = blob.digest_algorithm if not digest_algorithm: digest_algorithm = get_digest_algorithm(blob.digest) if not digest_algorithm: raise UnknownDigest(blob.digest) self.local.set_remote_id( dir_path, digest_algorithm.encode("utf-8"), name="nxdirecteditdigestalgorithm", ) self.local.set_remote_id(dir_path, filename, name="nxdirecteditname") safe_rename(tmp_file, file_path) timing = current_milli_time() - start_time self.openDocument.emit(filename, timing) return file_path
def from_dict(fs_item: Dict[str, Any]) -> "RemoteFileInfo": """Convert Automation file system item description to RemoteFileInfo""" try: uid = fs_item["id"] parent_uid = fs_item["parentId"] path = fs_item["path"] name = unicodedata.normalize("NFC", fs_item["name"]) except KeyError: raise DriveError( f"This item is missing mandatory information: {fs_item}") folderish = fs_item.get("folder", False) def to_date(timestamp: Optional[int]) -> Optional[datetime]: if not isinstance(timestamp, int): return None try: return datetime.fromtimestamp(timestamp // 1000) except (OSError, OverflowError): # OSError: [Errno 22] Invalid argument (Windows 7, see NXDRIVE-1600) # OverflowError: timestamp out of range for platform time_t return None last_update = to_date(fs_item.get("lastModificationDate")) creation = to_date(fs_item.get("creationDate")) if folderish: digest = None digest_algorithm = None download_url = None can_update = False can_create_child = fs_item.get("canCreateChild", False) # Scroll API availability can_scroll = fs_item.get("canScrollDescendants", False) can_scroll_descendants = can_scroll else: digest = fs_item["digest"] digest_algorithm = fs_item.get("digestAlgorithm") if digest_algorithm: digest_algorithm = digest_algorithm.lower().replace("-", "") else: digest_algorithm = get_digest_algorithm(digest) if not digest_algorithm: raise UnknownDigest(digest) download_url = fs_item.get("downloadURL") can_update = fs_item.get("canUpdate", False) can_create_child = False can_scroll_descendants = False # Lock info lock_info = fs_item.get("lockInfo") lock_owner = lock_created = None if lock_info: lock_owner = lock_info.get("owner") lock_created = lock_info.get("created") if lock_created: lock_created = datetime.fromtimestamp(lock_created // 1000) return RemoteFileInfo( name, uid, parent_uid, path, folderish, last_update, creation, fs_item.get("lastContributor"), digest, digest_algorithm, download_url, fs_item.get("canRename", False), fs_item.get("canDelete", False), can_update, can_create_child, lock_owner, lock_created, can_scroll_descendants, )
def check_integrity_simple(self, digest: str, file: Path) -> None: """Check the integrity of a relatively small downloaded file.""" digester = get_digest_algorithm(digest) computed_digest = compute_digest(file, digester) if digest != computed_digest: raise CorruptedFile(file, digest, computed_digest)
def _prepare_edit( self, server_url: str, doc_id: str, user: str = None, download_url: str = None ) -> Optional[Path]: start_time = current_milli_time() engine = self._get_engine(server_url, doc_id=doc_id, user=user) if not engine: return None # Avoid any link with the engine, remote_doc are not cached so we # can do that info = self._get_info(engine, doc_id) if not info: return None url = None url_info: Dict[str, str] = {} if download_url: import re urlmatch = re.match( r"([^\/]+\/){3}(?P<xpath>.+)\/(?P<filename>[^\?]*).*", download_url, re.I, ) if urlmatch: url_info = urlmatch.groupdict() url = server_url if not url.endswith("/"): url += "/" url += download_url xpath = url_info.get("xpath") if not xpath: if info.doc_type == "Note": xpath = "note:note" else: xpath = "file:content" elif xpath == "blobholder:0": xpath = "file:content" blob = info.get_blob(xpath) if not blob: log.warning(f"No blob associated with xpath {xpath} for file {info.path}") return None filename = blob.name self.directEditStarting.emit(engine.hostname, filename) # Create local structure folder_name = safe_os_filename(f"{doc_id}_{xpath}") dir_path = self._folder / folder_name dir_path.mkdir(exist_ok=True) log.info(f"Editing {filename!r}") file_path = dir_path / filename # Download the file FileAction("Download", file_path, size=0, reporter=QApplication.instance()) try: tmp_file = self._download(engine, info, file_path, blob, xpath, url=url) if tmp_file is None: log.warning("Download failed") return None finally: FileAction.finish_action() # Set the remote_id dir_path = self.local.get_path(dir_path) self.local.set_remote_id(dir_path, doc_id) self.local.set_remote_id(dir_path, server_url, name="nxdirectedit") if user: self.local.set_remote_id(dir_path, user, name="nxdirectedituser") if xpath: self.local.set_remote_id(dir_path, xpath, name="nxdirecteditxpath") if blob.digest: self.local.set_remote_id(dir_path, blob.digest, name="nxdirecteditdigest") # Set digest algorithm if not sent by the server digest_algorithm = blob.digest_algorithm if not digest_algorithm: digest_algorithm = get_digest_algorithm(blob.digest) if not digest_algorithm: raise UnknownDigest(blob.digest) self.local.set_remote_id( dir_path, digest_algorithm.encode("utf-8"), name="nxdirecteditdigestalgorithm", ) self.local.set_remote_id(dir_path, filename, name="nxdirecteditname") # Rename to final filename # Under Windows first need to delete target file if exists, # otherwise will get a 183 WindowsError if WINDOWS and file_path.exists(): file_path.unlink() tmp_file.rename(file_path) self._last_action_timing = current_milli_time() - start_time self.openDocument.emit(filename) return file_path
:param str local_digest: Digest of the local document. Set to None to force digest computation. :param str remote_digest: Digest of the remote document. :param str local_path: Local path of the document. :param str remote_digest_algorithm: Remote document digest algorithm :return bool: Digest are equals. """ if local_digest == remote_digest: return True if remote_digest_algorithm is None: if not remote_digest: return False remote_digest_algorithm = get_digest_algorithm(remote_digest) if not remote_digest_algorithm: raise UnknownDigest(str(remote_digest)) file_info = self.try_get_info(local_path) if not file_info: return False digest = file_info.get_digest(digest_func=remote_digest_algorithm) return digest == remote_digest def is_ignored(self, parent_ref: Path, file_name: str, /) -> bool: """ Note: added parent_ref to be able to filter on size if needed. """ file_name = safe_filename(force_decode(file_name.lower())) if file_name.endswith(