Ejemplo n.º 1
0
 def download_file(
     self,
     destination: Path,
     show_progress: bool = False,
     digests: Optional[dict] = None
 ):
     get_dxpy().download_dxfile(
         self.file_id,
         str(destination),
         show_progress=show_progress,
         project=self.project_id
     )
     if digests:
         verify_digests(destination, digests)
Ejemplo n.º 2
0
 def verify(self, path: Path) -> bool:
     if not super().verify(path):
         return False
     if self.digests:
         try:
             verify_digests(path, self.digests)
         except DigestsNotEqualError:
             LOG.exception(
                 "%s already exists but its digest does not match the expected "
                 "digest; deleting the existing file and re-downloading from "
                 "%s", str(path), self.url)
             path.unlink()
             return False
     return True
Ejemplo n.º 3
0
    def download_file(
        self,
        destination: Path,
        show_progress: bool = False,
        digests: Optional[dict] = None
    ):
        total_size = self.get_content_length()
        block_size = 16 * 1024
        if total_size and total_size < block_size:
            block_size = total_size

        if show_progress and progress:
            progress_bar = progress(
                total=total_size,
                unit="b",
                unit_scale=True,
                unit_divisor=1024,
                desc=f"Localizing {destination.name}"
            )

            def progress_reader():
                b = self.read(block_size)
                if b:
                    progress_bar.update(block_size)
                else:
                    progress_bar.close()
                return b

            reader = progress_reader
        else:
            reader = functools.partial(self.read, block_size)

        downloaded_size = 0

        with open(destination, "wb") as out:
            while True:
                buf = reader()
                if not buf:
                    break
                downloaded_size += len(buf)
                out.write(buf)

        if downloaded_size != total_size:  # TODO: test this
            raise AssertionError(
                f"Size of downloaded file {destination} does not match expected size "
                f"{total_size}"
            )

        if digests:
            verify_digests(destination, digests)
Ejemplo n.º 4
0
    def download_file(
        self,
        destination: Path,
        show_progress: bool = False,
        digests: Optional[dict] = None
    ):
        destination.parent.mkdir(parents=True, exist_ok=True)

        with login():
            dxpy.download_dxfile(
                self.file_id,
                str(destination),
                show_progress=show_progress,
                project=self.project_id
            )

        if digests:
            verify_digests(destination, digests)