コード例 #1
0
ファイル: hdfs_hdfscli.py プロジェクト: lbtanh/dbnd
    def open_read(self, path, mode="r"):
        local_tmp_file = get_local_tempfile("download-%s" %
                                            os.path.basename(path))
        # We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
        self.download(path, local_tmp_file)

        return _DeleteOnCloseFile(local_tmp_file, mode)
コード例 #2
0
    def open_read(self, path, mode="r"):
        account, container, blob = self._path_to_account_container_and_blob(path)
        assert self.account == account

        local_tmp_file = get_local_tempfile("download-%s" % os.path.basename(path))
        # We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
        self.download_as_file(container, blob, local_tmp_file)

        return _DeleteOnCloseFile(local_tmp_file, mode)
コード例 #3
0
ファイル: gcs.py プロジェクト: ipattarapong/dbnd
    def _open_read(
        self,
        remote_path,
        local_path=None,
        delete_file_on_close=True,
        chunksize=None,
        chunk_callback=lambda _: False,
    ):
        """Downloads the object contents to local file system.

        Optionally stops after the first chunk for which chunk_callback returns True.
        """
        chunksize = chunksize or self.chunksize
        bucket, obj = self._path_to_bucket_and_key(remote_path)

        tmp_file_path = local_path or get_local_tempfile(os.path.basename(remote_path))
        with open(tmp_file_path, "wb") as fp:
            # We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
            if delete_file_on_close:
                return_fp = _DeleteOnCloseFile(tmp_file_path, "r")
            else:
                return_fp = fp

            # Special case empty files because chunk-based downloading doesn't work.
            result = self.client.objects().get(bucket=bucket, object=obj).execute()
            if int(result["size"]) == 0:
                return return_fp

            request = self.client.objects().get_media(bucket=bucket, object=obj)
            downloader = http.MediaIoBaseDownload(fp, request, chunksize=chunksize)

            attempts = 0
            done = False
            while not done:
                error = None
                try:
                    _, done = downloader.next_chunk()
                    if chunk_callback(fp):
                        done = True
                except errors.HttpError as err:
                    error = err
                    if err.resp.status < 500:
                        raise
                    logger.warning("Error downloading file, retrying", exc_info=True)
                except RETRYABLE_ERRORS as err:
                    logger.warning("Error downloading file, retrying", exc_info=True)
                    error = err

                if error:
                    attempts += 1
                    if attempts >= NUM_RETRIES:
                        raise error
                else:
                    attempts = 0

        return return_fp