def _download(self, make_dest_dirs=True): with self.httpr(stream=True) as httpr: if self.exists(): # Find out if the source file is gzip compressed in order to keep # compression intact after the download. # Per default requests decompresses .gz files. # More detials can be found here: https://stackoverflow.com/questions/25749345/how-to-download-gz-files-with-requests-in-python-without-decoding-it?noredirect=1&lq=1 # Since data transferred with HTTP compression need to be decompressed automatically # check the header and decode if the content is encoded. if (not self.name.endswith(".gz") and httpr.headers.get("Content-Encoding") == "gzip"): # Decode non-gzipped sourcefiles automatically. # This is needed to decompress uncompressed files that are compressed # for the transfer by HTTP compression. httpr.raw.decode_content = True # if the destination path does not exist if make_dest_dirs: os.makedirs(os.path.dirname(self.local_path), exist_ok=True) with open(self.local_path, "wb") as f: shutil.copyfileobj(httpr.raw, f) os_sync() # ensure flush to disk else: raise HTTPFileException( "The file does not seem to exist remotely: %s" % self.remote_file())
def download(self, make_dest_dirs=True): if self.exists(): # if the destination path does not exist, make it if make_dest_dirs: os.makedirs(os.path.dirname(self.local_file()), exist_ok=True) self._dropboxc.files_download_to_file(self.local_file(), self.dropbox_file()) os_sync() # ensure flush to disk else: raise DropboxFileException( "The file does not seem to exist remotely: %s" % self.dropbox_file())
def download(self, make_dest_dirs=True): if self.exists(): # if the destination path does not exist, make it if make_dest_dirs: os.makedirs(os.path.dirname(self.local_file()), exist_ok=True) with self.webdavc() as webdavc: self.loop.run_until_complete( self.conn.download(self.webdav_file, self.local_file())) os_sync() # ensure flush to disk else: raise WorkflowError( "The file does not seem to exist remotely: %s" % self.webdav_file)
def download(self, make_dest_dirs=True): with self.connection_pool.item() as sftpc: if self.exists(): # if the destination path does not exist if make_dest_dirs: os.makedirs(os.path.dirname(self.local_path), exist_ok=True) sftpc.get( remotepath=self.remote_path, localpath=self.local_path, preserve_mtime=True, ) os_sync() # ensure flush to disk else: raise SFTPFileException( "The file does not seem to exist remotely: %s" % self.local_file() )
def download(self): if self.exists(): if self.size() == 0: # Globus erroneously thinks that a transfer is incomplete if a # file is empty. Hence we manually touch the local file. self.local_touch_or_create() return self.local_file() # Download file. Wait for staging. source = self.remote_file() target = "file://" + os.path.abspath(self.local_file()) self._globus("-parallel", "4", "-create-dest", "-recurse", "-dp", source, target) os_sync() return self.local_file() return None
def download(self, make_dest_dirs=True): with self.ftpc() as ftpc: if self.exists(): # if the destination path does not exist if make_dest_dirs: os.makedirs(os.path.dirname(self.local_path), exist_ok=True) try: # requires write access ftpc.synchronize_times() except: pass ftpc.download(source=self.remote_path, target=self.local_path) os_sync() # ensure flush to disk else: raise FTPFileException( "The file does not seem to exist remotely: %s" % self.local_file())
def download(self): if self.exists(): if self.size() == 0: # Globus erroneously thinks that a transfer is incomplete if a # file is empty. Hence we manually touch the local file. self.local_touch_or_create() return self.local_file() # Download file. Wait for staging. source = self.remote_file() target = "file://" + os.path.abspath(self.local_file()) # disable all timeouts (file transfers can take a long time) self._gfal("copy", "-p", "-f", "-n", "4", "-t", "0", "-T", "0", source, target) os_sync() return self.local_file() return None
def download(self, make_dest_dirs=True): if self.exists(): if make_dest_dirs: os.makedirs(os.path.dirname(self.local_path), exist_ok=True) # force irods to overwrite existing file if this option is set opt = {} if self.kwargs.get("overwrite"): opt[kw.FORCE_FLAG_KW] = "" # get object and change timestamp obj = self._irods_session.data_objects.get(self.remote_path, self.local_path, options=opt) os.utime(self.local_path, (self.atime(), self.mtime())) os_sync() else: raise WorkflowError( "The file does not seem to exist remotely: %s" % self.local_file())
def download(self): if not self.exists(): return None # Create the directory for the intended file os.makedirs(os.path.dirname(self.local_file()), exist_ok=True) # ideally we could calculate hash while streaming to file with provided function # https://github.com/googleapis/python-storage/issues/29 with open(self.local_file(), "wb") as blob_file: parser = Crc32cCalculator(blob_file) self.blob.download_to_file(parser) os_sync() # Compute local hash and verify correct if parser.hexdigest() != self.blob.crc32c: os.remove(self.local_file()) raise CheckSumMismatchException( "The checksum of %s does not match." % self.local_file()) return self.local_file()
def download(self): self._s3c.download_from_s3(self.s3_bucket, self.s3_key, self.local_file()) os_sync() # ensure flush to disk