def open(self, mode): """Open the FileSystem target. This method returns a file-like object which can either be read from or written to depending on the specified mode. :param str mode: the mode `r` opens the FileSystemTarget in read-only mode, whereas `w` will open the FileSystemTarget in write mode. Subclasses can implement additional options. """ if mode == 'w': if self.format: return self.format.pipe_writer( AtomicFtpfile(self._fs, self.path)) else: return AtomicFtpfile(self._fs, self.path) elif mode == 'r': self.__tmp_path = self.path + '-luigi-tmp-%09d' % random.randrange( 0, 1e10) # download file to local self._fs.get(self.path, self.__tmp_path) # manage tmp file fileobj = FileWrapper(open(self.__tmp_path, 'r')) if self.format: return self.format.pipe_reader(fileobj) return fileobj else: raise Exception('mode must be r/w')
def open(self, mode): """ Open the FileSystem target. This method returns a file-like object which can either be read from or written to depending on the specified mode. :param mode: the mode `r` opens the FileSystemTarget in read-only mode, whereas `w` will open the FileSystemTarget in write mode. Subclasses can implement additional options. :type mode: str """ if mode == 'w': return self.format.pipe_writer(AtomicFtpFile(self._fs, self.path)) elif mode == 'r': temp_dir = os.path.join(tempfile.gettempdir(), 'luigi-contrib-ftp') self.__tmp_path = temp_dir + '/' + self.path.lstrip( '/') + '-luigi-tmp-%09d' % random.randrange(0, 1e10) # download file to local self._fs.get(self.path, self.__tmp_path) return self.format.pipe_reader( FileWrapper(io.BufferedReader(io.FileIO(self.__tmp_path, 'r')))) else: raise Exception("mode must be 'r' or 'w' (got: %s)" % mode)
def open(self, mode='r'): """ """ if mode not in ('r', 'w'): raise ValueError("Unsupported open mode '%s'" % mode) if mode == 'r': s3_key = self.fs.get_key(self.path) if s3_key: fileobj = ReadableS3File(s3_key) if self.format: self._tmp_extract_path = tempfile.mktemp( prefix='luigi_s3_') with open(self._tmp_extract_path, 'w') as f: f.write(fileobj.read()) try: with open(self._tmp_extract_path) as f: return self.format.pipe_reader(FileWrapper(f)) finally: os.remove(self._tmp_extract_path) return fileobj else: raise FileNotFoundException("Could not find file at %s" % self.path) else: if self.format: return self.format.pipe_writer(AtomicS3File( self.path, self.fs)) else: return AtomicS3File(self.path, self.fs)
def open(self, mode='r'): if mode == 'r': return self.format.pipe_reader( FileWrapper(io.BufferedReader(self.fs.download(self.path)))) elif mode == 'w': return self.format.pipe_writer(AtomicGCSFile(self.path, self.fs)) else: raise ValueError("Unsupported open mode '{}'".format(mode))
def open(self, mode='r'): if mode == 'w': self.makedirs() return self.format.pipe_writer(atomic_file(self.path)) elif mode == 'r': fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path, 'r'))) return self.format.pipe_reader(fileobj) else: raise Exception('mode must be r/w')
def open(self, mode='r'): rwmode = mode.replace('b', '').replace('t', '') if rwmode == 'w': self.makedirs() return self.format.pipe_writer(atomic_file(self.path)) elif rwmode == 'r': fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path, mode))) return self.format.pipe_reader(fileobj) else: raise Exception("mode must be 'r' or 'w' (got: %s)" % mode)
def open(self, mode='r'): if mode == 'w': self.makedirs() if self.format: return self.format.pipe_writer(atomic_file(self.path)) else: return atomic_file(self.path) elif mode == 'r': fileobj = FileWrapper(open(self.path, 'r')) if self.format: return self.format.pipe_reader(fileobj) return fileobj else: raise Exception('mode must be r/w')
def open(self, mode='r'): '''Modifying any code in LocalTarget to use self.atomic_provider rather than atomic_file ''' rwmode = mode.replace('b', '').replace('t', '') if rwmode == 'w': self.makedirs() return self.format.pipe_writer(self.atomic_provider(self.path)) elif rwmode == 'r': fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path, mode))) return self.format.pipe_reader(fileobj) else: raise Exception( f"The kwarg `mode` must be 'r' or 'w' (got: `{mode}`)")
def open(self, mode="r"): rwmode = mode.replace("b", "").replace("t", "") if rwmode == "w": self.makedirs() tmp_file_path = self.atomic_provider(self.path) print("tmp_file_path:", tmp_file_path.name) return self.format.pipe_writer(tmp_file_path) elif rwmode == "r": fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path, mode))) return self.format.pipe_reader(fileobj) else: raise Exception("mode must be 'r' or 'w' (got: {})".format(mode))
def open(self, mode='r'): if mode == 'w': # Create folder if it does not exist normpath = os.path.normpath(self.path) parentfolder = os.path.dirname(normpath) if parentfolder and not os.path.exists(parentfolder): os.makedirs(parentfolder) if self.format: return self.format.pipe_writer(atomic_file(self.path)) else: return atomic_file(self.path) elif mode == 'r': fileobj = FileWrapper(open(self.path, 'r')) if self.format: return self.format.pipe_reader(fileobj) return fileobj else: raise Exception('mode must be r/w')
def open(self, mode='r'): # leverage super() my_super = super(LocalTarget, self).__init__(self.path) try: # Modify LocalTarget.open() to use atomic_provider rather than atomic_file rwmode = mode.replace('b', '').replace('t', '') if rwmode == 'w': self.makedirs() ipath = self.format.pipe_writer(self.atomic_provider( self.path)) return ipath elif rwmode == 'r': fileobj = FileWrapper( io.BufferedReader(io.FileIO(self.path, mode))) return self.format.pipe_reader(fileobj) else: raise Exception("mode must be 'r' or 'w' (got: %s)" % mode) except Exception: traceback.print_exc()
def open(self, mode): """ Open the FileSystem target. This method returns a file-like object which can either be read from or written to depending on the specified mode. :param mode: the mode `r` opens the FileSystemTarget in read-only mode, whereas `w` will open the FileSystemTarget in write mode. Subclasses can implement additional options. :type mode: str """ if mode == 'w': return self.format.pipe_writer(AtomicFtpFile(self._fs, self.path)) elif mode == 'r': temppath = '{}-luigi-tmp-{:09d}'.format( self.path.lstrip('/'), random.randrange(0, 1e10) ) try: # store reference to the TemporaryDirectory because it will be removed on GC self.__temp_dir = tempfile.TemporaryDirectory( prefix="luigi-contrib-ftp_" ) except AttributeError: # TemporaryDirectory only available in Python3, use old behaviour in Python2 # this file will not be cleaned up automatically self.__tmp_path = os.path.join( tempfile.gettempdir(), 'luigi-contrib-ftp', temppath ) else: self.__tmp_path = os.path.join(self.__temp_dir.name, temppath) # download file to local self._fs.get(self.path, self.__tmp_path) return self.format.pipe_reader( FileWrapper(io.BufferedReader(io.FileIO(self.__tmp_path, 'r'))) ) else: raise Exception("mode must be 'r' or 'w' (got: %s)" % mode)