def open(self, mode):
"""Open the FileSystem target.
This method returns a file-like object which can either be read from or written to depending
on the specified mode.
:param str mode: the mode `r` opens the FileSystemTarget in read-only mode, whereas `w` will
open the FileSystemTarget in write mode. Subclasses can implement
additional options.
"""
if mode == 'w':
if self.format:
return self.format.pipe_writer(
AtomicFtpfile(self._fs, self.path))
else:
return AtomicFtpfile(self._fs, self.path)
elif mode == 'r':
self.__tmp_path = self.path + '-luigi-tmp-%09d' % random.randrange(
0, 1e10)
# download file to local
self._fs.get(self.path, self.__tmp_path)
# manage tmp file
fileobj = FileWrapper(open(self.__tmp_path, 'r'))
if self.format:
return self.format.pipe_reader(fileobj)
return fileobj
else:
raise Exception('mode must be r/w')
def open(self, mode):
"""
Open the FileSystem target.
This method returns a file-like object which can either be read from or written to depending
on the specified mode.
:param mode: the mode `r` opens the FileSystemTarget in read-only mode, whereas `w` will
open the FileSystemTarget in write mode. Subclasses can implement
additional options.
:type mode: str
"""
if mode == 'w':
return self.format.pipe_writer(AtomicFtpFile(self._fs, self.path))
elif mode == 'r':
temp_dir = os.path.join(tempfile.gettempdir(), 'luigi-contrib-ftp')
self.__tmp_path = temp_dir + '/' + self.path.lstrip(
'/') + '-luigi-tmp-%09d' % random.randrange(0, 1e10)
# download file to local
self._fs.get(self.path, self.__tmp_path)
return self.format.pipe_reader(
FileWrapper(io.BufferedReader(io.FileIO(self.__tmp_path,
'r'))))
else:
raise Exception("mode must be 'r' or 'w' (got: %s)" % mode)
def open(self, mode='r'):
"""
"""
if mode not in ('r', 'w'):
raise ValueError("Unsupported open mode '%s'" % mode)
if mode == 'r':
s3_key = self.fs.get_key(self.path)
if s3_key:
fileobj = ReadableS3File(s3_key)
if self.format:
self._tmp_extract_path = tempfile.mktemp(
prefix='luigi_s3_')
with open(self._tmp_extract_path, 'w') as f:
f.write(fileobj.read())
try:
with open(self._tmp_extract_path) as f:
return self.format.pipe_reader(FileWrapper(f))
finally:
os.remove(self._tmp_extract_path)
return fileobj
else:
raise FileNotFoundException("Could not find file at %s" %
self.path)
else:
if self.format:
return self.format.pipe_writer(AtomicS3File(
self.path, self.fs))
else:
return AtomicS3File(self.path, self.fs)
def open(self, mode='r'):
if mode == 'r':
return self.format.pipe_reader(
FileWrapper(io.BufferedReader(self.fs.download(self.path))))
elif mode == 'w':
return self.format.pipe_writer(AtomicGCSFile(self.path, self.fs))
else:
raise ValueError("Unsupported open mode '{}'".format(mode))
def open(self, mode='r'):
if mode == 'w':
self.makedirs()
return self.format.pipe_writer(atomic_file(self.path))
elif mode == 'r':
fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path, 'r')))
return self.format.pipe_reader(fileobj)
else:
raise Exception('mode must be r/w')
def open(self, mode='r'):
rwmode = mode.replace('b', '').replace('t', '')
if rwmode == 'w':
self.makedirs()
return self.format.pipe_writer(atomic_file(self.path))
elif rwmode == 'r':
fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path, mode)))
return self.format.pipe_reader(fileobj)
else:
raise Exception("mode must be 'r' or 'w' (got: %s)" % mode)
def open(self, mode='r'):
if mode == 'w':
self.makedirs()
if self.format:
return self.format.pipe_writer(atomic_file(self.path))
else:
return atomic_file(self.path)
elif mode == 'r':
fileobj = FileWrapper(open(self.path, 'r'))
if self.format:
return self.format.pipe_reader(fileobj)
return fileobj
else:
raise Exception('mode must be r/w')
def open(self, mode='r'):
'''Modifying any code in LocalTarget to use self.atomic_provider
rather than atomic_file
'''
rwmode = mode.replace('b', '').replace('t', '')
if rwmode == 'w':
self.makedirs()
return self.format.pipe_writer(self.atomic_provider(self.path))
elif rwmode == 'r':
fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path,
mode)))
return self.format.pipe_reader(fileobj)
else:
raise Exception(
f"The kwarg `mode` must be 'r' or 'w' (got: `{mode}`)")
def open(self, mode="r"):
rwmode = mode.replace("b", "").replace("t", "")
if rwmode == "w":
self.makedirs()
tmp_file_path = self.atomic_provider(self.path)
print("tmp_file_path:", tmp_file_path.name)
return self.format.pipe_writer(tmp_file_path)
elif rwmode == "r":
fileobj = FileWrapper(io.BufferedReader(io.FileIO(self.path,
mode)))
return self.format.pipe_reader(fileobj)
else:
raise Exception("mode must be 'r' or 'w' (got: {})".format(mode))
def open(self, mode='r'):
if mode == 'w':
# Create folder if it does not exist
normpath = os.path.normpath(self.path)
parentfolder = os.path.dirname(normpath)
if parentfolder and not os.path.exists(parentfolder):
os.makedirs(parentfolder)
if self.format:
return self.format.pipe_writer(atomic_file(self.path))
else:
return atomic_file(self.path)
elif mode == 'r':
fileobj = FileWrapper(open(self.path, 'r'))
if self.format:
return self.format.pipe_reader(fileobj)
return fileobj
else:
raise Exception('mode must be r/w')
def open(self, mode='r'):
# leverage super()
my_super = super(LocalTarget, self).__init__(self.path)
try:
# Modify LocalTarget.open() to use atomic_provider rather than atomic_file
rwmode = mode.replace('b', '').replace('t', '')
if rwmode == 'w':
self.makedirs()
ipath = self.format.pipe_writer(self.atomic_provider(
self.path))
return ipath
elif rwmode == 'r':
fileobj = FileWrapper(
io.BufferedReader(io.FileIO(self.path, mode)))
return self.format.pipe_reader(fileobj)
else:
raise Exception("mode must be 'r' or 'w' (got: %s)" % mode)
except Exception:
traceback.print_exc()
def open(self, mode):
"""
Open the FileSystem target.
This method returns a file-like object which can either be read from or written to depending
on the specified mode.
:param mode: the mode `r` opens the FileSystemTarget in read-only mode, whereas `w` will
open the FileSystemTarget in write mode. Subclasses can implement
additional options.
:type mode: str
"""
if mode == 'w':
return self.format.pipe_writer(AtomicFtpFile(self._fs, self.path))
elif mode == 'r':
temppath = '{}-luigi-tmp-{:09d}'.format(
self.path.lstrip('/'), random.randrange(0, 1e10)
)
try:
# store reference to the TemporaryDirectory because it will be removed on GC
self.__temp_dir = tempfile.TemporaryDirectory(
prefix="luigi-contrib-ftp_"
)
except AttributeError:
# TemporaryDirectory only available in Python3, use old behaviour in Python2
# this file will not be cleaned up automatically
self.__tmp_path = os.path.join(
tempfile.gettempdir(), 'luigi-contrib-ftp', temppath
)
else:
self.__tmp_path = os.path.join(self.__temp_dir.name, temppath)
# download file to local
self._fs.get(self.path, self.__tmp_path)
return self.format.pipe_reader(
FileWrapper(io.BufferedReader(io.FileIO(self.__tmp_path, 'r')))
)
else:
raise Exception("mode must be 'r' or 'w' (got: %s)" % mode)
