def __init__(self, path=None, is_tmp=False, **kwargs):
# handle tmp paths manually since luigi uses the env tmp dir
if not path:
if not is_tmp:
raise Exception("either path or is_tmp must be set")
# get the tmp dir from the config and ensure it exists
tmp_dir = os.path.realpath(Config.instance().get_expanded("target", "tmp_dir"))
if not self.fs.exists(tmp_dir):
perm = Config.instance().get("target", "tmp_dir_permission")
self.fs.mkdir(tmp_dir, perm=perm and int(perm))
# create a random path
while True:
path = os.path.join(tmp_dir, "luigi-tmp-%09d" % (random.randint(0, 999999999,)))
if not self.fs.exists(path):
break
# is_tmp might be an extension
if isinstance(is_tmp, six.string_types):
if is_tmp[0] != ".":
is_tmp = "." + is_tmp
path += is_tmp
else:
path = self.fs.abspath(os.path.expandvars(os.path.expanduser(remove_scheme(path))))
luigi.LocalTarget.__init__(self, path=path, is_tmp=is_tmp)
FileSystemTarget.__init__(self, self.path, **kwargs)
def __init__(self, path, fs, exists=None):
if not isinstance(fs, RemoteFileSystem):
raise ValueError("fs is not a RemoteFileSystem instance")
self.fs = fs
self._path = None
FileSystemTarget.__init__(self, path, exists=exists)
def __init__(self, path, fs, **kwargs):
if not isinstance(fs, RemoteFileSystem):
raise TypeError("fs must be a {} instance, is {}".format(RemoteFileSystem, fs))
self.fs = fs
self._path = None
FileSystemTarget.__init__(self, path, **kwargs)
def __init__(self,
path=None,
fs=LocalFileSystem.default_instance,
is_tmp=False,
tmp_dir=None,
**kwargs):
if isinstance(fs, six.string_types):
fs = LocalFileSystem(fs)
# handle tmp paths manually since luigi uses the env tmp dir
if not path:
if not is_tmp:
raise Exception("either path or is_tmp must be set")
# if not set, get the tmp dir from the config and ensure that it exists
cfg = Config.instance()
if tmp_dir:
tmp_dir = get_path(tmp_dir)
else:
tmp_dir = os.path.realpath(
cfg.get_expanded("target", "tmp_dir"))
if not fs.exists(tmp_dir):
perm = cfg.get_expanded_int("target", "tmp_dir_perm")
fs.mkdir(tmp_dir, perm=perm)
# create a random path
while True:
basename = "luigi-tmp-{:09d}".format(
random.randint(0, 999999999))
path = os.path.join(tmp_dir, basename)
if not fs.exists(path):
break
# is_tmp might be a file extension
if isinstance(is_tmp, six.string_types):
if is_tmp[0] != ".":
is_tmp = "." + is_tmp
path += is_tmp
else:
# ensure path is not a target and does not contain, then normalize
path = remove_scheme(get_path(path))
path = fs.abspath(os.path.expandvars(os.path.expanduser(path)))
luigi.LocalTarget.__init__(self, path=path, is_tmp=is_tmp)
FileSystemTarget.__init__(self, self.path, fs=fs, **kwargs)
def __init__(self, path=None, format=None, is_tmp=False, exists=None):
# handle tmp paths manually since luigi uses the env tmp dir
if not path:
if not is_tmp:
raise Exception("path or is_tmp must be set")
base = Config.instance().get("core", "target_tmp_dir")
base = os.path.expandvars(os.path.expanduser(base))
if not self.fs.exists(base):
self.fs.mkdir(base, mode=0o0777)
while True:
path = os.path.join(base, "luigi-tmp-%09d" % random.randint(0, 999999999))
if not os.path.exists(path):
break
luigi.LocalTarget.__init__(self, path=path, format=format, is_tmp=is_tmp)
FileSystemTarget.__init__(self, self.path, exists=exists)
self.path = self.fs.abspath(os.path.expandvars(os.path.expanduser(self.path)))
def _parent_args(self):
args, kwargs = FileSystemTarget._parent_args(self)
args += (self.fs, )
return args, kwargs
def _repr_flags(self):
flags = FileSystemTarget._repr_flags(self)
if self.is_tmp:
flags.append("temporary")
return flags
