def __init__(self, path, format=None, client=None):
super(S3Target, self).__init__(path)
if format is None:
format = get_default_format()
self.format = format
self.fs = client or S3Client()
def __init__(self, path, client=None, format=None):
super(WebHdfsTarget, self).__init__(path)
path = self.path
self.fs = client or WebHdfsClient()
if format is None:
format = get_default_format()
self.format = format
def __init__(self, path, format=None, client=None, **kwargs):
super(S3Target, self).__init__(path)
if format is None:
format = get_default_format()
self.path = path
self.format = format
self.fs = client or S3Client()
self.s3_options = kwargs
def __init__(self, path, format=None, client=None, **kwargs):
super(S3Target, self).__init__(path)
if format is None:
format = get_default_format()
self.path = path
self.format = format
self.fs = client or S3Client()
self.s3_options = kwargs
def __init__(self, path, format=None, client=None):
super(S3Target, self).__init__(path)
if format is None:
format = get_default_format()
# Allow to write unicode in file for retrocompatibility
if sys.version_info[:2] <= (2, 6):
format = format >> MixedUnicodeBytes
self.format = format
self.fs = client or S3Client()
def __init__(self, fn, is_tmp=None, mirror_on_stderr=False, format=None):
self._mirror_on_stderr = mirror_on_stderr
self.path = fn
if format is None:
format = get_default_format()
# Allow to write unicode in file for retrocompatibility
if six.PY2:
format = format >> MixedUnicodeBytes
self.format = format
def __init__(self, fn, is_tmp=None, mirror_on_stderr=False, format=None):
self._mirror_on_stderr = mirror_on_stderr
self.path = fn
if format is None:
format = get_default_format()
# Allow to write unicode in file for retrocompatibility
if six.PY2:
format = format >> MixedUnicodeBytes
self.format = format
def __init__(self, path, format=None, client=None):
super(S3Target, self).__init__(path)
if format is None:
format = get_default_format()
# Allow to write unicode in file for retrocompatibility
if sys.version_info[:2] <= (2, 6):
format = format >> MixedUnicodeBytes
self.format = format
self.fs = client or S3Client()
def __init__(self, path=None, format=None, is_tmp=False):
if format is None:
format = get_default_format()
if not path:
if not is_tmp:
raise Exception('path or is_tmp must be set')
path = os.path.join(tempfile.gettempdir(), 'luigi-tmp-%09d' % random.randint(0, 999999999))
super(LocalTarget, self).__init__(path)
self.format = format
self.is_tmp = is_tmp
def __init__(self, path, format=None, gcs_filesystem=None):
super(GcsTarget, self).__init__(path)
if format is None:
format = get_default_format()
# Allow to write unicode in file for retrocompatibility
if sys.version_info[:2] <= (2, 6):
format = format >> MixedUnicodeBytes
self.format = format
self.fs = gcs_filesystem or GcsFileSystem()
def __init__(self, path=None, format=None, is_tmp=False):
if format is None:
format = get_default_format()
if not path:
if not is_tmp:
raise Exception('path or is_tmp must be set')
path = os.path.join(tempfile.gettempdir(), 'luigi-tmp-%09d' % random.randint(0, 999999999))
super(LocalTarget, self).__init__(path)
self.format = format
self.is_tmp = is_tmp
def __init__(self, path=None, format=None, is_tmp=False):
if format is None:
format = get_default_format()
# Allow to write unicode in file for retrocompatibility
if sys.version_info[:2] <= (2, 6):
format = format >> MixedUnicodeBytes
if not path:
if not is_tmp:
raise Exception('path or is_tmp must be set')
path = os.path.join(tempfile.gettempdir(), 'luigi-tmp-%09d' % random.randint(0, 999999999))
super(LocalTarget, self).__init__(path)
self.format = format
self.is_tmp = is_tmp
def __init__(self, path=None, format=None, is_tmp=False):
if format is None:
format = get_default_format()
# Allow to write unicode in file for retrocompatibility
if sys.version_info[:2] <= (2, 6):
format = format >> MixedUnicodeBytes
if not path:
if not is_tmp:
raise Exception('path or is_tmp must be set')
path = os.path.join(tempfile.gettempdir(), 'luigi-tmp-%09d' % random.randint(0, 999999999))
super(LocalTarget, self).__init__(path)
self.format = format
self.is_tmp = is_tmp
def __init__(self, path, format=None, client=None, flag="_SUCCESS"):
"""
Initializes a S3FlagTarget.
:param path: the directory where the files are stored.
:type path: str
:param client:
:type client:
:param flag:
:type flag: str
"""
if format is None:
format = get_default_format()
if path[-1] != "/":
raise ValueError("S3FlagTarget requires the path to be to a " "directory. It must end with a slash ( / ).")
super(S3FlagTarget, self).__init__(path, format, client)
self.flag = flag
def __init__(self, path, format=None, client=None, flag='_SUCCESS'):
"""
Initializes a S3FlagTarget.
:param path: the directory where the files are stored.
:type path: str
:param client:
:type client:
:param flag:
:type flag: str
"""
if format is None:
format = get_default_format()
if path[-1] != "/":
raise ValueError("S3FlagTarget requires the path to be to a "
"directory. It must end with a slash ( / ).")
super(S3FlagTarget, self).__init__(path, format, client)
self.flag = flag
def __init__(self,
path,
format=None,
client=None,
remote_temp_write=False,
boto3_session_kwargs=None,
**kwargs):
"""
Creates an S3Target that can be opened for reading or writing.
Setting remote_temp_write to True will place the temporary file on S3
instead of the local file system. Requires boto3 >=1.4.0.
"""
super(S3Target, self).__init__(path)
if format is None:
format = get_default_format()
self.path = path
self.format = format
self.fs = client or S3Client()
self.s3_options = kwargs
self.remote_temp_write = remote_temp_write
self.boto3_session_kwargs = boto3_session_kwargs
def __init__(self, path, format=None, client=None):
super(S3Target, self).__init__(path)
if format is None:
format = get_default_format()
self.format = format
self.fs = client or S3Client()
def __init__(self, path=None, format=None, is_tmp=False):
if format is None:
format = get_default_format()
super(LocalTarget, self).__init__(path)
self.format = format
def __init__(self, fn, is_tmp=None, mirror_on_stderr=False, format=None):
self._mirror_on_stderr = mirror_on_stderr
self.path = fn
self.format = format or get_default_format()
def __init__(self, fn, is_tmp=None, mirror_on_stderr=False, format=None):
self._mirror_on_stderr = mirror_on_stderr
self._fn = fn
if format is None:
format = get_default_format()
self.format = format
