def from_python_std(cls, t_value, mode="wb", format=None):
"""
:param T t_value:
:param Text mode: Read or write mode of the object.
:param Text format:
:rtype: MultiPartBlob
"""
if isinstance(t_value, (str, _six.text_type)):
if _os.path.isdir(t_value):
# TODO: Infer format
blob = cls.create_at_any_location(mode=mode, format=format)
blob._directory = _utils.Directory(t_value)
blob.upload()
else:
blob = cls.create_at_known_location(t_value,
mode=mode,
format=format)
return blob
elif isinstance(t_value, cls):
return t_value
else:
raise _user_exceptions.FlyteTypeException(
type(t_value),
{str, _six.text_type, MultiPartBlob},
received_value=t_value,
additional_msg=
"Unable to create Blob from user-provided value.",
)
def from_python_std(cls, t_value, schema_type=None):
"""
:param T t_value:
:param SchemaType schema_type: [Optional] If specified, we will ensure
:rtype: Schema
"""
if isinstance(t_value, (str, _six.text_type)):
if _os.path.isdir(t_value):
schema = cls.create_at_any_location(schema_type=schema_type)
schema.multipart_blob._directory = _utils.Directory(t_value)
schema.upload()
else:
schema = cls.create_at_known_location(t_value,
schema_type=schema_type)
return schema
elif isinstance(t_value, cls):
return t_value
elif isinstance(t_value, _pd.DataFrame):
# Accepts a pandas dataframe and converts to a Schema object
o = cls.create_at_any_location(schema_type=schema_type)
with o as w:
w.write(t_value)
return o
elif isinstance(t_value, list):
# Accepts a list of pandas dataframe and converts to a Schema object
o = cls.create_at_any_location(schema_type=schema_type)
with o as w:
for x in t_value:
if isinstance(x, _pd.DataFrame):
w.write(x)
else:
raise _user_exceptions.FlyteTypeException(
type(t_value),
{str, _six.text_type, Schema},
received_value=x,
additional_msg=
"A Schema object can only be create from a pandas DataFrame or a list of pandas DataFrame.",
)
return o
else:
raise _user_exceptions.FlyteTypeException(
type(t_value),
{str, _six.text_type, Schema},
received_value=t_value,
additional_msg=
"Unable to create Schema from user-provided value.",
)
def download(self, local_path=None, overwrite=False):
"""
Forces the download of the remote multi-part blob to the local machine.
:param Text local_path: [Optional] If provided, the blob pieces will be downloaded to this path. This will
make the resulting file objects unmanaged and it will not be cleaned up by the system upon exiting the
context.
:param bool overwrite: If true and local_path is specified, we will download the blob pieces and
overwrite any existing files at that location. Default is False.
"""
if "r" not in self.mode:
raise _user_exceptions.FlyteAssertion(
"Cannot download a write-only object!")
if local_path:
self._is_managed = False
elif _data_proxy.LocalWorkingDirectoryContext.get() is None:
raise _user_exceptions.FlyteAssertion(
"No temporary file system is present. Either call this method from within the "
"context of a task or surround with a 'with LocalTestFileSystem():' block. Or "
"specify a path when calling this function. Note: Cleanup is not automatic when a "
"path is specified.")
else:
local_path = _data_proxy.LocalWorkingDirectoryContext.get(
).get_named_tempfile(_uuid.uuid4().hex)
path_exists = _os.path.exists(local_path)
if not path_exists or overwrite:
if path_exists:
_shutil.rmtree(local_path)
_os.makedirs(local_path)
self._directory = _utils.Directory(local_path)
_data_proxy.Data.get_data(self.remote_location,
self.local_path,
is_multipart=True)
else:
raise _user_exceptions.FlyteAssertion(
"Cannot download multi-part blob to a location that already exists when overwrite is not set to True. "
"Attempted download from {} -> {}".format(
self.remote_location, self.local_path))
def from_python_std(cls, t_value, schema_type=None):
"""
:param T t_value:
:param SchemaType schema_type: [Optional] If specified, we will ensure
:rtype: Schema
"""
if isinstance(t_value, (str, _six.text_type)):
if _os.path.isdir(t_value):
schema = cls.create_at_any_location(schema_type=schema_type)
schema.multipart_blob._directory = _utils.Directory(t_value)
schema.upload()
else:
schema = cls.create_at_known_location(t_value,
schema_type=schema_type)
return schema
elif isinstance(t_value, cls):
return t_value
else:
raise _user_exceptions.FlyteTypeException(
type(t_value), {str, _six.text_type, Schema},
received_value=t_value,
additional_msg=
"Unable to create Schema from user-provided value.")