def data_exists(cls, path):
"""
:param Text path:
:rtype: bool: whether the file exists or not
"""
with _common_utils.PerformanceTimer("Check file exists {}".format(path)):
proxy = cls._load_data_proxy_by_path(path)
return proxy.exists(path)
def get_data(self, remote_path: str, local_path: str, is_multipart=False):
"""
:param Text remote_path:
:param Text local_path:
:param bool is_multipart:
"""
try:
with _common_utils.PerformanceTimer("Copying ({} -> {})".format(
remote_path, local_path)):
if is_multipart:
self.download_directory(remote_path, local_path)
else:
self.download(remote_path, local_path)
except Exception as ex:
raise _user_exception.FlyteAssertion(
"Failed to get data from {remote_path} to {local_path} (recursive={is_multipart}).\n\n"
"Original exception: {error_string}".format(
remote_path=remote_path,
local_path=local_path,
is_multipart=is_multipart,
error_string=str(ex),
))
def put_data(cls, local_path, remote_path, is_multipart=False):
"""
:param Text local_path:
:param Text remote_path:
:param bool is_multipart:
"""
try:
with _common_utils.PerformanceTimer("Writing ({} -> {})".format(local_path, remote_path)):
proxy = cls._load_data_proxy_by_path(remote_path)
if is_multipart:
proxy.upload_directory(local_path, remote_path)
else:
proxy.upload(local_path, remote_path)
except Exception as ex:
raise _user_exception.FlyteAssertion(
"Failed to put data from {local_path} to {remote_path} (recursive={is_multipart}).\n\n"
"Original exception: {error_string}".format(
remote_path=remote_path,
local_path=local_path,
is_multipart=is_multipart,
error_string=_six.text_type(ex)
)
)
def put_data(self,
local_path: Union[str, os.PathLike],
remote_path: str,
is_multipart=False):
"""
The implication here is that we're always going to put data to the remote location, so we .remote to ensure
we don't use the true local proxy if the remote path is a file://
:param Text local_path:
:param Text remote_path:
:param bool is_multipart:
"""
try:
with _common_utils.PerformanceTimer("Writing ({} -> {})".format(
local_path, remote_path)):
if is_multipart:
self.remote.upload_directory(local_path, remote_path)
else:
self.remote.upload(local_path, remote_path)
except Exception as ex:
raise _user_exception.FlyteAssertion(
f"Failed to put data from {local_path} to {remote_path} (recursive={is_multipart}).\n\n"
f"Original exception: {str(ex)}") from ex