def _upload_local_file(self,
local_file,
name,
delete_after_upload=False,
override_filename=None,
override_filename_ext=None,
wait_on_upload=False):
# type: (str, str, bool, Optional[str], Optional[str], bool) -> str
"""
Upload local file and return uri of the uploaded file (uploading in the background)
"""
from clearml.storage import StorageManager
upload_uri = self._task.output_uri or self._task.get_logger(
).get_default_upload_destination()
if not isinstance(local_file, Path):
local_file = Path(local_file)
ev = UploadEvent(
metric='artifacts',
variant=name,
image_data=None,
upload_uri=upload_uri,
local_image_path=local_file.as_posix(),
delete_after_upload=delete_after_upload,
override_filename=override_filename,
override_filename_ext=override_filename_ext,
override_storage_key_prefix=self._get_storage_uri_prefix())
_, uri = ev.get_target_full_upload_uri(upload_uri, quote_uri=False)
# send for upload
# noinspection PyProtectedMember
if wait_on_upload:
StorageManager.upload_file(local_file.as_posix(),
uri,
wait_for_upload=True,
retries=ev.retries)
if delete_after_upload:
try:
os.unlink(local_file.as_posix())
except OSError:
LoggerRoot.get_base_logger().warning(
'Failed removing temporary {}'.format(local_file))
else:
self._task._reporter._report(ev)
_, quoted_uri = ev.get_target_full_upload_uri(upload_uri)
return quoted_uri
def get_local_copy(self,
extract_archive=True,
raise_on_error=False,
force_download=False):
# type: (bool, bool, bool) -> str
"""
:param bool extract_archive: If True and artifact is of type 'archive' (compressed folder)
The returned path will be a temporary folder containing the archive content
:param bool raise_on_error: If True and the artifact could not be downloaded,
raise ValueError, otherwise return None on failure and output log warning.
:param bool force_download: download file from remote even if exists in local cache
:raise: Raises error if local copy not found.
:return: A local path to a downloaded copy of the artifact.
"""
from clearml.storage import StorageManager
local_copy = StorageManager.get_local_copy(
remote_url=self.url,
extract_archive=extract_archive and self.type == 'archive',
name=self.name,
force_download=force_download)
if raise_on_error and local_copy is None:
raise ValueError(
"Could not retrieve a local copy of artifact {}, failed downloading {}"
.format(self.name, self.url))
return local_copy