def log_artifact(self, experiment, artifact, destination=None):
if isinstance(artifact, str):
if os.path.isfile(artifact):
target_name = (os.path.basename(artifact)
if destination is None else destination)
dest_path = self._get_dest_and_ext(target_name)
operation = alpha_operation.UploadFile(
path=dest_path,
ext="",
file_path=os.path.abspath(artifact),
)
elif os.path.isdir(artifact):
for path, file_destination in self._log_dir_artifacts(
artifact, destination):
self.log_artifact(experiment, path, file_destination)
return
else:
raise FileNotFound(artifact)
elif hasattr(artifact, "read"):
if not destination:
raise ValueError("destination is required for IO streams")
dest_path = self._get_dest_and_ext(destination)
data = artifact.read()
content = data.encode("utf-8") if isinstance(data, str) else data
operation = alpha_operation.UploadFileContent(
path=dest_path, ext="", file_content=base64_encode(content))
else:
raise ValueError("Artifact must be a local path or an IO object")
self._execute_upload_operations_with_400_retry(experiment, operation)
def prepare_output_download_reuqest(self, experiment, path):
try:
return self.backend_swagger_client.api.prepareForDownload(
experimentIdentity=experiment.internal_id,
resource='output',
path=path).response().result
except HTTPNotFound:
raise FileNotFound(path)
def validate_notebook_path(path):
if not path.endswith(".ipynb"):
raise InvalidNotebookPath(path)
if not os.path.exists(path):
raise FileNotFound(path)
if not os.path.isfile(path):
raise NotAFile(path)
def log_artifact(self, artifact, destination=None):
"""Save an artifact (file) in experiment storage.
Args:
artifact (:obj:`str` or :obj:`IO object`):
A path to the file in local filesystem or IO object. It can be open
file descriptor or in-memory buffer like `io.StringIO` or `io.BytesIO`.
destination (:obj:`str`, optional, default is ``None``):
A destination path.
If ``None`` is passed, an artifact file name will be used.
Note:
If you use in-memory buffers like `io.StringIO` or `io.BytesIO`, remember that in typical case when you
write to such a buffer, it's current position is set to the end of the stream, so in order to read it's
content, you need to move back it's position to the beginning.
We recommend to call seek(0) on the in-memory buffers before passing it to Neptune.
Additionally, if you provide `io.StringIO`, it will be encoded in 'utf-8' before sent to Neptune.
Raises:
`FileNotFound`: When ``artifact`` file was not found.
`StorageLimitReached`: When storage limit in the project has been reached.
Example:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`:
.. code:: python3
# simple use
experiment.log_artifact('images/wrong_prediction_1.png')
# save file in other directory
experiment.log_artifact('images/wrong_prediction_1.png', 'validation/images/wrong_prediction_1.png')
# save file under different name
experiment.log_artifact('images/wrong_prediction_1.png', 'images/my_image_1.png')
"""
if isinstance(artifact, str):
if os.path.exists(artifact):
target_name = os.path.basename(artifact) if destination is None else destination
upload_entry = UploadEntry(os.path.abspath(artifact), normalize_file_name(target_name))
else:
raise FileNotFound(artifact)
elif hasattr(artifact, 'read'):
if destination is not None:
upload_entry = UploadEntry(artifact, normalize_file_name(destination))
else:
raise ValueError("destination is required for file streams")
else:
raise ValueError("artifact is a local path or an IO object")
upload_to_storage(upload_entries=[upload_entry],
upload_api_fun=self._backend.upload_experiment_output,
upload_tar_api_fun=self._backend.extract_experiment_output,
experiment=self)
def send_artifact(self, artifact):
"""
Raises:
`StorageLimitReached`: When storage limit in the project has been reached.
"""
if not os.path.exists(artifact):
raise FileNotFound(artifact)
upload_to_storage(files_list=[(os.path.abspath(artifact), artifact)],
upload_api_fun=self._client.upload_experiment_output,
upload_tar_api_fun=self._client.extract_experiment_output,
experiment=self)
def get_image_content(image):
if isinstance(image, six.string_types):
if not os.path.exists(image):
raise FileNotFound(image)
with open(image, 'rb') as image_file:
return image_file.read()
elif isinstance(image, Image.Image):
with io.BytesIO() as image_buffer:
image.save(image_buffer, format='PNG')
return image_buffer.getvalue()
raise InvalidChannelValue(expected_type='image',
actual_type=type(image).__name__)
def upload_source_files(self, source_files):
"""
Raises:
`StorageLimitReached`: When storage limit in the project has been reached.
"""
files_list = []
for source_file in source_files:
if not os.path.exists(source_file):
raise FileNotFound(source_file)
files_list.append((os.path.abspath(source_file), source_file))
upload_to_storage(files_list=files_list,
upload_api_fun=self._client.upload_experiment_source,
upload_tar_api_fun=self._client.extract_experiment_source,
experiment=self)
def log_artifact(self, artifact, destination=None):
"""Save an artifact (file) in experiment storage.
Args:
artifact (:obj:`str`): A path to the file in local filesystem.
destination (:obj:`str`, optional, default is ``None``):
A destination path.
If ``None`` is passed, an artifact file name will be used.
Raises:
`FileNotFound`: When ``artifact`` file was not found.
`StorageLimitReached`: When storage limit in the project has been reached.
Example:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`:
.. code:: python3
# simple use
experiment.log_artifact('images/wrong_prediction_1.png')
# save file in other directory
experiment.log_artifact('images/wrong_prediction_1.png', 'validation/images/wrong_prediction_1.png')
# save file under different name
experiment.log_artifact('images/wrong_prediction_1.png', 'images/my_image_1.png')
"""
if not os.path.exists(artifact):
raise FileNotFound(artifact)
target_name = os.path.basename(
artifact) if destination is None else destination
upload_to_storage(
upload_entries=[
UploadEntry(os.path.abspath(artifact),
normalize_file_name(target_name))
],
upload_api_fun=self._backend.upload_experiment_output,
upload_tar_api_fun=self._backend.extract_experiment_output,
experiment=self)
def download_artifact(self, path, destination_dir=None):
"""Download an artifact (file) from the experiment storage.
Download a file indicated by ``path`` from the experiment artifacts and save it in ``destination_dir``.
Args:
path (:obj:`str`): Path to the file to be downloaded.
destination_dir (:obj:`str`):
The directory where the file will be downloaded.
If ``None`` is passed, the file will be downloaded to the current working directory.
Raises:
`NotADirectory`: When ``destination_dir`` is not a directory.
`FileNotFound`: If a path in experiment artifacts does not exist.
Examples:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`.
.. code:: python3
experiment.download_artifact('forest_results.pkl', '/home/user/files/')
"""
if not destination_dir:
destination_dir = os.getcwd()
project_storage_path = "/{exp_id}/output/{file}".format(exp_id=self.id,
file=path)
destination_path = os.path.join(destination_dir,
os.path.basename(path))
if not os.path.exists(destination_dir):
os.makedirs(destination_dir)
elif not os.path.isdir(destination_dir):
raise NotADirectory(destination_dir)
try:
self._backend.download_data(self._project, project_storage_path,
destination_path)
except PathInProjectNotFound:
raise FileNotFound(path)
def get_image_content(image):
if isinstance(image, six.string_types):
if not os.path.exists(image):
raise FileNotFound(image)
with open(image, 'rb') as image_file:
return image_file.read()
elif isinstance(image, numpy.ndarray):
shape = image.shape
if len(shape) == 2:
return _get_pil_image_data(
Image.fromarray(image.astype(numpy.uint8)))
if len(shape) == 3:
if shape[2] == 1:
array2d = numpy.array([[col[0] for col in row]
for row in image])
return _get_pil_image_data(
Image.fromarray(array2d.astype(numpy.uint8)))
if shape[2] in (3, 4):
return _get_pil_image_data(
Image.fromarray(image.astype(numpy.uint8)))
raise ValueError(
"Incorrect size of numpy.ndarray. Should be 2-dimensional or"
"3-dimensional with 3rd dimension of size 1, 3 or 4.")
elif isinstance(image, Image.Image):
return _get_pil_image_data(image)
else:
try:
from matplotlib import figure
if isinstance(image, figure.Figure):
return _get_pil_image_data(_figure_to_pil_image(image))
except ImportError:
pass
raise InvalidChannelValue(expected_type='image',
actual_type=type(image).__name__)
def delete_artifacts(self, path):
"""Removes an artifact(s) (file/directory) from the experiment storage.
Args:
path (:obj:`list` or :obj:`str`): Path or list of paths to remove from the experiment's output
Raises:
`FileNotFound`: If a path in experiment artifacts does not exist.
Examples:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`.
.. code:: python3
experiment.delete_artifacts('forest_results.pkl')
experiment.delete_artifacts(['forest_results.pkl', 'directory'])
experiment.delete_artifacts('')
"""
if path is None:
raise ValueError("path argument must not be None")
paths = path
if not isinstance(path, list):
paths = [path]
for path in paths:
if path is None:
raise ValueError("path argument must not be None")
normalized_path = os.path.normpath(path)
if normalized_path.startswith(".."):
raise ValueError("path to delete must be within project's directory")
if normalized_path == "." or normalized_path == "/" or not normalized_path:
raise ValueError("Cannot delete whole artifacts directory")
try:
for path in paths:
self._backend.rm_data(experiment=self, path=path)
except PathInProjectNotFound:
raise FileNotFound(path)
def get_image_content(image):
if isinstance(image, six.string_types):
if not os.path.exists(image):
raise FileNotFound(image)
with open(image, 'rb') as image_file:
return image_file.read()
elif isinstance(image, numpy.ndarray):
return _get_numpy_as_image(image)
elif isinstance(image, Image.Image):
return _get_pil_image_data(image)
else:
try:
from matplotlib import figure
if isinstance(image, figure.Figure):
return _get_figure_as_image(image)
except ImportError:
pass
try:
from torch import Tensor as TorchTensor
if isinstance(image, TorchTensor):
return _get_numpy_as_image(image.detach().numpy())
except ImportError:
pass
try:
from tensorflow import Tensor as TensorflowTensor
if isinstance(image, TensorflowTensor):
return _get_numpy_as_image(image.numpy())
except ImportError:
pass
raise InvalidChannelValue(expected_type='image', actual_type=type(image).__name__)