def log_artifact(self, artifact, destination=None):
"""Save an artifact (file) in experiment storage.
Args:
artifact (:obj:`str` or :obj:`IO object`):
A path to the file in local filesystem or IO object. It can be open
file descriptor or in-memory buffer like `io.StringIO` or `io.BytesIO`.
destination (:obj:`str`, optional, default is ``None``):
A destination path.
If ``None`` is passed, an artifact file name will be used.
Note:
If you use in-memory buffers like `io.StringIO` or `io.BytesIO`, remember that in typical case when you
write to such a buffer, it's current position is set to the end of the stream, so in order to read it's
content, you need to move back it's position to the beginning.
We recommend to call seek(0) on the in-memory buffers before passing it to Neptune.
Additionally, if you provide `io.StringIO`, it will be encoded in 'utf-8' before sent to Neptune.
Raises:
`FileNotFound`: When ``artifact`` file was not found.
`StorageLimitReached`: When storage limit in the project has been reached.
Example:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`:
.. code:: python3
# simple use
experiment.log_artifact('images/wrong_prediction_1.png')
# save file in other directory
experiment.log_artifact('images/wrong_prediction_1.png', 'validation/images/wrong_prediction_1.png')
# save file under different name
experiment.log_artifact('images/wrong_prediction_1.png', 'images/my_image_1.png')
"""
if isinstance(artifact, str):
if os.path.exists(artifact):
target_name = os.path.basename(artifact) if destination is None else destination
upload_entry = UploadEntry(os.path.abspath(artifact), normalize_file_name(target_name))
else:
raise FileNotFound(artifact)
elif hasattr(artifact, 'read'):
if destination is not None:
upload_entry = UploadEntry(artifact, normalize_file_name(destination))
else:
raise ValueError("destination is required for file streams")
else:
raise ValueError("artifact is a local path or an IO object")
upload_to_storage(upload_entries=[upload_entry],
upload_api_fun=self._backend.upload_experiment_output,
upload_tar_api_fun=self._backend.extract_experiment_output,
experiment=self)
def send_artifact(self, artifact):
"""
Raises:
`StorageLimitReached`: When storage limit in the project has been reached.
"""
if not os.path.exists(artifact):
raise FileNotFound(artifact)
upload_to_storage(files_list=[(os.path.abspath(artifact), artifact)],
upload_api_fun=self._client.upload_experiment_output,
upload_tar_api_fun=self._client.extract_experiment_output,
experiment=self)
def test_upload_small_sources_should_not_generate_warning(self, warning):
# GIVEN
entry = UploadEntry("/tmp/mocked/file", "some_file")
# WHEN
upload_to_storage(upload_entries=[entry],
upload_api_fun=MagicMock(),
upload_tar_api_fun=MagicMock(),
warn_limit=100 * 1024 * 1024)
# THEN
warning.assert_not_called()
def upload_source_files(self, source_files):
"""
Raises:
`StorageLimitReached`: When storage limit in the project has been reached.
"""
files_list = []
for source_file in source_files:
if not os.path.exists(source_file):
raise FileNotFound(source_file)
files_list.append((os.path.abspath(source_file), source_file))
upload_to_storage(files_list=files_list,
upload_api_fun=self._client.upload_experiment_source,
upload_tar_api_fun=self._client.extract_experiment_source,
experiment=self)
def test_upload_large_sources_should_generate_warning(self, warning):
# GIVEN
entry = UploadEntry("/tmp/mocked/file", "some_file")
# WHEN
upload_to_storage(upload_entries=[entry],
upload_api_fun=MagicMock(),
upload_tar_api_fun=MagicMock(),
warn_limit=100 * 1024 * 1024)
# THEN
warning.assert_any_call(
'You are sending %dMB of source code to Neptune. '
'It is pretty uncommon - please make sure it\'s what you wanted.',
101)
warning.assert_any_call(
'%d MB (100%%) of source code was sent to Neptune.', 101)
def log_artifact(self, artifact, destination=None):
"""Save an artifact (file) in experiment storage.
Args:
artifact (:obj:`str`): A path to the file in local filesystem.
destination (:obj:`str`, optional, default is ``None``):
A destination path.
If ``None`` is passed, an artifact file name will be used.
Raises:
`FileNotFound`: When ``artifact`` file was not found.
`StorageLimitReached`: When storage limit in the project has been reached.
Example:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`:
.. code:: python3
# simple use
experiment.log_artifact('images/wrong_prediction_1.png')
# save file in other directory
experiment.log_artifact('images/wrong_prediction_1.png', 'validation/images/wrong_prediction_1.png')
# save file under different name
experiment.log_artifact('images/wrong_prediction_1.png', 'images/my_image_1.png')
"""
if not os.path.exists(artifact):
raise FileNotFound(artifact)
target_name = os.path.basename(
artifact) if destination is None else destination
upload_to_storage(
upload_entries=[
UploadEntry(os.path.abspath(artifact),
normalize_file_name(target_name))
],
upload_api_fun=self._backend.upload_experiment_output,
upload_tar_api_fun=self._backend.extract_experiment_output,
experiment=self)
def _start(self,
upload_source_entries=None,
abort_callback=None,
logger=None,
upload_stdout=True,
upload_stderr=True,
send_hardware_metrics=True,
run_monitoring_thread=True,
handle_uncaught_exceptions=True):
upload_to_storage(
upload_entries=upload_source_entries,
upload_api_fun=self._backend.upload_experiment_source,
upload_tar_api_fun=self._backend.extract_experiment_source,
experiment=self)
self._execution_context.start(
abort_callback=abort_callback,
logger=logger,
upload_stdout=upload_stdout,
upload_stderr=upload_stderr,
send_hardware_metrics=send_hardware_metrics,
run_monitoring_thread=run_monitoring_thread,
handle_uncaught_exceptions=handle_uncaught_exceptions)
