def _to_proper_value_type(values: Union[str, Iterable[str]]) -> Iterable[str]:
if is_collection(values):
verify_collection_type("values", values, str)
return list(values)
else:
verify_type("values", values, str)
return [values]
def __init__(self, file_globs: Union[str, Iterable[str]]):
verify_type("file_globs", file_globs, (str, Iterable))
if isinstance(file_globs, str):
file_globs = [file_globs]
else:
verify_collection_type("file_globs", file_globs, str)
self.file_globs: List[str] = list(file_globs)
def upload_files(
self, globs: Union[str, Iterable[str]], wait: bool = False
) -> None:
if isinstance(globs, str):
globs = [globs]
else:
verify_collection_type("globs", globs, str)
self._enqueue_upload_operation(globs, reset=False, wait=wait)
def delete_files(
self, paths: Union[str, Iterable[str]], wait: bool = False
) -> None:
if isinstance(paths, str):
paths = [paths]
else:
verify_collection_type("paths", paths, str)
with self._container.lock():
self._enqueue_operation(DeleteFiles(self._path, set(paths)), wait)
def _as_list(
name: str,
value: Optional[Union[str,
Iterable[str]]]) -> Optional[Iterable[str]]:
verify_type(name, value, (type(None), str, Iterable))
if value is None:
return None
if isinstance(value, str):
return [value]
verify_collection_type(name, value, str)
return value
def assign(
self, value: Union[FileSetVal, str, Iterable[str]], wait: bool = False
) -> None:
verify_type("value", value, (FileSetVal, str, Iterable))
if isinstance(value, FileSetVal):
value = value.file_globs
elif isinstance(value, str):
value = [value]
else:
verify_collection_type("value", value, str)
self._enqueue_upload_operation(value, reset=True, wait=wait)
def upload_files(self,
value: Union[str, Iterable[str]],
wait: bool = False) -> None:
if is_collection(value):
verify_collection_type("value", value, str)
else:
verify_type("value", value, str)
with self._run.lock():
attr = self._run.get_attribute(self._path)
if not attr:
attr = FileSet(self._run, parse_path(self._path))
self._run.set_attribute(self._path, attr)
attr.upload_files(value, wait)
def init_run(
project: Optional[str] = None,
api_token: Optional[str] = None,
run: Optional[str] = None,
custom_run_id: Optional[str] = None,
mode: str = Mode.ASYNC.value,
name: Optional[str] = None,
description: Optional[str] = None,
tags: Optional[Union[List[str], str]] = None,
source_files: Optional[Union[List[str], str]] = None,
capture_stdout: bool = True,
capture_stderr: bool = True,
capture_hardware_metrics: bool = True,
fail_on_exception: bool = True,
monitoring_namespace: Optional[str] = None,
flush_period: float = 5,
proxies: Optional[dict] = None,
capture_traceback: bool = True,
**kwargs,
) -> Run:
"""Starts a new tracked run, and append it to the top of the Runs table view.
Args:
project(str, optional): Name of a project in a form of `namespace/project_name`. Defaults to `None`.
If `None`, the value of `NEPTUNE_PROJECT` environment variable will be taken.
api_token(str, optional): User’s API token. Defaults to `None`.
If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken.
.. note::
It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your
API token in plain text in your source code.
run (str, optional): An existing run's identifier like 'SAN-1' in case of resuming a tracked run.
Defaults to `None`.
A run with such identifier must exist. If `None` is passed, starts a new tracked run.
custom_run_id (str, optional): A unique identifier to be used when running Neptune in pipelines.
Defaults to `None`.
Make sure you are using the same identifier throughout the whole pipeline execution.
mode (str, optional): Connection mode in which the tracking will work. Defaults to `'async'`.
Possible values 'async', 'sync', 'offline', 'read-only' and 'debug'.
name (str, optional): Editable name of the run. Defaults to `'Untitled'`.
Name is displayed in the run's Details and in Runs table as a column.
description (str, optional): Editable description of the run. Defaults to `''`.
Description is displayed in the run's Details and can be displayed in the runs view as a column.
tags (list of str or str, optional): Tags of the run. Defaults to `[]`.
They are editable after run is created.
Tags are displayed in the run's Details and can be viewed in Runs table view as a column.
source_files (list of str or str, optional): List of source files to be uploaded.
Uploaded sources are displayed in the run’s Source code tab.
Unix style pathname pattern expansion is supported. For example, you can pass '*.py' to upload all python
source files from the current directory.
If `None` is passed, Python file from which run was created will be uploaded.
capture_stdout (bool, optional): Whether to send run's stdout. Defaults to `True`.
Tracked metadata will be stored inside `monitoring_namespace`.
capture_stderr (bool, optional): Whether to send run’s stderr. Defaults to `True`.
Tracked metadata will be stored inside `monitoring_namespace`.
capture_hardware_metrics (bool, optional): Whether to send hardware monitoring logs
(CPU, GPU, Memory utilization). Defaults to `True`.
Tracked metadata will be stored inside `monitoring_namespace`.
fail_on_exception (bool, optional): Whether to register an uncaught exception handler to this process and,
in case of an exception, set run's sys/failed to True. Exception is always logged
monitoring_namespace (str, optional): Namespace inside which all monitoring logs be stored.
Defaults to 'monitoring'.
flush_period (float, optional): In an asynchronous (default) connection mode how often asynchronous thread
should synchronize data with Neptune servers. Defaults to 5.
proxies (dict of str, optional): Argument passed to HTTP calls made via the Requests library.
For more information see
`their proxies section <https://2.python-requests.org/en/master/user/advanced/#proxies>`_.
capture_traceback (bool, optional): Whether to send run’s traceback in case of an exception.
Defaults to `True`.
Tracked metadata will be stored inside `monitoring/traceback`.
Returns:
``Run``: object that is used to manage the tracked run and log metadata to it.
Examples:
>>> import neptune.new as neptune
>>> # minimal invoke
... run = neptune.init()
>>> # create a tracked run with a name
... run = neptune.init(name='first-pytorch-ever')
>>> # create a tracked run with a name and a description, and no sources files uploaded
>>> run = neptune.init(name='neural-net-mnist',
... description='neural net trained on MNIST',
... source_files=[])
>>> # Send all py files in cwd (excluding hidden files with names beginning with a dot)
... run = neptune.init(source_files='*.py')
>>> # Send all py files from all subdirectories (excluding hidden files with names beginning with a dot)
... # Supported on Python 3.5 and later.
... run = neptune.init(source_files='**/*.py')
>>> # Send all files and directories in cwd (excluding hidden files with names beginning with a dot)
... run = neptune.init(source_files='*')
>>> # Send all files and directories in cwd including hidden files
... run = neptune.init(source_files=['*', '.*'])
>>> # Send files with names being a single character followed by '.py' extension.
... run = neptune.init(source_files='?.py')
>>> # larger example
... run = neptune.init(name='first-pytorch-ever',
... description='write longer description here',
... tags=['list-of', 'tags', 'goes-here', 'as-list-of-strings'],
... source_files=['training_with_pytorch.py', 'net.py'])
You may also want to check `init docs page`_.
.. _init docs page:
https://docs.neptune.ai/api-reference/neptune#.init
"""
_check_for_extra_kwargs(init_run.__name__, kwargs)
verify_type("project", project, (str, type(None)))
verify_type("api_token", api_token, (str, type(None)))
verify_type("run", run, (str, type(None)))
verify_type("custom_run_id", custom_run_id, (str, type(None)))
verify_type("mode", mode, str)
verify_type("name", name, (str, type(None)))
verify_type("description", description, (str, type(None)))
verify_type("capture_stdout", capture_stdout, bool)
verify_type("capture_stderr", capture_stderr, bool)
verify_type("capture_hardware_metrics", capture_hardware_metrics, bool)
verify_type("monitoring_namespace", monitoring_namespace,
(str, type(None)))
verify_type("flush_period", flush_period, (int, float))
verify_type("proxies", proxies, (dict, type(None)))
verify_type("capture_traceback", capture_hardware_metrics, bool)
if tags is not None:
if isinstance(tags, str):
tags = [tags]
else:
verify_collection_type("tags", tags, str)
if source_files is not None:
if isinstance(source_files, str):
source_files = [source_files]
else:
verify_collection_type("source_files", source_files, str)
name = "Untitled" if run is None and name is None else name
description = "" if run is None and description is None else description
hostname = get_hostname() if run is None else None
custom_run_id = custom_run_id or os.getenv(CUSTOM_RUN_ID_ENV_NAME)
monitoring_namespace = (monitoring_namespace
or os.getenv(MONITORING_NAMESPACE) or "monitoring")
if run and custom_run_id:
raise NeptuneRunResumeAndCustomIdCollision()
backend = get_backend(mode, api_token=api_token, proxies=proxies)
if mode == Mode.OFFLINE or mode == Mode.DEBUG:
project = "offline/project-placeholder"
project_obj = project_name_lookup(backend, project)
project = f"{project_obj.workspace}/{project_obj.name}"
if run:
api_run = backend.get_run(project + "/" + run)
else:
if mode == Mode.READ_ONLY:
raise NeedExistingRunForReadOnlyMode()
git_ref = get_git_info(discover_git_repo_location())
if custom_run_id_exceeds_length(custom_run_id):
custom_run_id = None
notebook_id, checkpoint_id = _create_notebook_checkpoint(backend)
api_run = backend.create_run(project_obj.id, git_ref, custom_run_id,
notebook_id, checkpoint_id)
run_lock = threading.RLock()
operation_processor = get_operation_processor(
mode,
container_id=api_run.id,
container_type=Run.container_type,
backend=backend,
lock=run_lock,
flush_period=flush_period,
)
stdout_path = "{}/stdout".format(monitoring_namespace)
stderr_path = "{}/stderr".format(monitoring_namespace)
traceback_path = "{}/traceback".format(monitoring_namespace)
background_jobs = []
if mode != Mode.READ_ONLY:
if capture_stdout:
background_jobs.append(
StdoutCaptureBackgroundJob(attribute_name=stdout_path))
if capture_stderr:
background_jobs.append(
StderrCaptureBackgroundJob(attribute_name=stderr_path))
if capture_hardware_metrics:
background_jobs.append(
HardwareMetricReportingJob(
attribute_namespace=monitoring_namespace))
websockets_factory = backend.websockets_factory(
project_obj.id, api_run.id)
if websockets_factory:
background_jobs.append(
WebsocketSignalsBackgroundJob(websockets_factory))
if capture_traceback:
background_jobs.append(
TracebackJob(traceback_path, fail_on_exception))
background_jobs.append(PingBackgroundJob())
_run = Run(
api_run.id,
backend,
operation_processor,
BackgroundJobList(background_jobs),
run_lock,
api_run.workspace,
api_run.project_name,
api_run.short_id,
project_obj.id,
monitoring_namespace,
)
if mode != Mode.OFFLINE:
_run.sync(wait=False)
if mode != Mode.READ_ONLY:
if name is not None:
_run[attr_consts.SYSTEM_NAME_ATTRIBUTE_PATH] = name
if description is not None:
_run[attr_consts.SYSTEM_DESCRIPTION_ATTRIBUTE_PATH] = description
if hostname is not None:
_run[attr_consts.SYSTEM_HOSTNAME_ATTRIBUTE_PATH] = hostname
if tags is not None:
_run[attr_consts.SYSTEM_TAGS_ATTRIBUTE_PATH].add(tags)
if run is None:
_run[attr_consts.SYSTEM_FAILED_ATTRIBUTE_PATH] = False
if capture_stdout and not _run.exists(stdout_path):
_run.define(stdout_path, StringSeries([]))
if capture_stderr and not _run.exists(stderr_path):
_run.define(stderr_path, StringSeries([]))
if run is None or source_files is not None:
# upload default sources ONLY if creating a new run
upload_source_code(source_files=source_files, run=_run)
# pylint: disable=protected-access
_run._startup(debug_mode=mode == Mode.DEBUG)
return _run
def test_verify_collection_type_failed_element(self):
with self.assertRaises(TypeError):
verify_collection_type("arg", ["string", 3, "a", 4.0, 1],
(int, str))
def test_verify_collection_type_failed(self):
with self.assertRaises(TypeError):
verify_collection_type("arg", "string", (int, str))
def test_verify_collection_type(self):
verify_collection_type("arg", ["string", "aaa", 5, 1, "q"], (int, str))