def _to_proper_value_type(values: Union[str, Iterable[str]]) -> Iterable[str]:
if is_collection(values):
verify_collection_type("values", values, str)
return list(values)
else:
verify_type("values", values, str)
return [values]
def log(self,
value: Union[Data, Iterable[Data]],
step: Optional[float] = None,
timestamp: Optional[float] = None,
wait: bool = False,
**kwargs) -> None:
if is_collection(value):
if step is not None and len(value) > 1:
raise ValueError(
"Collection of values are not supported for explicitly defined 'step'."
)
value = self._data_to_value(value, **kwargs)
else:
value = self._data_to_value([value], **kwargs)
if step is not None:
verify_type("step", step, (float, int))
if timestamp is not None:
verify_type("timestamp", timestamp, (float, int))
if not timestamp:
timestamp = time.time()
ops = self._get_log_operations_from_value(value, step, timestamp)
with self._container.lock():
for op in ops:
self._enqueue_operation(op, wait)
def add(self,
values: Union[str, Iterable[str]],
wait: bool = False) -> None:
"""Adds the provided tag or tags to the run's tags.
Args:
values (str or collection of str): Tag or tags to be added.
.. note::
If you want you can use emojis in your tags eg. "Exploration 🧪"
wait (bool, optional): If `True` the client will wait to send all tracked metadata to the server first.
This makes the call synchronous.
Defaults to `False`.
You may also want to check `add docs page`_.
.. _add types docs page:
https://docs.neptune.ai/api-reference/field-types#.add
"""
verify_type("values", values, (str, Iterable))
with self._run.lock():
attr = self._run.get_attribute(self._path)
if not attr:
attr = StringSet(self._run, parse_path(self._path))
self._run.set_attribute(self._path, attr)
attr.add(values, wait)
def __init__(self, file_globs: Union[str, Iterable[str]]):
verify_type("file_globs", file_globs, (str, Iterable))
if isinstance(file_globs, str):
file_globs = [file_globs]
else:
verify_collection_type("file_globs", file_globs, str)
self.file_globs: List[str] = list(file_globs)
def from_stream(stream: IOBase,
seek: Optional[int] = 0,
extension: Optional[str] = None) -> "File":
"""Factory method for creating File value objects directly from binary and text streams.
In the case of text stream, UTF-8 encoding will be used.
Args:
stream (IOBase): Stream to be converted.
seek (int, optional): See IOBase documentation.
Defaults to `0`.
extension (str, optional): Extension of the file created that will be used for interpreting the type
of content for visualization.
If `None` it will be bin for binary stream and txt for text stream.
Defaults to `None`.
Returns:
``File``: value object created from the stream.
You may also want to check `from_stream docs page`_ and `IOBase documentation`_.
.. _from_stream docs page:
https://docs.neptune.ai/api-reference/field-types#.from_stream
.. _IOBase documentation:
https://docs.python.org/3/library/io.html#io.IOBase
"""
verify_type("stream", stream, IOBase)
content, stream_default_ext = get_stream_content(stream, seek)
return File(content=content, extension=extension or stream_default_ext)
def get_project_list(api_token: Optional[str] = None) -> List[str]:
"""Get a list of projects you have access to.
Args:
api_token(str, optional): User’s API token. Defaults to `None`.
If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken.
.. note::
It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your
API token in plain text in your source code.
Returns:
``List[str]``: list of project names of a format 'WORKSPACE/PROJECT'
Examples:
>>> from neptune import management
>>> management.get_project_list()
You may also want to check `management API reference`_.
.. _management API reference:
https://docs.neptune.ai/api-reference/management
"""
verify_type("api_token", api_token, (str, type(None)))
backend_client = _get_backend_client(api_token=api_token)
params = {
"userRelation": "viewerOrHigher",
"sortBy": ["lastViewed"],
**DEFAULT_REQUEST_KWARGS,
}
projects = backend_client.api.listProjects(
**params).response().result.entries
return [
normalize_project_name(name=project.name,
workspace=project.organizationName)
for project in projects
]
def assign(self, value: StringSetVal, wait: bool = False):
verify_type("value", value, StringSetVal)
with self._container.lock():
if not value.values:
self._enqueue_operation(ClearStringSet(self._path), wait=wait)
else:
self._enqueue_operation(ClearStringSet(self._path), wait=False)
self._enqueue_operation(AddStrings(self._path, value.values), wait=wait)
def to_domain(role: str) -> str:
verify_type("role", role, str)
return {
ProjectMemberRoleDTO.VIEWER.value: ProjectMemberRole.VIEWER,
ProjectMemberRoleDTO.MANAGER.value: ProjectMemberRole.OWNER,
ProjectMemberRoleDTO.MEMBER.value: ProjectMemberRole.CONTRIBUTOR,
}.get(role)
def get_project(self, project_id: str) -> Project:
verify_type("project_id", project_id, str)
project_spec = re.search(PROJECT_QUALIFIED_NAME_PATTERN, project_id)
workspace, name = project_spec["workspace"], project_spec["project"]
try:
if not workspace:
available_projects = list(
filter(
lambda p: p.name == name,
self.get_available_projects(search_term=name),
)
)
if len(available_projects) == 1:
project = available_projects[0]
project_id = f"{project.workspace}/{project.name}"
elif len(available_projects) > 1:
raise ProjectNameCollision(
project_id=project_id, available_projects=available_projects
)
else:
raise ProjectNotFound(
project_id=project_id,
available_projects=self.get_available_projects(),
available_workspaces=self.get_available_workspaces(),
)
response = self.backend_client.api.getProject(
projectIdentifier=project_id,
**DEFAULT_REQUEST_KWARGS,
).response()
project = response.result
project_version = project.version if hasattr(project, "version") else 1
if project_version < 2:
raise NeptuneLegacyProjectException(project_id)
return Project(project.id, project.name, project.organizationName)
except HTTPNotFound:
available_workspaces = self.get_available_workspaces()
if workspace and not list(
filter(lambda aw: aw.name == workspace, available_workspaces)
):
# Could not found specified workspace, forces listing all projects
raise ProjectNotFound(
project_id=project_id,
available_projects=self.get_available_projects(),
available_workspaces=available_workspaces,
)
else:
raise ProjectNotFound(
project_id=project_id,
available_projects=self.get_available_projects(
workspace_id=workspace
),
)
def from_str(cls, visibility: str) -> "ProjectVisibilityDTO":
verify_type("visibility", visibility, str)
try:
return {
ProjectVisibility.PRIVATE: ProjectVisibilityDTO.PRIVATE,
ProjectVisibility.PUBLIC: ProjectVisibilityDTO.PUBLIC,
}[visibility]
except KeyError as e:
raise UnsupportedValue(enum=cls.__name__, value=visibility) from e
def create_run(
self,
project_id: str,
git_ref: Optional[GitRef] = None,
custom_run_id: Optional[str] = None,
notebook_id: Optional[str] = None,
checkpoint_id: Optional[str] = None,
) -> ApiRun:
verify_type("project_id", project_id, str)
git_info = (
{
"commit": {
"commitId": git_ref.commit_id,
"message": git_ref.message,
"authorName": git_ref.author_name,
"authorEmail": git_ref.author_email,
"commitDate": git_ref.commit_date,
},
"repositoryDirty": git_ref.dirty,
"currentBranch": git_ref.branch,
"remotes": git_ref.remotes,
}
if git_ref
else None
)
params = {
"projectIdentifier": project_id,
"cliVersion": str(neptune_client_version),
"gitInfo": git_info,
"customId": custom_run_id,
}
if notebook_id is not None and checkpoint_id is not None:
params["notebookId"] = notebook_id if notebook_id is not None else None
params["checkpointId"] = (
checkpoint_id if checkpoint_id is not None else None
)
kwargs = {
"experimentCreationParams": params,
"X-Neptune-CliVersion": str(neptune_client_version),
**DEFAULT_REQUEST_KWARGS,
}
try:
run = (
self.leaderboard_client.api.createExperiment(**kwargs).response().result
)
return ApiRun(
run.id, run.shortId, run.organizationName, run.projectName, run.trashed
)
except HTTPNotFound:
raise ProjectNotFound(project_id=project_id)
def from_str(cls, role: str) -> "ProjectMemberRoleDTO":
verify_type("role", role, str)
try:
return {
ProjectMemberRole.VIEWER: ProjectMemberRoleDTO.VIEWER,
ProjectMemberRole.CONTRIBUTOR: ProjectMemberRoleDTO.MEMBER,
ProjectMemberRole.OWNER: ProjectMemberRoleDTO.MANAGER,
}[role]
except KeyError as e:
raise UnsupportedValue(enum=cls.__name__, value=role) from e
def assign(
self, value: Union[FileSetVal, str, Iterable[str]], wait: bool = False
) -> None:
verify_type("value", value, (FileSetVal, str, Iterable))
if isinstance(value, FileSetVal):
value = value.file_globs
elif isinstance(value, str):
value = [value]
else:
verify_collection_type("value", value, str)
self._enqueue_upload_operation(value, reset=True, wait=wait)
def _as_list(
name: str,
value: Optional[Union[str,
Iterable[str]]]) -> Optional[Iterable[str]]:
verify_type(name, value, (type(None), str, Iterable))
if value is None:
return None
if isinstance(value, str):
return [value]
verify_collection_type(name, value, str)
return value
def assign(self,
value: typing.Union[DatetimeVal, datetime],
wait: bool = False):
verify_type("value", value, (DatetimeVal, datetime))
if isinstance(value, DatetimeVal):
value = value.value
else:
value = value.replace(microsecond=1000 *
int(value.microsecond / 1000))
with self._container.lock():
self._enqueue_operation(
self.create_assignment_operation(self._path, value), wait)
def upload_files(self,
value: Union[str, Iterable[str]],
wait: bool = False) -> None:
if is_collection(value):
verify_collection_type("value", value, str)
else:
verify_type("value", value, str)
with self._run.lock():
attr = self._run.get_attribute(self._path)
if not attr:
attr = FileSet(self._run, parse_path(self._path))
self._run.set_attribute(self._path, attr)
attr.upload_files(value, wait)
def __init__(self, run: 'neptune.Run', base_namespace=""):
expect_not_an_experiment(run)
verify_type("run", run, neptune.Run)
verify_type("base_namespace", base_namespace, str)
if base_namespace and not base_namespace.endswith("/"):
base_namespace += "/"
self._run = run
self._base_namespace = base_namespace
self.params_logged = False
self.feature_names_logged = False
self._run[INTEGRATION_VERSION_KEY] = __version__
def init_project(
*,
name: Optional[str] = None,
api_token: Optional[str] = None,
mode: str = Mode.ASYNC.value,
flush_period: float = 5,
proxies: Optional[dict] = None,
) -> Project:
verify_type("name", name, (str, type(None)))
verify_type("api_token", api_token, (str, type(None)))
verify_type("mode", mode, str)
verify_type("flush_period", flush_period, (int, float))
verify_type("proxies", proxies, (dict, type(None)))
if mode == Mode.OFFLINE:
raise NeptuneException("Project can't be initialized in OFFLINE mode")
backend = get_backend(mode, api_token=api_token, proxies=proxies)
project_obj = project_name_lookup(backend, name)
project_lock = threading.RLock()
operation_processor = get_operation_processor(
mode,
container_id=project_obj.id,
container_type=Project.container_type,
backend=backend,
lock=project_lock,
flush_period=flush_period,
)
background_jobs = []
project = Project(
project_obj.id,
backend,
operation_processor,
BackgroundJobList(background_jobs),
project_lock,
project_obj.workspace,
project_obj.name,
)
if mode != Mode.OFFLINE:
project.sync(wait=False)
# pylint: disable=protected-access
project._startup(debug_mode=mode == Mode.DEBUG)
return project
def project_name_lookup(backend: NeptuneBackend,
name: Optional[str] = None) -> Project:
verify_type("name", name, (str, type(None)))
if not name:
name = os.getenv(PROJECT_ENV_NAME)
if not name:
available_workspaces = backend.get_available_workspaces()
available_projects = backend.get_available_projects()
raise NeptuneMissingProjectNameException(
available_workspaces=available_workspaces,
available_projects=available_projects,
)
return backend.get_project(name)
def stop(self, seconds: Optional[Union[float, int]] = None) -> None:
verify_type("seconds", seconds, (float, int, type(None)))
if self._state != ContainerState.STARTED:
return
self._state = ContainerState.STOPPING
ts = time.time()
click.echo("Shutting down background jobs, please wait a moment...")
self._bg_job.stop()
self._bg_job.join(seconds)
click.echo("Done!")
with self._lock:
sec_left = None if seconds is None else seconds - (time.time() -
ts)
self._op_processor.stop(sec_left)
self._backend.close()
self._state = ContainerState.STOPPED
def assign(self, value: FileVal, wait: bool = False) -> None:
verify_type("value", value, FileVal)
if value.path is not None:
operation = UploadFile(self._path,
ext=value.extension,
file_path=os.path.abspath(value.path))
elif value.content is not None:
operation = UploadFileContent(
self._path,
ext=value.extension,
file_content=base64_encode(value.content),
)
else:
raise ValueError("File path and content are None")
with self._container.lock():
self._enqueue_operation(operation, wait)
def __init__(self, run: Run, base_namespace: Optional[str] = None):
super().__init__()
expect_not_an_experiment(run)
verify_type('run', run, Run)
verify_type('base_namespace', base_namespace, (str, type(None)))
self._base_namespace = ''
if base_namespace:
if base_namespace.endswith("/"):
self._base_namespace = base_namespace[:-1]
else:
self._base_namespace = base_namespace
if self._base_namespace:
self._metric_logger = run[self._base_namespace]
else:
self._metric_logger = run
run[INTEGRATION_VERSION_KEY] = __version__
def remove_project_member(
name: str,
username: str,
workspace: Optional[str] = None,
api_token: Optional[str] = None,
):
"""Removes member from the Neptune project.
Args:
name(str): The name of the project in Neptune in the format 'WORKSPACE/PROJECT'.
If workspace argument was set, it should only contain 'PROJECT' instead of 'WORKSPACE/PROJECT'.
username(str): name of the user you want to remove from the project.
workspace(str, optional): Name of your Neptune workspace.
If you specify change the format of the name argument to 'PROJECT' instead of 'WORKSPACE/PROJECT'.
If 'None' it will be parsed from the name argument.
api_token(str, optional): User’s API token. Defaults to `None`.
If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken.
.. note::
It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your
API token in plain text in your source code.
Examples:
>>> from neptune import management
>>> management.remove_project_member(name="awesome-team/amazing-project",
... username="******")
You may also want to check `management API reference`_.
.. _management API reference:
https://docs.neptune.ai/api-reference/management
"""
verify_type("name", name, str)
verify_type("username", username, str)
verify_type("workspace", workspace, (str, type(None)))
verify_type("api_token", api_token, (str, type(None)))
backend_client = _get_backend_client(api_token=api_token)
project_identifier = normalize_project_name(name=name, workspace=workspace)
params = {
"projectIdentifier": project_identifier,
"userId": username,
**DEFAULT_REQUEST_KWARGS,
}
try:
backend_client.api.deleteProjectMember(**params).response()
except HTTPNotFound as e:
raise ProjectNotFound(name=project_identifier) from e
except HTTPUnprocessableEntity as e:
raise UserNotExistsOrWithoutAccess(user=username,
project=project_identifier) from e
except HTTPForbidden as e:
raise AccessRevokedOnMemberRemoval(user=username,
project=project_identifier) from e
def __init__(
self,
path: Optional[str] = None,
content: Optional[bytes] = None,
extension: Optional[str] = None,
):
verify_type("path", path, (str, type(None)))
verify_type("content", content, (bytes, type(None)))
verify_type("extension", extension, (str, type(None)))
if path is not None and content is not None:
raise ValueError("path and content are mutually exclusive")
if path is None and content is None:
raise ValueError("path or content is required")
self.path = path
self.content = content
if extension is None and path is not None:
try:
ext = os.path.splitext(path)[1]
self.extension = ext[1:] if ext else ""
except ValueError:
self.extension = ""
else:
self.extension = extension or ""
def get_workspace_member_list(name: str,
api_token: Optional[str] = None
) -> Dict[str, str]:
"""Get a list of members of a workspace.
Args:
name(str, optional): Name of your Neptune workspace.
api_token(str, optional): User’s API token. Defaults to `None`.
If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken.
.. note::
It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your
API token in plain text in your source code.
Returns:
``Dict[str, str]``: Dictionary with usernames as keys and `WorkspaceMemberRole` ('member', 'admin') as values.
Examples:
>>> from neptune import management
>>> management.get_workspace_member_list(name="awesome-team")
You may also want to check `management API reference`_.
.. _management API reference:
https://docs.neptune.ai/api-reference/management
"""
verify_type("name", name, str)
verify_type("api_token", api_token, (str, type(None)))
backend_client = _get_backend_client(api_token=api_token)
params = {"organizationIdentifier": name, **DEFAULT_REQUEST_KWARGS}
try:
result = backend_client.api.listOrganizationMembers(
**params).response().result
return {
f"{m.registeredMemberInfo.username}":
WorkspaceMemberRoleDTO.to_domain(m.role)
for m in result
}
except HTTPNotFound as e:
raise WorkspaceNotFound(workspace=name) from e
def __init__(self,
run,
base_namespace="training",
log_model=True,
log_importance=True,
max_num_features=None,
log_tree=None,
tree_figsize=30):
expect_not_an_experiment(run)
verify_type("run", run, neptune.Run)
verify_type("base_namespace", base_namespace, str)
log_model is not None and verify_type("log_model", log_model, bool)
log_importance is not None and verify_type("log_importance", log_importance, bool)
max_num_features is not None and verify_type("max_num_features", max_num_features, int)
log_tree is not None and verify_type("log_tree", log_tree, list)
verify_type("tree_figsize", tree_figsize, int)
self.run = run[base_namespace]
self.log_model = log_model
self.log_importance = log_importance
self.max_num_features = max_num_features
self.log_tree = log_tree
self.cv = False
self.tree_figsize = tree_figsize
if self.log_tree:
try:
subprocess.call(["dot", "-V"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except OSError:
self.log_tree = None
message = "Graphviz executables not found, so trees will not be logged. " \
"Make sure the Graphviz executables are on your systems' PATH"
warnings.warn(message)
run[INTEGRATION_VERSION_KEY] = __version__
def __init__(self, *, run: Run, level=logging.NOTSET, path: str = None):
verify_type("run", run, Run)
verify_type("level", level, int)
if path is None:
path = f"{run.monitoring_namespace}/python_logger"
verify_type("path", path, str)
super().__init__(level=level)
self._run = run
self._logger = Logger(run, path)
self._thread_local = threading.local()
self._run[INTEGRATION_VERSION_KEY] = neptune_client_version
def configure(
self,
min: Optional[Union[float, int]] = None,
max: Optional[Union[float, int]] = None,
unit: Optional[str] = None,
wait: bool = False,
) -> None:
verify_type("min", min, (float, int))
verify_type("max", max, (float, int))
verify_type("unit", unit, str)
with self._container.lock():
self._enqueue_operation(
ConfigFloatSeries(self._path, min, max, unit), wait)
def get_project_member_list(name: str,
workspace: Optional[str] = None,
api_token: Optional[str] = None) -> Dict[str, str]:
"""Get a list of members for a project.
Args:
name(str): The name of the project in Neptune in the format 'WORKSPACE/PROJECT'.
If workspace argument was set it should only contain 'PROJECT' instead of 'WORKSPACE/PROJECT'.
workspace(str, optional): Name of your Neptune workspace.
If you specify change the format of the name argument to 'PROJECT' instead of 'WORKSPACE/PROJECT'.
If 'None' it will be parsed from the name argument.
api_token(str, optional): User’s API token. Defaults to `None`.
If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken.
.. note::
It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your
API token in plain text in your source code.
Returns:
``Dict[str, str]``: Dictionary with usernames as keys and ProjectMemberRoles
('owner', 'contributor', 'viewer') as values.
Examples:
>>> from neptune import management
>>> management.get_project_member_list(name="awesome-team/amazing-project")
You may also want to check `management API reference`_.
.. _management API reference:
https://docs.neptune.ai/api-reference/management
"""
verify_type("name", name, str)
verify_type("workspace", workspace, (str, type(None)))
verify_type("api_token", api_token, (str, type(None)))
backend_client = _get_backend_client(api_token=api_token)
project_identifier = normalize_project_name(name=name, workspace=workspace)
params = {
"projectIdentifier": project_identifier,
**DEFAULT_REQUEST_KWARGS
}
try:
result = backend_client.api.listProjectMembers(
**params).response().result
return {
f"{m.registeredMemberInfo.username}":
ProjectMemberRoleDTO.to_domain(m.role)
for m in result
}
except HTTPNotFound as e:
raise ProjectNotFound(name=project_identifier) from e
def pop(self, path: str = None, wait: bool = False) -> None:
if path:
verify_type("path", path, str)
self._run.pop(join_paths(self._path, path), wait)
else:
self._run.pop(self._path, wait)
