def _to_proper_value_type(values: Union[str, Iterable[str]]) -> Iterable[str]: if is_collection(values): verify_collection_type("values", values, str) return list(values) else: verify_type("values", values, str) return [values]
def log(self, value: Union[Data, Iterable[Data]], step: Optional[float] = None, timestamp: Optional[float] = None, wait: bool = False, **kwargs) -> None: if is_collection(value): if step is not None and len(value) > 1: raise ValueError( "Collection of values are not supported for explicitly defined 'step'." ) value = self._data_to_value(value, **kwargs) else: value = self._data_to_value([value], **kwargs) if step is not None: verify_type("step", step, (float, int)) if timestamp is not None: verify_type("timestamp", timestamp, (float, int)) if not timestamp: timestamp = time.time() ops = self._get_log_operations_from_value(value, step, timestamp) with self._container.lock(): for op in ops: self._enqueue_operation(op, wait)
def add(self, values: Union[str, Iterable[str]], wait: bool = False) -> None: """Adds the provided tag or tags to the run's tags. Args: values (str or collection of str): Tag or tags to be added. .. note:: If you want you can use emojis in your tags eg. "Exploration 🧪" wait (bool, optional): If `True` the client will wait to send all tracked metadata to the server first. This makes the call synchronous. Defaults to `False`. You may also want to check `add docs page`_. .. _add types docs page: https://docs.neptune.ai/api-reference/field-types#.add """ verify_type("values", values, (str, Iterable)) with self._run.lock(): attr = self._run.get_attribute(self._path) if not attr: attr = StringSet(self._run, parse_path(self._path)) self._run.set_attribute(self._path, attr) attr.add(values, wait)
def __init__(self, file_globs: Union[str, Iterable[str]]): verify_type("file_globs", file_globs, (str, Iterable)) if isinstance(file_globs, str): file_globs = [file_globs] else: verify_collection_type("file_globs", file_globs, str) self.file_globs: List[str] = list(file_globs)
def from_stream(stream: IOBase, seek: Optional[int] = 0, extension: Optional[str] = None) -> "File": """Factory method for creating File value objects directly from binary and text streams. In the case of text stream, UTF-8 encoding will be used. Args: stream (IOBase): Stream to be converted. seek (int, optional): See IOBase documentation. Defaults to `0`. extension (str, optional): Extension of the file created that will be used for interpreting the type of content for visualization. If `None` it will be bin for binary stream and txt for text stream. Defaults to `None`. Returns: ``File``: value object created from the stream. You may also want to check `from_stream docs page`_ and `IOBase documentation`_. .. _from_stream docs page: https://docs.neptune.ai/api-reference/field-types#.from_stream .. _IOBase documentation: https://docs.python.org/3/library/io.html#io.IOBase """ verify_type("stream", stream, IOBase) content, stream_default_ext = get_stream_content(stream, seek) return File(content=content, extension=extension or stream_default_ext)
def get_project_list(api_token: Optional[str] = None) -> List[str]: """Get a list of projects you have access to. Args: api_token(str, optional): User’s API token. Defaults to `None`. If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken. .. note:: It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your API token in plain text in your source code. Returns: ``List[str]``: list of project names of a format 'WORKSPACE/PROJECT' Examples: >>> from neptune import management >>> management.get_project_list() You may also want to check `management API reference`_. .. _management API reference: https://docs.neptune.ai/api-reference/management """ verify_type("api_token", api_token, (str, type(None))) backend_client = _get_backend_client(api_token=api_token) params = { "userRelation": "viewerOrHigher", "sortBy": ["lastViewed"], **DEFAULT_REQUEST_KWARGS, } projects = backend_client.api.listProjects( **params).response().result.entries return [ normalize_project_name(name=project.name, workspace=project.organizationName) for project in projects ]
def assign(self, value: StringSetVal, wait: bool = False): verify_type("value", value, StringSetVal) with self._container.lock(): if not value.values: self._enqueue_operation(ClearStringSet(self._path), wait=wait) else: self._enqueue_operation(ClearStringSet(self._path), wait=False) self._enqueue_operation(AddStrings(self._path, value.values), wait=wait)
def to_domain(role: str) -> str: verify_type("role", role, str) return { ProjectMemberRoleDTO.VIEWER.value: ProjectMemberRole.VIEWER, ProjectMemberRoleDTO.MANAGER.value: ProjectMemberRole.OWNER, ProjectMemberRoleDTO.MEMBER.value: ProjectMemberRole.CONTRIBUTOR, }.get(role)
def get_project(self, project_id: str) -> Project: verify_type("project_id", project_id, str) project_spec = re.search(PROJECT_QUALIFIED_NAME_PATTERN, project_id) workspace, name = project_spec["workspace"], project_spec["project"] try: if not workspace: available_projects = list( filter( lambda p: p.name == name, self.get_available_projects(search_term=name), ) ) if len(available_projects) == 1: project = available_projects[0] project_id = f"{project.workspace}/{project.name}" elif len(available_projects) > 1: raise ProjectNameCollision( project_id=project_id, available_projects=available_projects ) else: raise ProjectNotFound( project_id=project_id, available_projects=self.get_available_projects(), available_workspaces=self.get_available_workspaces(), ) response = self.backend_client.api.getProject( projectIdentifier=project_id, **DEFAULT_REQUEST_KWARGS, ).response() project = response.result project_version = project.version if hasattr(project, "version") else 1 if project_version < 2: raise NeptuneLegacyProjectException(project_id) return Project(project.id, project.name, project.organizationName) except HTTPNotFound: available_workspaces = self.get_available_workspaces() if workspace and not list( filter(lambda aw: aw.name == workspace, available_workspaces) ): # Could not found specified workspace, forces listing all projects raise ProjectNotFound( project_id=project_id, available_projects=self.get_available_projects(), available_workspaces=available_workspaces, ) else: raise ProjectNotFound( project_id=project_id, available_projects=self.get_available_projects( workspace_id=workspace ), )
def from_str(cls, visibility: str) -> "ProjectVisibilityDTO": verify_type("visibility", visibility, str) try: return { ProjectVisibility.PRIVATE: ProjectVisibilityDTO.PRIVATE, ProjectVisibility.PUBLIC: ProjectVisibilityDTO.PUBLIC, }[visibility] except KeyError as e: raise UnsupportedValue(enum=cls.__name__, value=visibility) from e
def create_run( self, project_id: str, git_ref: Optional[GitRef] = None, custom_run_id: Optional[str] = None, notebook_id: Optional[str] = None, checkpoint_id: Optional[str] = None, ) -> ApiRun: verify_type("project_id", project_id, str) git_info = ( { "commit": { "commitId": git_ref.commit_id, "message": git_ref.message, "authorName": git_ref.author_name, "authorEmail": git_ref.author_email, "commitDate": git_ref.commit_date, }, "repositoryDirty": git_ref.dirty, "currentBranch": git_ref.branch, "remotes": git_ref.remotes, } if git_ref else None ) params = { "projectIdentifier": project_id, "cliVersion": str(neptune_client_version), "gitInfo": git_info, "customId": custom_run_id, } if notebook_id is not None and checkpoint_id is not None: params["notebookId"] = notebook_id if notebook_id is not None else None params["checkpointId"] = ( checkpoint_id if checkpoint_id is not None else None ) kwargs = { "experimentCreationParams": params, "X-Neptune-CliVersion": str(neptune_client_version), **DEFAULT_REQUEST_KWARGS, } try: run = ( self.leaderboard_client.api.createExperiment(**kwargs).response().result ) return ApiRun( run.id, run.shortId, run.organizationName, run.projectName, run.trashed ) except HTTPNotFound: raise ProjectNotFound(project_id=project_id)
def from_str(cls, role: str) -> "ProjectMemberRoleDTO": verify_type("role", role, str) try: return { ProjectMemberRole.VIEWER: ProjectMemberRoleDTO.VIEWER, ProjectMemberRole.CONTRIBUTOR: ProjectMemberRoleDTO.MEMBER, ProjectMemberRole.OWNER: ProjectMemberRoleDTO.MANAGER, }[role] except KeyError as e: raise UnsupportedValue(enum=cls.__name__, value=role) from e
def assign( self, value: Union[FileSetVal, str, Iterable[str]], wait: bool = False ) -> None: verify_type("value", value, (FileSetVal, str, Iterable)) if isinstance(value, FileSetVal): value = value.file_globs elif isinstance(value, str): value = [value] else: verify_collection_type("value", value, str) self._enqueue_upload_operation(value, reset=True, wait=wait)
def _as_list( name: str, value: Optional[Union[str, Iterable[str]]]) -> Optional[Iterable[str]]: verify_type(name, value, (type(None), str, Iterable)) if value is None: return None if isinstance(value, str): return [value] verify_collection_type(name, value, str) return value
def assign(self, value: typing.Union[DatetimeVal, datetime], wait: bool = False): verify_type("value", value, (DatetimeVal, datetime)) if isinstance(value, DatetimeVal): value = value.value else: value = value.replace(microsecond=1000 * int(value.microsecond / 1000)) with self._container.lock(): self._enqueue_operation( self.create_assignment_operation(self._path, value), wait)
def upload_files(self, value: Union[str, Iterable[str]], wait: bool = False) -> None: if is_collection(value): verify_collection_type("value", value, str) else: verify_type("value", value, str) with self._run.lock(): attr = self._run.get_attribute(self._path) if not attr: attr = FileSet(self._run, parse_path(self._path)) self._run.set_attribute(self._path, attr) attr.upload_files(value, wait)
def __init__(self, run: 'neptune.Run', base_namespace=""): expect_not_an_experiment(run) verify_type("run", run, neptune.Run) verify_type("base_namespace", base_namespace, str) if base_namespace and not base_namespace.endswith("/"): base_namespace += "/" self._run = run self._base_namespace = base_namespace self.params_logged = False self.feature_names_logged = False self._run[INTEGRATION_VERSION_KEY] = __version__
def init_project( *, name: Optional[str] = None, api_token: Optional[str] = None, mode: str = Mode.ASYNC.value, flush_period: float = 5, proxies: Optional[dict] = None, ) -> Project: verify_type("name", name, (str, type(None))) verify_type("api_token", api_token, (str, type(None))) verify_type("mode", mode, str) verify_type("flush_period", flush_period, (int, float)) verify_type("proxies", proxies, (dict, type(None))) if mode == Mode.OFFLINE: raise NeptuneException("Project can't be initialized in OFFLINE mode") backend = get_backend(mode, api_token=api_token, proxies=proxies) project_obj = project_name_lookup(backend, name) project_lock = threading.RLock() operation_processor = get_operation_processor( mode, container_id=project_obj.id, container_type=Project.container_type, backend=backend, lock=project_lock, flush_period=flush_period, ) background_jobs = [] project = Project( project_obj.id, backend, operation_processor, BackgroundJobList(background_jobs), project_lock, project_obj.workspace, project_obj.name, ) if mode != Mode.OFFLINE: project.sync(wait=False) # pylint: disable=protected-access project._startup(debug_mode=mode == Mode.DEBUG) return project
def project_name_lookup(backend: NeptuneBackend, name: Optional[str] = None) -> Project: verify_type("name", name, (str, type(None))) if not name: name = os.getenv(PROJECT_ENV_NAME) if not name: available_workspaces = backend.get_available_workspaces() available_projects = backend.get_available_projects() raise NeptuneMissingProjectNameException( available_workspaces=available_workspaces, available_projects=available_projects, ) return backend.get_project(name)
def stop(self, seconds: Optional[Union[float, int]] = None) -> None: verify_type("seconds", seconds, (float, int, type(None))) if self._state != ContainerState.STARTED: return self._state = ContainerState.STOPPING ts = time.time() click.echo("Shutting down background jobs, please wait a moment...") self._bg_job.stop() self._bg_job.join(seconds) click.echo("Done!") with self._lock: sec_left = None if seconds is None else seconds - (time.time() - ts) self._op_processor.stop(sec_left) self._backend.close() self._state = ContainerState.STOPPED
def assign(self, value: FileVal, wait: bool = False) -> None: verify_type("value", value, FileVal) if value.path is not None: operation = UploadFile(self._path, ext=value.extension, file_path=os.path.abspath(value.path)) elif value.content is not None: operation = UploadFileContent( self._path, ext=value.extension, file_content=base64_encode(value.content), ) else: raise ValueError("File path and content are None") with self._container.lock(): self._enqueue_operation(operation, wait)
def __init__(self, run: Run, base_namespace: Optional[str] = None): super().__init__() expect_not_an_experiment(run) verify_type('run', run, Run) verify_type('base_namespace', base_namespace, (str, type(None))) self._base_namespace = '' if base_namespace: if base_namespace.endswith("/"): self._base_namespace = base_namespace[:-1] else: self._base_namespace = base_namespace if self._base_namespace: self._metric_logger = run[self._base_namespace] else: self._metric_logger = run run[INTEGRATION_VERSION_KEY] = __version__
def remove_project_member( name: str, username: str, workspace: Optional[str] = None, api_token: Optional[str] = None, ): """Removes member from the Neptune project. Args: name(str): The name of the project in Neptune in the format 'WORKSPACE/PROJECT'. If workspace argument was set, it should only contain 'PROJECT' instead of 'WORKSPACE/PROJECT'. username(str): name of the user you want to remove from the project. workspace(str, optional): Name of your Neptune workspace. If you specify change the format of the name argument to 'PROJECT' instead of 'WORKSPACE/PROJECT'. If 'None' it will be parsed from the name argument. api_token(str, optional): User’s API token. Defaults to `None`. If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken. .. note:: It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your API token in plain text in your source code. Examples: >>> from neptune import management >>> management.remove_project_member(name="awesome-team/amazing-project", ... username="******") You may also want to check `management API reference`_. .. _management API reference: https://docs.neptune.ai/api-reference/management """ verify_type("name", name, str) verify_type("username", username, str) verify_type("workspace", workspace, (str, type(None))) verify_type("api_token", api_token, (str, type(None))) backend_client = _get_backend_client(api_token=api_token) project_identifier = normalize_project_name(name=name, workspace=workspace) params = { "projectIdentifier": project_identifier, "userId": username, **DEFAULT_REQUEST_KWARGS, } try: backend_client.api.deleteProjectMember(**params).response() except HTTPNotFound as e: raise ProjectNotFound(name=project_identifier) from e except HTTPUnprocessableEntity as e: raise UserNotExistsOrWithoutAccess(user=username, project=project_identifier) from e except HTTPForbidden as e: raise AccessRevokedOnMemberRemoval(user=username, project=project_identifier) from e
def __init__( self, path: Optional[str] = None, content: Optional[bytes] = None, extension: Optional[str] = None, ): verify_type("path", path, (str, type(None))) verify_type("content", content, (bytes, type(None))) verify_type("extension", extension, (str, type(None))) if path is not None and content is not None: raise ValueError("path and content are mutually exclusive") if path is None and content is None: raise ValueError("path or content is required") self.path = path self.content = content if extension is None and path is not None: try: ext = os.path.splitext(path)[1] self.extension = ext[1:] if ext else "" except ValueError: self.extension = "" else: self.extension = extension or ""
def get_workspace_member_list(name: str, api_token: Optional[str] = None ) -> Dict[str, str]: """Get a list of members of a workspace. Args: name(str, optional): Name of your Neptune workspace. api_token(str, optional): User’s API token. Defaults to `None`. If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken. .. note:: It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your API token in plain text in your source code. Returns: ``Dict[str, str]``: Dictionary with usernames as keys and `WorkspaceMemberRole` ('member', 'admin') as values. Examples: >>> from neptune import management >>> management.get_workspace_member_list(name="awesome-team") You may also want to check `management API reference`_. .. _management API reference: https://docs.neptune.ai/api-reference/management """ verify_type("name", name, str) verify_type("api_token", api_token, (str, type(None))) backend_client = _get_backend_client(api_token=api_token) params = {"organizationIdentifier": name, **DEFAULT_REQUEST_KWARGS} try: result = backend_client.api.listOrganizationMembers( **params).response().result return { f"{m.registeredMemberInfo.username}": WorkspaceMemberRoleDTO.to_domain(m.role) for m in result } except HTTPNotFound as e: raise WorkspaceNotFound(workspace=name) from e
def __init__(self, run, base_namespace="training", log_model=True, log_importance=True, max_num_features=None, log_tree=None, tree_figsize=30): expect_not_an_experiment(run) verify_type("run", run, neptune.Run) verify_type("base_namespace", base_namespace, str) log_model is not None and verify_type("log_model", log_model, bool) log_importance is not None and verify_type("log_importance", log_importance, bool) max_num_features is not None and verify_type("max_num_features", max_num_features, int) log_tree is not None and verify_type("log_tree", log_tree, list) verify_type("tree_figsize", tree_figsize, int) self.run = run[base_namespace] self.log_model = log_model self.log_importance = log_importance self.max_num_features = max_num_features self.log_tree = log_tree self.cv = False self.tree_figsize = tree_figsize if self.log_tree: try: subprocess.call(["dot", "-V"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except OSError: self.log_tree = None message = "Graphviz executables not found, so trees will not be logged. " \ "Make sure the Graphviz executables are on your systems' PATH" warnings.warn(message) run[INTEGRATION_VERSION_KEY] = __version__
def __init__(self, *, run: Run, level=logging.NOTSET, path: str = None): verify_type("run", run, Run) verify_type("level", level, int) if path is None: path = f"{run.monitoring_namespace}/python_logger" verify_type("path", path, str) super().__init__(level=level) self._run = run self._logger = Logger(run, path) self._thread_local = threading.local() self._run[INTEGRATION_VERSION_KEY] = neptune_client_version
def configure( self, min: Optional[Union[float, int]] = None, max: Optional[Union[float, int]] = None, unit: Optional[str] = None, wait: bool = False, ) -> None: verify_type("min", min, (float, int)) verify_type("max", max, (float, int)) verify_type("unit", unit, str) with self._container.lock(): self._enqueue_operation( ConfigFloatSeries(self._path, min, max, unit), wait)
def get_project_member_list(name: str, workspace: Optional[str] = None, api_token: Optional[str] = None) -> Dict[str, str]: """Get a list of members for a project. Args: name(str): The name of the project in Neptune in the format 'WORKSPACE/PROJECT'. If workspace argument was set it should only contain 'PROJECT' instead of 'WORKSPACE/PROJECT'. workspace(str, optional): Name of your Neptune workspace. If you specify change the format of the name argument to 'PROJECT' instead of 'WORKSPACE/PROJECT'. If 'None' it will be parsed from the name argument. api_token(str, optional): User’s API token. Defaults to `None`. If `None`, the value of `NEPTUNE_API_TOKEN` environment variable will be taken. .. note:: It is strongly recommended to use `NEPTUNE_API_TOKEN` environment variable rather than placing your API token in plain text in your source code. Returns: ``Dict[str, str]``: Dictionary with usernames as keys and ProjectMemberRoles ('owner', 'contributor', 'viewer') as values. Examples: >>> from neptune import management >>> management.get_project_member_list(name="awesome-team/amazing-project") You may also want to check `management API reference`_. .. _management API reference: https://docs.neptune.ai/api-reference/management """ verify_type("name", name, str) verify_type("workspace", workspace, (str, type(None))) verify_type("api_token", api_token, (str, type(None))) backend_client = _get_backend_client(api_token=api_token) project_identifier = normalize_project_name(name=name, workspace=workspace) params = { "projectIdentifier": project_identifier, **DEFAULT_REQUEST_KWARGS } try: result = backend_client.api.listProjectMembers( **params).response().result return { f"{m.registeredMemberInfo.username}": ProjectMemberRoleDTO.to_domain(m.role) for m in result } except HTTPNotFound as e: raise ProjectNotFound(name=project_identifier) from e
def pop(self, path: str = None, wait: bool = False) -> None: if path: verify_type("path", path, str) self._run.pop(join_paths(self._path, path), wait) else: self._run.pop(self._path, wait)