def get_float_series_values( self, container_id: str, container_type: ContainerType, path: List[str], offset: int, limit: int, ) -> FloatSeriesValues: params = { "experimentId": container_id, "attribute": path_to_str(path), "limit": limit, "offset": offset, **DEFAULT_REQUEST_KWARGS, } try: result = ( self.leaderboard_client.api.getFloatSeriesValues(**params) .response() .result ) return FloatSeriesValues( result.totalItemCount, [ FloatPointValue(v.timestampMillis, v.step, v.value) for v in result.values ], ) except HTTPNotFound: raise FetchAttributeNotFoundException(path_to_str(path))
def get(self, path: List[str]) -> Union[T, Node, None]: ref = self._structure for index, part in enumerate(path): if not isinstance(ref, self._node_type): raise MetadataInconsistency( "Cannot access path '{}': '{}' is already defined as an attribute, " "not a namespace".format(path_to_str(path), path_to_str(path[:index]))) if part not in ref: return None ref = ref[part] return ref
def get_file_attribute( self, container_id: str, container_type: ContainerType, path: List[str] ) -> FileAttribute: params = { "experimentId": container_id, "attribute": path_to_str(path), **DEFAULT_REQUEST_KWARGS, } try: result = ( self.leaderboard_client.api.getFileAttribute(**params).response().result ) return FileAttribute(name=result.name, ext=result.ext, size=result.size) except HTTPNotFound: raise FetchAttributeNotFoundException(path_to_str(path))
def get_bool_attribute( self, container_id: str, container_type: ContainerType, path: List[str] ) -> BoolAttribute: params = { "experimentId": container_id, "attribute": path_to_str(path), **DEFAULT_REQUEST_KWARGS, } try: result = ( self.leaderboard_client.api.getBoolAttribute(**params).response().result ) return BoolAttribute(result.value) except HTTPNotFound: raise FetchAttributeNotFoundException(path_to_str(path))
def _execute_operations( self, container_id: str, container_type: ContainerType, operations: List[Operation], ) -> List[MetadataInconsistency]: kwargs = { "experimentId": container_id, "operations": [ { "path": path_to_str(op.path), OperationApiNameVisitor() .visit(op): OperationApiObjectConverter() .convert(op), } for op in operations ], **DEFAULT_REQUEST_KWARGS, } try: result = ( self.leaderboard_client.api.executeOperations(**kwargs) .response() .result ) return [MetadataInconsistency(err.errorDescription) for err in result] except HTTPNotFound as e: raise_container_not_found(container_id, container_type, from_exception=e) except (HTTPPaymentRequired, HTTPUnprocessableEntity) as e: raise NeptuneLimitExceedException( reason=e.response.json().get("title", "Unknown reason") ) from e
def _get_file_set_download_request( self, container_id: str, container_type: ContainerType, path: List[str] ): params = { "experimentId": container_id, "attribute": path_to_str(path), **DEFAULT_REQUEST_KWARGS, } try: return ( self.leaderboard_client.api.prepareForDownloadFileSetAttributeZip( **params ) .response() .result ) except HTTPNotFound: raise FetchAttributeNotFoundException(path_to_str(path))
def setUp(self): self.monkeypatch = MonkeyPatch() self.wait = self._random_wait() self.op_processor = MagicMock() self.exp = self._create_run(processor=self.op_processor) self.path = self._random_path() self.path_str = path_to_str(self.path) self.artifact_hash = ( "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") self.artifact_files = [ ArtifactFileData( file_path="fname.txt", file_hash= "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", type="test", size=213, metadata={}, ), ArtifactFileData( file_path="subdir/other.mp3", file_hash= "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", type="test", metadata={}, ), ] self.exp.set_attribute(self.path_str, Artifact(self.exp, self.path)) self.exp._backend._containers[(self.exp._id, ContainerType.RUN)].set( self.path, ArtifactAttr(self.artifact_hash)) self.exp._backend._artifacts[self.exp._project_id, self.artifact_hash] = self.artifact_files self._downloads = set() class TestArtifactDriver(ArtifactDriver): @classmethod def get_type(cls): return "test" @classmethod def matches(cls, path: str) -> bool: return False @classmethod def get_tracked_files(cls, path, destination=None): return [] @classmethod def download_file(cls, destination: pathlib.Path, file_definition: ArtifactFileData): destination.touch() self.test_artifact_driver = TestArtifactDriver
def download_file( self, container_id: str, container_type: ContainerType, path: List[str], destination: Optional[str] = None, ): try: download_file_attribute( swagger_client=self.leaderboard_client, container_id=container_id, attribute=path_to_str(path), destination=destination, ) except ClientHttpError as e: if e.status == HTTPNotFound.status_code: raise FetchAttributeNotFoundException(path_to_str(path)) else: raise
def process(self, operations: List[Operation]): for op in operations: path_str = path_to_str(op.path) self._accumulators.setdefault(path_str, _OperationsAccumulator( op.path)).visit(op) result = [] for acc in self._accumulators.values(): result.extend(acc.get_operations()) return result
def _execute_upload_operation(self, experiment: Experiment, upload_operation: alpha_operation.Operation): experiment_id = experiment.internal_id try: if isinstance(upload_operation, alpha_operation.UploadFile): alpha_hosted_file_operations.upload_file_attribute( swagger_client=self.leaderboard_swagger_client, container_id=experiment_id, attribute=alpha_path_utils.path_to_str( upload_operation.path), source=upload_operation.file_path, ext=upload_operation.ext, multipart_config=self._client_config.multipart_config, ) elif isinstance(upload_operation, alpha_operation.UploadFileContent): alpha_hosted_file_operations.upload_file_attribute( swagger_client=self.leaderboard_swagger_client, container_id=experiment_id, attribute=alpha_path_utils.path_to_str( upload_operation.path), source=base64_decode(upload_operation.file_content), ext=upload_operation.ext, multipart_config=self._client_config.multipart_config, ) elif isinstance(upload_operation, alpha_operation.UploadFileSet): alpha_hosted_file_operations.upload_file_set_attribute( swagger_client=self.leaderboard_swagger_client, container_id=experiment_id, attribute=alpha_path_utils.path_to_str( upload_operation.path), file_globs=upload_operation.file_globs, reset=upload_operation.reset, multipart_config=self._client_config.multipart_config, ) else: raise NeptuneException( "Upload operation in neither File or FileSet") except alpha_exceptions.NeptuneException as e: raise NeptuneException(e) from e return None
def _pop_impl(self, ref, sub_path: List[str], attr_path: List[str]): if not sub_path: return head, tail = sub_path[0], sub_path[1:] if head not in ref: raise MetadataInconsistency( "Cannot delete {}. Attribute not found.".format( path_to_str(attr_path))) if not tail: if isinstance(ref[head], self._node_type): raise MetadataInconsistency( "Cannot delete {}. It's a namespace, not an attribute.". format(path_to_str(attr_path))) del ref[head] else: self._pop_impl(ref[head], tail, attr_path) if not ref[head]: del ref[head]
def fetch_atom_attribute_values( self, container_id: str, container_type: ContainerType, path: List[str]) -> List[Tuple[str, AttributeType, Any]]: run = self._get_container(container_id, container_type) values = self._get_attribute_values(run.get(path), path) namespace_prefix = path_to_str(path) if namespace_prefix: # don't want to catch "ns/attribute/other" while looking for "ns/attr" namespace_prefix += "/" return [(full_path, attr_type, attr_value) for (full_path, attr_type, attr_value) in values if full_path.startswith(namespace_prefix)]
def download_file_series_by_index( self, container_id: str, container_type: ContainerType, path: List[str], index: int, destination: str, ): try: download_image_series_element( swagger_client=self.leaderboard_client, container_id=container_id, attribute=path_to_str(path), index=index, destination=destination, ) except ClientHttpError as e: if e.status == HTTPNotFound.status_code: raise FetchAttributeNotFoundException(path_to_str(path)) else: raise
def set(self, path: List[str], attr: T) -> None: ref = self._structure location, attribute_name = path[:-1], path[-1] for idx, part in enumerate(location): if part not in ref: ref[part] = self._node_factory(location[:idx + 1]) ref = ref[part] if not isinstance(ref, self._node_type): raise MetadataInconsistency( "Cannot access path '{}': '{}' is already defined as an attribute, " "not a namespace".format(path_to_str(path), part)) if attribute_name in ref and isinstance(ref[attribute_name], self._node_type): if isinstance(attr, self._node_type): # in-between nodes are auto-created, so ignore it's OK unless we want to change the type return raise MetadataInconsistency( "Cannot set attribute '{}'. It's a namespace".format( path_to_str(path))) ref[attribute_name] = attr
def get_image_series_values( self, container_id: str, container_type: ContainerType, path: List[str], offset: int, limit: int, ) -> ImageSeriesValues: params = { "experimentId": container_id, "attribute": path_to_str(path), "limit": limit, "offset": offset, **DEFAULT_REQUEST_KWARGS, } try: result = ( self.leaderboard_client.api.getImageSeriesValues(**params) .response() .result ) return ImageSeriesValues(result.totalItemCount) except HTTPNotFound: raise FetchAttributeNotFoundException(path_to_str(path))
def _get_attribute( self, container_id: str, container_type: ContainerType, path: List[str], expected_type: Type[Val], ) -> Val: run = self._get_container(container_id, container_type) value: Optional[Value] = run.get(path) str_path = path_to_str(path) if value is None: raise MetadataInconsistency( "Attribute {} not found".format(str_path)) if isinstance(value, expected_type): return value raise MetadataInconsistency("Attribute {} is not {}".format( str_path, type.__name__))
def _process_config_op(self, expected_type: _DataType, op: Operation) -> None: if self._type and self._type != expected_type: # This case should never happen since inconsistencies on data types are verified on user api. # So such operations should not appear in the queue without delete operation between them. # Still we want to support this case to avoid some unclear dependencies and assumptions. self._errors.append( MetadataInconsistency( "Cannot perform {} operation on {}: Attribute is not a {}". format( op.__class__.__name__, path_to_str(self._path), expected_type.value, ))) else: self._type = expected_type self._config_ops = [op]
def _execute_operations(self, experiment: Experiment, operations: List[alpha_operation.Operation]): experiment_id = experiment.internal_id file_operations = ( alpha_operation.UploadFile, alpha_operation.UploadFileContent, alpha_operation.UploadFileSet, ) if any(isinstance(op, file_operations) for op in operations): raise NeptuneException( "File operations must be handled directly by `_execute_upload_operation`," " not by `_execute_operations` function call.") kwargs = { "experimentId": experiment_id, "operations": [{ "path": alpha_path_utils.path_to_str(op.path), AlphaOperationApiNameVisitor().visit(op): AlphaOperationApiObjectConverter().convert(op), } for op in operations], } try: result = (self.leaderboard_swagger_client.api.executeOperations( **kwargs).response().result) errors = [ alpha_exceptions.MetadataInconsistency(err.errorDescription) for err in result ] if errors: raise ExperimentOperationErrors(errors=errors) return None except HTTPNotFound as e: # pylint: disable=protected-access raise ExperimentNotFound( experiment_short_id=experiment.id, project_qualified_name=experiment._project.full_id, ) from e
def fetch_atom_attribute_values( self, container_id: str, container_type: ContainerType, path: List[str] ) -> List[Tuple[str, AttributeType, Any]]: params = { "experimentId": container_id, } try: namespace_prefix = path_to_str(path) if namespace_prefix: # don't want to catch "ns/attribute/other" while looking for "ns/attr" namespace_prefix += "/" result = ( self.leaderboard_client.api.getExperimentAttributes(**params) .response() .result ) return [ (attr.name, attr.type, map_attribute_result_to_value(attr)) for attr in result.attributes if attr.name.startswith(namespace_prefix) ] except HTTPNotFound as e: raise_container_not_found(container_id, container_type, from_exception=e)
def iterate_subpaths(self, path_prefix: List[str]): root = self.get(path_prefix) for path in self._iterate_node(root or {}, path_prefix): yield path_to_str(path)
def __init__(self, container: "AttributeContainer", path: List[str]): Attribute.__init__(self, container, path) self._attributes = {} self._str_path = path_to_str(path)
def _execute_upload_operations( self, container_id: str, container_type: ContainerType, upload_operations: List[Operation], ) -> List[NeptuneException]: errors = list() if self._client_config.has_feature(OptionalFeatures.MULTIPART_UPLOAD): multipart_config = self._client_config.multipart_config # collect delete operations and execute them attributes_to_reset = [ DeleteAttribute(op.path) for op in upload_operations if isinstance(op, UploadFileSet) and op.reset ] if attributes_to_reset: errors.extend( self._execute_operations( container_id, container_type, operations=attributes_to_reset ) ) else: multipart_config = None for op in upload_operations: if isinstance(op, UploadFile): upload_errors = upload_file_attribute( swagger_client=self.leaderboard_client, container_id=container_id, attribute=path_to_str(op.path), source=op.file_path, ext=op.ext, multipart_config=multipart_config, ) if upload_errors: errors.extend(upload_errors) elif isinstance(op, UploadFileContent): upload_errors = upload_file_attribute( swagger_client=self.leaderboard_client, container_id=container_id, attribute=path_to_str(op.path), source=base64_decode(op.file_content), ext=op.ext, multipart_config=multipart_config, ) if upload_errors: errors.extend(upload_errors) elif isinstance(op, UploadFileSet): upload_errors = upload_file_set_attribute( swagger_client=self.leaderboard_client, container_id=container_id, attribute=path_to_str(op.path), file_globs=op.file_globs, reset=op.reset, multipart_config=multipart_config, ) if upload_errors: errors.extend(upload_errors) else: raise InternalClientError("Upload operation in neither File or FileSet") return errors