def get_float_series_values(
self,
container_id: str,
container_type: ContainerType,
path: List[str],
offset: int,
limit: int,
) -> FloatSeriesValues:
params = {
"experimentId": container_id,
"attribute": path_to_str(path),
"limit": limit,
"offset": offset,
**DEFAULT_REQUEST_KWARGS,
}
try:
result = (
self.leaderboard_client.api.getFloatSeriesValues(**params)
.response()
.result
)
return FloatSeriesValues(
result.totalItemCount,
[
FloatPointValue(v.timestampMillis, v.step, v.value)
for v in result.values
],
)
except HTTPNotFound:
raise FetchAttributeNotFoundException(path_to_str(path))
def get(self, path: List[str]) -> Union[T, Node, None]:
ref = self._structure
for index, part in enumerate(path):
if not isinstance(ref, self._node_type):
raise MetadataInconsistency(
"Cannot access path '{}': '{}' is already defined as an attribute, "
"not a namespace".format(path_to_str(path),
path_to_str(path[:index])))
if part not in ref:
return None
ref = ref[part]
return ref
def get_file_attribute(
self, container_id: str, container_type: ContainerType, path: List[str]
) -> FileAttribute:
params = {
"experimentId": container_id,
"attribute": path_to_str(path),
**DEFAULT_REQUEST_KWARGS,
}
try:
result = (
self.leaderboard_client.api.getFileAttribute(**params).response().result
)
return FileAttribute(name=result.name, ext=result.ext, size=result.size)
except HTTPNotFound:
raise FetchAttributeNotFoundException(path_to_str(path))
def get_bool_attribute(
self, container_id: str, container_type: ContainerType, path: List[str]
) -> BoolAttribute:
params = {
"experimentId": container_id,
"attribute": path_to_str(path),
**DEFAULT_REQUEST_KWARGS,
}
try:
result = (
self.leaderboard_client.api.getBoolAttribute(**params).response().result
)
return BoolAttribute(result.value)
except HTTPNotFound:
raise FetchAttributeNotFoundException(path_to_str(path))
def _execute_operations(
self,
container_id: str,
container_type: ContainerType,
operations: List[Operation],
) -> List[MetadataInconsistency]:
kwargs = {
"experimentId": container_id,
"operations": [
{
"path": path_to_str(op.path),
OperationApiNameVisitor()
.visit(op): OperationApiObjectConverter()
.convert(op),
}
for op in operations
],
**DEFAULT_REQUEST_KWARGS,
}
try:
result = (
self.leaderboard_client.api.executeOperations(**kwargs)
.response()
.result
)
return [MetadataInconsistency(err.errorDescription) for err in result]
except HTTPNotFound as e:
raise_container_not_found(container_id, container_type, from_exception=e)
except (HTTPPaymentRequired, HTTPUnprocessableEntity) as e:
raise NeptuneLimitExceedException(
reason=e.response.json().get("title", "Unknown reason")
) from e
def _get_file_set_download_request(
self, container_id: str, container_type: ContainerType, path: List[str]
):
params = {
"experimentId": container_id,
"attribute": path_to_str(path),
**DEFAULT_REQUEST_KWARGS,
}
try:
return (
self.leaderboard_client.api.prepareForDownloadFileSetAttributeZip(
**params
)
.response()
.result
)
except HTTPNotFound:
raise FetchAttributeNotFoundException(path_to_str(path))
def setUp(self):
self.monkeypatch = MonkeyPatch()
self.wait = self._random_wait()
self.op_processor = MagicMock()
self.exp = self._create_run(processor=self.op_processor)
self.path = self._random_path()
self.path_str = path_to_str(self.path)
self.artifact_hash = (
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
self.artifact_files = [
ArtifactFileData(
file_path="fname.txt",
file_hash=
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
type="test",
size=213,
metadata={},
),
ArtifactFileData(
file_path="subdir/other.mp3",
file_hash=
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
type="test",
metadata={},
),
]
self.exp.set_attribute(self.path_str, Artifact(self.exp, self.path))
self.exp._backend._containers[(self.exp._id, ContainerType.RUN)].set(
self.path, ArtifactAttr(self.artifact_hash))
self.exp._backend._artifacts[self.exp._project_id,
self.artifact_hash] = self.artifact_files
self._downloads = set()
class TestArtifactDriver(ArtifactDriver):
@classmethod
def get_type(cls):
return "test"
@classmethod
def matches(cls, path: str) -> bool:
return False
@classmethod
def get_tracked_files(cls, path, destination=None):
return []
@classmethod
def download_file(cls, destination: pathlib.Path,
file_definition: ArtifactFileData):
destination.touch()
self.test_artifact_driver = TestArtifactDriver
def download_file(
self,
container_id: str,
container_type: ContainerType,
path: List[str],
destination: Optional[str] = None,
):
try:
download_file_attribute(
swagger_client=self.leaderboard_client,
container_id=container_id,
attribute=path_to_str(path),
destination=destination,
)
except ClientHttpError as e:
if e.status == HTTPNotFound.status_code:
raise FetchAttributeNotFoundException(path_to_str(path))
else:
raise
def process(self, operations: List[Operation]):
for op in operations:
path_str = path_to_str(op.path)
self._accumulators.setdefault(path_str,
_OperationsAccumulator(
op.path)).visit(op)
result = []
for acc in self._accumulators.values():
result.extend(acc.get_operations())
return result
def _execute_upload_operation(self, experiment: Experiment,
upload_operation: alpha_operation.Operation):
experiment_id = experiment.internal_id
try:
if isinstance(upload_operation, alpha_operation.UploadFile):
alpha_hosted_file_operations.upload_file_attribute(
swagger_client=self.leaderboard_swagger_client,
container_id=experiment_id,
attribute=alpha_path_utils.path_to_str(
upload_operation.path),
source=upload_operation.file_path,
ext=upload_operation.ext,
multipart_config=self._client_config.multipart_config,
)
elif isinstance(upload_operation,
alpha_operation.UploadFileContent):
alpha_hosted_file_operations.upload_file_attribute(
swagger_client=self.leaderboard_swagger_client,
container_id=experiment_id,
attribute=alpha_path_utils.path_to_str(
upload_operation.path),
source=base64_decode(upload_operation.file_content),
ext=upload_operation.ext,
multipart_config=self._client_config.multipart_config,
)
elif isinstance(upload_operation, alpha_operation.UploadFileSet):
alpha_hosted_file_operations.upload_file_set_attribute(
swagger_client=self.leaderboard_swagger_client,
container_id=experiment_id,
attribute=alpha_path_utils.path_to_str(
upload_operation.path),
file_globs=upload_operation.file_globs,
reset=upload_operation.reset,
multipart_config=self._client_config.multipart_config,
)
else:
raise NeptuneException(
"Upload operation in neither File or FileSet")
except alpha_exceptions.NeptuneException as e:
raise NeptuneException(e) from e
return None
def _pop_impl(self, ref, sub_path: List[str], attr_path: List[str]):
if not sub_path:
return
head, tail = sub_path[0], sub_path[1:]
if head not in ref:
raise MetadataInconsistency(
"Cannot delete {}. Attribute not found.".format(
path_to_str(attr_path)))
if not tail:
if isinstance(ref[head], self._node_type):
raise MetadataInconsistency(
"Cannot delete {}. It's a namespace, not an attribute.".
format(path_to_str(attr_path)))
del ref[head]
else:
self._pop_impl(ref[head], tail, attr_path)
if not ref[head]:
del ref[head]
def fetch_atom_attribute_values(
self, container_id: str, container_type: ContainerType,
path: List[str]) -> List[Tuple[str, AttributeType, Any]]:
run = self._get_container(container_id, container_type)
values = self._get_attribute_values(run.get(path), path)
namespace_prefix = path_to_str(path)
if namespace_prefix:
# don't want to catch "ns/attribute/other" while looking for "ns/attr"
namespace_prefix += "/"
return [(full_path, attr_type, attr_value)
for (full_path, attr_type, attr_value) in values
if full_path.startswith(namespace_prefix)]
def download_file_series_by_index(
self,
container_id: str,
container_type: ContainerType,
path: List[str],
index: int,
destination: str,
):
try:
download_image_series_element(
swagger_client=self.leaderboard_client,
container_id=container_id,
attribute=path_to_str(path),
index=index,
destination=destination,
)
except ClientHttpError as e:
if e.status == HTTPNotFound.status_code:
raise FetchAttributeNotFoundException(path_to_str(path))
else:
raise
def set(self, path: List[str], attr: T) -> None:
ref = self._structure
location, attribute_name = path[:-1], path[-1]
for idx, part in enumerate(location):
if part not in ref:
ref[part] = self._node_factory(location[:idx + 1])
ref = ref[part]
if not isinstance(ref, self._node_type):
raise MetadataInconsistency(
"Cannot access path '{}': '{}' is already defined as an attribute, "
"not a namespace".format(path_to_str(path), part))
if attribute_name in ref and isinstance(ref[attribute_name],
self._node_type):
if isinstance(attr, self._node_type):
# in-between nodes are auto-created, so ignore it's OK unless we want to change the type
return
raise MetadataInconsistency(
"Cannot set attribute '{}'. It's a namespace".format(
path_to_str(path)))
ref[attribute_name] = attr
def get_image_series_values(
self,
container_id: str,
container_type: ContainerType,
path: List[str],
offset: int,
limit: int,
) -> ImageSeriesValues:
params = {
"experimentId": container_id,
"attribute": path_to_str(path),
"limit": limit,
"offset": offset,
**DEFAULT_REQUEST_KWARGS,
}
try:
result = (
self.leaderboard_client.api.getImageSeriesValues(**params)
.response()
.result
)
return ImageSeriesValues(result.totalItemCount)
except HTTPNotFound:
raise FetchAttributeNotFoundException(path_to_str(path))
def _get_attribute(
self,
container_id: str,
container_type: ContainerType,
path: List[str],
expected_type: Type[Val],
) -> Val:
run = self._get_container(container_id, container_type)
value: Optional[Value] = run.get(path)
str_path = path_to_str(path)
if value is None:
raise MetadataInconsistency(
"Attribute {} not found".format(str_path))
if isinstance(value, expected_type):
return value
raise MetadataInconsistency("Attribute {} is not {}".format(
str_path, type.__name__))
def _process_config_op(self, expected_type: _DataType,
op: Operation) -> None:
if self._type and self._type != expected_type:
# This case should never happen since inconsistencies on data types are verified on user api.
# So such operations should not appear in the queue without delete operation between them.
# Still we want to support this case to avoid some unclear dependencies and assumptions.
self._errors.append(
MetadataInconsistency(
"Cannot perform {} operation on {}: Attribute is not a {}".
format(
op.__class__.__name__,
path_to_str(self._path),
expected_type.value,
)))
else:
self._type = expected_type
self._config_ops = [op]
def _execute_operations(self, experiment: Experiment,
operations: List[alpha_operation.Operation]):
experiment_id = experiment.internal_id
file_operations = (
alpha_operation.UploadFile,
alpha_operation.UploadFileContent,
alpha_operation.UploadFileSet,
)
if any(isinstance(op, file_operations) for op in operations):
raise NeptuneException(
"File operations must be handled directly by `_execute_upload_operation`,"
" not by `_execute_operations` function call.")
kwargs = {
"experimentId":
experiment_id,
"operations": [{
"path":
alpha_path_utils.path_to_str(op.path),
AlphaOperationApiNameVisitor().visit(op):
AlphaOperationApiObjectConverter().convert(op),
} for op in operations],
}
try:
result = (self.leaderboard_swagger_client.api.executeOperations(
**kwargs).response().result)
errors = [
alpha_exceptions.MetadataInconsistency(err.errorDescription)
for err in result
]
if errors:
raise ExperimentOperationErrors(errors=errors)
return None
except HTTPNotFound as e:
# pylint: disable=protected-access
raise ExperimentNotFound(
experiment_short_id=experiment.id,
project_qualified_name=experiment._project.full_id,
) from e
def fetch_atom_attribute_values(
self, container_id: str, container_type: ContainerType, path: List[str]
) -> List[Tuple[str, AttributeType, Any]]:
params = {
"experimentId": container_id,
}
try:
namespace_prefix = path_to_str(path)
if namespace_prefix:
# don't want to catch "ns/attribute/other" while looking for "ns/attr"
namespace_prefix += "/"
result = (
self.leaderboard_client.api.getExperimentAttributes(**params)
.response()
.result
)
return [
(attr.name, attr.type, map_attribute_result_to_value(attr))
for attr in result.attributes
if attr.name.startswith(namespace_prefix)
]
except HTTPNotFound as e:
raise_container_not_found(container_id, container_type, from_exception=e)
def iterate_subpaths(self, path_prefix: List[str]):
root = self.get(path_prefix)
for path in self._iterate_node(root or {}, path_prefix):
yield path_to_str(path)
def __init__(self, container: "AttributeContainer", path: List[str]):
Attribute.__init__(self, container, path)
self._attributes = {}
self._str_path = path_to_str(path)
def _execute_upload_operations(
self,
container_id: str,
container_type: ContainerType,
upload_operations: List[Operation],
) -> List[NeptuneException]:
errors = list()
if self._client_config.has_feature(OptionalFeatures.MULTIPART_UPLOAD):
multipart_config = self._client_config.multipart_config
# collect delete operations and execute them
attributes_to_reset = [
DeleteAttribute(op.path)
for op in upload_operations
if isinstance(op, UploadFileSet) and op.reset
]
if attributes_to_reset:
errors.extend(
self._execute_operations(
container_id, container_type, operations=attributes_to_reset
)
)
else:
multipart_config = None
for op in upload_operations:
if isinstance(op, UploadFile):
upload_errors = upload_file_attribute(
swagger_client=self.leaderboard_client,
container_id=container_id,
attribute=path_to_str(op.path),
source=op.file_path,
ext=op.ext,
multipart_config=multipart_config,
)
if upload_errors:
errors.extend(upload_errors)
elif isinstance(op, UploadFileContent):
upload_errors = upload_file_attribute(
swagger_client=self.leaderboard_client,
container_id=container_id,
attribute=path_to_str(op.path),
source=base64_decode(op.file_content),
ext=op.ext,
multipart_config=multipart_config,
)
if upload_errors:
errors.extend(upload_errors)
elif isinstance(op, UploadFileSet):
upload_errors = upload_file_set_attribute(
swagger_client=self.leaderboard_client,
container_id=container_id,
attribute=path_to_str(op.path),
file_globs=op.file_globs,
reset=op.reset,
multipart_config=multipart_config,
)
if upload_errors:
errors.extend(upload_errors)
else:
raise InternalClientError("Upload operation in neither File or FileSet")
return errors
