def save(self):
if self.obj is None: # create
logger.debug("Saving result with data {}".format(self.data))
timestamp = datetime.now()
additional_data = [
AnalysisResult(name="{} @ {}".format(uri,
timestamp.isoformat()),
result_file=Distribution(uri),
timestamp=timestamp)
for uri in self.data["results_storage"]
]
for ad in additional_data:
ad.save(self.client)
self.obj = ValidationResult(
name=
"Validation results for model {} and test {} with timestamp {}"
.format(self.data["model_version_id"],
self.data["test_code_id"], timestamp.isoformat()),
generated_by=None,
description=None,
score=self.data["score"],
normalized_score=self.data["normalized_score"],
passed=self.data["passed"],
timestamp=timestamp,
additional_data=additional_data,
collab_id=self.data["project"])
self.obj.save(self.client)
test_definition = self.data["test_script"].test_definition.resolve(
self.client, api="nexus")
reference_data = Collection(
"Reference data for {}".format(test_definition.name),
members=[
item.resolve(self.client, api="nexus")
for item in as_list(test_definition.reference_data)
])
reference_data.save(self.client)
activity = ValidationActivity(
model_instance=self.data["model_instance"],
test_script=self.data["test_script"],
reference_data=reference_data,
timestamp=timestamp,
result=self.obj)
activity.save(self.client)
self.obj.generated_by = activity
self.obj.save(self.client)
else: # update
raise NotImplementedError()
return self.obj
async def query_results_extended(
passed: List[bool] = Query(None),
project_id: List[int] = Query(None),
model_instance_id: List[UUID] = Query(
None), # todo: rename this 'model_instance_id' for consistency
test_instance_id: List[UUID] = Query(None),
model_id: List[UUID] = Query(None),
test_id: List[UUID] = Query(None),
model_alias: List[str] = Query(None),
test_alias: List[str] = Query(None),
score_type: List[ScoreType] = None,
size: int = Query(100),
from_index: int = Query(0),
# from header
token: HTTPAuthorizationCredentials = Depends(auth),
):
filter_query, context = build_result_filters(
model_instance_id,
test_instance_id,
model_id,
test_id,
model_alias,
test_alias,
score_type,
passed,
project_id,
kg_client,
)
if len(filter_query["value"]) > 0:
logger.info(
f"Searching for ValidationResult with the following query: {filter_query}"
)
# note that from_index is not currently supported by KGQuery.resolve
query = KGQuery(ValidationResultKG, {"nexus": filter_query}, context)
results = query.resolve(kg_client, api="nexus", size=size)
else:
results = ValidationResultKG.list(kg_client,
api="nexus",
size=size,
from_index=from_index)
response = []
for result in results:
try:
obj = await ValidationResultWithTestAndModel.from_kg_object(
result, kg_client, token)
except ConsistencyError as err: # todo: count these and report them in the response
logger.warning(str(err))
else:
response.append(obj)
return response
async def delete_result(result_id: UUID,
token: HTTPAuthorizationCredentials = Depends(auth)):
# todo: handle non-existent UUID
result = ValidationResultKG.from_uuid(str(result_id),
kg_client,
api="nexus")
if not await is_admin(token.credentials):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Deleting validation results is restricted to admins",
)
for item in as_list(result.additional_data):
item.delete(kg_client)
# todo: check whether the result has been used in further analysis
# if so, we should probably disallow deletion unless forced
result.generated_by.delete(kg_client)
result.delete(kg_client)
def get_result(result_id: UUID,
token: HTTPAuthorizationCredentials = Depends(auth)):
result = ValidationResultKG.from_uuid(str(result_id),
kg_client,
api="nexus")
if result:
try:
obj = ValidationResult.from_kg_object(result, kg_client)
except ConsistencyError as err:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=str(err))
else:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Validation result {result_id} not found.",
)
return obj
def _query_results(passed, project_id, model_instance_id, test_instance_id,
model_id, test_id, model_alias, test_alias, score_type,
size, from_index, token):
filter_query, context = build_result_filters(
model_instance_id,
test_instance_id,
model_id,
test_id,
model_alias,
test_alias,
score_type,
passed,
project_id,
kg_client,
)
if len(filter_query["value"]) > 0:
logger.info(
f"Searching for ValidationResult with the following query: {filter_query}"
)
# note that from_index is not currently supported by KGQuery.resolve
query = KGQuery(ValidationResultKG, {"nexus": filter_query}, context)
results = query.resolve(kg_client, api="nexus", size=size)
else:
results = ValidationResultKG.list(kg_client,
api="nexus",
size=size,
from_index=from_index)
response = []
for result in results:
try:
obj = ValidationResult.from_kg_object(result, kg_client)
except ConsistencyError as err: # todo: count these and report them in the response
logger.warning(str(err))
else:
response.append(obj)
return response
def migrate_validation_results(self):
result_objects = ValidationTestResult.objects.all()
storage_token = os.environ["HBP_STORAGE_TOKEN"]
storage_client = StorageClient.new(storage_token)
for ro in result_objects[800:]:
model_instance = lookup_model_instance(
str(ro.model_version.id),
NAR_client) # use oldUUID (stored in nsg:providerId)
test_script = lookup_test_script(str(ro.test_code.id), NAR_client)
if not model_instance:
logger.error("Model instance for {} not found in KG".format(
ro.model_version))
continue
if not test_script:
logger.error("Test script for {} not found in KG".format(
ro.test_code))
continue
test_definition = test_script.test_definition.resolve(NAR_client)
assert test_definition
additional_data = [
AnalysisResult(name="{} @ {}".format(uri,
ro.timestamp.isoformat()),
result_file=Distribution(uri),
timestamp=ro.timestamp)
for uri in get_file_list(ro.results_storage, storage_client)
]
for ad in additional_data:
ad.save(NAR_client)
result_kg = ValidationResult(
name="Result of running '{}' on model '{}' at {}".format(
test_script.name, model_instance.name, ro.timestamp),
generated_by=None,
description=ro.
platform, # temporary location pending integration in KG
score=ro.score,
normalized_score=ro.normalized_score,
passed=ro.passed,
timestamp=ro.timestamp,
additional_data=additional_data,
old_uuid=str(ro.id),
collab_id=ro.project)
result_kg.save(NAR_client)
logger.info("ValidationResult saved: %s", result_kg)
reference_data = Collection(
"Reference data for {}".format(test_definition.name),
members=test_definition.reference_data.resolve(NAR_client))
reference_data.save(NAR_client)
validation_activity = ValidationActivity(
model_instance=model_instance,
test_script=test_script,
reference_data=reference_data,
timestamp=ro.timestamp,
result=result_kg,
started_by=None)
validation_activity.save(NAR_client)
logger.info("ValidationActivity saved: %s", validation_activity)
result_kg.generated_by = validation_activity
result_kg.save(NAR_client)
class ValidationTestResultKGSerializer(BaseKGSerializer):
def is_valid(self):
return True # todo
def serialize(self, obj):
# todo: rewrite all this using KG Query API, to avoid doing all the individual resolves.
validation_activity = obj.generated_by.resolve(self.client,
api="nexus")
model_version_id = validation_activity.model_instance.uuid
test_code_id = validation_activity.test_script.uuid
logger.debug("Serializing validation test result")
logger.debug("Additional data for {}:\n{}".format(
obj.id, obj.additional_data))
additional_data_urls = []
for item in as_list(obj.additional_data):
item = item.resolve(self.client, api="nexus")
if item:
additional_data_urls.append(item.result_file.location)
else:
logger.warning("Couldn't resolve {}".format(item))
data = {
"uri":
obj.id,
"id":
obj.uuid,
"old_uuid":
obj.old_uuid,
"model_version_id":
model_version_id,
"test_code_id":
test_code_id,
"results_storage": [
serialize_additional_data(url, self.user_token)
for url in additional_data_urls
],
"score":
obj.score,
"passed":
obj.passed,
"timestamp":
obj.timestamp,
"project":
obj.collab_id,
"normalized_score":
obj.normalized_score,
# the following are temporary. Ideally the client should do a lookup using the IDs above
"model_version":
ScientificModelInstanceKGSerializer(
validation_activity.model_instance, self.client).data,
"test_code":
ValidationTestCodeKGSerializer(validation_activity.test_script,
self.client).data,
"activity_uuid":
validation_activity.uuid
}
return data
def save(self):
if self.obj is None: # create
logger.debug("Saving result with data {}".format(self.data))
timestamp = datetime.now()
additional_data = [
AnalysisResult(name="{} @ {}".format(uri,
timestamp.isoformat()),
result_file=Distribution(uri),
timestamp=timestamp)
for uri in self.data["results_storage"]
]
for ad in additional_data:
ad.save(self.client)
self.obj = ValidationResult(
name=
"Validation results for model {} and test {} with timestamp {}"
.format(self.data["model_version_id"],
self.data["test_code_id"], timestamp.isoformat()),
generated_by=None,
description=None,
score=self.data["score"],
normalized_score=self.data["normalized_score"],
passed=self.data["passed"],
timestamp=timestamp,
additional_data=additional_data,
collab_id=self.data["project"])
self.obj.save(self.client)
test_definition = self.data["test_script"].test_definition.resolve(
self.client, api="nexus")
reference_data = Collection(
"Reference data for {}".format(test_definition.name),
members=[
item.resolve(self.client, api="nexus")
for item in as_list(test_definition.reference_data)
])
reference_data.save(self.client)
activity = ValidationActivity(
model_instance=self.data["model_instance"],
test_script=self.data["test_script"],
reference_data=reference_data,
timestamp=timestamp,
result=self.obj)
activity.save(self.client)
self.obj.generated_by = activity
self.obj.save(self.client)
else: # update
raise NotImplementedError()
return self.obj