def create_simulation(simulation: Simulation, token: HTTPAuthorizationCredentials = Depends(auth)): logger.info("Beginning post simulation") kg_objects = simulation.to_kg_objects(kg_client, token) logger.info("Created objects") for label in ('person', 'config', 'outputs', 'hardware', 'dependencies', 'env', 'activity'): for obj in as_list(kg_objects[label]): obj.save(kg_client) for obj in as_list(kg_objects['outputs']): obj.generated_by = kg_objects['activity'] obj.save(kg_client) logger.info("Saved objects") return Simulation.from_kg_object(kg_objects['activity'], kg_client)
def print_model_project(project, index): owner = as_list(project.owners)[0] if project.model_of: scope = project.model_of.label else: scope = "unknown" print("\n{:3} {:100} {:30} {:25} {} {}".format( index, project.name, owner.resolve(client).full_name, scope, project.uuid, project.old_uuid)) for instance_proxy in as_list(project.instances): instance = instance_proxy.resolve(client) print(" - {}".format(instance.name)) print(" {}".format( instance.main_script.resolve(client).code_location))
async def query_released_live_papers(): lps = fairgraph.livepapers.LivePaper.list(kg_client, api="query", scope="latest", size=1000) # to do: change "latest" to "release" once we're out of testing return [ LivePaperSummary.from_kg_object(lp) for lp in as_list(lps) ]
def __init__(self, objects, client, data=None, many=False, context=None, user_token=None): self.client = client # used for KG access self.user_token = user_token # used for accessing other services (e.g. collab storage) if isinstance(objects, (KGProxy, KGQuery)): objects = objects.resolve(self.client, api="nexus") if many: self.objects = objects if data: self.data = data else: self.data = [self.serialize(obj) for obj in as_list(objects)] else: self.obj = objects if data: self.data = data else: self.data = self.serialize(self.obj) self.context = context self.errors = []
def get_uniminds_person_list(neuroshapes_person_list, client): people = [] for ns_person in as_list(neuroshapes_person_list): ns_person = ns_person.resolve(client) filter = { "op": "and", "value": [ { "path": "schema:familyName", "op": "eq", "value": ns_person.family_name }, { "path": "schema:givenName", "op": "eq", "value": ns_person.given_name } ] } context = {"schema": "http://schema.org/"} u_person = KGQuery(uPerson, filter, context).resolve(client) if u_person: people.append(u_person) else: #raise Exception("cannot find {}".format(ns_person)) print("Warning: cannot find {}".format(ns_person)) people = [] return people
def print_model_information(index): model = models[index] instance = as_list(model.instances)[-1].resolve(client) code = instance.main_script.resolve(client) morph = instance.morphology.resolve(client) print(instance.name, instance.timestamp.isoformat()) print(code.code_location) print(morph.morphology_file)
async def get_latest_model_instance_given_model_id( model_id: str, token: HTTPAuthorizationCredentials = Depends(auth)): model_project = await _get_model_by_id_or_alias(model_id, token) model_instances = [ ModelInstance.from_kg_object(inst, kg_client, model_project.uuid) for inst in as_list(model_project.instances) ] latest = sorted(model_instances, key=lambda inst: inst["timestamp"])[-1] return latest
def contributor_names(self, client, api="query"): names = [] for person in as_list(self.contributor): person = person.resolve(client, api=api) if person: names.append(person.name) else: pass # todo: warning return ", ".join(names)
def query_simulations( model_id: UUID = None, model_instance_id: UUID = None, size: int = Query(100), from_index: int = Query(0), # from header token: HTTPAuthorizationCredentials = Depends(auth), ): user = get_person_from_token(kg_client, token) kwargs = {"size": size, "from_index": from_index, "api": "nexus"} if user: kwargs["started_by"] = user else: return [] model_instances = [] if model_instance_id: model_instance = fairgraph.brainsimulation.ModelInstance.from_uuid( str(model_instance_id), kg_client, api="nexus") # todo: handle MEModel, not just ModelInstance # todo: if model_id is given, check if it is consistent and return an error if not if model_instance: model_instances.append(model_instance) else: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Model instance with id {model_instance_id} not found.", ) elif model_id: model_project = fairgraph.brainsimulation.ModelProject.from_uuid( str(model_id), kg_client, api="nexus") model_instances = as_list(model_project.instances) if len(model_instances) > 0: simulations = [] for model_instance in model_instances: kwargs["model_instance_id"] = model_instance.id simulations.extend( as_list( fairgraph.brainsimulation.Simulation.list( kg_client, **kwargs))) else: simulations = fairgraph.brainsimulation.Simulation.list( kg_client, **kwargs) return [Simulation.from_kg_object(sim, kg_client) for sim in simulations]
async def _delete_model_instance(model_instance_id, model_project): model_instances = as_list(model_project.instances) for model_instance in model_instances[:]: # todo: we should possibly also delete emodels, modelscripts, morphologies, # but need to check they're not shared with other instances if model_instance.uuid == str(model_instance_id): model_instance.delete(kg_client) model_instances.remove(model_instance) break model_project.instances = model_instances model_project.save(kg_client)
def _check_test_script_uniqueness(test_definition, test_script, kg_client): other_scripts = test_definition.scripts.resolve(kg_client, api="nexus") for other_script in as_list(other_scripts): if (test_script.version == other_script.version and test_script.parameters == other_script.parameters and test_script.id != other_script.id): raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail= "Version and parameters match those of an existing test instance", )
def save(self): if self.obj is None: # create logger.debug("Saving result with data {}".format(self.data)) timestamp = datetime.now() additional_data = [ AnalysisResult(name="{} @ {}".format(uri, timestamp.isoformat()), result_file=Distribution(uri), timestamp=timestamp) for uri in self.data["results_storage"] ] for ad in additional_data: ad.save(self.client) self.obj = ValidationResult( name= "Validation results for model {} and test {} with timestamp {}" .format(self.data["model_version_id"], self.data["test_code_id"], timestamp.isoformat()), generated_by=None, description=None, score=self.data["score"], normalized_score=self.data["normalized_score"], passed=self.data["passed"], timestamp=timestamp, additional_data=additional_data, collab_id=self.data["project"]) self.obj.save(self.client) test_definition = self.data["test_script"].test_definition.resolve( self.client, api="nexus") reference_data = Collection( "Reference data for {}".format(test_definition.name), members=[ item.resolve(self.client, api="nexus") for item in as_list(test_definition.reference_data) ]) reference_data.save(self.client) activity = ValidationActivity( model_instance=self.data["model_instance"], test_script=self.data["test_script"], reference_data=reference_data, timestamp=timestamp, result=self.obj) activity.save(self.client) self.obj.generated_by = activity self.obj.save(self.client) else: # update raise NotImplementedError() return self.obj
def serialize(self, obj): # todo: rewrite all this using KG Query API, to avoid doing all the individual resolves. validation_activity = obj.generated_by.resolve(self.client, api="nexus") model_version_id = validation_activity.model_instance.uuid test_code_id = validation_activity.test_script.uuid logger.debug("Serializing validation test result") logger.debug("Additional data for {}:\n{}".format( obj.id, obj.additional_data)) additional_data_urls = [] for item in as_list(obj.additional_data): item = item.resolve(self.client, api="nexus") if item: additional_data_urls.append(item.result_file.location) else: logger.warning("Couldn't resolve {}".format(item)) data = { "uri": obj.id, "id": obj.uuid, "old_uuid": obj.old_uuid, "model_version_id": model_version_id, "test_code_id": test_code_id, "results_storage": [ serialize_additional_data(url, self.user_token) for url in additional_data_urls ], "score": obj.score, "passed": obj.passed, "timestamp": obj.timestamp, "project": obj.collab_id, "normalized_score": obj.normalized_score, # the following are temporary. Ideally the client should do a lookup using the IDs above "model_version": ScientificModelInstanceKGSerializer( validation_activity.model_instance, self.client).data, "test_code": ValidationTestCodeKGSerializer(validation_activity.test_script, self.client).data, "activity_uuid": validation_activity.uuid } return data
def get_test_instance_given_test_id( test_id: str, test_instance_id: UUID, token: HTTPAuthorizationCredentials = Depends(auth)): test_definition = _get_test_by_id_or_alias(test_id, token) for inst in as_list(test_definition.scripts.resolve(kg_client, api="nexus")): if UUID(inst.uuid) == test_instance_id: return ValidationTestInstance.from_kg_object(inst, kg_client) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Test ID and test instance ID are inconsistent", )
async def get_model_instance_given_model_id( model_id: str, model_instance_id: UUID, token: HTTPAuthorizationCredentials = Depends(auth)): model_project = await _get_model_by_id_or_alias(model_id, token) for inst in as_list(model_project.instances): if UUID(inst.uuid) == model_instance_id: return ModelInstance.from_kg_object(inst, kg_client, model_project.uuid) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Model ID/alias and model instance ID are inconsistent", )
async def delete_test(test_id: UUID, token: HTTPAuthorizationCredentials = Depends(auth)): # todo: handle non-existent UUID test_definition = ValidationTestDefinition.from_uuid(str(test_id), kg_client, api="nexus") if not await is_admin(token.credentials): # todo: replace this check with a group membership check for Collab v2 raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Deleting tests is restricted to admins") test_definition.delete(kg_client) for test_script in as_list( test_definition.scripts.resolve(kg_client, api="nexus")): test_script.delete(kg_client)
def get_test_instances(test_id: str, version: str = Query(None), token: HTTPAuthorizationCredentials = Depends(auth)): test_definition = _get_test_by_id_or_alias(test_id, token) test_instances = [ ValidationTestInstance.from_kg_object(inst, kg_client) for inst in as_list( test_definition.scripts.resolve(kg_client, api="nexus")) ] if version: test_instances = [ inst for inst in test_instances if inst.version == version ] return test_instances
async def get_model_instances( model_id: str, version: str = None, token: HTTPAuthorizationCredentials = Depends(auth)): model_project = await _get_model_by_id_or_alias(model_id, token) model_instances = [ ModelInstance.from_kg_object(inst, kg_client, model_project.uuid) for inst in as_list(model_project.instances) ] if version is not None: model_instances = [ inst for inst in model_instances if inst.version == version ] return model_instances
def _are_test_code_version_unique_kg(testcode_json, kg_client): """ Check if versions of test code are unique :param testcode_json: datas of test code :type testcode_json: dict :returns: response :rtype: boolean """ new_version_name = testcode_json['version'] test_definition = testcode_json['test_definition'] all_instances_versions_name = [script.version for script in as_list(test_definition.scripts.resolve(kg_client))] logger.debug("all versions: {} new version: {}".format(all_instances_versions_name, new_version_name)) if new_version_name in all_instances_versions_name: return False return True
def _are_model_instance_version_unique_kg(instance_json, kg_client): """ Check if versions of model instance are unique :param instance_json: datas of instance :type instance_json: dict :returns: response :rtype: boolean """ new_version_name = instance_json['version'] model_project = ModelProject.from_uuid(instance_json['model_id'], kg_client) if model_project.instances: all_instances_versions_name = [inst.resolve(kg_client).version for inst in as_list(model_project.instances)] if new_version_name in all_instances_versions_name: return False return True
def get_latest_test_instance_given_test_id( test_id: str, token: HTTPAuthorizationCredentials = Depends(auth)): test_definition = _get_test_by_id_or_alias(test_id, token) test_instances = [ ValidationTestInstance.from_kg_object(inst, kg_client) for inst in as_list( test_definition.scripts.resolve(kg_client, api="nexus")) ] if len(test_instances) == 0: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Test definition {test_id} has no code associated with it", ) latest = sorted(test_instances, key=lambda inst: inst.timestamp)[-1] return latest
async def delete_result(result_id: UUID, token: HTTPAuthorizationCredentials = Depends(auth)): # todo: handle non-existent UUID result = ValidationResultKG.from_uuid(str(result_id), kg_client, api="nexus") if not await is_admin(token.credentials): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Deleting validation results is restricted to admins", ) for item in as_list(result.additional_data): item.delete(kg_client) # todo: check whether the result has been used in further analysis # if so, we should probably disallow deletion unless forced result.generated_by.delete(kg_client) result.delete(kg_client)
def __init__(self, objects, client, data=None, many=False, context=None): self.client = client if isinstance(objects, (KGProxy, KGQuery)): objects = objects.resolve(self.client) if many: self.objects = objects if data: self.data = data else: self.data = [self.serialize(obj) for obj in as_list(objects)] else: self.obj = objects if data: self.data = data else: self.data = self.serialize(self.obj) self.context = context self.errors = []
async def delete_model(model_id: UUID, token: HTTPAuthorizationCredentials = Depends(auth)): # todo: handle non-existent UUID model_project = ModelProject.from_uuid(str(model_id), kg_client, api="nexus") if not (await is_collab_member(model_project.collab_id, token.credentials) or await is_admin(token.credentials)): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail= f"Access to this model is restricted to members of Collab #{model_project.collab_id}", ) model_project.delete(kg_client) for model_instance in as_list(model_project.instances): # todo: we should possibly also delete emodels, modelscripts, morphologies, # but need to check they're not shared with other instances model_instance.delete(kg_client)
async def query_live_papers( editable: bool = False, token: HTTPAuthorizationCredentials = Depends(auth) ): lps = fairgraph.livepapers.LivePaper.list(kg_client, api="nexus", size=1000) if editable: # include only those papers the user can edit editable_collabs = await get_editable_collabs(token.credentials) accessible_lps = [ lp for lp in lps if lp.collab_id in editable_collabs ] else: # include all papers the user can view accessible_lps = [] for lp in lps: if await can_view_collab(lp.collab_id, token.credentials): accessible_lps.append(lp) return [ LivePaperSummary.from_kg_object(lp) for lp in as_list(accessible_lps) ]
def serialize(self, obj): # todo: rewrite all this using KG Query API, to avoid doing all the individual resolves. validation_activity = obj.generated_by.resolve(self.client) model_version_id = validation_activity.model_instance.uuid test_code_id = validation_activity.test_script.uuid data = { "uri": obj.id, "id": obj.uuid, "old_uuid": obj.old_uuid, "model_version_id": model_version_id, "test_code_id": test_code_id, "results_storage": [ serialize_additional_data( item.resolve(self.client).result_file.location, self.client) for item in as_list(obj.additional_data) ], "score": obj.score, "passed": obj.passed, "timestamp": obj.timestamp, "project": obj.collab_id, "normalized_score": obj.normalized_score, # the following are temporary. Ideally the client should do a lookup using the IDs above "model_version": ScientificModelInstanceKGSerializer( validation_activity.model_instance, self.client).data, "test_code": ValidationTestCodeKGSerializer(validation_activity.test_script, self.client).data } return data
async def create_model_instance( model_id: str, model_instance: ModelInstance, token: HTTPAuthorizationCredentials = Depends(auth), ): model_project = await _get_model_by_id_or_alias(model_id, token) # check permissions for this model if model_project.collab_id and not (await is_collab_member( model_project.collab_id, token.credentials) or await is_admin(token.credentials)): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail= f"This account is not a member of Collab #{model_project.project_id}", ) kg_objects = model_instance.to_kg_objects(model_project) model_instance_kg = kg_objects[-1] # check if an identical model instance already exists, raise an error if so if model_instance_kg.exists(kg_client, api="any"): raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=f"Another model instance with the same name already exists.", ) # otherwise save to KG for obj in kg_objects: obj.save(kg_client) # not sure the following is needed. # Should just be able to leave the existing model instances as KGProxy objects? model_project.instances = [ inst.resolve(kg_client, api="nexus") for inst in as_list(model_project.instances) ] model_project.instances.append(model_instance_kg) model_project.save(kg_client) return ModelInstance.from_kg_object(model_instance_kg, kg_client, model_project.uuid)
def save(self, allow_update=True): if self.obj is None: # create for key in ("author", "owner"): if isinstance(self.data[key], dict): self.data[key] = [self.data[key]] self.obj = ModelProject( self.data["name"], [ Person(p["family_name"], p["given_name"], p.get("email", None)) for p in as_list(self.data["owner"]) ], [ Person(p["family_name"], p["given_name"], p.get("email", None)) for p in as_list(self.data["author"]) ], # need to update person representation in clients, self.data.get("description"), datetime.now(), self.data.get("private", True), self.context["collab_id"], self.data.get("alias"), Organization(self.data["organization"]) if self.data.get( "organization", False) else None, pla_components=None, brain_region=self._get_ontology_obj(BrainRegion, "brain_region"), species=self._get_ontology_obj(Species, "species"), celltype=self._get_ontology_obj(CellType, "cell_type"), abstraction_level=self._get_ontology_obj( AbstractionLevel, "abstraction_level"), model_of=self._get_ontology_obj(ModelScope, "model_scope"), old_uuid=self.data.get("old_uuid"), images=self.data.get("images")) else: # update if "name" in self.data: self.obj.name = self.data["name"] if "alias" in self.data: self.obj.alias = self.data["alias"] if "author" in self.data: self.obj.authors = [ Person(p["family_name"], p["given_name"], p.get("email", None)) for p in as_list(self.data["author"]) ] # need to update person representation in clients if "owner" in self.data: self.obj.owners = [ Person(p["family_name"], p["given_name"], p.get("email", None)) for p in as_list(self.data["owner"]) ] # need to update person representation in clients if "app" in self.data: self.obj.collab_id = self.data["app"]["collab_id"] if "organization" in self.data and self.data[ "organization"] is not None: self.obj.organization = Organization(self.data["organization"]) if "private" in self.data: self.obj.private = self.data["private"] if "cell_type" in self.data: self.obj.celltype = self._get_ontology_obj( CellType, "cell_type") if "model_scope" in self.data: self.obj.model_of = self._get_ontology_obj( ModelScope, "model_scope") if "abstraction_level" in self.data: self.obj.abstraction_level = self._get_ontology_obj( AbstractionLevel, "abstraction_level") if "brain_region" in self.data: self.obj.brain_region = self._get_ontology_obj( BrainRegion, "brain_region") if "species" in self.data: self.obj.species = self._get_ontology_obj(Species, "species") if "description" in self.data: self.obj.description = self.data["description"] if "old_uuid" in self.data: self.obj.old_uuid = self.data["old_uuid"] if "images" in self.data: self.obj.images = self.data["images"] # now save people, organization, model. No easy way to make this atomic, I don't think. for person in chain(as_list(self.obj.authors), as_list(self.obj.owners)): if not isinstance(person, KGProxy): # no need to save if we have a proxy object, as # that means the person hasn't been updated person.save(self.client) if self.obj.organization and not isinstance(self.obj.organization, KGProxy): self.obj.organization.save(self.client) self.obj.save(self.client) return self.obj
def save(self): if self.obj is None: # create reference_data = [ AnalysisResult( name="Reference data #{} for validation test '{}'".format( i, self.data["name"]), result_file=Distribution(url)) for i, url in enumerate(as_list(self.data["data_location"])) ] for item in reference_data: try: item.save(self.client) except Exception as err: logger.error( "error saving reference data. name = {}, urls={}". format(self.data["name"], self.data["data_location"])) raise authors = self.data["author"] # if not isinstance(authors, list): # authors = [authors] self.obj = ValidationTestDefinition( name=self.data["name"], alias=self.data.get("alias"), status=self.data.get("status", "proposal"), species=self._get_ontology_obj(Species, "species"), brain_region=self._get_ontology_obj(BrainRegion, "brain_region"), celltype=self._get_ontology_obj(CellType, "cell_type"), reference_data=reference_data, data_type=self.data.get("data_type"), recording_modality=self.data.get("data_modality"), test_type=self.data.get("test_type"), score_type=self.data.get("score_type"), description=self.data.get("protocol"), authors=[ Person(p["family_name"], p["given_name"], p.get("email", None)) for p in as_list(authors) ], date_created=datetime.now()) for author in self.obj.authors: author.save(self.client) else: # update logger.debug("Updating test {} with data {}".format( self.obj.id, self.data)) if "name" in self.data: self.obj.name = self.data["name"] if "alias" in self.data: self.obj.alias = self.data["alias"] if "status" in self.data: self.obj.status = self.data["status"] if "species" in self.data: self.obj.species = self._get_ontology_obj(Species, "species") if "brain_region" in self.data: self.obj.brain_region = self._get_ontology_obj( BrainRegion, "brain_region") if "cell_type" in self.data: self.obj.celltype = self._get_ontology_obj( CellType, "cell_type") if "data_type" in self.data: self.obj.data_type = self.data["data_type"] if "data_modality" in self.data: self.obj.recording_modality = self.data["data_modality"] if "test_type" in self.data: self.obj.test_type = self.data["test_type"] if "score_type" in self.data: self.obj.score_type = self.data["score_type"] if "protocol" in self.data: self.obj.description = self.data["protocol"] if "data_location" in self.data: self.obj.reference_data = [ AnalysisResult( name="Reference data #{} for validation test '{}'". format(i, self.data["name"]), result_file=Distribution(url)) for i, url in enumerate( as_list(self.data["data_location"])) ] if "author" in self.data: self.obj.authors = [ Person(p["family_name"], p["given_name"], p.get("email", None)) for p in as_list(self.data["author"]) ] # now save people, ref data, test. No easy way to make this atomic, I don't think. for person in as_list(self.obj.authors): if not isinstance(person, KGProxy): # no need to save if we have a proxy object, as # that means the person hasn't been updated # although in fact the authors are saved when the test is saved # need to make this consistent person.save(self.client) for ref_data in as_list(self.obj.reference_data): if not isinstance(person, KGProxy): ref_data.save(self.client) self.obj.save(self.client) return self.obj
def serialize(self, test): # todo: rewrite all this using KG Query API, to avoid doing all the individual resolves. def serialize_person(p): if isinstance(p, KGProxy): pr = p.resolve(self.client, api="nexus") else: pr = p return {"given_name": pr.given_name, "family_name": pr.family_name} data = { 'id': test.uuid, # extract uuid from uri? 'uri': test.id, 'name': test.name, 'alias': test.alias, 'status': test.status, 'species': test.species.label if test.species else None, 'brain_region': test.brain_region.label if test.brain_region else None, 'cell_type': test.celltype.label if test.celltype else None, #'age': # todo 'data_location': [ item.resolve(self.client, api="nexus").result_file.location for item in as_list(test.reference_data) ][0], # to fix: reference_data should never really be a list 'data_type': test.data_type, 'data_modality': test.recording_modality, 'test_type': test.test_type, 'score_type': test.score_type or "Other", 'protocol': test.description, 'author': [serialize_person(au) for au in as_list(test.authors)], 'creation_date': test.date_created, #'publication': test.publication, 'old_uuid': test.old_uuid, 'codes': [], # unclear if this should be "codes" or "test_codes" } logger.debug("!!! {}".format(test.scripts)) for script in as_list(test.scripts.resolve(self.client, api="nexus")): if isinstance(script.repository, dict): repo = script.repository["@id"] elif isinstance(script.repository, IRI): repo = script.repository.value else: repo = script.repository data['codes'].append({ "uri": script.id, "id": script.uuid, "old_uuid": script.old_uuid, "repository": repo, "version": script.version, "description": script.description, "parameters": script.parameters, "path": script.test_class, "timestamp": script.date_created }) logger.debug("Serialized {} to {}".format(test, data)) return data