async def _get_model_by_id_or_alias(model_id, token):
try:
model_id = UUID(model_id)
model_project = ModelProject.from_uuid(str(model_id), kg_client, api="nexus")
except ValueError:
model_alias = str(model_id)
model_project = ModelProject.from_alias(model_alias, kg_client, api="nexus")
if not model_project:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Model with ID or alias '{model_id}' not found.",
)
# todo: fairgraph should accept UUID object as well as str
await _check_model_access(model_project, token)
return model_project
def test_list_kgquery(self, kg_client):
models = ModelProject.list(kg_client,
api="query",
scope="latest",
size=10,
resolved=True)
assert len(models) == 10
def check_if_alias_is_already_taken(name):
print('comparing with KG database [...]')
client = KGClient(token,
nexus_endpoint="https://nexus.humanbrainproject.org/v0")
MODELS = ModelProject.list(client, size=10000)
NAMES = [project.name for project in MODELS]
if name in NAMES:
i0 = int(np.argwhere(np.array(NAMES, dtype=str) == name))
if type(MODELS[i0].authors) is list:
author_list = ''
for nn in list(MODELS[i0].authors):
author_list += nn.resolve(client).full_name + '; '
else:
author_list = MODELS[i0].authors.resolve(client).full_name
print('/!\ --> The alias "%s"' % name,
'is redundant with the following entry:')
print('''
- Description:
%s
- Author(s):
%s
''' % (MODELS[i0].resolve(client).description[:200]+' [...] '+\
MODELS[i0].resolve(client).description[-200:],
author_list))
else:
print('--> The alias "%s"' % name, 'is valid')
def test_list_kgquery_with_filter(self, kg_client):
models = ModelProject.list(kg_client,
api="query",
scope="latest",
size=10,
resolved=True,
species="Rattus norvegicus")
assert len(models) == 2
async def update_model(
model_id: UUID,
model_patch: ScientificModelPatch,
token: HTTPAuthorizationCredentials = Depends(auth),
):
# if payload contains a project_id, check permissions for that id
if model_patch.project_id and not (await is_collab_member(
model_patch.project_id, token.credentials)
or await is_admin(token.credentials)):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=
f"This account is not a member of Collab #{model_patch.project_id}",
)
# retrieve stored model
model_project = ModelProject.from_uuid(str(model_id),
kg_client,
api="nexus")
stored_model = ScientificModel.from_kg_object(model_project, kg_client)
# if retrieved project_id is different to payload id, check permissions for that id
if stored_model.project_id != model_patch.project_id and not (
await is_collab_member(stored_model.project_id, token.credentials)
or await is_admin(token.credentials)):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=
f"Access to this model is restricted to members of Collab #{stored_model.project_id}",
)
# if alias changed, check uniqueness of new alias
if (model_patch.alias and model_patch.alias != stored_model.alias
and model_alias_exists(model_patch.alias, kg_client)):
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=
f"Another model with alias '{model_patch.alias}' already exists.",
)
# todo: if model id provided in payload, check it matches the model_id parameter
# todo: if model uri provided in payload, check it matches the id
# here we are updating the pydantic model `stored_model`, then recreating the kg objects
# from this. It might be more efficient to directly update `model_project`.
# todo: profile both possible implementations
update_data = model_patch.dict(exclude_unset=True)
for field, value in update_data.items():
if field in ("author", "owner"):
update_data[field] = [Person(**p) for p in update_data[field]]
updated_model = stored_model.copy(update=update_data)
kg_objects = updated_model.to_kg_objects()
for obj in kg_objects:
obj.save(kg_client)
model_project = kg_objects[-1]
assert isinstance(model_project, ModelProject)
return ScientificModel.from_kg_object(model_project, kg_client)
def test_existence_query(self, kg_client):
obj = ModelProject(name="foo",
owners=Person("Holmes", "Sherlock"),
authors=Person("Holmes", "Sherlock"),
description="",
private=True,
date_created=datetime(2000, 1, 1))
expected = {
"op":
"and",
"value": [{
"path": "schema:name",
"op": "eq",
"value": "foo"
}, {
"path": "schema:dateCreated",
"op": "eq",
"value": "2000-01-01T00:00:00"
}]
}
generated = obj._build_existence_query(api="nexus")
assert expected == generated
def _are_model_instance_version_unique_kg(instance_json, kg_client):
"""
Check if versions of model instance are unique
:param instance_json: datas of instance
:type instance_json: dict
:returns: response
:rtype: boolean
"""
new_version_name = instance_json['version']
model_project = ModelProject.from_uuid(instance_json['model_id'], kg_client)
if model_project.instances:
all_instances_versions_name = [inst.resolve(kg_client).version for inst in as_list(model_project.instances)]
if new_version_name in all_instances_versions_name:
return False
return True
async def delete_model_instance(
model_id: UUID,
model_instance_id: UUID,
token: HTTPAuthorizationCredentials = Depends(auth)):
# todo: handle non-existent UUID
model_project = ModelProject.from_uuid(str(model_id),
kg_client,
api="nexus")
if not (await is_collab_member(model_project.collab_id, token.credentials)
or await is_admin(token.credentials)):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=
f"Access to this model is restricted to members of Collab #{model_project.collab_id}",
)
await _delete_model_instance(model_instance_id, model_project)
async def delete_model(model_id: UUID,
token: HTTPAuthorizationCredentials = Depends(auth)):
# todo: handle non-existent UUID
model_project = ModelProject.from_uuid(str(model_id),
kg_client,
api="nexus")
if not (await is_collab_member(model_project.collab_id, token.credentials)
or await is_admin(token.credentials)):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=
f"Access to this model is restricted to members of Collab #{model_project.collab_id}",
)
model_project.delete(kg_client)
for model_instance in as_list(model_project.instances):
# todo: we should possibly also delete emodels, modelscripts, morphologies,
# but need to check they're not shared with other instances
model_instance.delete(kg_client)
def is_valid(self):
# check alias is unique
if "alias" in self.data and self.data["alias"]:
if self.obj and self.obj.alias == self.data["alias"]:
return True
logger.debug("Checking for model with same alias")
model_with_same_alias = ModelProject.from_alias(
self.data["alias"], self.client)
if bool(model_with_same_alias):
self.errors.append(
"Another model with this alias already exists.")
return False
if "private" in self.data:
if not isinstance(self.data["private"], bool):
self.errors.append("'private' must be a boolean")
return False
if "author" not in self.data or not self.data["author"]:
self.errors.append("This field may not be blank.")
return False
return True # todo
def save(self):
# todo: Create/update EModel, MEModel and Morphology where model_scope is "single cell"
if self.obj is None: # create
model_project = ModelProject.from_uuid(self.data["model_id"],
self.client,
api="nexus")
script = ModelScript(name="ModelScript for {} @ {}".format(
model_project.name, self.data["version"]),
code_format=self.data.get("code_format"),
code_location=self.data["source"],
license=self.data.get("license"))
script.save(self.client)
if model_project.model_of and model_project.model_of.label == "single cell" and "morphology" in self.data:
e_model = EModel(name="EModel for {} @ {}".format(
model_project.name, self.data["version"]),
brain_region=model_project.brain_region,
species=model_project.species,
model_of=None,
main_script=None,
release=None)
e_model.save(self.client)
morph = Morphology(name="Morphology for {} @ {}".format(
model_project.name, self.data["version"]),
cell_type=model_project.celltype,
morphology_file=self.data["morphology"])
morph.save(self.client)
minst = MEModel(name="ModelInstance for {} @ {}".format(
model_project.name, self.data["version"]),
description=self.data.get("description", ""),
brain_region=model_project.brain_region,
species=model_project.species,
model_of=None,
main_script=script,
e_model=e_model,
morphology=morph,
version=self.data["version"],
parameters=self.data.get("parameters"),
timestamp=datetime.now(),
release=None)
else:
minst = ModelInstance(name="ModelInstance for {} @ {}".format(
model_project.name, self.data["version"]),
description=self.data.get(
"description", ""),
brain_region=model_project.brain_region,
species=model_project.species,
model_of=None,
main_script=script,
version=self.data["version"],
parameters=self.data.get("parameters"),
timestamp=datetime.now(),
release=None)
minst.save(self.client)
self.obj = minst
if model_project.instances:
if not isinstance(model_project.instances, list):
model_project.instances = [model_project.instances]
model_project.instances.append(minst)
else:
model_project.instances = [minst]
model_project.save(self.client)
else: # update
instance_changed = False
script_changed = False
morphology_changed = False
def resolve_obj(obj):
if isinstance(obj, KGProxy):
return obj.resolve(self.client, api="nexus")
else:
return obj
if "name" in self.data:
self.obj.name = self.data["name"]
# todo: also update e_model and morphology
instance_changed = True
if "description" in self.data:
self.obj.description = self.data.get("description", "")
instance_changed = True
if "version" in self.data:
self.obj.version = self.data["version"]
instance_changed = True
if "parameters" in self.data:
self.obj.parameters = self.data.get("parameters")
instance_changed = True
if "code_format" in self.data:
self.obj.main_script = resolve_obj(self.obj.main_script)
self.obj.main_script.code_format = self.data.get("code_format")
script_changed = True
if "source" in self.data:
self.obj.main_script = resolve_obj(self.obj.main_script)
self.obj.main_script.code_location = self.data["source"]
script_changed = True
if "license" in self.data:
self.obj.main_script = resolve_obj(self.obj.main_script)
self.obj.main_script.license = self.data.get("license")
script_changed = True
if "morphology" in self.data and self.data[
"morphology"] is not None:
self.obj.morphology = resolve_obj(self.obj.morphology)
self.obj.morphology.morphology_file = self.data["morphology"]
morphology_changed = True
logger.debug("Morphology changed: {}".format(morphology_changed))
logger.debug("Script changed: {}".format(script_changed))
logger.debug("Instance changed: {}".format(instance_changed))
if morphology_changed:
self.obj.morphology.save(self.client)
if script_changed:
self.obj.main_script.save(self.client)
if instance_changed:
self.obj.save(self.client)
return self.obj
def test_list_nexus_with_filter(self, kg_client):
models = ModelProject.list(kg_client,
api="nexus",
size=10,
species="Rattus norvegicus")
assert len(models) == 6
def test_count_kgquery(self, kg_client):
count = ModelProject.count(kg_client, api='query')
assert count == 351
def test_count_nexus(self, kg_client):
count = ModelProject.count(kg_client, api='nexus')
assert count == 351
def model_alias_exists(alias, client):
if alias:
model_with_same_alias = ModelProject.from_alias(alias, client, api="nexus")
return bool(model_with_same_alias)
return False
class ScientificModelKGSerializer(BaseKGSerializer):
def is_valid(self):
# check alias is unique
if "alias" in self.data and self.data["alias"]:
if self.obj and self.obj.alias == self.data["alias"]:
return True
logger.debug("Checking for model with same alias")
model_with_same_alias = ModelProject.from_alias(self.data["alias"],
self.client,
api="nexus")
if bool(model_with_same_alias):
self.errors.append(
"Another model with this alias already exists.")
return False
if "private" in self.data:
if not isinstance(self.data["private"], bool):
self.errors.append("'private' must be a boolean")
return False
if "author" not in self.data or not self.data["author"]:
self.errors.append("This field may not be blank.")
return False
return True # todo
def save(self, allow_update=True):
if self.obj is None: # create
for key in ("author", "owner"):
if isinstance(self.data[key], dict):
self.data[key] = [self.data[key]]
self.obj = ModelProject(
self.data["name"],
[
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["owner"])
],
[
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["author"])
], # need to update person representation in clients,
self.data.get("description"),
datetime.now(),
self.data.get("private", True),
self.context["collab_id"],
self.data.get("alias"),
Organization(self.data["organization"]) if self.data.get(
"organization", False) else None,
pla_components=None,
brain_region=self._get_ontology_obj(BrainRegion,
"brain_region"),
species=self._get_ontology_obj(Species, "species"),
celltype=self._get_ontology_obj(CellType, "cell_type"),
abstraction_level=self._get_ontology_obj(
AbstractionLevel, "abstraction_level"),
model_of=self._get_ontology_obj(ModelScope, "model_scope"),
old_uuid=self.data.get("old_uuid"),
images=self.data.get("images"))
else: # update
if "name" in self.data:
self.obj.name = self.data["name"]
if "alias" in self.data:
self.obj.alias = self.data["alias"]
if "author" in self.data:
self.obj.authors = [
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["author"])
] # need to update person representation in clients
if "owner" in self.data:
self.obj.owners = [
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["owner"])
] # need to update person representation in clients
if "app" in self.data:
self.obj.collab_id = self.data["app"]["collab_id"]
if "organization" in self.data and self.data[
"organization"] is not None:
self.obj.organization = Organization(self.data["organization"])
if "private" in self.data:
self.obj.private = self.data["private"]
if "cell_type" in self.data:
self.obj.celltype = self._get_ontology_obj(
CellType, "cell_type")
if "model_scope" in self.data:
self.obj.model_of = self._get_ontology_obj(
ModelScope, "model_scope")
if "abstraction_level" in self.data:
self.obj.abstraction_level = self._get_ontology_obj(
AbstractionLevel, "abstraction_level")
if "brain_region" in self.data:
self.obj.brain_region = self._get_ontology_obj(
BrainRegion, "brain_region")
if "species" in self.data:
self.obj.species = self._get_ontology_obj(Species, "species")
if "description" in self.data:
self.obj.description = self.data["description"]
if "old_uuid" in self.data:
self.obj.old_uuid = self.data["old_uuid"]
if "images" in self.data:
self.obj.images = self.data["images"]
# now save people, organization, model. No easy way to make this atomic, I don't think.
for person in chain(as_list(self.obj.authors),
as_list(self.obj.owners)):
if not isinstance(person, KGProxy):
# no need to save if we have a proxy object, as
# that means the person hasn't been updated
person.save(self.client)
if self.obj.organization and not isinstance(self.obj.organization,
KGProxy):
self.obj.organization.save(self.client)
self.obj.save(self.client)
return self.obj
def serialize(self, model):
# todo: rewrite all this using KG Query API, to avoid doing all the individual resolves.
def serialize_person(p):
if isinstance(p, KGProxy):
pr = p.resolve(self.client, api="nexus")
else:
pr = p
return {"given_name": pr.given_name, "family_name": pr.family_name}
data = {
'id':
model.uuid, # extract uuid from uri?
'uri':
model.id,
'name':
model.name,
'alias':
model.alias,
'author': [serialize_person(au) for au in as_list(model.authors)],
'owner': [serialize_person(ow) for ow in as_list(model.owners)],
'app': {
'collab_id': model.collab_id
},
'organization':
model.organization.resolve(self.client, api="nexus").name
if model.organization else None,
'private':
model.private,
'cell_type':
model.celltype.label if model.celltype else None,
'model_scope':
model.model_of.label if model.model_of else None,
'abstraction_level':
model.abstraction_level.label if model.abstraction_level else None,
'brain_region':
model.brain_region.label if model.brain_region else None,
'species':
model.species.label if model.species else None,
'description':
model.description,
'images':
model.images,
'old_uuid':
model.old_uuid,
'instances': [],
#"raw_data": model.instance.data
}
for instance in as_list(model.instances):
instance = instance.resolve(self.client, api="nexus")
if not instance: # if we have a stale reference to a deprecated instance
continue
main_script = instance.main_script.resolve(self.client,
api="nexus")
instance_data = {
"id":
instance.uuid,
"uri":
instance.id,
#"old_uuid": instance.old_uuid
"version":
instance.version,
"description":
instance.description or '',
"parameters":
instance.parameters or '',
"code_format":
'',
"source":
'',
"license":
'',
"hash":
'',
"timestamp":
instance.timestamp.isoformat() if instance.timestamp else None
}
if main_script:
instance_data.update({
"code_format": main_script.code_format or '',
"source": main_script.code_location or '',
"license": main_script.license or '',
})
if hasattr(instance, "morphology"):
morph = instance.morphology.resolve(self.client, api="nexus")
instance_data["morphology"] = morph.morphology_file
data['instances'].append(instance_data)
return data
def load_model_instances(show_ignore=False,
size=100000,
scope='inferred',
verbose=False,
api='nexus'):
client = KGClient(os.environ["HBP_token"])
MODEL_INSTANCES = []
models = ModelProject.list(client,
api=api,
scope=scope,
size=size)
for model in models:
if verbose:
print('\n##########################################')
print('------ %s ------' % model.name)
minst = template.copy()
minst['name'] = model.name
minst['owner'] = get_author_details(model.owners.resolve(client))
if type(model.authors) is list:
minst['author(s)'] = [get_author_details(auth.resolve(client)) for auth in model.authors]
else:
minst['author(s)'] = [get_author_details(model.authors.resolve(client))]
try:
minst['authors_str'] = model.authors_str(client)
except TypeError:
minst['authors_str'] = ''
for a in minst['author(s)']:
minst['authors_str'] += a['family_name']+', '+a['given_name']+';'
minst['description'] = model.description
minst['private'] = model.private
minst['collab_id'] = model.collab_id
minst['alias'] = model.alias
for key, quant in zip(['brain_region', 'species', 'cell_type', 'abstraction_level', 'model_scope'],
[model.brain_region, model.species, model.celltype, model.abstraction_level, model.model_of]):
if quant is not None:
minst[key] = (quant.label, '')
elif verbose:
print('[!!] "%s" missing for %s ' % (key, model.name))
for key, quant in zip(['organization'],
[model.organization]):
if quant is not None:
minst[key] = (quant.resolve(client).name, '')
elif verbose:
print('[!!] "%s" missing for %s ' % (key, model.name))
if type(model.instances) is list:
for modelI in model.instances:
MODEL_INSTANCES.append(minst)
# pprint.pprint(MODEL_INSTANCES[-1])
# print('')
add_version_details_to_model(MODEL_INSTANCES[-1], modelI.resolve(client), client, verbose=verbose)
elif type(model.instances) is KGProxy:
modelI = model.instances
MODEL_INSTANCES.append(minst.copy())
add_version_details_to_model(MODEL_INSTANCES[-1], modelI.resolve(client), client, verbose=verbose)
elif show_ignore:
print('Ignoring %s @ %s' % (model.name, model.date_created))
pass # we don't care about models without specific version
DATES = np.array([time.mktime(minst['date_created'].timetuple()) for minst in MODEL_INSTANCES])
return [MODEL_INSTANCES[i] for i in np.argsort(DATES)[::-1]]
async def query_models(
alias: List[str] = Query(
None,
description="A list of model aliases (short names) to search for"),
id: List[UUID] = Query(
None, description="A list of specific model IDs to search for"),
name: List[str] = Query(None,
description="Model name(s) to search for"),
brain_region: List[BrainRegion] = Query(
None,
description=
"Find models intended to represent this/these brain region(s)"),
species: List[Species] = Query(
None,
description="Find models intended to represent this/these species"
),
cell_type: List[CellType] = Query(
None, description="Find models of this/these cell type(s)"),
model_scope: ModelScope = Query(
None, description="Find models with a certain scope"),
abstraction_level: AbstractionLevel = Query(
None, description="Find models with a certain abstraction level"),
author: List[str] = Query(
None, description="Find models by author (family name)"),
owner: List[str] = Query(
None, description="Find models by owner (family name)"),
organization: List[str] = Query(
None, description="Find models by organization"),
project_id: List[str] = Query(
None,
description="Find models belonging to a specific project/projects"
),
private: bool = Query(
None, description="Limit the search to public or private models"),
summary: bool = Query(
False,
description="Return only summary information about each model"),
size: int = Query(100, description="Maximum number of responses"),
from_index: int = Query(
0, description="Index of the first response returned"),
# from header
token: HTTPAuthorizationCredentials = Depends(auth),
):
"""
Search the model catalog for specific models (identitified by their unique ID or by a short name / alias),
and/or search by attributes of the models (e.g. the cell types being modelled, the type of model, the model author).
"""
# If project_id is provided:
# - private = None: both public and private models from that project (collab), if the user is a member
# - private = True: only private models from that project, if that user is a member
# - private = False: only public models from that project
# If project_id is not provided:
# - private = None: only public models, from all projects
# - private = True: 400? error "To see private models, you must specify the project/collab"
# - private = False: only public models, from all projects
if private:
if project_id:
for collab_id in project_id:
if not await is_collab_member(collab_id, token.credentials):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="You are not a member of project #{collab_id}",
)
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=
"To see private models, you must specify the project/collab id",
)
# get the values of of the Enums
if brain_region:
brain_region = [item.value for item in brain_region]
if species:
species = [item.value for item in species]
if cell_type:
cell_type = [item.value for item in cell_type]
if model_scope:
model_scope = model_scope.value
if abstraction_level:
abstraction_level = abstraction_level.value
filter_query, context = build_model_project_filters(
alias,
id,
name,
brain_region,
species,
cell_type,
model_scope,
abstraction_level,
author,
owner,
organization,
project_id,
private,
)
if len(filter_query["value"]) > 0:
logger.info(
"Searching for ModelProject with the following query: {}".format(
filter_query))
# note that from_index is not currently supported by KGQuery.resolve
model_projects = KGQuery(ModelProject, {
"nexus": filter_query
}, context).resolve(kg_client, api="nexus", size=size)
else:
model_projects = ModelProject.list(kg_client,
api="nexus",
size=size,
from_index=from_index)
if summary:
cls = ScientificModelSummary
else:
cls = ScientificModel
return [
cls.from_kg_object(model_project, kg_client)
for model_project in as_list(model_projects)
]
def save(self, allow_update=True):
if self.obj is None: # create
for key in ("author", "owner"):
if isinstance(self.data[key], dict):
self.data[key] = [self.data[key]]
self.obj = ModelProject(
self.data["name"],
[
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["owner"])
],
[
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["author"])
], # need to update person representation in clients,
self.data.get("description"),
datetime.now(),
self.data.get("private", True),
self.context["collab_id"],
self.data.get("alias"),
Organization(self.data["organization"]) if self.data.get(
"organization", False) else None,
pla_components=None,
brain_region=self._get_ontology_obj(BrainRegion,
"brain_region"),
species=self._get_ontology_obj(Species, "species"),
celltype=self._get_ontology_obj(CellType, "cell_type"),
abstraction_level=self._get_ontology_obj(
AbstractionLevel, "abstraction_level"),
model_of=self._get_ontology_obj(ModelScope, "model_scope"),
old_uuid=self.data.get("old_uuid"),
images=self.data.get("images"))
else: # update
if "name" in self.data:
self.obj.name = self.data["name"]
if "alias" in self.data:
self.obj.alias = self.data["alias"]
if "author" in self.data:
self.obj.authors = [
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["author"])
] # need to update person representation in clients
if "owner" in self.data:
self.obj.owners = [
Person(p["family_name"], p["given_name"],
p.get("email", None))
for p in as_list(self.data["owner"])
] # need to update person representation in clients
if "app" in self.data:
self.obj.collab_id = self.data["app"]["collab_id"]
if "organization" in self.data and self.data[
"organization"] is not None:
self.obj.organization = Organization(self.data["organization"])
if "private" in self.data:
self.obj.private = self.data["private"]
if "cell_type" in self.data:
self.obj.celltype = self._get_ontology_obj(
CellType, "cell_type")
if "model_scope" in self.data:
self.obj.model_of = self._get_ontology_obj(
ModelScope, "model_scope")
if "abstraction_level" in self.data:
self.obj.abstraction_level = self._get_ontology_obj(
AbstractionLevel, "abstraction_level")
if "brain_region" in self.data:
self.obj.brain_region = self._get_ontology_obj(
BrainRegion, "brain_region")
if "species" in self.data:
self.obj.species = self._get_ontology_obj(Species, "species")
if "description" in self.data:
self.obj.description = self.data["description"]
if "old_uuid" in self.data:
self.obj.old_uuid = self.data["old_uuid"]
if "images" in self.data:
self.obj.images = self.data["images"]
# now save people, organization, model. No easy way to make this atomic, I don't think.
for person in chain(as_list(self.obj.authors),
as_list(self.obj.owners)):
if not isinstance(person, KGProxy):
# no need to save if we have a proxy object, as
# that means the person hasn't been updated
person.save(self.client)
if self.obj.organization and not isinstance(self.obj.organization,
KGProxy):
self.obj.organization.save(self.client)
self.obj.save(self.client)
return self.obj
assert len(name) > 6
bundle = FileBundle(identifier=hashlib.md5(script.code_location.encode('utf-8')).hexdigest(),
name=name,
description=model_instance.description,
url=script.code_location,
#usageNotes,
modelInstance=model_release) # check this is showing in KG editor
print(bundle)
bundle.save(client)
if __name__ == "__main__":
client = KGClient(token, nexus_endpoint="https://nexus.humanbrainproject.org/v0")
projects = ModelProject.list(client, size=10000)
single_cell_projects = [p for p in projects if p.model_of and p.model_of.label == "single cell" and "emodel" not in p.name]
single_cell_no_memodel = [p for p in single_cell_projects if "CA1_pyr" not in p.name and "CA1_int" not in p.name]
#for model_project in ModelProject.list(client, size=10000):
for model_project in single_cell_no_memodel:
#if model_project.model_of and "network" in model_project.model_of.label:
#if model_project.model_of and "single cell" not in model_project.model_of.label:
if True:
contributors = get_uniminds_person_list(model_project.authors, client)
custodians = get_uniminds_person_list(model_project.owners, client)
for model_instance in as_list(model_project.instances):
model_instance = model_instance.resolve(client)
script = model_instance.main_script.resolve(client)
release = create_or_update_model_release(model_project, model_instance, contributors, custodians, script, client)
def migrate_models(self):
models = ScientificModel.objects.all()
for model in models:
authors = self._get_people_from_Persons_table(model.author)
for author in authors:
author.save(NAR_client)
owners = self._get_people_from_Persons_table(model.owner)
for owner in owners:
owner.save(NAR_client)
if len(owners) == 0:
owners = authors[-1:]
if len(owners) > 1:
owners = owners[
0] # temporary, need to fix schema to remove maxCount: 1
organization = self.get_organization(model.organization)
brain_region = self.get_parameters("brain_region",
model.brain_region)
species = self.get_parameters("species", model.species)
cell_type = self.get_parameters("cell_type", model.cell_type)
abstraction_level = self.get_parameters("abstraction_level",
model.abstraction_level)
model_scope = self.get_parameters("model_scope", model.model_scope)
model_project = ModelProject(
name=model.name,
owners=owners,
authors=authors,
description=model.description,
date_created=model.creation_date,
private=model.private,
collab_id=model.app.collab_id,
alias=model.alias,
organization=organization,
pla_components=model.pla_components,
brain_region=brain_region,
species=species,
celltype=cell_type,
abstraction_level=abstraction_level,
model_of=model_scope, # to fix
images=[{
"url": im.url,
"caption": im.caption
} for im in model.images.all()],
old_uuid=str(model.id))
#print ("--authors",model_project.authors)
#print ("--organization",model_project.organization)
#print ("--brain_region",model_project.brain_region)
#print ("--species",model_project.species)
try:
model_project.save(NAR_client)
except Exception as err:
if "internal server error" in err.response.text:
logger.error(err)
else:
raise
else:
logger.info("ModelProject saved: %s", model_project)
print(model_project)
# models_with_parents = ScientificModel.objects.filter(parents__isnull=False)
# for model in models_with_parents:
# mp = ModelProject.by_name(model.name, NAR_client)
# for parent_obj in model.parents.all():
# parent_kg = ModelProject.by_name(parent_obj.name, NAR_client)
# mp.parents = as_list(mp.parents) + [parent_kg]
# mp.save(NAR_client)
return ''
def test_all():
ModelProject.set_strict_mode(False) # check: will this interfere with other tests?
MEModel.set_strict_mode(False)
## Get a list of possible filter terms
for cls in (Species, CellType, BrainRegion, AbstractionLevel, ModelScope):
print("\nPossible values of {}:\n - ".format(cls.__name__))
print("\n - ".join(cls.iri_map.keys()))
## Find models of hippocampus pyramidal neurons
models = ModelProject.list(client,
brain_region="hippocampus",
model_of="single cell",
species="Rattus norvegicus")
len(models)
columns = ("name", "brain_region", "celltype", "authors")
data = []
for model in models:
row = {}
for attr in columns:
value = getattr(model, attr)
if hasattr(value, "label"):
row[attr] = value.label
elif attr == "authors":
row[attr] = ", ".join("{au.given_name} {au.family_name}".format(au=obj.resolve(client))
for obj in as_list(value))
else:
row[attr] = value
data.append(row)
df = pd.DataFrame(data=data, columns=columns) #.sort_values(by=['name'], kind="mergesort")
dfStyler = df.style.set_properties(**{'text-align': 'left'})
dfStyler.set_table_styles([dict(selector='th', props=[('text-align', 'left')])])
# Exploring a single model
# take the last model from the list above, and show its versions
example_model = models[-1]
example_model.instances
# take the last version (not necessarily the most recent)
instance = as_list(example_model.instances)[-1].resolve(client)
instance
instance.timestamp.isoformat()
code = instance.main_script.resolve(client)
code.code_format
code.code_location
morph = instance.morphology.resolve(client)
morph.morphology_file
def print_model_information(index):
model = models[index]
instance = as_list(model.instances)[-1].resolve(client)
code = instance.main_script.resolve(client)
morph = instance.morphology.resolve(client)
print(instance.name, instance.timestamp.isoformat())
print(code.code_location)
print(morph.morphology_file)
print_model_information(99)
# not all models have morphologies, I'm not sure if this is expected or not.
# Here is a list of models with morphologies
for model in models:
for instance in as_list(model.instances):
instance = instance.resolve(client)
morph_file = instance.morphology.resolve(client).morphology_file
if morph_file:
print(instance.name, morph_file)
