def fetch_models(api='query', scope='inferred', n=100000):
client = KGClient(os.environ["HBP_token"])
if api is 'nexus':
KGmodels = ModelInstance.list(client, api=api, size=n, resolved=True)
else:
KGmodels = ModelInstance.list(client,
api=api,
size=n,
resolved=True,
scope=scope)
models = []
for m in KGmodels:
models.append({})
for key, val in m.__dict__.items():
if (type(val) is str) or (type(val) is list):
models[-1][key] = val
elif key == 'instance' or (val is None):
models[-1][key] = str(val)
else:
try:
models[-1][key] = val.name
except AttributeError:
print(key)
return models
def check_if_alias_is_already_taken(name):
print('comparing with KG database [...]')
client = KGClient(token,
nexus_endpoint="https://nexus.humanbrainproject.org/v0")
MODELS = ModelProject.list(client, size=10000)
NAMES = [project.name for project in MODELS]
if name in NAMES:
i0 = int(np.argwhere(np.array(NAMES, dtype=str) == name))
if type(MODELS[i0].authors) is list:
author_list = ''
for nn in list(MODELS[i0].authors):
author_list += nn.resolve(client).full_name + '; '
else:
author_list = MODELS[i0].authors.resolve(client).full_name
print('/!\ --> The alias "%s"' % name,
'is redundant with the following entry:')
print('''
- Description:
%s
- Author(s):
%s
''' % (MODELS[i0].resolve(client).description[:200]+' [...] '+\
MODELS[i0].resolve(client).description[-200:],
author_list))
else:
print('--> The alias "%s"' % name, 'is valid')
def get_kg_client():
global kg_client
if kg_client is None:
kg_client = KGClient(
client_id=settings.KG_SERVICE_ACCOUNT_CLIENT_ID,
client_secret=settings.KG_SERVICE_ACCOUNT_SECRET,
oidc_host=settings.OIDC_HOST,
nexus_endpoint=settings.NEXUS_ENDPOINT,
)
return kg_client
def suggest_KG_entries_and_pick_one(models, i, desired_key):
"""
"""
client = KGClient(os.environ["HBP_token"])
print('\nHere is the list of available entries for that entity:')
for key, cls in zip([
'abstraction_level', 'brain_region', 'cell_type', 'license',
'code_format', 'model_scope'
], [
AbstractionLevel, BrainStructure, CellularTarget, License,
ModelFormat, ModelScope
]):
if key == desired_key:
KEY_LIST = cls.list(client,
size=10000,
api='query',
resolve=True,
scope='released')
for i, k in enumerate(KEY_LIST):
print(' - %i) %s' % (i, k.name))
print('\nHere is the list of available entries for that entity:')
print(
'/!\ if that entry is not available in the list, create it in the KG-editor and release it /!\ '
)
values = input(
'\nDo you want to add one ? (use either "3", or "2,3,4" or "2,58,3", ...)\n'
)
value_list = values.split(',')
if (len(value_list) == 1) and (value_list[0] == ''):
return models[i][desired_key]
elif len(value_list) == 1:
return (KEY_LIST[int(value_list[0])].name,
KEY_LIST[int(value_list[0])].identifier)
else:
OUTPUT = []
for value in value_list:
OUTPUT.append(
(KEY_LIST[int(value)].name, KEY_LIST[int(value)].identifier))
return OUTPUT
}
if len(sys.argv)<2:
print("""
need to provide an argument, either:
- get-KG-released
- ANNEX-B
- ANNEX-C
- ANNEX-D
""")
elif sys.argv[-1]=='get-KG-released':
client = KGClient(os.environ['HBP_token'])
models = ModelInstance.list(client, size=1000, api='query', scope='released', resolve=True)
MODELS = {}
for i, model in enumerate(models):
print("%i) %s" % (i, model.name))
MODELS[model.name] = {'url':KG_url_prefix['model']+model.identifier}
for key in ['brain_structure', 'custodian', 'contributor', 'modelscope', 'study_target', 'custodian']:
name = ''
if getattr(model, key) is not None:
if type(getattr(model, key)) is list:
for n in getattr(model, key):
name += n.resolve(client).name+'; '
import requests
import json
import datetime
import logging
from time import sleep
try:
raw_input
except NameError:
raw_input = input
logger = logging.getLogger("kg_migration")
nexus_token = os.environ['HBP_token']
# nexus_endpoint = "https://nexus-int.humanbrainproject.org/v0"
nexus_endpoint = "https://nexus.humanbrainproject.org/v0"
NAR_client = KGClient(nexus_token, nexus_endpoint)
cell_type_map = {
"L2/3 Chandelier cell": "L2/3 Chandelier cell",
"Fast spiking interneuron": "fast spiking interneuron",
"Purkinje Cell": "Purkinje cell",
"Spiny stellate neuron": "spiny stellate neuron",
"Medium spiny neuron": "medium spiny neuron",
"L5 Tufted pyramidal cell": "L5 tufted pyramidal cell",
"Interneuron": "interneuron",
"L2/3 Pyramidal cell": "L2/3 Pyramidal cell",
"Golgi Cell": "Golgi cell",
"Medium spiny neuron (D2 type)": "medium spiny neuron (D2 type)",
"L6 Inverted pyramidal cell": "L6 Inverted pyramidal cell",
"L4 Martinotti cell": "L4 Martinotti cell",
"Medium spiny neuron (D1 type)": "medium spiny neuron (D1 type)",
if (person.family_name is not None) and :
# if str(person.given_name)+' '+str(person.family_name)==author:
print(person.name)
if __name__ == '__main__':
# sys.path.append(str(pathlib.Path(__file__).resolve().parents[1]))
# # from processing.entries import refactor_model_entries
# from processing.names import resolve_name
# from src import local_db
client = KGClient(os.environ["HBP_token"])
Person_list = Person.list(client, api='query',
scope='inferred', resolved=True,
size=100000)
# email=get_email_from_collab('Yann', 'Zerlaut')
# print(email)
# save_person_in_KG(client, 'Yann', 'Zerlaut', email)
# print(get_email_from_collab('Yann', 'Zerlaut'))
# person = Person(family_name='Zerlaut',
# given_name ='Yann',
# email='*****@*****.**')
# print(person.exists(client, api='query'))
find_author_in_KG('Maurizio Mattia')
name = "code for {} @ {}".format(model_project.name, model_instance.version)
assert len(name) > 6
bundle = FileBundle(identifier=hashlib.md5(script.code_location.encode('utf-8')).hexdigest(),
name=name,
description=model_instance.description,
url=script.code_location,
#usageNotes,
modelInstance=model_release) # check this is showing in KG editor
print(bundle)
bundle.save(client)
if __name__ == "__main__":
client = KGClient(token, nexus_endpoint="https://nexus.humanbrainproject.org/v0")
projects = ModelProject.list(client, size=10000)
single_cell_projects = [p for p in projects if p.model_of and p.model_of.label == "single cell" and "emodel" not in p.name]
single_cell_no_memodel = [p for p in single_cell_projects if "CA1_pyr" not in p.name and "CA1_int" not in p.name]
#for model_project in ModelProject.list(client, size=10000):
for model_project in single_cell_no_memodel:
#if model_project.model_of and "network" in model_project.model_of.label:
#if model_project.model_of and "single cell" not in model_project.model_of.label:
if True:
contributors = get_uniminds_person_list(model_project.authors, client)
custodians = get_uniminds_person_list(model_project.owners, client)
for model_instance in as_list(model_project.instances):
model_instance = model_instance.resolve(client)
def create_new_instance(model):
prerequisite_ok = True # switch to false below if not
# check
for key in [
'owner', 'abstraction_level', 'brain_region', 'cell_type',
'license', 'code_format', 'model_scope'
]:
if model[key][1] == '':
prerequisite_ok = False
print('Need to fetch the UUID of the entry "%s" for the key "%s" ' %\
(model[key][0], key))
for author in model['author(s)']:
if author[1] == '':
prerequisite_ok = False
print('Need to fetch the UUID of the author "%s" ' % author[0])
if model['code_location'] == '':
prerequisite_ok = False
print('no code location')
if not prerequisite_ok:
print('===========================================================')
print(' the entry *can not* be pushed to the Knowledge Graph')
print(' fix the metadata first')
print('===========================================================')
else:
pprint.pprint(model)
print('===========================================================')
print(' the entry *can* be pushed to the Knowledge Graph')
print(' please review the above information carefully')
print('===========================================================')
if input('are you sure that the above informations are correct ? y/[n]\n'
) in ['y', 'yes']:
client = KGClient(os.environ["HBP_token"])
custodian = Person.by_name(model['owner'][0], client, api='query')
list_of_authors = [Person.by_name(auth[0], client, api='query')\
for auth in model['author(s)']]
al = AbstractionLevel.from_uuid(model['abstraction_level'][1],
client,
api='query')
bs = BrainStructure.from_uuid(model['brain_region'][1],
client,
api='query')
ct = CellularTarget.from_uuid(model['cell_type'][1],
client,
api='query')
lic = License.from_uuid(model['license'][1], client, api='query')
mf = ModelFormat.from_uuid(model['code_format'][1],
client,
api='query')
ms = ModelScope.from_uuid(model['model_scope'][1], client, api='query')
UUID = hashlib.md5("{} @ {}".format(
model['name'], model['version']).encode('utf-8')).hexdigest()
minst = ModelInstance(name=model['name'],
version=model['version'],
abstraction_level=al,
brain_structure=bs,
cellular_target=ct,
model_format=mf,
model_scope=ms,
license=lic,
description=model['description'],
custodian=custodian,
contributor=list_of_authors,
identifier=UUID)
minst.save(client)
print('[ok] created the ModelInstance')
minst = ModelInstance.by_name(model['name'], client, api='query')
if minst is not None:
name = 'filebundle for the code of %s @ %s' % (model['name'],
model['version'])
fb = FileBundle(name=name,
description='file bundle for model ' + name,
identifier=hashlib.sha1(
name.encode('utf-8')).hexdigest(),
url=model['code_location'],
model_instance=minst)
try:
fb.save(client)
print('[ok] created the FileBundle')
except BaseException as e:
print('------------- ERROR --------------------')
print(e)
print('------------- ERROR --------------------')
print('Need to manually create the FileBundle:')
print('- name:', name)
print('- description:', name)
print('- identifier:',
hashlib.sha1(name.encode('utf-8')).hexdigest())
print('- url:', model['code_location'])
print('- model_instance:', minst.name)
else:
print('Model instance not found !')
print('KG update takes some time, retry in a few minutes...')
print('Need to manually create the FileBundle:')
print('- name:', name)
print('- description:', name)
print('- identifier:',
hashlib.sha1(name.encode('utf-8')).hexdigest())
print('- url:', model['code_location'])
print('- model_instance:', minst.name)
else:
print('Model instance not found !')
print('KG update takes some time, retry in a few minutes...')
if __name__ == '__main__':
client = KGClient()
print(ModelInstance.from_id('c3862631-1905-4375-b477-699e94f5f6ea',
client))
# for model in Dataset.list(client, api='query', scope='inferred', size=100):
# print(model)
# print(AbstractionLevel.from_uuid('33cd7f9e-0f6e-46af-8a55-1e3fbebbb0f0', client))
# from src import local_db
# models = local_db.load_models()
# for m in models[11:15]:
# print('-------------------------------')
# for author in m['author(s)']:
# first_name, last_name = names.resolve_name(author[0])
def load_model_instances(show_ignore=False,
size=100000,
scope='inferred',
verbose=False,
api='nexus'):
client = KGClient(os.environ["HBP_token"])
MODEL_INSTANCES = []
models = ModelProject.list(client,
api=api,
scope=scope,
size=size)
for model in models:
if verbose:
print('\n##########################################')
print('------ %s ------' % model.name)
minst = template.copy()
minst['name'] = model.name
minst['owner'] = get_author_details(model.owners.resolve(client))
if type(model.authors) is list:
minst['author(s)'] = [get_author_details(auth.resolve(client)) for auth in model.authors]
else:
minst['author(s)'] = [get_author_details(model.authors.resolve(client))]
try:
minst['authors_str'] = model.authors_str(client)
except TypeError:
minst['authors_str'] = ''
for a in minst['author(s)']:
minst['authors_str'] += a['family_name']+', '+a['given_name']+';'
minst['description'] = model.description
minst['private'] = model.private
minst['collab_id'] = model.collab_id
minst['alias'] = model.alias
for key, quant in zip(['brain_region', 'species', 'cell_type', 'abstraction_level', 'model_scope'],
[model.brain_region, model.species, model.celltype, model.abstraction_level, model.model_of]):
if quant is not None:
minst[key] = (quant.label, '')
elif verbose:
print('[!!] "%s" missing for %s ' % (key, model.name))
for key, quant in zip(['organization'],
[model.organization]):
if quant is not None:
minst[key] = (quant.resolve(client).name, '')
elif verbose:
print('[!!] "%s" missing for %s ' % (key, model.name))
if type(model.instances) is list:
for modelI in model.instances:
MODEL_INSTANCES.append(minst)
# pprint.pprint(MODEL_INSTANCES[-1])
# print('')
add_version_details_to_model(MODEL_INSTANCES[-1], modelI.resolve(client), client, verbose=verbose)
elif type(model.instances) is KGProxy:
modelI = model.instances
MODEL_INSTANCES.append(minst.copy())
add_version_details_to_model(MODEL_INSTANCES[-1], modelI.resolve(client), client, verbose=verbose)
elif show_ignore:
print('Ignoring %s @ %s' % (model.name, model.date_created))
pass # we don't care about models without specific version
DATES = np.array([time.mktime(minst['date_created'].timetuple()) for minst in MODEL_INSTANCES])
return [MODEL_INSTANCES[i] for i in np.argsort(DATES)[::-1]]