def fetch_models(api='query', scope='inferred', n=100000): client = KGClient(os.environ["HBP_token"]) if api is 'nexus': KGmodels = ModelInstance.list(client, api=api, size=n, resolved=True) else: KGmodels = ModelInstance.list(client, api=api, size=n, resolved=True, scope=scope) models = [] for m in KGmodels: models.append({}) for key, val in m.__dict__.items(): if (type(val) is str) or (type(val) is list): models[-1][key] = val elif key == 'instance' or (val is None): models[-1][key] = str(val) else: try: models[-1][key] = val.name except AttributeError: print(key) return models
def check_if_alias_is_already_taken(name): print('comparing with KG database [...]') client = KGClient(token, nexus_endpoint="https://nexus.humanbrainproject.org/v0") MODELS = ModelProject.list(client, size=10000) NAMES = [project.name for project in MODELS] if name in NAMES: i0 = int(np.argwhere(np.array(NAMES, dtype=str) == name)) if type(MODELS[i0].authors) is list: author_list = '' for nn in list(MODELS[i0].authors): author_list += nn.resolve(client).full_name + '; ' else: author_list = MODELS[i0].authors.resolve(client).full_name print('/!\ --> The alias "%s"' % name, 'is redundant with the following entry:') print(''' - Description: %s - Author(s): %s ''' % (MODELS[i0].resolve(client).description[:200]+' [...] '+\ MODELS[i0].resolve(client).description[-200:], author_list)) else: print('--> The alias "%s"' % name, 'is valid')
def get_kg_client(): global kg_client if kg_client is None: kg_client = KGClient( client_id=settings.KG_SERVICE_ACCOUNT_CLIENT_ID, client_secret=settings.KG_SERVICE_ACCOUNT_SECRET, oidc_host=settings.OIDC_HOST, nexus_endpoint=settings.NEXUS_ENDPOINT, ) return kg_client
def suggest_KG_entries_and_pick_one(models, i, desired_key): """ """ client = KGClient(os.environ["HBP_token"]) print('\nHere is the list of available entries for that entity:') for key, cls in zip([ 'abstraction_level', 'brain_region', 'cell_type', 'license', 'code_format', 'model_scope' ], [ AbstractionLevel, BrainStructure, CellularTarget, License, ModelFormat, ModelScope ]): if key == desired_key: KEY_LIST = cls.list(client, size=10000, api='query', resolve=True, scope='released') for i, k in enumerate(KEY_LIST): print(' - %i) %s' % (i, k.name)) print('\nHere is the list of available entries for that entity:') print( '/!\ if that entry is not available in the list, create it in the KG-editor and release it /!\ ' ) values = input( '\nDo you want to add one ? (use either "3", or "2,3,4" or "2,58,3", ...)\n' ) value_list = values.split(',') if (len(value_list) == 1) and (value_list[0] == ''): return models[i][desired_key] elif len(value_list) == 1: return (KEY_LIST[int(value_list[0])].name, KEY_LIST[int(value_list[0])].identifier) else: OUTPUT = [] for value in value_list: OUTPUT.append( (KEY_LIST[int(value)].name, KEY_LIST[int(value)].identifier)) return OUTPUT
} if len(sys.argv)<2: print(""" need to provide an argument, either: - get-KG-released - ANNEX-B - ANNEX-C - ANNEX-D """) elif sys.argv[-1]=='get-KG-released': client = KGClient(os.environ['HBP_token']) models = ModelInstance.list(client, size=1000, api='query', scope='released', resolve=True) MODELS = {} for i, model in enumerate(models): print("%i) %s" % (i, model.name)) MODELS[model.name] = {'url':KG_url_prefix['model']+model.identifier} for key in ['brain_structure', 'custodian', 'contributor', 'modelscope', 'study_target', 'custodian']: name = '' if getattr(model, key) is not None: if type(getattr(model, key)) is list: for n in getattr(model, key): name += n.resolve(client).name+'; '
import requests import json import datetime import logging from time import sleep try: raw_input except NameError: raw_input = input logger = logging.getLogger("kg_migration") nexus_token = os.environ['HBP_token'] # nexus_endpoint = "https://nexus-int.humanbrainproject.org/v0" nexus_endpoint = "https://nexus.humanbrainproject.org/v0" NAR_client = KGClient(nexus_token, nexus_endpoint) cell_type_map = { "L2/3 Chandelier cell": "L2/3 Chandelier cell", "Fast spiking interneuron": "fast spiking interneuron", "Purkinje Cell": "Purkinje cell", "Spiny stellate neuron": "spiny stellate neuron", "Medium spiny neuron": "medium spiny neuron", "L5 Tufted pyramidal cell": "L5 tufted pyramidal cell", "Interneuron": "interneuron", "L2/3 Pyramidal cell": "L2/3 Pyramidal cell", "Golgi Cell": "Golgi cell", "Medium spiny neuron (D2 type)": "medium spiny neuron (D2 type)", "L6 Inverted pyramidal cell": "L6 Inverted pyramidal cell", "L4 Martinotti cell": "L4 Martinotti cell", "Medium spiny neuron (D1 type)": "medium spiny neuron (D1 type)",
if (person.family_name is not None) and : # if str(person.given_name)+' '+str(person.family_name)==author: print(person.name) if __name__ == '__main__': # sys.path.append(str(pathlib.Path(__file__).resolve().parents[1])) # # from processing.entries import refactor_model_entries # from processing.names import resolve_name # from src import local_db client = KGClient(os.environ["HBP_token"]) Person_list = Person.list(client, api='query', scope='inferred', resolved=True, size=100000) # email=get_email_from_collab('Yann', 'Zerlaut') # print(email) # save_person_in_KG(client, 'Yann', 'Zerlaut', email) # print(get_email_from_collab('Yann', 'Zerlaut')) # person = Person(family_name='Zerlaut', # given_name ='Yann', # email='*****@*****.**') # print(person.exists(client, api='query')) find_author_in_KG('Maurizio Mattia')
name = "code for {} @ {}".format(model_project.name, model_instance.version) assert len(name) > 6 bundle = FileBundle(identifier=hashlib.md5(script.code_location.encode('utf-8')).hexdigest(), name=name, description=model_instance.description, url=script.code_location, #usageNotes, modelInstance=model_release) # check this is showing in KG editor print(bundle) bundle.save(client) if __name__ == "__main__": client = KGClient(token, nexus_endpoint="https://nexus.humanbrainproject.org/v0") projects = ModelProject.list(client, size=10000) single_cell_projects = [p for p in projects if p.model_of and p.model_of.label == "single cell" and "emodel" not in p.name] single_cell_no_memodel = [p for p in single_cell_projects if "CA1_pyr" not in p.name and "CA1_int" not in p.name] #for model_project in ModelProject.list(client, size=10000): for model_project in single_cell_no_memodel: #if model_project.model_of and "network" in model_project.model_of.label: #if model_project.model_of and "single cell" not in model_project.model_of.label: if True: contributors = get_uniminds_person_list(model_project.authors, client) custodians = get_uniminds_person_list(model_project.owners, client) for model_instance in as_list(model_project.instances): model_instance = model_instance.resolve(client)
def create_new_instance(model): prerequisite_ok = True # switch to false below if not # check for key in [ 'owner', 'abstraction_level', 'brain_region', 'cell_type', 'license', 'code_format', 'model_scope' ]: if model[key][1] == '': prerequisite_ok = False print('Need to fetch the UUID of the entry "%s" for the key "%s" ' %\ (model[key][0], key)) for author in model['author(s)']: if author[1] == '': prerequisite_ok = False print('Need to fetch the UUID of the author "%s" ' % author[0]) if model['code_location'] == '': prerequisite_ok = False print('no code location') if not prerequisite_ok: print('===========================================================') print(' the entry *can not* be pushed to the Knowledge Graph') print(' fix the metadata first') print('===========================================================') else: pprint.pprint(model) print('===========================================================') print(' the entry *can* be pushed to the Knowledge Graph') print(' please review the above information carefully') print('===========================================================') if input('are you sure that the above informations are correct ? y/[n]\n' ) in ['y', 'yes']: client = KGClient(os.environ["HBP_token"]) custodian = Person.by_name(model['owner'][0], client, api='query') list_of_authors = [Person.by_name(auth[0], client, api='query')\ for auth in model['author(s)']] al = AbstractionLevel.from_uuid(model['abstraction_level'][1], client, api='query') bs = BrainStructure.from_uuid(model['brain_region'][1], client, api='query') ct = CellularTarget.from_uuid(model['cell_type'][1], client, api='query') lic = License.from_uuid(model['license'][1], client, api='query') mf = ModelFormat.from_uuid(model['code_format'][1], client, api='query') ms = ModelScope.from_uuid(model['model_scope'][1], client, api='query') UUID = hashlib.md5("{} @ {}".format( model['name'], model['version']).encode('utf-8')).hexdigest() minst = ModelInstance(name=model['name'], version=model['version'], abstraction_level=al, brain_structure=bs, cellular_target=ct, model_format=mf, model_scope=ms, license=lic, description=model['description'], custodian=custodian, contributor=list_of_authors, identifier=UUID) minst.save(client) print('[ok] created the ModelInstance') minst = ModelInstance.by_name(model['name'], client, api='query') if minst is not None: name = 'filebundle for the code of %s @ %s' % (model['name'], model['version']) fb = FileBundle(name=name, description='file bundle for model ' + name, identifier=hashlib.sha1( name.encode('utf-8')).hexdigest(), url=model['code_location'], model_instance=minst) try: fb.save(client) print('[ok] created the FileBundle') except BaseException as e: print('------------- ERROR --------------------') print(e) print('------------- ERROR --------------------') print('Need to manually create the FileBundle:') print('- name:', name) print('- description:', name) print('- identifier:', hashlib.sha1(name.encode('utf-8')).hexdigest()) print('- url:', model['code_location']) print('- model_instance:', minst.name) else: print('Model instance not found !') print('KG update takes some time, retry in a few minutes...')
print('Need to manually create the FileBundle:') print('- name:', name) print('- description:', name) print('- identifier:', hashlib.sha1(name.encode('utf-8')).hexdigest()) print('- url:', model['code_location']) print('- model_instance:', minst.name) else: print('Model instance not found !') print('KG update takes some time, retry in a few minutes...') if __name__ == '__main__': client = KGClient() print(ModelInstance.from_id('c3862631-1905-4375-b477-699e94f5f6ea', client)) # for model in Dataset.list(client, api='query', scope='inferred', size=100): # print(model) # print(AbstractionLevel.from_uuid('33cd7f9e-0f6e-46af-8a55-1e3fbebbb0f0', client)) # from src import local_db # models = local_db.load_models() # for m in models[11:15]: # print('-------------------------------') # for author in m['author(s)']: # first_name, last_name = names.resolve_name(author[0])
def load_model_instances(show_ignore=False, size=100000, scope='inferred', verbose=False, api='nexus'): client = KGClient(os.environ["HBP_token"]) MODEL_INSTANCES = [] models = ModelProject.list(client, api=api, scope=scope, size=size) for model in models: if verbose: print('\n##########################################') print('------ %s ------' % model.name) minst = template.copy() minst['name'] = model.name minst['owner'] = get_author_details(model.owners.resolve(client)) if type(model.authors) is list: minst['author(s)'] = [get_author_details(auth.resolve(client)) for auth in model.authors] else: minst['author(s)'] = [get_author_details(model.authors.resolve(client))] try: minst['authors_str'] = model.authors_str(client) except TypeError: minst['authors_str'] = '' for a in minst['author(s)']: minst['authors_str'] += a['family_name']+', '+a['given_name']+';' minst['description'] = model.description minst['private'] = model.private minst['collab_id'] = model.collab_id minst['alias'] = model.alias for key, quant in zip(['brain_region', 'species', 'cell_type', 'abstraction_level', 'model_scope'], [model.brain_region, model.species, model.celltype, model.abstraction_level, model.model_of]): if quant is not None: minst[key] = (quant.label, '') elif verbose: print('[!!] "%s" missing for %s ' % (key, model.name)) for key, quant in zip(['organization'], [model.organization]): if quant is not None: minst[key] = (quant.resolve(client).name, '') elif verbose: print('[!!] "%s" missing for %s ' % (key, model.name)) if type(model.instances) is list: for modelI in model.instances: MODEL_INSTANCES.append(minst) # pprint.pprint(MODEL_INSTANCES[-1]) # print('') add_version_details_to_model(MODEL_INSTANCES[-1], modelI.resolve(client), client, verbose=verbose) elif type(model.instances) is KGProxy: modelI = model.instances MODEL_INSTANCES.append(minst.copy()) add_version_details_to_model(MODEL_INSTANCES[-1], modelI.resolve(client), client, verbose=verbose) elif show_ignore: print('Ignoring %s @ %s' % (model.name, model.date_created)) pass # we don't care about models without specific version DATES = np.array([time.mktime(minst['date_created'].timetuple()) for minst in MODEL_INSTANCES]) return [MODEL_INSTANCES[i] for i in np.argsort(DATES)[::-1]]