def main(argv): #create new nidm-experiment document with project kwargs={Constants.NIDM_PROJECT_NAME:"FBIRN_PhaseII",Constants.NIDM_PROJECT_IDENTIFIER:9610,Constants.NIDM_PROJECT_DESCRIPTION:"Test investigation"} project = Project(attributes=kwargs) #test add string attribute with existing namespace #nidm_doc.addLiteralAttribute("nidm","isFun","ForMe") project.add_attributes({Constants.NIDM["isFun"]:"ForMe"}) #test adding string attribute with new namespace/term project.addLiteralAttribute("fred","notFound","in namespaces","www.fred.org/") #test add float attribute project.addLiteralAttribute("nidm", "float", float(2.34)) #test adding attributes in bulk with mix of existing and new namespaces #nidm_doc.addAttributesWithNamespaces(nidm_doc.getProject(),[{"prefix":"nidm", "uri":nidm_doc.namespaces["nidm"], "term":"score", "value":int(15)}, \ # {"prefix":"dave", "uri":"http://www.davidkeator.com/", "term":"isAwesome", "value":"15"}, \ # {"prefix":"nidm", "uri":nidm_doc.namespaces["nidm"], "term":"value", "value":float(2.34)}]) #nidm_doc.addAttributes(nidm_doc.getProject(),{"nidm:test":int(15), "ncit:isTerminology":"15","ncit:joker":float(1)}) #test add PI to investigation project_PI = project.add_person(role=Constants.NIDM_PI, attributes={Constants.NIDM_FAMILY_NAME:"Keator", Constants.NIDM_GIVEN_NAME:"David"}) #test add session to graph and associate with project session = Session(project) project.add_sessions(session) #test add acquisition activity to graph and associate with session acq_act = Acquisition(session=session) #test add acquisition object entity to graph associated with participant role NIDM_PARTICIPANT acq_entity = MRAcquisitionObject(acquisition=acq_act) acq_entity.add_person(role=Constants.NIDM_PARTICIPANT,attributes={Constants.NIDM_GIVEN_NAME:"George"}) #save a turtle file with open("test.ttl",'w') as f: f.write (project.serializeTurtle()) #save a DOT graph as PDF project.save_DotGraph("test.png",format="png")
def main(argv): parser = ArgumentParser( description= 'This program will convert a BIDS MRI dataset to a NIDM-Experiment \ RDF document. It will parse phenotype information and simply store variables/values \ and link to the associated json data dictionary file.') parser.add_argument('-d', dest='directory', required=True, help="Path to BIDS dataset directory") parser.add_argument('-o', dest='outputfile', default="nidm.ttl", help="NIDM output turtle file") args = parser.parse_args() directory = args.directory outputfile = args.outputfile #importlib.reload(sys) #sys.setdefaultencoding('utf8') #Parse dataset_description.json file in BIDS directory with open(os.path.join(directory, 'dataset_description.json')) as data_file: dataset = json.load(data_file) #print(dataset_data) #create project / nidm-exp doc project = Project() #add various attributes if they exist in BIDS dataset for key in dataset: #print(key) #if key from dataset_description file is mapped to term in BIDS_Constants.py then add to NIDM object if key in BIDS_Constants.dataset_description: if type(dataset[key]) is list: project.add_attributes({ BIDS_Constants.dataset_description[key]: "".join(dataset[key]) }) else: project.add_attributes( {BIDS_Constants.dataset_description[key]: dataset[key]}) #create empty dictinary for sessions where key is subject id and used later to link scans to same session as demographics session = {} #Parse participants.tsv file in BIDS directory and create study and acquisition objects with open(os.path.join(directory, 'participants.tsv')) as csvfile: participants_data = csv.DictReader(csvfile, delimiter='\t') #print(participants_data.fieldnames) for row in participants_data: #create session object for subject to be used for participant metadata and image data #parse subject id from "sub-XXXX" string subjid = row['participant_id'].split("-") session[subjid[1]] = Session(project) #add acquisition object acq = Acquisition(session=session[subjid[1]]) acq_entity = DemographicsAcquisitionObject(acquisition=acq) participant = acq.add_person(role=Constants.NIDM_PARTICIPANT, attributes=({ Constants.NIDM_SUBJECTID: row['participant_id'] })) for key, value in row.items(): #for now only convert variables in participants.tsv file who have term mappings in BIDS_Constants.py if key in BIDS_Constants.participants: acq_entity.add_attributes( {BIDS_Constants.participants[key]: value}) #get BIDS layout bids_layout = BIDSLayout(directory) #create acquisition objects for each scan for each subject #loop through all subjects in dataset for subject_id in bids_layout.get_subjects(): #skip .git directories...added to support datalad datasets if subject_id.startswith("."): continue for file_tpl in bids_layout.get(subject=subject_id, extensions=['.nii', '.nii.gz']): #create an acquisition activity acq = Acquisition(session[subject_id]) #print(file_tpl.type) if file_tpl.modality == 'anat': #do something with anatomicals acq_obj = MRAcquisitionObject(acq) acq_obj.add_attributes( {PROV_TYPE: BIDS_Constants.scans[file_tpl.modality]}) #add file link #make relative link to acq_obj.add_attributes( {Constants.NIDM_FILENAME: file_tpl.filename}) #get associated JSON file if exists json_data = bids_layout.get_metadata(file_tpl.filename) if json_data: for key in json_data: if key in BIDS_Constants.json_keys: if type(json_data[key]) is list: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: ''.join(str(e) for e in json_data[key]) }) else: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: json_data[key] }) elif file_tpl.modality == 'func': #do something with functionals acq_obj = MRAcquisitionObject(acq) acq_obj.add_attributes( {PROV_TYPE: BIDS_Constants.scans[file_tpl.modality]}) #add file link acq_obj.add_attributes( {Constants.NIDM_FILENAME: file_tpl.filename}) if 'run' in file_tpl._fields: acq_obj.add_attributes( {BIDS_Constants.json_keys["run"]: file_tpl.run}) #get associated JSON file if exists json_data = bids_layout.get_metadata(file_tpl.filename) if json_data: for key in json_data: if key in BIDS_Constants.json_keys: if type(json_data[key]) is list: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: ''.join(str(e) for e in json_data[key]) }) else: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: json_data[key] }) #get associated events TSV file if 'run' in file_tpl._fields: events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'], modality=file_tpl.modality, task=file_tpl.task, run=file_tpl.run) else: events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'], modality=file_tpl.modality, task=file_tpl.task) #if there is an events file then this is task-based so create an acquisition object for the task file and link if events_file: #for now create acquisition object and link it to the associated scan events_obj = AcquisitionObject(acq) #add prov type, task name as prov:label, and link to filename of events file events_obj.add_attributes({ PROV_TYPE: Constants.NIDM_MRI_BOLD_EVENTS, BIDS_Constants.json_keys["TaskName"]: json_data["TaskName"], Constants.NFO["filename"]: events_file[0].filename }) #link it to appropriate MR acquisition entity events_obj.wasAttributedTo(acq_obj) elif file_tpl.modality == 'dwi': #do stuff with with dwi scans... acq_obj = MRAcquisitionObject(acq) acq_obj.add_attributes( {PROV_TYPE: BIDS_Constants.scans[file_tpl.modality]}) #add file link acq_obj.add_attributes( {Constants.NIDM_FILENAME: file_tpl.filename}) if 'run' in file_tpl._fields: acq_obj.add_attributes( {BIDS_Constants.json_keys["run"]: file_tpl.run}) #get associated JSON file if exists json_data = bids_layout.get_metadata(file_tpl.filename) if json_data: for key in json_data: if key in BIDS_Constants.json_keys: if type(json_data[key]) is list: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: ''.join(str(e) for e in json_data[key]) }) else: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: json_data[key] }) #for bval and bvec files, what to do with those? #for now, create new generic acquisition objects, link the files, and associate with the one for the DWI scan? acq_obj_bval = AcquisitionObject(acq) acq_obj_bval.add_attributes( {PROV_TYPE: BIDS_Constants.scans["bval"]}) #add file link to bval files acq_obj_bval.add_attributes({ Constants.NIDM_FILENAME: bids_layout.get_bval(file_tpl.filename) }) acq_obj_bvec = AcquisitionObject(acq) acq_obj_bvec.add_attributes( {PROV_TYPE: BIDS_Constants.scans["bvec"]}) #add file link to bvec files acq_obj_bvec.add_attributes({ Constants.NIDM_FILENAME: bids_layout.get_bvec(file_tpl.filename) }) #link bval and bvec acquisition object entities together or is their association with enclosing activity enough? #Added temporarily to support phenotype files #for each *.tsv / *.json file pair in the phenotypes directory for tsv_file in glob.glob(os.path.join(directory, "phenotype", "*.tsv")): #for now, open the TSV file, extract the row for this subject, store it in an acquisition object and link to #the associated JSON data dictionary file with open(tsv_file) as phenofile: pheno_data = csv.DictReader(phenofile, delimiter='\t') for row in pheno_data: subjid = row['participant_id'].split("-") if not subjid[1] == subject_id: continue else: #add acquisition object acq = Acquisition(session=session[subjid[1]]) acq_entity = AssessmentAcquisitionObject( acquisition=acq) participant = acq.add_person( role=Constants.NIDM_PARTICIPANT, attributes=({ Constants.NIDM_SUBJECTID: row['participant_id'] })) for key, value in row.items(): if not key == "participant_id": #for now we're using a placeholder namespace for BIDS and simply the variable names as the concept IDs.. acq_entity.add_attributes( {Constants.BIDS[key]: value}) #link TSV file acq_entity.add_attributes( {Constants.NIDM_FILENAME: tsv_file}) #link associated JSON file if it exists data_dict = os.path.join( directory, "phenotype", os.path.splitext(os.path.basename(tsv_file))[0] + ".json") if os.path.isfile(data_dict): acq_entity.add_attributes( {Constants.BIDS["data_dictionary"]: data_dict}) #serialize graph #print(project.graph.get_provn()) with open(outputfile, 'w') as f: f.write(project.serializeTurtle()) #f.write(project.graph.get_provn()) #save a DOT graph as PNG project.save_DotGraph(str(outputfile + ".png"), format="png")
def main(argv): parser = ArgumentParser(description='This program will load in a CSV file and iterate over the header \ variable names performing an elastic search of https://scicrunch.org/ for NIDM-ReproNim \ tagged terms that fuzzy match the variable names. The user will then interactively pick \ a term to associate with the variable name. The resulting annotated CSV data will \ then be written to a NIDM data file.') parser.add_argument('-csv', dest='csv_file', required=True, help="Path to CSV file to convert") parser.add_argument('-ilxkey', dest='key', required=True, help="Interlex/SciCrunch API key to use for query") parser.add_argument('-json_map', dest='json_map',required=False,help="User-suppled JSON file containing variable-term mappings.") parser.add_argument('-nidm', dest='nidm_file', required=False, help="Optional NIDM file to add CSV->NIDM converted graph to") #parser.add_argument('-owl', action='store_true', required=False, help='Optionally searches NIDM OWL files...internet connection required') parser.add_argument('-png', action='store_true', required=False, help='Optional flag, when set a PNG image file of RDF graph will be produced') parser.add_argument('-jsonld', action='store_true', required=False, help='Optional flag, when set NIDM files are saved as JSON-LD instead of TURTLE') parser.add_argument('-out', dest='output_file', required=True, help="Filename to save NIDM file") args = parser.parse_args() #open CSV file and load into df = pd.read_csv(args.csv_file) #maps variables in CSV file to terms #if args.owl is not False: # column_to_terms = map_variables_to_terms(df=df, apikey=args.key, directory=dirname(args.output_file), output_file=args.output_file, json_file=args.json_map, owl_file=args.owl) #else: column_to_terms = map_variables_to_terms(df=df, apikey=args.key, directory=dirname(args.output_file), output_file=args.output_file, json_file=args.json_map) #If user has added an existing NIDM file as a command line parameter then add to existing file for subjects who exist in the NIDM file if args.nidm_file: print("Adding to NIDM file...") #read in NIDM file project = read_nidm(args.nidm_file) #get list of session objects session_objs=project.get_sessions() #look at column_to_terms dictionary for NIDM URL for subject id (Constants.NIDM_SUBJECTID) id_field=None for key, value in column_to_terms.items(): if Constants.NIDM_SUBJECTID._str == column_to_terms[key]['label']: id_field=key #make sure id_field is a string for zero-padded subject ids #re-read data file with constraint that key field is read as string #df = pd.read_csv(args.csv_file,dtype={id_field : str}) #if we couldn't find a subject ID field in column_to_terms, ask user if id_field is None: option=1 for column in df.columns: print("%d: %s" %(option,column)) option=option+1 selection=input("Please select the subject ID field from the list above: ") id_field=df.columns[int(selection)-1] #make sure id_field is a string for zero-padded subject ids #re-read data file with constraint that key field is read as string #df = pd.read_csv(args.csv_file,dtype={id_field : str}) #use RDFLib here for temporary graph making query easier rdf_graph = Graph() rdf_graph_parse = rdf_graph.parse(source=StringIO(project.serializeTurtle()),format='turtle') #find subject ids and sessions in NIDM document query = """SELECT DISTINCT ?session ?nidm_subj_id ?agent WHERE { ?activity prov:wasAssociatedWith ?agent ; dct:isPartOf ?session . ?agent rdf:type prov:Agent ; ndar:src_subject_id ?nidm_subj_id . }""" #print(query) qres = rdf_graph_parse.query(query) for row in qres: print('%s \t %s' %(row[0],row[1])) #find row in CSV file with subject id matching agent from NIDM file #csv_row = df.loc[df[id_field]==type(df[id_field][0])(row[1])] #find row in CSV file with matching subject id to the agent in the NIDM file #be carefull about data types...simply type-change dataframe subject id column and query to strings. #here we're removing the leading 0's from IDs because pandas.read_csv strips those unless you know ahead of #time which column is the subject id.... csv_row = df.loc[df[id_field].astype('str').str.contains(str(row[1]).lstrip("0"))] #if there was data about this subject in the NIDM file already (i.e. an agent already exists with this subject id) #then add this CSV assessment data to NIDM file, else skip it.... if (not (len(csv_row.index)==0)): #NIDM document sesssion uuid session_uuid = row[0] #temporary list of string-based URIs of session objects from API temp = [o.identifier._uri for o in session_objs] #get session object from existing NIDM file that is associated with a specific subject id #nidm_session = (i for i,x in enumerate([o.identifier._uri for o in session_objs]) if x == str(session_uuid)) nidm_session = session_objs[temp.index(str(session_uuid))] #for nidm_session in session_objs: # if nidm_session.identifier._uri == str(session_uuid): #add an assessment acquisition for the phenotype data to session and associate with agent acq=AssessmentAcquisition(session=nidm_session) #add acquisition entity for assessment acq_entity = AssessmentObject(acquisition=acq) #add qualified association with existing agent acq.add_qualified_association(person=row[2],role=Constants.NIDM_PARTICIPANT) #store other data from row with columns_to_term mappings for row_variable in csv_row: #check if row_variable is subject id, if so skip it if row_variable==id_field: continue else: if not csv_row[row_variable].values[0]: continue #get column_to_term mapping uri and add as namespace in NIDM document #provNamespace(Core.safe_string(None,string=str(row_variable)), column_to_terms[row_variable]["url"]) acq_entity.add_attributes({QualifiedName(provNamespace(Core.safe_string(None,string=str(row_variable)), column_to_terms[row_variable]["url"]), ""):csv_row[row_variable].values[0]}) continue #serialize NIDM file with open(args.nidm_file,'w') as f: print("Writing NIDM file...") if args.jsonld: f.write(project.serializeJSONLD()) else: f.write(project.serializeTurtle()) project.save_DotGraph(str(args.nidm_file + ".png"), format="png") else: print("Creating NIDM file...") #If user did not choose to add this data to an existing NIDM file then create a new one for the CSV data #create empty project project=Project() #simply add name of file to project since we don't know anything about it project.add_attributes({Constants.NIDM_FILENAME:args.csv_file}) #look at column_to_terms dictionary for NIDM URL for subject id (Constants.NIDM_SUBJECTID) id_field=None for key, value in column_to_terms.items(): if Constants.NIDM_SUBJECTID._str == column_to_terms[key]['label']: id_field=key #make sure id_field is a string for zero-padded subject ids #re-read data file with constraint that key field is read as string #df = pd.read_csv(args.csv_file,dtype={id_field : str}) #if we couldn't find a subject ID field in column_to_terms, ask user if id_field is None: option=1 for column in df.columns: print("%d: %s" %(option,column)) option=option+1 selection=input("Please select the subject ID field from the list above: ") id_field=df.columns[int(selection)-1] #iterate over rows and store in NIDM file for csv_index, csv_row in df.iterrows(): #create a session object session=Session(project) #create and acquisition activity and entity acq=AssessmentAcquisition(session) acq_entity=AssessmentObject(acq) #store other data from row with columns_to_term mappings for row_variable,row_data in csv_row.iteritems(): if not row_data: continue #check if row_variable is subject id, if so skip it if row_variable==id_field: #add qualified association with person acq.add_qualified_association(person= acq.add_person(attributes=({Constants.NIDM_SUBJECTID:row_data})),role=Constants.NIDM_PARTICIPANT) continue else: #get column_to_term mapping uri and add as namespace in NIDM document acq_entity.add_attributes({QualifiedName(provNamespace(Core.safe_string(None,string=str(row_variable)), column_to_terms[row_variable]["url"]),""):row_data}) #print(project.serializeTurtle()) #serialize NIDM file with open(args.output_file,'w') as f: print("Writing NIDM file...") if args.jsonld: f.write(project.serializeJSONLD()) else: f.write(project.serializeTurtle()) if args.png: project.save_DotGraph(str(args.output_file + ".png"), format="png")
def main(argv): parser = ArgumentParser() parser.add_argument('-d', dest='directory', required=True, help="Path to BIDS dataset directory") parser.add_argument('-o', dest='outputfile', default="nidm.ttl", help="NIDM output turtle file") args = parser.parse_args() directory = args.directory outputfile = args.outputfile #Parse dataset_description.json file in BIDS directory with open(directory + '/' + 'dataset_description.json') as data_file: dataset = json.load(data_file) #print(dataset_data) #create project / nidm-exp doc project = Project() #add various attributes if they exist in BIDS dataset for key in dataset: #print(key) #if key from dataset_description file is mapped to term in BIDS_Constants.py then add to NIDM object if key in BIDS_Constants.dataset_description: if type(dataset[key]) is list: project.add_attributes({ BIDS_Constants.dataset_description[key]: "".join(dataset[key]) }) else: project.add_attributes( {BIDS_Constants.dataset_description[key]: dataset[key]}) #create empty dictinary for sessions where key is subject id and used later to link scans to same session as demographics session = {} #Parse participants.tsv file in BIDS directory and create study and acquisition objects with open(directory + '/' + 'participants.tsv') as csvfile: participants_data = csv.DictReader(csvfile, delimiter='\t') #print(participants_data.fieldnames) for row in participants_data: #create session object for subject to be used for participant metadata and image data #parse subject id from "sub-XXXX" string subjid = row['participant_id'].split("-") session[subjid[1]] = Session(project) #add acquisition object acq = Acquisition(session=session[subjid[1]]) acq_entity = DemographicsAcquisitionObject(acquisition=acq) participant = acq.add_person(role=Constants.NIDM_PARTICIPANT, attributes=({ Constants.NIDM_SUBJECTID: row['participant_id'] })) for key, value in row.items(): #for now only convert variables in participants.tsv file who have term mappings in BIDS_Constants.py if key in BIDS_Constants.participants: acq_entity.add_attributes( {BIDS_Constants.participants[key]: value}) #get BIDS layout bids_layout = BIDSLayout(directory) #create acquisition objects for each scan for each subject #loop through all subjects in dataset for subject_id in bids_layout.get_subjects(): for file_tpl in bids_layout.get(subject=subject_id, extensions=['.nii', '.nii.gz']): #create an acquisition activity acq = Acquisition(session[subject_id]) #print(file_tpl.type) if file_tpl.modality == 'anat': #do something with anatomicals acq_obj = MRAcquisitionObject(acq) acq_obj.add_attributes( {PROV_TYPE: BIDS_Constants.scans[file_tpl.modality]}) #add file link acq_obj.add_attributes( {Constants.NFO["filename"]: file_tpl.filename}) #get associated JSON file if exists for json_file in bids_layout.get(subject=subject_id, extensions=['.json'], modality=file_tpl.modality): #open json file, grab key-value pairs, map them to terms and add to acquisition object with open(json_file[0]) as data_file: json_data = json.load(data_file) for key in json_data: if key in BIDS_Constants.json_keys: if type(json_data[key]) is list: project.add_attributes({ BIDS_Constants.json_keys[key]: "".join(json_data[key]) }) else: project.add_attributes({ BIDS_Constants.json_keys[key]: json_data[key] }) #if we want to do something further if T1w or t2, etc #if file_tpl.type == 'T1w': #elif file_tpl.type == 'inplaneT2': elif file_tpl.modality == 'func': #do something with functionals acq_obj = MRAcquisitionObject(acq) acq_obj.add_attributes( {PROV_TYPE: BIDS_Constants.scans[file_tpl.modality]}) #add file link acq_obj.add_attributes({ Constants.NFO["filename"]: file_tpl.filename, BIDS_Constants.json_keys["run"]: file_tpl.run }) #add attributes for task description keys from task JSON file for task_desc in bids_layout.get(extensions=['.json'], task=file_tpl.task): with open(task_desc[0]) as data_file: json_data = json.load(data_file) for key in json_data: if key in BIDS_Constants.json_keys: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: json_data[key] }) #get associated events TSV file events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'], modality=file_tpl.modality, task=file_tpl.task, run=file_tpl.run) #for now create acquisition object and link it to the associated scan events_obj = AcquisitionObject(acq) #add prov type, task name as prov:label, and link to filename of events file events_obj.add_attributes({ PROV_TYPE: Constants.NIDM_MRI_BOLD_EVENTS, BIDS_Constants.json_keys["TaskName"]: json_data["TaskName"], Constants.NFO["filename"]: events_file[0].filename }) #link it to appropriate MR acquisition entity events_obj.wasAttributedTo(acq_obj) elif file_tpl.modality == 'dwi': #do stuff with with dwi scans... acq_obj = MRAcquisitionObject(acq) acq_obj.add_attributes( {PROV_TYPE: BIDS_Constants.scans[file_tpl.modality]}) #add file link acq_obj.add_attributes({ Constants.NFO["filename"]: file_tpl.filename, BIDS_Constants.json_keys["run"]: file_tpl.run }) #add attributes for task description keys from task JSON file for task_desc in bids_layout.get(extensions=['.json'], task=file_tpl.task): with open(task_desc[0]) as data_file: json_data = json.load(data_file) for key in json_data: if key in BIDS_Constants.json_keys: acq_obj.add_attributes({ BIDS_Constants.json_keys[key]: json_data[key] }) #for bval and bvec files, what to do with those? #for now, create new generic acquisition objects, link the files, and associate with the one for the DWI scan? acq_obj = AcquisitionObject(acq) acq_obj.add_attributes( {PROV_TYPE: BIDS_Constants.scans["bval"]}) for bval in bids_layout.get(extensions=['.bval'], task=file_tpl.task): #add file link acq_obj.add_attributes( {Constants.NFO["filename"]: bval.filename}) for bvec in bids_layout.get(extensions=['.bvec'], task=file_tpl.task): #add file link acq_obj.add_attributes( {Constants.NFO["filename"]: bvec.filename}) #serialize graph #print(project.graph.get_provn()) with open(outputfile, 'w') as f: f.write(project.serializeTurtle()) #f.write(project.graph.get_provn()) #save a DOT graph as PNG project.save_DotGraph(str(outputfile + ".png"), format="png")