def test_jsonld_exports(): kwargs = { Constants.NIDM_PROJECT_NAME: "FBIRN_PhaseII", Constants.NIDM_PROJECT_IDENTIFIER: 9610, Constants.NIDM_PROJECT_DESCRIPTION: "Test investigation" } project = Project(uuid="_123456", attributes=kwargs) #save a turtle file with open("test.json", 'w') as f: f.write(project.serializeJSONLD()) #load in JSON file with open("test.json") as json_file: data = json.load(json_file) assert (data["Identifier"]['@value'] == "9610")
def main(argv): parser = ArgumentParser(description='This program will load in a CSV file and iterate over the header \ variable names performing an elastic search of https://scicrunch.org/ for NIDM-ReproNim \ tagged terms that fuzzy match the variable names. The user will then interactively pick \ a term to associate with the variable name. The resulting annotated CSV data will \ then be written to a NIDM data file.') parser.add_argument('-csv', dest='csv_file', required=True, help="Path to CSV file to convert") parser.add_argument('-ilxkey', dest='key', required=True, help="Interlex/SciCrunch API key to use for query") parser.add_argument('-json_map', dest='json_map',required=False,help="User-suppled JSON file containing variable-term mappings.") parser.add_argument('-nidm', dest='nidm_file', required=False, help="Optional NIDM file to add CSV->NIDM converted graph to") #parser.add_argument('-owl', action='store_true', required=False, help='Optionally searches NIDM OWL files...internet connection required') parser.add_argument('-png', action='store_true', required=False, help='Optional flag, when set a PNG image file of RDF graph will be produced') parser.add_argument('-jsonld', action='store_true', required=False, help='Optional flag, when set NIDM files are saved as JSON-LD instead of TURTLE') parser.add_argument('-out', dest='output_file', required=True, help="Filename to save NIDM file") args = parser.parse_args() #open CSV file and load into df = pd.read_csv(args.csv_file) #maps variables in CSV file to terms #if args.owl is not False: # column_to_terms = map_variables_to_terms(df=df, apikey=args.key, directory=dirname(args.output_file), output_file=args.output_file, json_file=args.json_map, owl_file=args.owl) #else: column_to_terms = map_variables_to_terms(df=df, apikey=args.key, directory=dirname(args.output_file), output_file=args.output_file, json_file=args.json_map) #If user has added an existing NIDM file as a command line parameter then add to existing file for subjects who exist in the NIDM file if args.nidm_file: print("Adding to NIDM file...") #read in NIDM file project = read_nidm(args.nidm_file) #get list of session objects session_objs=project.get_sessions() #look at column_to_terms dictionary for NIDM URL for subject id (Constants.NIDM_SUBJECTID) id_field=None for key, value in column_to_terms.items(): if Constants.NIDM_SUBJECTID._str == column_to_terms[key]['label']: id_field=key #make sure id_field is a string for zero-padded subject ids #re-read data file with constraint that key field is read as string #df = pd.read_csv(args.csv_file,dtype={id_field : str}) #if we couldn't find a subject ID field in column_to_terms, ask user if id_field is None: option=1 for column in df.columns: print("%d: %s" %(option,column)) option=option+1 selection=input("Please select the subject ID field from the list above: ") id_field=df.columns[int(selection)-1] #make sure id_field is a string for zero-padded subject ids #re-read data file with constraint that key field is read as string #df = pd.read_csv(args.csv_file,dtype={id_field : str}) #use RDFLib here for temporary graph making query easier rdf_graph = Graph() rdf_graph_parse = rdf_graph.parse(source=StringIO(project.serializeTurtle()),format='turtle') #find subject ids and sessions in NIDM document query = """SELECT DISTINCT ?session ?nidm_subj_id ?agent WHERE { ?activity prov:wasAssociatedWith ?agent ; dct:isPartOf ?session . ?agent rdf:type prov:Agent ; ndar:src_subject_id ?nidm_subj_id . }""" #print(query) qres = rdf_graph_parse.query(query) for row in qres: print('%s \t %s' %(row[0],row[1])) #find row in CSV file with subject id matching agent from NIDM file #csv_row = df.loc[df[id_field]==type(df[id_field][0])(row[1])] #find row in CSV file with matching subject id to the agent in the NIDM file #be carefull about data types...simply type-change dataframe subject id column and query to strings. #here we're removing the leading 0's from IDs because pandas.read_csv strips those unless you know ahead of #time which column is the subject id.... csv_row = df.loc[df[id_field].astype('str').str.contains(str(row[1]).lstrip("0"))] #if there was data about this subject in the NIDM file already (i.e. an agent already exists with this subject id) #then add this CSV assessment data to NIDM file, else skip it.... if (not (len(csv_row.index)==0)): #NIDM document sesssion uuid session_uuid = row[0] #temporary list of string-based URIs of session objects from API temp = [o.identifier._uri for o in session_objs] #get session object from existing NIDM file that is associated with a specific subject id #nidm_session = (i for i,x in enumerate([o.identifier._uri for o in session_objs]) if x == str(session_uuid)) nidm_session = session_objs[temp.index(str(session_uuid))] #for nidm_session in session_objs: # if nidm_session.identifier._uri == str(session_uuid): #add an assessment acquisition for the phenotype data to session and associate with agent acq=AssessmentAcquisition(session=nidm_session) #add acquisition entity for assessment acq_entity = AssessmentObject(acquisition=acq) #add qualified association with existing agent acq.add_qualified_association(person=row[2],role=Constants.NIDM_PARTICIPANT) #store other data from row with columns_to_term mappings for row_variable in csv_row: #check if row_variable is subject id, if so skip it if row_variable==id_field: continue else: if not csv_row[row_variable].values[0]: continue #get column_to_term mapping uri and add as namespace in NIDM document #provNamespace(Core.safe_string(None,string=str(row_variable)), column_to_terms[row_variable]["url"]) acq_entity.add_attributes({QualifiedName(provNamespace(Core.safe_string(None,string=str(row_variable)), column_to_terms[row_variable]["url"]), ""):csv_row[row_variable].values[0]}) continue #serialize NIDM file with open(args.nidm_file,'w') as f: print("Writing NIDM file...") if args.jsonld: f.write(project.serializeJSONLD()) else: f.write(project.serializeTurtle()) project.save_DotGraph(str(args.nidm_file + ".png"), format="png") else: print("Creating NIDM file...") #If user did not choose to add this data to an existing NIDM file then create a new one for the CSV data #create empty project project=Project() #simply add name of file to project since we don't know anything about it project.add_attributes({Constants.NIDM_FILENAME:args.csv_file}) #look at column_to_terms dictionary for NIDM URL for subject id (Constants.NIDM_SUBJECTID) id_field=None for key, value in column_to_terms.items(): if Constants.NIDM_SUBJECTID._str == column_to_terms[key]['label']: id_field=key #make sure id_field is a string for zero-padded subject ids #re-read data file with constraint that key field is read as string #df = pd.read_csv(args.csv_file,dtype={id_field : str}) #if we couldn't find a subject ID field in column_to_terms, ask user if id_field is None: option=1 for column in df.columns: print("%d: %s" %(option,column)) option=option+1 selection=input("Please select the subject ID field from the list above: ") id_field=df.columns[int(selection)-1] #iterate over rows and store in NIDM file for csv_index, csv_row in df.iterrows(): #create a session object session=Session(project) #create and acquisition activity and entity acq=AssessmentAcquisition(session) acq_entity=AssessmentObject(acq) #store other data from row with columns_to_term mappings for row_variable,row_data in csv_row.iteritems(): if not row_data: continue #check if row_variable is subject id, if so skip it if row_variable==id_field: #add qualified association with person acq.add_qualified_association(person= acq.add_person(attributes=({Constants.NIDM_SUBJECTID:row_data})),role=Constants.NIDM_PARTICIPANT) continue else: #get column_to_term mapping uri and add as namespace in NIDM document acq_entity.add_attributes({QualifiedName(provNamespace(Core.safe_string(None,string=str(row_variable)), column_to_terms[row_variable]["url"]),""):row_data}) #print(project.serializeTurtle()) #serialize NIDM file with open(args.output_file,'w') as f: print("Writing NIDM file...") if args.jsonld: f.write(project.serializeJSONLD()) else: f.write(project.serializeTurtle()) if args.png: project.save_DotGraph(str(args.output_file + ".png"), format="png")