def test_GetParticipantDetails():
import time
start = time.time()
restParser = RestParser()
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
project = projects[0]
import time
start = time.time()
subjects = restParser.run(BRAIN_VOL_FILES,
'/projects/{}/subjects'.format(project))
subject = subjects['uuid'][0]
Query.GetParticipantInstrumentData(BRAIN_VOL_FILES, project, subject)
details = Query.GetParticipantDetails(BRAIN_VOL_FILES, project, subject)
assert ('uuid' in details)
assert ('id' in details)
assert ('activity' in details)
assert ('instruments' in details)
assert ('derivatives' in details)
end = time.time()
runtime = end - start
def test_uri_projects_subjects_id():
global test_person_uuid
restParser = RestParser()
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
result = restParser.run(['./cmu_a.nidm.ttl'], '/projects')
project = result[0]
result = restParser.run(['./cmu_a.nidm.ttl'],
'/projects/{}/subjects'.format(project))
subject = result['uuid'][0]
uri = '/projects/{}/subjects/{}'.format(project, subject)
result = restParser.run(['./cmu_a.nidm.ttl'], uri)
assert type(result) == dict
assert result['uuid'] == subject
assert len(result['instruments']) > 2
instruments = result['instruments'].values()
all_keys = []
for i in instruments:
all_keys += i.keys()
assert 'AGE_AT_SCAN' in all_keys
assert len(result['derivatives']) > 0
def setup():
global cmu_test_project_uuid, cmu_test_subject_uuid
if Path(REST_TEST_FILE).is_file():
os.remove(REST_TEST_FILE)
makeTestFile(filename=REST_TEST_FILE,
params={
'PROJECT_UUID': 'p1',
'PROJECT2_UUID': 'p2'
})
for f in ['./cmu_a.nidm.ttl', 'caltech.nidm.ttl']:
if Path(f).is_file():
os.remove(f)
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT)
projects = restParser.run(['./cmu_a.nidm.ttl'], '/projects')
cmu_test_project_uuid = projects[0]
subjects = restParser.run(
['./cmu_a.nidm.ttl'],
'/projects/{}/subjects'.format(cmu_test_project_uuid))
cmu_test_subject_uuid = subjects['uuid'][0]
if not Path('./caltech.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/Caltech/nidm.ttl",
"caltech.nidm.ttl")
def setup():
global cmu_test_project_uuid
global cmu_test_subject_uuid
for f in ['./cmu_a.nidm.ttl', 'caltech.nidm.ttl']:
if Path(f).is_file():
os.remove(f)
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
if not Path('./caltech.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/Caltech/nidm.ttl",
"caltech.nidm.ttl")
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT)
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
for p in projects:
proj_info = restParser.run(BRAIN_VOL_FILES, '/projects/{}'.format(p))
if type(proj_info) == dict and 'dctypes:title' in proj_info.keys(
) and proj_info['dctypes:title'] == 'ABIDE CMU_a Site':
cmu_test_project_uuid = p
break
subjects = restParser.run(
BRAIN_VOL_FILES, '/projects/{}/subjects'.format(cmu_test_project_uuid))
cmu_test_subject_uuid = subjects['uuid'][0]
def test_dataelement_list():
rest_parser = RestParser(output_format=RestParser.OBJECT_FORMAT)
result = rest_parser.run(OPENNEURO_FILES, '/dataelements')
assert type(result) == dict
assert "data_elements" in result
assert 'uuid' in result["data_elements"]
assert 'label' in result["data_elements"]
assert 'data_type_info' in result["data_elements"]
assert len(result["data_elements"]["label"]) != 0
assert len(result["data_elements"]["label"]) == len(
result["data_elements"]["uuid"])
assert len(result["data_elements"]["label"]) == len(
result["data_elements"]["data_type_info"])
for label in result["data_elements"]["label"]:
assert label in [
str(x["label"]) for x in result["data_elements"]["data_type_info"]
]
for uuid in result["data_elements"]["uuid"]:
assert uuid in [
str(x["dataElementURI"])
for x in result["data_elements"]["data_type_info"]
]
# now check for derivatives
result = rest_parser.run(BRAIN_VOL_FILES, '/dataelements')
assert type(result) == dict
assert 'Left-WM-hypointensities Volume_mm3 (mm^3)' in result[
'data_elements']['label']
def test_CheckSubjectMatchesFilter():
restParser = RestParser()
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
project = projects[0]
subjects = restParser.run(BRAIN_VOL_FILES,
'/projects/{}/subjects'.format(project))
subject = subjects['uuid'][0]
derivatives = Query.GetDerivativesDataForSubject(BRAIN_VOL_FILES, project,
subject)
for skey in derivatives:
for vkey in derivatives[skey]['values']:
dt = vkey
val = derivatives[skey]['values'][vkey]['value']
if (dt and val):
break
# find an actual stat and build a matching filter to make sure our matcher passes it
filter = "derivatives.{} eq {}".format(dt, val)
assert Query.CheckSubjectMatchesFilter(BRAIN_VOL_FILES, project, subject,
filter)
instruments = Query.GetParticipantInstrumentData(BRAIN_VOL_FILES, project,
subject)
for (i, inst) in instruments.items():
if 'AGE_AT_SCAN' in inst:
age = inst['AGE_AT_SCAN']
older = str(float(age) + 1)
younger = str(float(age) - 1)
assert Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN eq {}".format(str(age)))
assert (Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN lt {}".format(younger)) == False)
assert (Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN gt {}".format(younger)) == True)
assert Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN lt {}".format(older))
assert (Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN gt {}".format(older)) == False)
eq__format = "instruments.{} eq '{}'".format('WISC_IV_VOCAB_SCALED', 'nan')
assert Query.CheckSubjectMatchesFilter(BRAIN_VOL_FILES, project, subject,
eq__format)
eq__format = "instruments.{} eq '{}'".format('WISC_IV_VOCAB_SCALED',
'not a match')
assert (Query.CheckSubjectMatchesFilter(BRAIN_VOL_FILES, project, subject,
eq__format) == False)
def test_uri_project_id():
# try with the real brain volume files
restParser = RestParser()
result = restParser.run(OPENNEURO_FILES, '/projects')
project = result[0]
result = restParser.run(OPENNEURO_FILES, '/projects/{}'.format(project))
assert 'dctypes:title' in result
assert 'sio:Identifier' in result
assert 'subjects' in result
assert len(result['subjects']['uuid']) > 2
assert 'data_elements' in result
assert len(result['data_elements']['uuid']) > 1
def test_ExtremeFilters():
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT)
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
project = projects[0]
details = restParser.run(BRAIN_VOL_FILES, '/projects/{}?filter=AGE_AT_SCAN gt 200'.format(project))
assert len(details['subjects']['uuid']) == 0
assert len(details['data_elements']['uuid']) > 0
details = restParser.run(BRAIN_VOL_FILES, '/projects/{}?filter=instruments.AGE_AT_SCAN gt 0'.format(project))
assert len(details['subjects']['uuid']) > 0
assert len(details['data_elements']['uuid']) > 0
def go():
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT,
verbosity_level=5)
result = restParser.run(
["ttl/cmu_a.nidm.ttl"],
"projects/dad0f09c-49ec-11ea-9fd8-003ee1ce9545?fields=fs_000003")
print(result)
def test_uri_project_list():
import uuid
kwargs={Constants.NIDM_PROJECT_NAME:"FBIRN_PhaseII",Constants.NIDM_PROJECT_IDENTIFIER:9610,Constants.NIDM_PROJECT_DESCRIPTION:"Test investigation"}
proj1_uuid = str(uuid.uuid1())
proj2_uuid = str(uuid.uuid1())
project = Project(uuid=proj1_uuid,attributes=kwargs)
#save a turtle file
with open("uritest.ttl",'w') as f:
f.write(project.serializeTurtle())
kwargs={Constants.NIDM_PROJECT_NAME:"FBIRN_PhaseIII",Constants.NIDM_PROJECT_IDENTIFIER:1200,Constants.NIDM_PROJECT_DESCRIPTION:"Test investigation2"}
project = Project(uuid=proj2_uuid,attributes=kwargs)
#save a turtle file
with open("uritest2.ttl",'w') as f:
f.write(project.serializeTurtle())
restParser = RestParser()
result = restParser.run(['uritest.ttl', 'uritest2.ttl'], '/projects')
project_uuids = []
for uuid in result:
project_uuids.append(uuid)
assert type(result) == list
assert len(project_uuids) >= 2
assert proj1_uuid in project_uuids
assert proj2_uuid in project_uuids
os.remove("uritest.ttl")
os.remove("uritest2.ttl")
def test_uri_subject_list():
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT)
result = restParser.run(ALL_FILES, '/subjects')
assert type(result) == dict
assert type(result['subject']) == list
assert len(result['subject']) > 10
def test_brain_vols():
restParser = RestParser()
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
project = (restParser.run(BRAIN_VOL_FILES, '/projects'))[0]
subjects = restParser.run(BRAIN_VOL_FILES,
'/projects/{}/subjects'.format(project))
subject = subjects['uuid'][0]
data = Query.GetDerivativesDataForSubject(BRAIN_VOL_FILES, None, subject)
assert (len(data) > 0)
for key in data:
assert ('StatCollectionType' in data[key])
assert ('URI' in data[key])
assert ('values' in data[key])
def go():
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT,
verbosity_level=5)
result = restParser.run([
"ttl/caltech.ttl"
], '/projects/e059fc5e-67aa-11ea-84b4-003ee1ce9545/subjects?filter=instruments.ADOS_MODULE gt 2'
)
print(result)
def test_Filter_Flexibility():
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT)
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
project = projects[0]
synonyms = Query.GetDatatypeSynonyms(tuple(BRAIN_VOL_FILES),project, 'ADOS_MODULE')
real_synonyms = [x for x in synonyms if len(x) > 1]
assert len(real_synonyms) > 1
for syn in real_synonyms:
if ' ' in syn:
continue
details = restParser.run(BRAIN_VOL_FILES, '/projects/{}?filter=instruments.{} gt 2'.format(project, syn))
assert len(details['subjects']['uuid']) > 0
assert len(details['data_elements']['uuid']) > 0
def test_rest_sub_id():
restParser = RestParser()
restParser.setOutputFormat(RestParser.OBJECT_FORMAT)
result = restParser.run(ALL_FILES,
'/projects/{}'.format(cmu_test_project_uuid))
sub_id = result['subjects']['subject id'][5]
sub_uuid = result['subjects']['uuid'][5]
result2 = restParser.run(ALL_FILES, '/subjects/{}'.format(sub_id))
pp = pprint.PrettyPrinter()
pp.pprint('/subjects/{}'.format(sub_id))
# make sure we got the same UUID when looking up by sub id
assert result2['uuid'] == sub_uuid
assert len(result2['instruments']) > 0
def test_project_fields_instruments():
rest_parser = RestParser(verbosity_level=0)
projects = rest_parser.run(BRAIN_VOL_FILES, '/projects')
proj_uuid = projects[0]
rest_parser.setOutputFormat(RestParser.OBJECT_FORMAT)
field = 'ncidb:Age'
project = rest_parser.run(
BRAIN_VOL_FILES, "/projects/{}?fields={}".format(proj_uuid, field))
# edited by DBK to account for only field values being returned
#assert( 'field_values' in project )
assert (len(project) > 0)
#fv = project['field_values']
fv = project
assert (type(fv) == list)
fields_used = set([i.label for i in fv])
assert field in fields_used
def test_dataelement_details_in_projects_field():
rest_parser = RestParser(output_format=RestParser.OBJECT_FORMAT)
# result = rest_parser.run(OPENNEURO_FILES, '/dataelements')
# dti = rest_parser.run(OPENNEURO_FILES, '/dataelements/{}'.format(result["data_elements"]["label"][0]))
# assert len(dti['inProjects']) >= 1
# find a data element that we are using for at least one subject
data_element_label = 'Right-non-WM-hypointensities normMax (MR)'
dti = rest_parser.run(BRAIN_VOL_FILES,
'/dataelements/{}'.format(data_element_label))
assert len(dti['inProjects']) >= 1
def test_uri_projects_subjects_1():
global test_p2_subject_uuids
proj_uuid = 'p2'
restParser = RestParser()
result = restParser.run([REST_TEST_FILE], '/projects/{}/subjects'.format(proj_uuid))
assert type(result) == dict
assert len(result['uuid']) == 2
assert test_p2_subject_uuids[0] in result['uuid']
assert test_p2_subject_uuids[1] in result['uuid']
def test_uri_projects_subjects_id():
global test_person_uuid
restParser = RestParser()
# result = restParser.run(OPENNEURO_FILES, '/projects')
project = OPENNEURO_PROJECT_URI
result = restParser.run(OPENNEURO_FILES, '/projects/{}/subjects'.format(project))
subject = result['uuid'][0]
uri = '/projects/{}/subjects/{}'.format(project,subject)
result = restParser.run(OPENNEURO_FILES, uri)
assert type(result) == dict
assert result['uuid'] == subject
assert len(result['instruments']) > 2
instruments = result['instruments'].values()
all_keys = []
for i in instruments:
all_keys += i.keys()
assert 'age' in all_keys
def test_uri_subjects():
global cmu_test_subject_uuid
restParser = RestParser()
restParser.setOutputFormat(RestParser.OBJECT_FORMAT)
result = restParser.run(BRAIN_VOL_FILES, '/subjects/{}'.format(cmu_test_subject_uuid))
assert type(result) == dict
assert 'uuid' in result
assert 'instruments' in result
assert 'derivatives' in result
assert cmu_test_subject_uuid == result['uuid']
def test_project_fields_not_found():
# test that things don't break if the field isn't in project
rest_parser = RestParser(verbosity_level=0)
rest_parser.setOutputFormat(RestParser.OBJECT_FORMAT)
field = 'not_real_field'
project = rest_parser.run( BRAIN_VOL_FILES, "/projects/{}?fields={}".format(cmu_test_project_uuid, field) )
print (project)
keys = set( [ i for i in project ] )
assert "error" in keys
def test_project_fields_instruments():
rest_parser = RestParser(verbosity_level=0)
rest_parser.setOutputFormat(RestParser.OBJECT_FORMAT)
field = 'AGE_AT_SCAN'
project = rest_parser.run(
BRAIN_VOL_FILES,
"/projects/{}?fields={}".format(cmu_test_project_uuid, field))
assert ('field_values' in project)
fv = project['field_values']
assert (type(fv) == list)
fields_used = set([i["field"] for i in fv])
assert field in fields_used
def test_project_fields_not_found():
# test that things don't break if the field isn't in project
rest_parser = RestParser(verbosity_level=0)
rest_parser.setOutputFormat(RestParser.OBJECT_FORMAT)
field = 'not_real_field'
project = rest_parser.run(
BRAIN_VOL_FILES,
"/projects/{}?fields={}".format(cmu_test_project_uuid, field))
assert ('field_values' in project)
fv = project['field_values']
assert (type(fv) == list)
assert len(fv) == 0
def test_project_fields_deriv():
rest_parser = RestParser(verbosity_level=0)
rest_parser.setOutputFormat(RestParser.OBJECT_FORMAT)
field = 'fs_000003'
project = rest_parser.run( BRAIN_VOL_FILES, "/projects/{}?fields={}".format(cmu_test_project_uuid, field) )
# edited by DBK to account for only field values being returned
#assert( 'field_values' in project )
assert (len(project) > 0)
#fv = project['field_values']
fv = project
assert( type( fv ) == list )
fields_used = set( [ i.label for i in fv ] )
assert 'Brain Segmentation Volume (mm^3)' in fields_used
def test_odd_isabout_uris():
rest_parser = RestParser(verbosity_level=0)
# rest_parser.setOutputFormat(RestParser.CLI_FORMAT)
rest_parser.setOutputFormat(RestParser.OBJECT_FORMAT)
field = 'http://www.cognitiveatlas.org/ontology/cogat.owl#CAO_00962'
fields = rest_parser.run( BRAIN_VOL_FILES, "/projects?fields={}".format(field) )
# edited by DBK to account for only field values being returned
#assert( 'field_values' in project )
assert (len(fields) > 0)
#fv = project['field_values']
print (fields)
fv = fields
assert( type( fv ) == list )
fields_used = set( [ i.label for i in fv ] )
assert 'ADOS_TOTAL' in fields_used
def test_multiple_project_fields():
rest_parser = RestParser(verbosity_level=0)
# rest_parser.setOutputFormat(RestParser.CLI_FORMAT)
rest_parser.setOutputFormat(RestParser.OBJECT_FORMAT)
field = 'fs_000003,ilx_0100400' # ilx0100400 is 'isAbout' age
fields = rest_parser.run( BRAIN_VOL_FILES, "/projects?fields={}".format(field) )
# edited by DBK to account for only field values being returned
#assert( 'field_values' in project )
assert (len(fields) > 0)
#fv = project['field_values']
print (fields)
fv = fields
assert( type( fv ) == list )
fields_used = set( [ i.label for i in fv ] )
assert 'Brain Segmentation Volume (mm^3)' in fields_used
assert 'age at scan' in fields_used
def setup():
global cmu_test_project_uuid, cmu_test_subject_uuid, OPENNEURO_PROJECT_URI, OPENNEURO_SUB_URI
if Path(REST_TEST_FILE).is_file():
os.remove(REST_TEST_FILE)
makeTestFile(filename=REST_TEST_FILE,
params={
'PROJECT_UUID': 'p1',
'PROJECT2_UUID': 'p2'
})
for f in ['./cmu_a.nidm.ttl', 'caltech.nidm.ttl']:
if Path(f).is_file():
os.remove(f)
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
if not Path('./caltech.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/Caltech/nidm.ttl",
"caltech.nidm.ttl")
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT)
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
for p in projects:
proj_info = restParser.run(BRAIN_VOL_FILES, '/projects/{}'.format(p))
if 'dctypes:title' in proj_info.keys(
) and proj_info['dctypes:title'] == 'ABIDE CMU_a Site':
cmu_test_project_uuid = p
break
subjects = restParser.run(
BRAIN_VOL_FILES, '/projects/{}/subjects'.format(cmu_test_project_uuid))
cmu_test_subject_uuid = subjects['uuid'][0]
if not Path('./ds000168.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/openneuro/ds000168/nidm.ttl",
"ds000168.nidm.ttl")
projects2 = restParser.run(OPENNEURO_FILES, '/projects')
for p in projects2:
proj_info = restParser.run(OPENNEURO_FILES, '/projects/{}'.format(p))
if 'dctypes:title' in proj_info.keys() and proj_info[
'dctypes:title'] == 'Offline Processing in Associative Learning':
OPENNEURO_PROJECT_URI = p
subjects = restParser.run(
OPENNEURO_FILES, '/projects/{}/subjects'.format(OPENNEURO_PROJECT_URI))
OPENNEURO_SUB_URI = subjects['uuid'][0]
def test_dataelement_details():
rest_parser = RestParser(output_format=RestParser.OBJECT_FORMAT)
# result = rest_parser.run(OPENNEURO_FILES, '/dataelements')
#
# dti = rest_parser.run(OPENNEURO_FILES, '/dataelements/{}'.format(result["data_elements"]["label"][0]))
#
# assert "label" in dti
# assert "description" in dti
# assert "isAbout" in dti
# assert "inProjects" in dti
#
# # make sure the text formatter doesn't fail horribly
# rest_parser.setOutputFormat(RestParser.CLI_FORMAT)
# txt = rest_parser.run(OPENNEURO_FILES, '/dataelements/{}'.format(result["data_elements"]["label"][0]))
#
dti = rest_parser.run(
OPENNEURO_FILES,
'/dataelements/Left-WM-hypointensities Volume_mm3 (mm^3)')
print(dti)
def test_uri_subject_list_with_fields():
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT)
result = restParser.run(ALL_FILES, '/subjects?fields=ilx_0100400,MagneticFieldStrength') # ilx_0100400 "is about" age
assert type(result) == dict
assert type(result['subject']) == list
assert len(result['subject']) > 10
assert type(result['fields']) == dict
all_fields = []
for uuid in result['fields']:
assert type(result['fields']) == dict
for sub in result['fields']:
assert type(result['fields'][sub]) == dict
for activity in result['fields'][sub]:
all_fields.append(result['fields'][sub][activity].label)
if result['fields'][sub][activity].value != 'n/a':
assert float(result['fields'][sub][activity].value) > 0
assert float(result['fields'][sub][activity].value) < 125
assert 'age' in all_fields
assert 'MagneticFieldStrength' in all_fields
def get(self, all):
query_bits = []
for a in request.args.keys():
query_bits.append("{}={}".format(a, request.args.get(a)))
query = "&".join(query_bits)
files = getTTLFiles()
if len(files) == 0:
return ({
'error':
'No NIDM files found. You may need to add NIDM ttl files to ~/PyNIDM/ttl'
})
restParser = RestParser(output_format=RestParser.OBJECT_FORMAT,
verbosity_level=5)
json_str = simplejson.dumps(restParser.run(files,
"{}?{}".format(all, query)),
indent=2)
response = app.response_class(response=json_str,
status=200,
mimetype='application/json')
return response