def getFieldInfoForSubject(self, project, subject):
'''
Returns a dictionary of activities where the subject has matching field data
The result[activity] is the full data_element so to get the value you would use result[activity].value
Note that a subject could match the same field in multiple activities.
:param project:
:param subject:
:return:
'''
result = {}
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
# get all the synonyms for all the fields - we can seach for them all at once
field_synonyms = functools.reduce(operator.iconcat, [
Query.GetDatatypeSynonyms(self.nidm_files, project, x)
for x in self.query['fields']
], [])
# print (field_synonyms)
for activity in Navigate.getActivities(self.nidm_files, subject):
activity_data = Navigate.getActivityData(
self.nidm_files, activity)
# print ([ x.label for x in activity.data])
for data_element in activity_data.data:
if not set([
data_element.dataElement, data_element.label,
data_element.isAbout
]).isdisjoint(set(field_synonyms)):
result[Query.URITail(activity)] = data_element
return result
def setup():
global cmu_test_project_uuid, PROJECT_URI, OPENNEURO_PROJECT_URI
for f in BRAIN_VOL_FILES:
if Path(f).is_file():
os.remove(f)
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
if not Path('./caltech.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/Caltech/nidm.ttl",
"caltech.nidm.ttl")
projects = Navigate.getProjects(BRAIN_VOL_FILES)
PROJECT_URI = projects[0]
if not Path('./ds000168.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/openneuro/ds000168/nidm.ttl",
"ds000168.nidm.ttl")
projects2 = Navigate.getProjects(OPENNEURO_FILES)
OPENNEURO_PROJECT_URI = projects2[0]
def subjectSummary(self):
match = re.match(r"^/?subjects/([^/]+)/?$", self.command)
self.restLog("Returning info about subject {}".format(match.group(1)),
2)
id = match.group(1)
# if we were passed in a sub_id rather than a UUID, lookup the associated UUID. (we might get multiple!)
if validate_uuid(id):
sub_ids = id
else:
sub_ids = Navigate.getSubjectUUIDsfromID(self.nidm_files, id)
if len(sub_ids) == 1:
sub_ids = sub_ids[0]
activities = Navigate.getActivities(self.nidm_files, id)
activityData = []
for a in activities:
data = Navigate.getActivityData(self.nidm_files, a)
activityData.append(data)
return self.subjectSummaryFormat_v2({
'uuid':
sub_ids,
'instruments':
list(filter(lambda x: x.category == 'instrument', activityData)),
'derivatives':
list(filter(lambda x: x.category == 'derivative', activityData))
})
def projects(self):
result = []
field_values = []
self.restLog("Returning all projects", 2)
projects = Query.GetProjectsUUID(self.nidm_files)
for uuid in projects:
result.append(str(uuid).replace(Constants.NIIRI, ""))
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
subjects_set = set()
dataelements_set = set()
self.restLog("Using fields {}".format(self.query['fields']), 2)
# result['field_values'] = []
for proj in projects:
# get all the synonyms for all the fields
field_synonyms = functools.reduce(operator.iconcat, [
Query.GetDatatypeSynonyms(self.nidm_files, proj, x)
for x in self.query['fields']
], [])
files = self.nidm_files
all_subjects = Query.GetParticipantUUIDsForProject(
self.nidm_files, proj, self.query['filter']
) # nidm_file_list= files, project_id=proj['uuid'], filter=self.query['filter']):
for sub in all_subjects['uuid']:
for activity in Navigate.getActivities(
self.nidm_files, sub):
activity = Navigate.getActivityData(
self.nidm_files, activity)
for data_element in activity.data:
if data_element.dataElement in field_synonyms:
field_values.append(
data_element._replace(subject=sub))
subjects_set.add(sub)
dataelements_set.add((data_element.datumType,
data_element.label))
if len(field_values) == 0:
raise ValueError("Supplied field not found. (" +
", ".join(self.query['fields']) + ")")
else:
summary_result = {}
summary_result['subjects'] = {"uuid": [], "subject id": []}
for sub in subjects_set:
summary_result["subjects"]["uuid"].append(sub)
summary_result["subjects"]["subject id"].append("")
summary_result['data_elements'] = {"uuid": [], "lable": []}
for de in dataelements_set:
summary_result['data_elements']["uuid"] = de[0]
summary_result['data_elements']["label"] = de[1]
summary_result['field_values'] = field_values
return self.projectSummaryFormat(summary_result)
return self.format(result, ["UUID"])
def test_navigate_get_subjects_for_acquisition():
subjects = set([])
sessions = Navigate.getSessions(BRAIN_VOL_FILES, PROJECT_URI)
for s in sessions:
acquisitions = Navigate.getAcquisitions(BRAIN_VOL_FILES, s)
for acq in acquisitions:
sub = Navigate.getSubject(BRAIN_VOL_FILES, acq)
assert sub != None
subjects.add(sub)
assert len(subjects) > 5
def ExpandProjectMetaData(self, meta_data):
"""
Takes in the meta_data from GetProjectsMetadata() and adds
the following statistics about each project to the existing
meta_data structure:
age_max (float)
age_min (float)
handedness of subjects (list)
genders of subjects (list)
number of subjects (int)
:param meta_data:
:return:
"""
for project_id in meta_data['projects']:
project_uuid = str(project_id)[6:] if (
str(project_id).startswith("niiri:")) else project_id
project = meta_data['projects'][project_id]
ages = set()
hands = set()
genders = set()
for session in Navigate.getSessions(self.nidm_files, project_uuid):
for acq in Navigate.getAcquisitions(self.nidm_files, session):
act_data = Navigate.getActivityData(self.nidm_files, acq)
for de in act_data.data:
if de.isAbout == "http://uri.interlex.org/ilx_0100400" or de.isAbout == "http://uri.interlex.org/base/ilx_0100400":
ages.add(float(de.value))
elif de.isAbout == "http://uri.interlex.org/ilx_0101292" or de.isAbout == "http://uri.interlex.org/base/ilx_0101292"\
or de.isAbout == "http://uri.interlex.org/ilx_0738439" or de.isAbout == \
"https://ndar.nih.gov/api/datadictionary/v2/dataelement/gender":
genders.add(de.value)
elif de.isAbout == "http://purl.obolibrary.org/obo/PATO_0002201":
hands.add(de.value)
print(
Query.GetParticipantUUIDsForProject(self.nidm_files,
project_uuid))
project['age_max'] = max(ages) if len(ages) > 0 else 0
project['age_min'] = min(ages) if len(ages) > 0 else 0
project[Query.matchPrefix(str(
Constants.NIDM_NUMBER_OF_SUBJECTS))] = len(
(Query.GetParticipantUUIDsForProject(
self.nidm_files, project_uuid))['uuid'])
project[str(Constants.NIDM_GENDER)] = list(genders)
project[str(Constants.NIDM_HANDEDNESS)] = list(hands)
def dataelementsSummary(self):
path = (urlparse(self.command)).path
match = re.match(r"^/?dataelements/([^/\?]+)", path)
dataelement = parse.unquote(str(match.group(1)))
result = Navigate.GetDataelementDetails(self.nidm_files, dataelement)
return self.dataElementDetailsFormat(result)
def test_navigate_get_sub_uuid_from_id():
uuids = Navigate.getSubjectUUIDsfromID(nidm_file_tuples=BRAIN_VOL_FILES,
sub_id='50653')
assert len(uuids) == 1
assert re.match(
"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
uuids[0]) # check that it's a UUID
def subjectSummary(self):
match = re.match(r"^/?subjects/([^/]+)/?$", self.command)
self.restLog("Returning info about subject {}".format(match.group(1)),
2)
activities = Navigate.getActivities(self.nidm_files, match.group(1))
activityData = []
for a in activities:
data = Navigate.getActivityData(self.nidm_files, a)
activityData.append(data)
return self.subjectSummaryFormat_v2({
'uuid':
match.group(1),
'instruments':
list(filter(lambda x: x.category == 'instrument', activityData)),
'derivatives':
list(filter(lambda x: x.category == 'derivative', activityData))
})
def test_navigate_get_acquisition_data_by_session():
set_of_keys_returned = set([])
set_of_activities = set([])
sessions = Navigate.getSessions(BRAIN_VOL_FILES, PROJECT_URI)
assert len(sessions) > 0
for s in sessions:
acquisitions = Navigate.getAcquisitions(BRAIN_VOL_FILES, s)
assert len(acquisitions) > 0
for a in acquisitions:
set_of_activities.add(str(a))
ad = Navigate.getActivityData(BRAIN_VOL_FILES, a)
assert len(ad.data) > 5
for vt in ad.data:
set_of_keys_returned.add(vt.label)
assert 'AGE_AT_SCAN' in set_of_keys_returned
assert 'hadAcquisitionModality' in set_of_keys_returned
def projectSubjectSummary(self):
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)/?$",
self.command)
subject = Navigate.normalizeSingleSubjectToUUID(
self.nidm_files, match.group(2))
self.restLog("Returning info about subject {}".format(match.group(2)),
2)
return self.subjectSummaryFormat(
Query.GetParticipantDetails(self.nidm_files, match.group(1),
subject))
def subjects(self):
self.restLog("Returning info about subjects", 2)
projects = Navigate.getProjects(self.nidm_files)
result = {'subject': []}
if 'fields' in self.query and len(self.query['fields']) > 0:
result['fields'] = {}
for proj in projects:
subs = Navigate.getSubjects(self.nidm_files, proj)
for s in subs:
result['subject'].append([
Query.URITail(s),
Navigate.getSubjectIDfromUUID(self.nidm_files, s)
])
# print ("getting info for " + str(s))
x = self.getFieldInfoForSubject(proj, s)
if x != {}:
result['fields'][Query.URITail(s)] = x
return self.subjectFormat(result)
def test_navigate_get_acquisition_data_by_session():
set_of_keys_returned = set([])
set_of_activities = set([])
sessions = Navigate.getSessions(OPENNEURO_FILES, OPENNEURO_PROJECT_URI)
assert len(sessions) > 0
for s in sessions:
acquisitions = Navigate.getAcquisitions(OPENNEURO_FILES, s)
assert len(acquisitions) > 0
for a in acquisitions:
set_of_activities.add(str(a))
ad = Navigate.getActivityData(OPENNEURO_FILES, a)
assert len(ad.data) > 5
for vt in ad.data:
set_of_keys_returned.add(vt.label)
print(set_of_keys_returned)
assert 'age' in set_of_keys_returned
assert 'hadAcquisitionModality' in set_of_keys_returned
def subjectsList(self):
match = re.match(r"^/?projects/([^/]+)/subjects/?$", self.command)
project = match.group((1))
self.restLog(
"Returning all agents matching filter '{}' for project {}".format(
self.query['filter'], project), 2)
# result = Query.GetParticipantUUIDsForProject(self.nidm_files, project, self.query['filter'], None)
all_subjects = Navigate.getSubjects(self.nidm_files, project)
result = {}
result['uuid'] = []
result['subject id'] = []
for sub_uuid in all_subjects:
if Query.CheckSubjectMatchesFilter(self.nidm_files, project,
sub_uuid, self.query['filter']):
uuid_string = (str(sub_uuid)).split('/')[
-1] # srip off the http://whatever/whatever/
result['uuid'].append(uuid_string)
sid = Navigate.getSubjectIDfromUUID(self.nidm_files, sub_uuid)
result['subject id'].append(str(sid))
return self.format(result)
def test_navigate_get_acquisition_data_by_subject():
set_of_keys_returned = set([])
set_of_activities = set([])
subjects = Navigate.getSubjects(OPENNEURO_FILES, OPENNEURO_PROJECT_URI)
assert len(subjects) > 0
for s in subjects:
activities = Navigate.getActivities(nidm_file_tuples=OPENNEURO_FILES,
subject_id=s)
assert len(activities) > 0
for a in activities:
set_of_activities.add(str(a))
ad = Navigate.getActivityData(OPENNEURO_FILES, a)
assert len(ad.data) > 5
for vt in ad.data:
set_of_keys_returned.add(vt.label)
assert 'age' in set_of_keys_returned
assert 'InversionTime' in set_of_keys_returned
assert 'hadAcquisitionModality' in set_of_keys_returned
def test_navigate_get_acquisition_data_by_subject():
set_of_keys_returned = set([])
set_of_activities = set([])
subjects = Navigate.getSubjects(BRAIN_VOL_FILES, PROJECT_URI)
assert len(subjects) > 0
for s in subjects:
activities = Navigate.getActivities(nidm_file_list=BRAIN_VOL_FILES,
subject_id=s)
assert len(activities) > 0
for a in activities:
set_of_activities.add(str(a))
ad = Navigate.getActivityData(BRAIN_VOL_FILES, a)
assert len(ad.data) > 5
for vt in ad.data:
set_of_keys_returned.add(vt.label)
assert 'AGE_AT_SCAN' in set_of_keys_returned
assert 'Left-Accumbens-area (voxels)' in set_of_keys_returned
assert 'hadAcquisitionModality' in set_of_keys_returned
def derivativesList(self):
result = []
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)", self.command)
self.restLog(
"Returning derivatives in subject {}".format(match.group(2)), 2)
subject = Navigate.normalizeSingleSubjectToUUID(
self.nidm_files, match.group(2))
derivatives = Query.GetDerivativesDataForSubject(
self.nidm_files, match.group(1), subject)
for s in derivatives:
result.append(s)
return self.format(result)
def instrumentSummary(self):
match = re.match(
r"^/?projects/([^/]+)/subjects/([^/]+)/instruments/([^/]+)$",
self.command)
self.restLog(
"Returning instrument {} in subject {}".format(
match.group(3), match.group(2)), 2)
subject = Navigate.normalizeSingleSubjectToUUID(
self.nidm_files, match.group(2))
instruments = Query.GetParticipantInstrumentData(
self.nidm_files, match.group(1), subject)
return self.format(instruments[match.group(3)],
headers=["Category", "Value"])
def projectSummary(self):
match = re.match(r"^/?projects/([^/]+)$", self.command)
id = parse.unquote(str(match.group(1)))
self.restLog("Returing project {} summary".format(id), 2)
result = nidm.experiment.Navigate.GetProjectAttributes(self.nidm_files,
project_id=id)
result['subjects'] = Query.GetParticipantUUIDsForProject(
self.nidm_files, project_id=id, filter=self.query['filter'])
result['data_elements'] = Query.GetProjectDataElements(self.nidm_files,
project_id=id)
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
self.restLog("Using fields {}".format(self.query['fields']), 2)
result['field_values'] = []
# get all the synonyms for all the fields
field_synonyms = functools.reduce(operator.iconcat, [
Query.GetDatatypeSynonyms(self.nidm_files, id, x)
for x in self.query['fields']
], [])
for sub in result['subjects']['uuid']:
for activity in Navigate.getActivities(self.nidm_files, sub):
activity = Navigate.getActivityData(
self.nidm_files, activity)
for data_element in activity.data:
if data_element.dataElement in field_synonyms:
result['field_values'].append(
data_element._replace(subject=sub))
if len(result['field_values']) == 0:
raise ValueError("Supplied field not found. (" +
", ".join(self.query['fields']) + ")")
return self.projectSummaryFormat(result)
def GetProjectsComputedMetadata(nidm_file_list):
'''
:param nidm_file_list: List of one or more NIDM files to query across for list of Projects
:return: Dictionay or projects, each project having a dictionary of project stats
including age_max, age_min, gender list, and handedness list.
'''
meta_data = {"projects": {}}
projects = Navigate.getProjects(tuple(nidm_file_list))
for p in projects:
proj_id = nidm.experiment.tools.rest.RestParser.getTailOfURI(str(p))
meta_data["projects"][proj_id] = {"age_max": 0, "age_min": sys.maxsize, "gender": [], "handedness": [] }
meta_data["projects"][proj_id].update(Navigate.GetProjectAttributes(tuple(nidm_file_list), p))
gender_set = set()
hand_set = set()
subjects = Navigate.getSubjects(tuple(nidm_file_list), p)
for s in subjects:
activities = Navigate.getActivities(tuple(nidm_file_list), s)
meta_data["projects"][proj_id]["number_of_subjects"] = len(subjects)
for a in activities:
data = Navigate.getActivityData(tuple(nidm_file_list), a)
if type(data) == nidm.experiment.Navigate.ActivityData:
for x in data.data:
if x.isAbout == Constants.NIDM_IS_ABOUT_AGE:
if float(x.value) > meta_data["projects"][proj_id]["age_max"]:
meta_data["projects"][proj_id]["age_max"] = float(x.value)
if float(x.value) < meta_data["projects"][proj_id]["age_min"]:
meta_data["projects"][proj_id]["age_min"] = float(x.value)
if x.isAbout == Constants.NIDM_IS_ABOUT_GENDER:
gender_set.add(str(x.value))
if x.isAbout == Constants.NIDM_IS_ABOUT_HANDEDNESS:
hand_set.add(str(x.value))
meta_data["projects"][proj_id]["gender"] = list(gender_set)
meta_data["projects"][proj_id]["handedness"] = list(hand_set)
return meta_data
def instrumentsList(self):
result = []
match = re.match(
r"^/?projects/([^/]+)/subjects/([^/]+)/instruments/?$",
self.command)
self.restLog(
"Returning instruments in subject {}".format(match.group(2)), 2)
subject = Navigate.normalizeSingleSubjectToUUID(
self.nidm_files, match.group(2))
instruments = Query.GetParticipantInstrumentData(
self.nidm_files, match.group(1), subject)
for i in instruments:
result.append(i)
return self.format(result)
def derivativeSummary(self):
match = re.match(
r"^/?projects/([^/]+)/subjects/([^/]+)/derivatives/([^/]+)",
self.command)
subject = Navigate.normalizeSingleSubjectToUUID(
self.nidm_files, match.group(2))
uri = match.group(3)
self.restLog(
"Returning stat {} in subject {}".format(uri, match.group(2)), 2)
derivatives = Query.GetDerivativesDataForSubject(
self.nidm_files, match.group(1), subject)
single_derivative = {uri: derivatives[uri]}
self.restLog("Formatting single derivative", 5)
return self.formatDerivatives(single_derivative)
def dataelements(self):
result = Navigate.GetDataelements(self.nidm_files)
return self.dataElementsFormat(result)
def test_navigate_get_sessions():
sessions = Navigate.getSessions(BRAIN_VOL_FILES, PROJECT_URI)
assert len(sessions) > 0
def test_navigate_get_projects():
projects = Navigate.getProjects(BRAIN_VOL_FILES)
assert len(projects) == 2
def test_navigate_get_acquisitions_for_session():
sessions = Navigate.getSessions(BRAIN_VOL_FILES, PROJECT_URI)
for s in sessions:
acquisitions = Navigate.getAcquisitions(BRAIN_VOL_FILES, sessions[0])
assert len(acquisitions) > 0
