def ExpandProjectMetaData(self, meta_data):
"""
Takes in the meta_data from GetProjectsMetadata() and adds
the following statistics about each project to the existing
meta_data structure:
age_max (float)
age_min (float)
handedness of subjects (list)
genders of subjects (list)
number of subjects (int)
:param meta_data:
:return:
"""
for project_id in meta_data['projects']:
project_uuid = str(project_id)[6:] if (
str(project_id).startswith("niiri:")) else project_id
project = meta_data['projects'][project_id]
ages = set()
hands = set()
genders = set()
for session in Navigate.getSessions(self.nidm_files, project_uuid):
for acq in Navigate.getAcquisitions(self.nidm_files, session):
act_data = Navigate.getActivityData(self.nidm_files, acq)
for de in act_data.data:
if de.isAbout == "http://uri.interlex.org/ilx_0100400" or de.isAbout == "http://uri.interlex.org/base/ilx_0100400":
ages.add(float(de.value))
elif de.isAbout == "http://uri.interlex.org/ilx_0101292" or de.isAbout == "http://uri.interlex.org/base/ilx_0101292"\
or de.isAbout == "http://uri.interlex.org/ilx_0738439" or de.isAbout == \
"https://ndar.nih.gov/api/datadictionary/v2/dataelement/gender":
genders.add(de.value)
elif de.isAbout == "http://purl.obolibrary.org/obo/PATO_0002201":
hands.add(de.value)
print(
Query.GetParticipantUUIDsForProject(self.nidm_files,
project_uuid))
project['age_max'] = max(ages) if len(ages) > 0 else 0
project['age_min'] = min(ages) if len(ages) > 0 else 0
project[Query.matchPrefix(str(
Constants.NIDM_NUMBER_OF_SUBJECTS))] = len(
(Query.GetParticipantUUIDsForProject(
self.nidm_files, project_uuid))['uuid'])
project[str(Constants.NIDM_GENDER)] = list(genders)
project[str(Constants.NIDM_HANDEDNESS)] = list(hands)
def restParser(nidm_files, command, verbosity_level=0):
restLog("parsing command " + command, 1, verbosity_level)
restLog("Files to read:" + str(nidm_files), 1, verbosity_level)
text = ('On', 'Disable',
'unsetting') if os.getenv('PYNIDM_GRAPH_CACHE') != None else (
'Off', 'Enable', 'setting')
restLog("Graph cache is {}".format(text[0]), 1, verbosity_level)
restLog(
"{} graph cache by {} the PYNIDM_GRAPH_CACHE environment variable".
format(text[1], text[2]), 1, verbosity_level)
result = []
if re.match(r"^/?projects/?$", command):
restLog("Returning all projects", 2, verbosity_level)
projects = Query.GetProjectsUUID(nidm_files)
for uuid in projects:
result.append(str(uuid).replace(Constants.NIIRI, ""))
elif re.match(r"^/?projects/[^/]+$", command):
restLog("Returing metadata ", 2, verbosity_level)
match = re.match(r"^/?projects/([^/]+)$", command)
id = parse.unquote(str(match.group(1)))
restLog("computing metadata", 5, verbosity_level)
projects = Query.GetProjectsComputedMetadata(nidm_files)
for pid in projects['projects'].keys():
restLog("comparng " + str(pid) + " with " + str(id), 5,
verbosity_level)
restLog("comparng " + str(pid) + " with " + Constants.NIIRI + id,
5, verbosity_level)
restLog("comparng " + str(pid) + " with niiri:" + id, 5,
verbosity_level)
if pid == id or pid == Constants.NIIRI + id or pid == "niiri:" + id:
result = projects['projects'][pid]
elif re.match(r"^/?projects/[^/]+/subjects/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/?$", command)
project = match.group((1))
restLog("Returning all agents for project {}".format(project), 2,
verbosity_level)
agents = Query.GetParticipantUUIDsForProject(nidm_files, project)
result = []
vals = agents.values
for x in vals:
result.append(str(x[0]).replace("http://iri.nidash.org/", ""))
elif re.match(r"^/?projects/[^/]+/subjects/[^/]+/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)/?$", command)
restLog("Returning info about subject {}".format(match.group(2)), 2,
verbosity_level)
result = Query.GetParticipantDetails(nidm_files, match.group(1),
match.group(2))
else:
restLog("NO MATCH!", 2, verbosity_level)
return result
def projects(self):
result = []
field_values = []
self.restLog("Returning all projects", 2)
projects = Query.GetProjectsUUID(self.nidm_files)
for uuid in projects:
result.append(str(uuid).replace(Constants.NIIRI, ""))
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
subjects_set = set()
dataelements_set = set()
self.restLog("Using fields {}".format(self.query['fields']), 2)
# result['field_values'] = []
for proj in projects:
# get all the synonyms for all the fields
field_synonyms = functools.reduce(operator.iconcat, [
Query.GetDatatypeSynonyms(self.nidm_files, proj, x)
for x in self.query['fields']
], [])
files = self.nidm_files
all_subjects = Query.GetParticipantUUIDsForProject(
self.nidm_files, proj, self.query['filter']
) # nidm_file_list= files, project_id=proj['uuid'], filter=self.query['filter']):
for sub in all_subjects['uuid']:
for activity in Navigate.getActivities(
self.nidm_files, sub):
activity = Navigate.getActivityData(
self.nidm_files, activity)
for data_element in activity.data:
if data_element.dataElement in field_synonyms:
field_values.append(
data_element._replace(subject=sub))
subjects_set.add(sub)
dataelements_set.add((data_element.datumType,
data_element.label))
if len(field_values) == 0:
raise ValueError("Supplied field not found. (" +
", ".join(self.query['fields']) + ")")
else:
summary_result = {}
summary_result['subjects'] = {"uuid": [], "subject id": []}
for sub in subjects_set:
summary_result["subjects"]["uuid"].append(sub)
summary_result["subjects"]["subject id"].append("")
summary_result['data_elements'] = {"uuid": [], "lable": []}
for de in dataelements_set:
summary_result['data_elements']["uuid"] = de[0]
summary_result['data_elements']["label"] = de[1]
summary_result['field_values'] = field_values
return self.projectSummaryFormat(summary_result)
return self.format(result, ["UUID"])
def subjectsList(self):
match = re.match(r"^/?projects/([^/]+)/subjects/?$", self.command)
project = match.group((1))
self.restLog(
"Returning all agents matching filter '{}' for project {}".format(
self.query['filter'], project), 2)
result = Query.GetParticipantUUIDsForProject(self.nidm_files, project,
self.query['filter'],
None)
return self.format(result)
def projectStats(self):
result = dict()
subjects = None
path = (urlparse(self.command)).path
match = re.match(r"^/?statistics/projects/([^/]+)\??$", path)
id = parse.unquote(str(match.group(1)))
self.restLog("Returing project {} stats metadata".format(id), 2)
meta_data = Query.GetProjectsMetadata(self.nidm_files)
self.ExpandProjectMetaData(meta_data)
projects = Query.compressForJSONResponse(meta_data)
for pid in projects['projects'].keys():
self.restLog("comparng " + str(pid) + " with " + str(id), 5)
self.restLog(
"comparng " + str(pid) + " with " + Constants.NIIRI + id, 5)
self.restLog("comparng " + str(pid) + " with niiri:" + id, 5)
if pid == id or pid == Constants.NIIRI + id or pid == "niiri:" + id:
# stip off prefixes to make it more human readable
for key in projects['projects'][pid]:
short_key = key
possible_prefix = re.sub(':.*', '', short_key)
if possible_prefix in Constants.namespaces:
short_key = re.sub('^.*:', '', short_key)
result[short_key] = projects['projects'][pid][key]
# now get any fields they reqested
for field in self.query['fields']:
if subjects == None:
subjects = Query.GetParticipantUUIDsForProject(
tuple(self.nidm_files),
project_id=id,
filter=self.query['filter'])
result['subjects'] = subjects['uuid']
bits = field.split('.')
if len(bits) > 1:
stat_type = self.getStatType(
bits[0]
) # should be either instruments or derivatives for now.
self.addFieldStats(result, id, subjects['uuid'], bits[1],
stat_type) # bits[1] will be the ID
return self.dictFormat(result)
def projectSummary(self):
match = re.match(r"^/?projects/([^/]+)$", self.command)
id = parse.unquote(str(match.group(1)))
self.restLog("Returing project {} summary".format(id), 2)
result = nidm.experiment.Navigate.GetProjectAttributes(self.nidm_files,
project_id=id)
result['subjects'] = Query.GetParticipantUUIDsForProject(
self.nidm_files, project_id=id, filter=self.query['filter'])
result['data_elements'] = Query.GetProjectDataElements(self.nidm_files,
project_id=id)
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
self.restLog("Using fields {}".format(self.query['fields']), 2)
result['field_values'] = []
# get all the synonyms for all the fields
field_synonyms = functools.reduce(operator.iconcat, [
Query.GetDatatypeSynonyms(self.nidm_files, id, x)
for x in self.query['fields']
], [])
for sub in result['subjects']['uuid']:
for activity in Navigate.getActivities(self.nidm_files, sub):
activity = Navigate.getActivityData(
self.nidm_files, activity)
for data_element in activity.data:
if data_element.dataElement in field_synonyms:
result['field_values'].append(
data_element._replace(subject=sub))
if len(result['field_values']) == 0:
raise ValueError("Supplied field not found. (" +
", ".join(self.query['fields']) + ")")
return self.projectSummaryFormat(result)
def restParser(nidm_files, command, verbosity_level=0):
restLog("parsing command " + command, 1, verbosity_level)
restLog("Files to read:" + str(nidm_files), 1, verbosity_level)
restLog("Using {} as the graph cache directory".format(gettempdir()), 1,
verbosity_level)
filter = ""
if str(command).find('?') != -1:
(command, query) = str(command).split('?')
for q in query.split('&'):
if len(q.split('=')) == 2:
left, right = q.split('=')[0], q.split('=')[1]
if left == 'filter':
filter = right
result = []
if re.match(r"^/?projects/?$", command):
restLog("Returning all projects", 2, verbosity_level)
projects = Query.GetProjectsUUID(nidm_files)
for uuid in projects:
result.append(str(uuid).replace(Constants.NIIRI, ""))
elif re.match(r"^/?projects/[^/]+$", command):
restLog("Returing metadata ", 2, verbosity_level)
match = re.match(r"^/?projects/([^/]+)$", command)
id = parse.unquote(str(match.group(1)))
restLog("computing metadata", 5, verbosity_level)
projects = Query.GetProjectsComputedMetadata(nidm_files)
for pid in projects['projects'].keys():
restLog("comparng " + str(pid) + " with " + str(id), 5,
verbosity_level)
restLog("comparng " + str(pid) + " with " + Constants.NIIRI + id,
5, verbosity_level)
restLog("comparng " + str(pid) + " with niiri:" + id, 5,
verbosity_level)
if pid == id or pid == Constants.NIIRI + id or pid == "niiri:" + id:
result = projects['projects'][pid]
elif re.match(r"^/?projects/[^/]+/subjects/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/?$", command)
project = match.group((1))
restLog(
"Returning all agents matching filter '{}' for project {}".format(
filter, project), 2, verbosity_level)
result = Query.GetParticipantUUIDsForProject(nidm_files, project,
filter, None)
elif re.match(r"^/?projects/[^/]+/subjects/[^/]+/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)/?$", command)
restLog("Returning info about subject {}".format(match.group(2)), 2,
verbosity_level)
result = Query.GetParticipantDetails(nidm_files, match.group(1),
match.group(2))
elif re.match(r"^/?projects/[^/]+/subjects/[^/]+/instruments/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)", command)
restLog("Returning instruments in subject {}".format(match.group(2)),
2, verbosity_level)
instruments = Query.GetParticipantInstrumentData(
nidm_files, match.group(1), match.group(2))
for i in instruments:
result.append(i)
elif re.match(r"^/?projects/[^/]+/subjects/[^/]+/instruments/[^/]+/?$",
command):
match = re.match(
r"^/?projects/([^/]+)/subjects/([^/]+)/instruments/([^/]+)",
command)
restLog(
"Returning instrument {} in subject {}".format(
match.group(3), match.group(2)), 2, verbosity_level)
instruments = Query.GetParticipantInstrumentData(
nidm_files, match.group(1), match.group(2))
result = instruments[match.group(3)]
elif re.match(r"^/?projects/[^/]+/subjects/[^/]+/derivatives/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)", command)
restLog("Returning derivatives in subject {}".format(match.group(2)),
2, verbosity_level)
derivatives = Query.GetDerivativesDataForSubject(
nidm_files, match.group(1), match.group(2))
for s in derivatives:
result.append(s)
elif re.match(r"^/?projects/[^/]+/subjects/[^/]+/derivatives/[^/]+/?$",
command):
match = re.match(
r"^/?projects/([^/]+)/subjects/([^/]+)/derivatives/([^/]+)",
command)
restLog(
"Returning stat {} in subject {}".format(match.group(3),
match.group(2)), 2,
verbosity_level)
derivatives = Query.GetDerivativesDataForSubject(
nidm_files, match.group(1), match.group(2))
result = derivatives[match.group(3)]
else:
restLog("NO MATCH!", 2, verbosity_level)
return result
def projectSummary(self):
GetParticipantDetails_Cached = memory.cache(
Query.GetParticipantDetails)
match = re.match(r"^/?projects/([^/]+)$", self.command)
id = parse.unquote(str(match.group(1)))
self.restLog("Returing project {} summary".format(id), 2)
result = nidm.experiment.Navigate.GetProjectAttributes(self.nidm_files,
project_id=id)
result['subjects'] = Query.GetParticipantUUIDsForProject(
self.nidm_files, project_id=id, filter=self.query['filter'])
result['data_elements'] = Query.GetProjectDataElements(self.nidm_files,
project_id=id)
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
self.restLog("Using fields {}".format(self.query['fields']), 2)
result['field_values'] = []
for sub in result['subjects']:
sub_data = GetParticipantDetails_Cached(
self.nidm_files, id, sub)
for study_type in ['derivatives']:
if study_type in sub_data:
for key, deriv in sub_data[study_type].items():
for element_uri, data_row in deriv['values'].items(
):
field = self.getTailOfURI(element_uri)
if field in self.query['fields']:
result['field_values'].append({
'subject':
sub,
'field':
field,
'datumType':
data_row['datumType'],
'label':
data_row['label'],
'value':
data_row['value'],
'units':
data_row['units']
})
for study_type in ['instruments']:
if study_type in sub_data:
for instrument_uri, instrument_dict in sub_data[
study_type].items():
for instrument_field, instument_value in instrument_dict.items(
):
if instrument_field in self.query['fields']:
result['field_values'].append({
'subject': sub,
'field': instrument_field,
'datumType': '',
'label': '',
'value': instument_value,
'units': ''
})
return self.projectSummaryFormat(result)