def test_GetProjectsComputedMetadata():
p1 = makeProjectTestFile("testfile.ttl")
p2 = makeProjectTestFile2("testfile2.ttl")
files = ["testfile.ttl", "testfile2.ttl"]
if USE_GITHUB_DATA:
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
files.append("cmu_a.nidm.ttl")
parsed = Query.GetProjectsComputedMetadata(files)
if USE_GITHUB_DATA:
for project_id in parsed['projects']:
print("looking at {}".format(project_id))
if project_id != p1 and project_id != p2:
p3 = project_id
print("got p3 of {}".format(p3))
assert parsed['projects'][p3][str(
Constants.NIDM_PROJECT_NAME)] == "ABIDE CMU_a Site"
assert parsed['projects'][p3][Query.matchPrefix(
str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 14
assert parsed['projects'][p3]["age_min"] == 21
assert parsed['projects'][p3]["age_max"] == 33
assert parsed['projects'][p3][str(Constants.NIDM_GENDER)] == ['1', '2']
def getFieldInfoForSubject(self, project, subject):
'''
Returns a dictionary of activities where the subject has matching field data
The result[activity] is the full data_element so to get the value you would use result[activity].value
Note that a subject could match the same field in multiple activities.
:param project:
:param subject:
:return:
'''
result = {}
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
# get all the synonyms for all the fields - we can seach for them all at once
field_synonyms = functools.reduce(operator.iconcat, [
Query.GetDatatypeSynonyms(self.nidm_files, project, x)
for x in self.query['fields']
], [])
# print (field_synonyms)
for activity in Navigate.getActivities(self.nidm_files, subject):
activity_data = Navigate.getActivityData(
self.nidm_files, activity)
# print ([ x.label for x in activity.data])
for data_element in activity_data.data:
if not set([
data_element.dataElement, data_element.label,
data_element.isAbout
]).isdisjoint(set(field_synonyms)):
result[Query.URITail(activity)] = data_element
return result
def test_GetParticipantDetails():
import time
start = time.time()
restParser = RestParser()
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
project = projects[0]
import time
start = time.time()
subjects = restParser.run(BRAIN_VOL_FILES,
'/projects/{}/subjects'.format(project))
subject = subjects['uuid'][0]
Query.GetParticipantInstrumentData(BRAIN_VOL_FILES, project, subject)
details = Query.GetParticipantDetails(BRAIN_VOL_FILES, project, subject)
assert ('uuid' in details)
assert ('id' in details)
assert ('activity' in details)
assert ('instruments' in details)
assert ('derivatives' in details)
end = time.time()
runtime = end - start
def test_GetProjectsMetadata():
p1 = makeProjectTestFile("testfile.ttl")
p2 = makeProjectTestFile2("testfile2.ttl")
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
p3 = "nidm:_504a60b0-7e86-11e9-ae20-003ee1ce9545" # from the cmu_a ttl file
json_response = Query.GetProjectsMetadata(
["testfile.ttl", "testfile2.ttl", "cmu_a.nidm.ttl"])
parsed = json.loads(json_response)
assert parsed['projects'][p1][str(
Constants.NIDM_PROJECT_DESCRIPTION)] == "Test investigation"
assert parsed['projects'][p2][str(
Constants.NIDM_PROJECT_DESCRIPTION)] == "More Scans"
assert parsed['projects'][p3][str(
Constants.NIDM_PROJECT_NAME)] == "ABIDE CMU_a Site"
# we shouldn't have the computed metadata in this result
assert parsed['projects'][p1].get(
Query.matchPrefix(str(Constants.NIDM_NUMBER_OF_SUBJECTS)), -1) == -1
def test_GetProjectsComputedMetadata():
p1 = makeProjectTestFile("testfile.ttl")
p2 = makeProjectTestFile2("testfile2.ttl")
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
#DBK: this is a bit unsafe as the TTL files in the github repo above can change and the UUID will change since they are randomly
#generated at this point. It's probably more robust to explicitly create these files for the time being and explicitly set the
#UUID in the test file:
# For example: kwargs={Constants.NIDM_PROJECT_NAME:"FBIRN_PhaseIII",Constants.NIDM_PROJECT_IDENTIFIER:1200,Constants.NIDM_PROJECT_DESCRIPTION:"Test investigation2"}
# project = Project(uuid="_654321",attributes=kwargs)
#
p3 = "nidm:_504a60b0-7e86-11e9-ae20-003ee1ce9545" # from the cmu_a ttl file
json_response = Query.GetProjectsComputedMetadata(
["testfile.ttl", "testfile2.ttl", "cmu_a.nidm.ttl"])
parsed = json.loads(json_response)
assert parsed['projects'][p1][str(
Constants.NIDM_PROJECT_DESCRIPTION)] == "Test investigation"
assert parsed['projects'][p2][str(
Constants.NIDM_PROJECT_DESCRIPTION)] == "More Scans"
assert parsed['projects'][p3][str(
Constants.NIDM_PROJECT_NAME)] == "ABIDE CMU_a Site"
assert parsed['projects'][p2][Query.matchPrefix(
str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 0
#DBK - this isn't working
#assert parsed['projects'][p3][Query.matchPrefix(str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 14
assert parsed['projects'][p1][Query.matchPrefix(
str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 0
def test_CMU_GetProjectsComputedMetadata():
'''
This will run the getProjectsComputedMetadata on a downloaded ttl file from the ABIDE dataset
This is for dev purposes only and should be skipped during a normal test
'''
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
json_response = Query.GetProjectsComputedMetadata(["cmu_a.nidm.ttl"])
parsed = json.loads(json_response)
print(parsed)
# should be only one
p1 = list(parsed['projects'].keys())[0]
assert parsed['projects'][p1][Query.matchPrefix(
str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 14
assert parsed['projects'][p1]["age_min"] == 21
assert parsed['projects'][p1]["age_max"] == 33
assert parsed['projects'][p1][str(Constants.NIDM_GENDER)] == ['1', '2']
assert parsed['projects'][p1][str(
Constants.NIDM_HANDEDNESS)] == ['R', 'L', 'Ambi']
def test_OpenGraph():
g = Query.OpenGraph(BRAIN_VOL_FILES[0])
assert isinstance(g, rdflib.graph.Graph)
# if you call OpenGraph with something that is already a graph, it should send it back
g2 = Query.OpenGraph(g)
assert isinstance(g, rdflib.graph.Graph)
def test_CheckSubjectMatchesFilter():
restParser = RestParser()
if cmu_test_project_uuid:
project = cmu_test_project_uuid
else:
projects = restParser.run(BRAIN_VOL_FILES, '/projects')
project = projects[0]
subjects = restParser.run(BRAIN_VOL_FILES,
'/projects/{}/subjects'.format(project))
subject = subjects['uuid'][0]
derivatives = Query.GetDerivativesDataForSubject(BRAIN_VOL_FILES, project,
subject)
for skey in derivatives:
for vkey in derivatives[skey]['values']:
dt = vkey
val = derivatives[skey]['values'][vkey]['value']
if (dt and val):
break
# find an actual stat and build a matching filter to make sure our matcher passes it
filter = "derivatives.{} eq {}".format(dt, val)
assert Query.CheckSubjectMatchesFilter(BRAIN_VOL_FILES, project, subject,
filter)
instruments = Query.GetParticipantInstrumentData(BRAIN_VOL_FILES, project,
subject)
for (i, inst) in instruments.items():
if 'AGE_AT_SCAN' in inst:
age = inst['AGE_AT_SCAN']
older = str(float(age) + 1)
younger = str(float(age) - 1)
assert Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN eq {}".format(str(age)))
assert (Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN lt {}".format(younger)) == False)
assert (Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN gt {}".format(younger)) == True)
assert Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN lt {}".format(older))
assert (Query.CheckSubjectMatchesFilter(
BRAIN_VOL_FILES, project, subject,
"instruments.AGE_AT_SCAN gt {}".format(older)) == False)
eq__format = "instruments.{} eq '{}'".format('WISC_IV_VOCAB_SCALED', 'nan')
assert Query.CheckSubjectMatchesFilter(BRAIN_VOL_FILES, project, subject,
eq__format)
eq__format = "instruments.{} eq '{}'".format('WISC_IV_VOCAB_SCALED',
'not a match')
assert (Query.CheckSubjectMatchesFilter(BRAIN_VOL_FILES, project, subject,
eq__format) == False)
def restParser(nidm_files, command, verbosity_level=0):
restLog("parsing command " + command, 1, verbosity_level)
restLog("Files to read:" + str(nidm_files), 1, verbosity_level)
text = ('On', 'Disable',
'unsetting') if os.getenv('PYNIDM_GRAPH_CACHE') != None else (
'Off', 'Enable', 'setting')
restLog("Graph cache is {}".format(text[0]), 1, verbosity_level)
restLog(
"{} graph cache by {} the PYNIDM_GRAPH_CACHE environment variable".
format(text[1], text[2]), 1, verbosity_level)
result = []
if re.match(r"^/?projects/?$", command):
restLog("Returning all projects", 2, verbosity_level)
projects = Query.GetProjectsUUID(nidm_files)
for uuid in projects:
result.append(str(uuid).replace(Constants.NIIRI, ""))
elif re.match(r"^/?projects/[^/]+$", command):
restLog("Returing metadata ", 2, verbosity_level)
match = re.match(r"^/?projects/([^/]+)$", command)
id = parse.unquote(str(match.group(1)))
restLog("computing metadata", 5, verbosity_level)
projects = Query.GetProjectsComputedMetadata(nidm_files)
for pid in projects['projects'].keys():
restLog("comparng " + str(pid) + " with " + str(id), 5,
verbosity_level)
restLog("comparng " + str(pid) + " with " + Constants.NIIRI + id,
5, verbosity_level)
restLog("comparng " + str(pid) + " with niiri:" + id, 5,
verbosity_level)
if pid == id or pid == Constants.NIIRI + id or pid == "niiri:" + id:
result = projects['projects'][pid]
elif re.match(r"^/?projects/[^/]+/subjects/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/?$", command)
project = match.group((1))
restLog("Returning all agents for project {}".format(project), 2,
verbosity_level)
agents = Query.GetParticipantUUIDsForProject(nidm_files, project)
result = []
vals = agents.values
for x in vals:
result.append(str(x[0]).replace("http://iri.nidash.org/", ""))
elif re.match(r"^/?projects/[^/]+/subjects/[^/]+/?$", command):
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)/?$", command)
restLog("Returning info about subject {}".format(match.group(2)), 2,
verbosity_level)
result = Query.GetParticipantDetails(nidm_files, match.group(1),
match.group(2))
else:
restLog("NO MATCH!", 2, verbosity_level)
return result
def projects(self):
result = []
field_values = []
self.restLog("Returning all projects", 2)
projects = Query.GetProjectsUUID(self.nidm_files)
for uuid in projects:
result.append(str(uuid).replace(Constants.NIIRI, ""))
# if we got fields, drill into each subject and pull out the field data
# subject details -> derivitives / instrument -> values -> element
if 'fields' in self.query and len(self.query['fields']) > 0:
subjects_set = set()
dataelements_set = set()
self.restLog("Using fields {}".format(self.query['fields']), 2)
# result['field_values'] = []
for proj in projects:
# get all the synonyms for all the fields
field_synonyms = functools.reduce(operator.iconcat, [
Query.GetDatatypeSynonyms(self.nidm_files, proj, x)
for x in self.query['fields']
], [])
files = self.nidm_files
all_subjects = Query.GetParticipantUUIDsForProject(
self.nidm_files, proj, self.query['filter']
) # nidm_file_list= files, project_id=proj['uuid'], filter=self.query['filter']):
for sub in all_subjects['uuid']:
for activity in Navigate.getActivities(
self.nidm_files, sub):
activity = Navigate.getActivityData(
self.nidm_files, activity)
for data_element in activity.data:
if data_element.dataElement in field_synonyms:
field_values.append(
data_element._replace(subject=sub))
subjects_set.add(sub)
dataelements_set.add((data_element.datumType,
data_element.label))
if len(field_values) == 0:
raise ValueError("Supplied field not found. (" +
", ".join(self.query['fields']) + ")")
else:
summary_result = {}
summary_result['subjects'] = {"uuid": [], "subject id": []}
for sub in subjects_set:
summary_result["subjects"]["uuid"].append(sub)
summary_result["subjects"]["subject id"].append("")
summary_result['data_elements'] = {"uuid": [], "lable": []}
for de in dataelements_set:
summary_result['data_elements']["uuid"] = de[0]
summary_result['data_elements']["label"] = de[1]
summary_result['field_values'] = field_values
return self.projectSummaryFormat(summary_result)
return self.format(result, ["UUID"])
def test_prefix_helpers():
assert Query.expandNIDMAbbreviation(
"ndar:src_subject_id"
) == "https://ndar.nih.gov/api/datadictionary/v2/dataelement/src_subject_id"
assert Query.matchPrefix("http://purl.org/nidash/nidm#abc") == "nidm:abc"
assert Query.matchPrefix("http://www.w3.org/ns/prov#123") == "prov:123"
assert Query.matchPrefix("http://purl.org/nidash/fsl#xyz") == "fsl:xyz"
assert Query.matchPrefix("http://purl.org/nidash/fsl#xyz",
short=True) == "fsl"
def test_getProjectAcquisitionObjects():
if not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
files = ['cmu_a.nidm.ttl']
project_list = Query.GetProjectsUUID(files)
project_uuid = str(project_list[0])
objects = Query.getProjectAcquisitionObjects(files, project_uuid)
assert isinstance(objects, list)
def setup():
global cmu_test_project_uuid, cmu_test_subject_uuid
projects = Query.GetProjectsUUID(ABIDE_FILES)
for p in projects:
proj_info = nidm.experiment.Navigate.GetProjectAttributes(
ABIDE_FILES, p)
if 'dctypes:title' in proj_info.keys(
) and proj_info['dctypes:title'] == 'ABIDE CMU_a Site':
cmu_test_project_uuid = p
break
subjects = Query.GetParticipantIDs(ABIDE_FILES)
cmu_test_subject_uuid = subjects['uuid'][0]
def ExpandProjectMetaData(self, meta_data):
"""
Takes in the meta_data from GetProjectsMetadata() and adds
the following statistics about each project to the existing
meta_data structure:
age_max (float)
age_min (float)
handedness of subjects (list)
genders of subjects (list)
number of subjects (int)
:param meta_data:
:return:
"""
for project_id in meta_data['projects']:
project_uuid = str(project_id)[6:] if (
str(project_id).startswith("niiri:")) else project_id
project = meta_data['projects'][project_id]
ages = set()
hands = set()
genders = set()
for session in Navigate.getSessions(self.nidm_files, project_uuid):
for acq in Navigate.getAcquisitions(self.nidm_files, session):
act_data = Navigate.getActivityData(self.nidm_files, acq)
for de in act_data.data:
if de.isAbout == "http://uri.interlex.org/ilx_0100400" or de.isAbout == "http://uri.interlex.org/base/ilx_0100400":
ages.add(float(de.value))
elif de.isAbout == "http://uri.interlex.org/ilx_0101292" or de.isAbout == "http://uri.interlex.org/base/ilx_0101292"\
or de.isAbout == "http://uri.interlex.org/ilx_0738439" or de.isAbout == \
"https://ndar.nih.gov/api/datadictionary/v2/dataelement/gender":
genders.add(de.value)
elif de.isAbout == "http://purl.obolibrary.org/obo/PATO_0002201":
hands.add(de.value)
print(
Query.GetParticipantUUIDsForProject(self.nidm_files,
project_uuid))
project['age_max'] = max(ages) if len(ages) > 0 else 0
project['age_min'] = min(ages) if len(ages) > 0 else 0
project[Query.matchPrefix(str(
Constants.NIDM_NUMBER_OF_SUBJECTS))] = len(
(Query.GetParticipantUUIDsForProject(
self.nidm_files, project_uuid))['uuid'])
project[str(Constants.NIDM_GENDER)] = list(genders)
project[str(Constants.NIDM_HANDEDNESS)] = list(hands)
def test_GetProjectMetadata():
kwargs={Constants.NIDM_PROJECT_NAME:"FBIRN_PhaseII",Constants.NIDM_PROJECT_IDENTIFIER:9610,Constants.NIDM_PROJECT_DESCRIPTION:"Test investigation"}
project = Project(uuid="_123456",attributes=kwargs)
session = Session(project)
acq = AssessmentAcquisition(session)
kwargs={Constants.NIDM_HANDEDNESS:"Left", Constants.NIDM_AGE:"90"}
acq_obj = AssessmentObject(acq,kwargs)
#save a turtle file
with open("test.ttl",'w') as f:
f.write(project.serializeTurtle())
kwargs={Constants.NIDM_PROJECT_NAME:"FBIRN_PhaseIII",Constants.NIDM_PROJECT_IDENTIFIER:1200,Constants.NIDM_PROJECT_DESCRIPTION:"Test investigation"}
project = Project(uuid="_654321",attributes=kwargs)
session = Session(project)
acq = AssessmentAcquisition(session)
kwargs={Constants.NIDM_HANDEDNESS:"Right", Constants.NIDM_AGE:"75"}
acq_obj = AssessmentObject(acq,kwargs)
#save a turtle file
with open("test2.ttl",'w') as f:
f.write(project.serializeTurtle())
test = Query.GetProjectMetadata(["test.ttl", "test2.ttl"])
def test_GetParticipantIDs():
kwargs = {
Constants.NIDM_PROJECT_NAME: "FBIRN_PhaseII",
Constants.NIDM_PROJECT_IDENTIFIER: 9610,
Constants.NIDM_PROJECT_DESCRIPTION: "Test investigation"
}
project = Project(uuid="_123456", attributes=kwargs)
session = Session(uuid="_13579", project=project)
acq = Acquisition(uuid="_15793", session=session)
acq2 = Acquisition(uuid="_15795", session=session)
person = acq.add_person(attributes=({Constants.NIDM_SUBJECTID: "9999"}))
acq.add_qualified_association(person=person,
role=Constants.NIDM_PARTICIPANT)
person2 = acq2.add_person(attributes=({Constants.NIDM_SUBJECTID: "8888"}))
acq2.add_qualified_association(person=person2,
role=Constants.NIDM_PARTICIPANT)
#save a turtle file
with open("test.ttl", 'w') as f:
f.write(project.serializeTurtle())
participant_list = Query.GetParticipantIDs(["test.ttl"])
remove("test.ttl")
assert (participant_list['ID'].str.contains('9999').any())
assert (participant_list['ID'].str.contains('8888').any())
def projects(self):
result = []
self.restLog("Returning all projects", 2)
projects = Query.GetProjectsUUID(self.nidm_files)
for uuid in projects:
result.append(str(uuid).replace(Constants.NIIRI, ""))
return self.format(result)
def test_GetProjectsMetadata():
p1 = makeProjectTestFile("testfile.ttl")
p2 = makeProjectTestFile2("testfile2.ttl")
files = ["testfile.ttl", "testfile2.ttl"]
if USE_GITHUB_DATA and not Path('./cmu_a.nidm.ttl').is_file():
urllib.request.urlretrieve(
"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
"cmu_a.nidm.ttl")
files.append("cmu_a.nidm.ttl")
parsed = Query.GetProjectsMetadata(files)
# assert parsed['projects'][p1][str(Constants.NIDM_PROJECT_DESCRIPTION)] == "Test investigation"
# assert parsed['projects'][p2][str(Constants.NIDM_PROJECT_DESCRIPTION)] == "More Scans"
# we shouldn't have the computed metadata in this result
# assert parsed['projects'][p1].get (Query.matchPrefix(str(Constants.NIDM_NUMBER_OF_SUBJECTS)), -1) == -1
if USE_GITHUB_DATA:
# find the project ID from the CMU file
for project_id in parsed['projects']:
if project_id != p1 and project_id != p2:
p3 = project_id
assert parsed['projects'][p3][str(
Constants.NIDM_PROJECT_NAME)] == "ABIDE CMU_a Site"
def test_get_software_agents():
nidm_file = BRAIN_VOL_FILES[0]
rdf_graph = Query.OpenGraph(nidm_file)
agents = Query.getSoftwareAgents(rdf_graph)
assert len(agents) > 0
isa = URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#type')
count = 0
for a in agents:
for s, o, p in rdf_graph.triples((a, isa, Constants.PROV['Agent'])):
count += 1
assert (count == len(agents))
def projectSubjectSummary(self):
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)/?$",
self.command)
self.restLog("Returning info about subject {}".format(match.group(2)),
2)
return self.subjectSummaryFormat(
Query.GetParticipantDetails(self.nidm_files, match.group(1),
match.group(2)))
def test_download_cde_files():
cde_dir = Query.download_cde_files()
assert cde_dir == tempfile.gettempdir()
fcount = 0
for url in Constants.CDE_FILE_LOCATIONS:
fname = url.split('/')[-1]
assert path.isfile("{}/{}".format(cde_dir, fname))
fcount += 1
assert fcount > 0
def projectStats(self):
result = dict()
subjects = None
path = (urlparse(self.command)).path
match = re.match(r"^/?statistics/projects/([^/]+)\??$", path)
id = parse.unquote(str(match.group(1)))
self.restLog("Returing project {} stats metadata".format(id), 2)
meta_data = Query.GetProjectsMetadata(self.nidm_files)
self.ExpandProjectMetaData(meta_data)
projects = Query.compressForJSONResponse(meta_data)
for pid in projects['projects'].keys():
self.restLog("comparng " + str(pid) + " with " + str(id), 5)
self.restLog(
"comparng " + str(pid) + " with " + Constants.NIIRI + id, 5)
self.restLog("comparng " + str(pid) + " with niiri:" + id, 5)
if pid == id or pid == Constants.NIIRI + id or pid == "niiri:" + id:
# stip off prefixes to make it more human readable
for key in projects['projects'][pid]:
short_key = key
possible_prefix = re.sub(':.*', '', short_key)
if possible_prefix in Constants.namespaces:
short_key = re.sub('^.*:', '', short_key)
result[short_key] = projects['projects'][pid][key]
# now get any fields they reqested
for field in self.query['fields']:
if subjects == None:
subjects = Query.GetParticipantUUIDsForProject(
tuple(self.nidm_files),
project_id=id,
filter=self.query['filter'])
result['subjects'] = subjects['uuid']
bits = field.split('.')
if len(bits) > 1:
stat_type = self.getStatType(
bits[0]
) # should be either instruments or derivatives for now.
self.addFieldStats(result, id, subjects['uuid'], bits[1],
stat_type) # bits[1] will be the ID
return self.dictFormat(result)
def derivativesList(self):
result = []
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)", self.command)
self.restLog(
"Returning derivatives in subject {}".format(match.group(2)), 2)
derivatives = Query.GetDerivativesDataForSubject(
self.nidm_files, match.group(1), match.group(2))
for s in derivatives:
result.append(s)
return self.format(result)
def test_GetProjectsComputedMetadata():
files = []
rest = RestParser()
rest.nidm_files = tuple(BRAIN_VOL_FILES)
meta_data = Query.GetProjectsMetadata(BRAIN_VOL_FILES)
rest.ExpandProjectMetaData(meta_data)
parsed = Query.compressForJSONResponse(meta_data)
for project_id in parsed['projects']:
if parsed['projects'][project_id][str(Constants.NIDM_PROJECT_NAME)] == "ABIDE CMU_a Site":
p3 = project_id
break
assert parsed['projects'][p3][str(Constants.NIDM_PROJECT_NAME)] == "ABIDE CMU_a Site"
assert parsed['projects'][p3][Query.matchPrefix(str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 14
assert parsed['projects'][p3]["age_min"] == 21.0
assert parsed['projects'][p3]["age_max"] == 33.0
assert set(parsed['projects'][p3][str(Constants.NIDM_GENDER)]) == set(['1', '2'])
def test_GetProjectsComputedMetadata():
p1 = makeProjectTestFile("testfile.ttl")
p2 = makeProjectTestFile2("testfile2.ttl")
json_response = Query.GetProjectsComputedMetadata(
["testfile.ttl", "testfile2.ttl"])
parsed = json.loads(json_response)
assert parsed['projects'][p1][str(
Constants.NIDM_PROJECT_DESCRIPTION)] == "Test investigation"
assert parsed['projects'][p2][str(
Constants.NIDM_PROJECT_DESCRIPTION)] == "More Scans"
assert parsed['projects'][p2][Query.matchPrefix(
str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 0
assert parsed['projects'][p1][Query.matchPrefix(
str(Constants.NIDM_NUMBER_OF_SUBJECTS))] == 0
def subjectsList(self):
match = re.match(r"^/?projects/([^/]+)/subjects/?$", self.command)
project = match.group((1))
self.restLog(
"Returning all agents matching filter '{}' for project {}".format(
self.query['filter'], project), 2)
result = Query.GetParticipantUUIDsForProject(self.nidm_files, project,
self.query['filter'],
None)
return self.format(result)
def addFieldStats(self, result, project, subjects, field, type):
'''
Geneerates basic stats on a group of subjects and adds it to the result
:param result:
:param subjects:
:param field:
:return:
'''
values = []
for s in subjects:
if type == self.STAT_TYPE_INSTRUMENTS:
data = Query.GetParticipantInstrumentData(
tuple(self.nidm_files), project, s)
for i in data:
if field in data[i]:
values.append(float(data[i][field]))
# derivatives are of the form [UUID]['values'][URI]{datumType, label, values, units}
if type == self.STAT_TYPE_DERIVATIVES:
data = Query.GetDerivativesDataForSubject(
tuple(self.nidm_files), project, s)
for deriv in data:
for URI in data[deriv]['values']:
measures = data[deriv]['values'][URI]
if field == measures[
'label'] or field == self.getTailOfURI(URI):
values.append(float(measures['value']))
if len(values) > 0:
med = median(values)
avg = mean(values)
st = std(values)
mn = min(values)
mx = max(values)
else:
med = avg = st = mn = mx = None
result[field] = {
"max": mx,
"min": mn,
"median": med,
"mean": avg,
"standard_deviation": st
}
def instrumentsList(self):
result = []
match = re.match(r"^/?projects/([^/]+)/subjects/([^/]+)$",
self.command)
self.restLog(
"Returning instruments in subject {}".format(match.group(2)), 2)
instruments = Query.GetParticipantInstrumentData(
self.nidm_files, match.group(1), match.group(2))
for i in instruments:
result.append(i)
return self.format(result)
def instrumentSummary(self):
match = re.match(
r"^/?projects/([^/]+)/subjects/([^/]+)/instruments/([^/]+)$",
self.command)
self.restLog(
"Returning instrument {} in subject {}".format(
match.group(3), match.group(2)), 2)
instruments = Query.GetParticipantInstrumentData(
self.nidm_files, match.group(1), match.group(2))
return self.format(instruments[match.group(3)],
headers=["Category", "Value"])
def subjects(self):
self.restLog("Returning info about subjects", 2)
projects = Navigate.getProjects(self.nidm_files)
result = {'subject': []}
if 'fields' in self.query and len(self.query['fields']) > 0:
result['fields'] = {}
for proj in projects:
subs = Navigate.getSubjects(self.nidm_files, proj)
for s in subs:
result['subject'].append([
Query.URITail(s),
Navigate.getSubjectIDfromUUID(self.nidm_files, s)
])
# print ("getting info for " + str(s))
x = self.getFieldInfoForSubject(proj, s)
if x != {}:
result['fields'][Query.URITail(s)] = x
return self.subjectFormat(result)