def import_subject(dsc, xnatSubject, project=None, timezone='UTC'): """ Insert a single XNAT subject """ sourceID = xnat_api(xnatSubject.id) #print(sourceID) _log.info(' Importing subject: ' + sourceID) ctx = dsc.getContext() r = ctx.sourceForInsertion([sourceID], ['xnat:subjectURI'], [xnatSubject._uri]) src = r.getSource() if r.isNew(): _import_entity_common(src, xnatSubject) expIDs = xnat_api(xnatSubject.experiments().get) for experimentID in expIDs: xnatExp = xnat_api(xnatSubject.experiment, experimentID) import_session(dsc, src, project, xnatExp, timezone=timezone) return src
def import_project(dsc, xnatProject, timezone='UTC', importProjectTree=True): """ Import a single XNAT project. """ projectID = xnat_api(xnatProject.id) _log.info('Importing project: ' + projectID) name = xnat_api(xnatProject.attrs.get,'xnat:projectData/name') purpose = xnat_api(xnatProject.attrs.get, 'xnat:projectData/description') if not purpose: purpose = "<None>" # Find the earliest session date in the project xnat = xnatProject._intf sessionTypes = xnat.inspect.experiment_types() if len(sessionTypes) == 0: sessionTypes = ('xnat:mrSessionData', 'xnat:ctSessionData') #raise XnatImportError("No session types defined in database") minSessionDate = None for sessionType in sessionTypes: columns = (sessionType + '/DATE', sessionType + '/PROJECT') xnat_api_pause() query = xnat.select(sessionType, columns=columns).where([(sessionType + '/Project', '=', projectID), 'AND']) sessionDates = [ datetime.fromtimestamp(mktime(strptime(row['date'], DATE_FORMAT))) for row in query if len(row['date']) > 0] if len(sessionDates) > 0: for sd in sessionDates: if minSessionDate is not None: if sd < minSessionDate: minSessionDate = sd else: minSessionDate = sd if minSessionDate is not None: startTime = to_joda_datetime(minSessionDate, timezone) else: startTime = api.datetime() ctx = dsc.getContext() project = ctx.insertProject(name, purpose, startTime) _import_entity_common(project, xnatProject) if importProjectTree: _log.info(" Importing project tree...") subjectIDs = xnat_api(xnatProject.subjects().get) for subjectID in subjectIDs: s = xnat_api(xnatProject.subject, subjectID) import_subject(dsc, s, timezone=timezone, project=project) return project
def import_project(dsc, xnatProject, timezone='UTC', importProjectTree=True): """ Import a single XNAT project. """ projectID = xnat_api(xnatProject.id) _log.info('Importing project: ' + projectID) name = xnat_api(xnatProject.attrs.get, 'xnat:projectData/name') purpose = xnat_api(xnatProject.attrs.get, 'xnat:projectData/description') if not purpose: purpose = "<None>" # Find the earliest session date in the project xnat = xnatProject._intf sessionTypes = xnat.inspect.experiment_types() if len(sessionTypes) == 0: sessionTypes = ('xnat:mrSessionData', 'xnat:ctSessionData') #raise XnatImportError("No session types defined in database") minSessionDate = None for sessionType in sessionTypes: columns = (sessionType + '/DATE', sessionType + '/PROJECT') xnat_api_pause() query = xnat.select(sessionType, columns=columns).where([ (sessionType + '/Project', '=', projectID), 'AND' ]) sessionDates = [ datetime.fromtimestamp(mktime(strptime(row['date'], DATE_FORMAT))) for row in query if len(row['date']) > 0 ] if len(sessionDates) > 0: for sd in sessionDates: if minSessionDate is not None: if sd < minSessionDate: minSessionDate = sd else: minSessionDate = sd if minSessionDate is not None: startTime = to_joda_datetime(minSessionDate, timezone) else: startTime = api.datetime() ctx = dsc.getContext() project = ctx.insertProject(name, purpose, startTime) _import_entity_common(project, xnatProject) if importProjectTree: _log.info(" Importing project tree...") subjectIDs = xnat_api(xnatProject.subjects().get) for subjectID in subjectIDs: s = xnat_api(xnatProject.subject, subjectID) import_subject(dsc, s, timezone=timezone, project=project) return project
def should_set_subject_datatype_property(self): xnatProject = self._import_first_project()[0] ctx = self.dsc.getContext() for s in iterate_entity_collection(xnatProject.subjects): subjectID = xnat_api(s.id) sources = ctx.getSources(subjectID) eq_(1, len(sources)) eq_(sources[0].getOwnerProperty(DATATYPE_PROPERTY), xnat_api(s.datatype))
def should_import_attrs(self): self._init_xnat_connection() (subject, xnatSubject) = self._import_entity() attributes = xnat_api(xnatSubject.attrs) attrs = xnatSubject.attrs for attr in attributes: if is_atomic_attribute(xnatSubject, attrs): value = xnat_api(attrs.get, attr) eq_(subject.getOwnerProperty(attr), value)
def should_import_attrs(self): self._init_xnat_connection() (subject,xnatSubject) = self._import_entity() attributes = xnat_api(xnatSubject.attrs) attrs = xnatSubject.attrs for attr in attributes: if is_atomic_attribute(xnatSubject, attrs): value = xnat_api(attrs.get, attr) eq_(subject.getOwnerProperty(attr), value)
def _insert_entity_resources(ovEntity, xnatEntity): xnat = xnatEntity._intf for rsrc in iterate_entity_collection(xnatEntity.resources): label = xnat_api(rsrc.label) if not label: label = "<empty>" for f in iterate_entity_collection(rsrc.files): url, uti = file_info(xnat, f) ovEntity.addURLResource(uti, xnat_api(f.label) + ' (' + label + ')', url)
def should_import_attrs(self): projectName = 'PROJECT_NAME' xnatSubject = subject_mock('1', projectName) subject = import_subject(self.dsc, xnatSubject) attributes = xnat_api(xnatSubject.attrs) attrs = xnatSubject.attrs for attr in attributes: if is_atomic_attribute(attrs(), attrs): value = xnat_api(attrs.get, attr) eq_(subject.getOwnerProperty(attr), value)
def should_import_session_from_subject_as_experiments_with_one_epochgroup(self): xnatProject,projectURI = self._import_first_project() ctx = self.dsc.getContext() project = ctx.objectdWithURI(projectURI) for xnatSession in self.xnat.select('/projects/' + xnat_api(xnatProject.id) + '/subjects/*/experiments/*'): attrs = xnatSession.attrs dtype = xnat_api(xnatSession.datatype) datestr = xnat_api(attrs.get, dtype + '/date') + ' ' + xnat_api(attrs.get, dtype + '/time') startTime = to_joda_datetime(datestr, 'UTC') exps = project.getExperiments(startTime)
def import_project(dsc, xnatProject, timezone="UTC"): """ Import a single XNAT project. """ projectID = xnat_api(xnatProject.id) name = xnat_api(xnatProject.attrs.get, "xnat:projectData/name") purpose = xnat_api(xnatProject.attrs.get, "xnat:projectData/description") # Find the earliest session date in the project xnat = xnatProject._intf sessionTypes = xnat.inspect.experiment_types() if len(sessionTypes) == 0: sessionTypes = ("xnat:mrSessionData", "xnat:ctSessionData") # raise XnatImportError("No session types defined in database") minSessionDate = None for sessionType in sessionTypes: columns = (sessionType + "/DATE", sessionType + "/PROJECT") xnat_api_pause() query = xnat.select(sessionType, columns=columns).where([(sessionType + "/Project", "=", projectID), "AND"]) sessionDates = [ datetime.fromtimestamp(mktime(strptime(row["date"], DATE_FORMAT))) for row in query if len(row["date"]) > 0 ] if len(sessionDates) > 0: for sd in sessionDates: if minSessionDate is not None: if sd < minSessionDate: minSessionDate = sd else: minSessionDate = sd if minSessionDate is not None: startTime = to_joda_datetime(minSessionDate, timezone) else: startTime = api.datetime() ctx = dsc.getContext() project = ctx.insertProject(name, purpose, startTime) _import_entity_common(project, xnatProject) for s in iterate_entity_collection(xnatProject.subjects): src = import_source(dsc, s) for session in iterate_entity_collection(s.sessions): import_session(dsc, src, project, session) return project
def should_import_session_from_subject_as_experiments_with_one_epochgroup( self): xnatProject, projectURI = self._import_first_project() ctx = self.dsc.getContext() project = ctx.objectdWithURI(projectURI) for xnatSession in self.xnat.select('/projects/' + xnat_api(xnatProject.id) + '/subjects/*/experiments/*'): attrs = xnatSession.attrs dtype = xnat_api(xnatSession.datatype) datestr = xnat_api(attrs.get, dtype + '/date') + ' ' + xnat_api( attrs.get, dtype + '/time') startTime = to_joda_datetime(datestr, 'UTC') exps = project.getExperiments(startTime)
def should_set_project_datatype_property(self): xnatProject,projectURI = self._import_first_project() ctx = self.dsc.getContext() project = ctx.objectdWithURI(projectURI) eq_(project.getOwnerProperty(DATATYPE_PROPERTY), xnat_api(xnatProject.datatype))
def should_set_project_datatype_property(self): xnatProject, projectURI = self._import_first_project() ctx = self.dsc.getContext() project = ctx.objectdWithURI(projectURI) eq_(project.getOwnerProperty(DATATYPE_PROPERTY), xnat_api(xnatProject.datatype))
def _import_entity(self): ''' We could make this import any entity that we know has keywords, attributes, and resources ''' xnatEntity = xnat_api(self.xnat.select('/subjects').first) source = import_source(self.dsc, xnatEntity) return (source, xnatEntity)
def should_set_subject_datatype_property(self, xnatProject): import_project(self.dsc, xnatProject, importProjectTree=True) ctx = self.dsc.getContext() for subjectID in xnatProject.subjects().get(): sources = ctx.getSources(subjectID) s = xnatProject.subject(subjectID) eq_(1, len(sources)) eq_(sources[0].getOwnerProperty(DATATYPE_PROPERTY), xnat_api(s.datatype))
def should_import_all_subjects_for_project(self): xnatProject = self._import_first_project()[0] ctx = self.dsc.getContext() for s in iterate_entity_collection(xnatProject.subjects): subjectID = xnat_api(s.id) sources = ctx.getSources(subjectID) eq_(1, len(sources)) eq_(sources[0].getOwnerProperty('xnat:subjectURI'), s._uri)
def import_session(dsc, src, project, xnatSession, timezone='UTC'): if project == None: return try: attrs = xnatSession.attrs dtype = xnat_api(xnatSession.datatype) #print(dtype) _log.info(' Importing session: ' + dtype) dateString = xnat_api(attrs.get, dtype + '/date') timeString = xnat_api(attrs.get, dtype + '/time') purpose = xnat_api(attrs.get, dtype + '/note') if not purpose: #None or len 0 purpose = 'XNAT experiment ' + xnatSession._uri dateTimeString = dateString + ' ' + timeString if timeString: format = DATETIME_FORMAT if timeString.find('.') > 0 else DATETIMESHORT_FORMAT else: format = DATE_FORMAT startTime = datetime.fromtimestamp(mktime(strptime(dateTimeString, format))) exp = project.insertExperiment(purpose, to_joda_datetime(startTime, timezone)) _import_entity_common(exp, xnatSession) scanIDs = xnat_api(xnatSession.scans().get) for scanID in scanIDs: import_scan(dsc, src, exp, xnat_api(xnatSession.scan, scanID), timezone=timezone) return exp except DatabaseError, e: _log.exception("Unable to import session: " + e.message) return None
def import_scan(dsc, src, exp, xnatScan, timezone='UTC'): dtype = xnat_api(xnatScan.datatype) _log.info(' Importing scan: ' + dtype) # dateString = xnat_api(xnatScan.attrs.get, dtype + '/date') # timeString = xnat_api(xnatScan.attrs.get, dtype + '/parameters/scanTime') # dateTimeString = dateString + ' ' + timeString # # if timeString: # format = DATETIME_FORMAT if timeString.find('.') > 0 else DATETIMESHORT_FORMAT # else: # format = DATE_FORMAT # # startTime = to_joda_datetime( # datetime.fromtimestamp(mktime(strptime(dateTimeString, format))), # timezone) startTime = exp.getStartTime() parentSession = xnat_api(xnatScan.parent) parentDtype = xnat_api(parentSession.datatype) scanAttrs = parentSession.attrs sessionScanner = xnat_api(scanAttrs.get, parentDtype + '/scanner') durationString = xnat_api(scanAttrs.get, parentDtype + '/duration') if durationString: duration = float(durationString) else: _log.warn("Scan duration is empty") duration = 1.0 endTime = startTime.plusMillis(int(duration*1000)) scanType = xnat_api(xnatScan.attrs.get, dtype + '/type') if not scanType: scanType = "<unknown type>" g = exp.insertEpochGroup(src, scanType, startTime, endTime) _import_entity_common(g, xnatScan, resources=False) try: parametersDict = {} # Retrieving each parameter may throw a DatabaseError, so we can't use a list comprehension for k in xnatScan.attrs(): try: if k.startswith(dtype + '/parameters/'): parametersDict[k] = xnat_api(xnatScan.attrs.get, k) except (DatabaseError,StopIteration): _log.error("Unable to retrieve parameter " + k) except DatabaseError, e: _log.error("Unable to set parameters") parametersDict = {}
def import_scan(dsc, src, exp, xnatScan, timezone='UTC'): dtype = xnat_api(xnatScan.datatype) _log.info(' Importing scan: ' + dtype) # dateString = xnat_api(xnatScan.attrs.get, dtype + '/date') # timeString = xnat_api(xnatScan.attrs.get, dtype + '/parameters/scanTime') # dateTimeString = dateString + ' ' + timeString # # if timeString: # format = DATETIME_FORMAT if timeString.find('.') > 0 else DATETIMESHORT_FORMAT # else: # format = DATE_FORMAT # # startTime = to_joda_datetime( # datetime.fromtimestamp(mktime(strptime(dateTimeString, format))), # timezone) startTime = exp.getStartTime() parentSession = xnat_api(xnatScan.parent) parentDtype = xnat_api(parentSession.datatype) scanAttrs = parentSession.attrs sessionScanner = xnat_api(scanAttrs.get, parentDtype + '/scanner') durationString = xnat_api(scanAttrs.get, parentDtype + '/duration') if durationString: duration = float(durationString) else: _log.warn("Scan duration is empty") duration = 1.0 endTime = startTime.plusMillis(int(duration * 1000)) scanType = xnat_api(xnatScan.attrs.get, dtype + '/type') if not scanType: scanType = "<unknown type>" g = exp.insertEpochGroup(src, scanType, startTime, endTime) _import_entity_common(g, xnatScan, resources=False) try: parametersDict = {} # Retrieving each parameter may throw a DatabaseError, so we can't use a list comprehension for k in xnatScan.attrs(): try: if k.startswith(dtype + '/parameters/'): parametersDict[k] = xnat_api(xnatScan.attrs.get, k) except (DatabaseError, StopIteration): _log.error("Unable to retrieve parameter " + k) except DatabaseError, e: _log.error("Unable to set parameters") parametersDict = {}
def import_source(dsc, xnatSubject): """ Insert a single XNAT subject """ sourceID = xnat_api(xnatSubject.id) ctx = dsc.getContext() r = ctx.sourceForInsertion([sourceID], ["xnat:subjectURI"], [xnatSubject._uri]) src = r.getSource() if r.isNew(): _import_entity_common(src, xnatSubject) return src
def import_source(dsc, xnatSubject): """ Insert a single XNAT subject """ sourceID = xnat_api(xnatSubject.id) ctx = dsc.getContext() r = ctx.sourceForInsertion([sourceID], ['xnat:subjectURI'], [xnatSubject._uri]) src = r.getSource() if r.isNew(): _import_entity_common(src, xnatSubject) return src
def should_set_project_start_date_from_earliest_session_date(self): xnatProject,projectURI = self._import_first_project() # Find the earliest session date in the project projectID = xnat_api(xnatProject.id) xnat = xnatProject._intf sessionTypes = xnat.inspect.experiment_types() if len(sessionTypes) == 0: sessionTypes = ('xnat:mrSessionData', 'xnat:ctSessionData') #raise XnatImportError("No session types defined in database") minSessionDate = None for sessionType in sessionTypes: columns = (sessionType + '/DATE', sessionType + '/PROJECT') xnat_api_pause() query = xnat.select(sessionType, columns=columns).where([(sessionType + '/Project', '=', projectID), 'AND']) sessionDates = [ datetime.fromtimestamp(mktime(strptime(time_str['date'], DATE_FORMAT))) for time_str in query if len(time_str['date']) > 0] if len(sessionDates) > 0: for sd in sessionDates: if minSessionDate is not None: if sd < minSessionDate: minSessionDate = sd else: minSessionDate = sd if minSessionDate is not None: startTime = to_joda_datetime(minSessionDate, 'UTC') else: self.fail("Unable to find project sessions' start date") ctx = self.dsc.getContext() project = ctx.objectdWithURI(projectURI) eq_(project.getStartTime(), startTime)
def should_set_project_start_date_from_earliest_session_date(self): xnatProject, projectURI = self._import_first_project() # Find the earliest session date in the project projectID = xnat_api(xnatProject.id) xnat = xnatProject._intf sessionTypes = xnat.inspect.experiment_types() if len(sessionTypes) == 0: sessionTypes = ('xnat:mrSessionData', 'xnat:ctSessionData') #raise XnatImportError("No session types defined in database") minSessionDate = None for sessionType in sessionTypes: columns = (sessionType + '/DATE', sessionType + '/PROJECT') xnat_api_pause() query = xnat.select(sessionType, columns=columns).where([ (sessionType + '/Project', '=', projectID), 'AND' ]) sessionDates = [ datetime.fromtimestamp( mktime(strptime(time_str['date'], DATE_FORMAT))) for time_str in query if len(time_str['date']) > 0 ] if len(sessionDates) > 0: for sd in sessionDates: if minSessionDate is not None: if sd < minSessionDate: minSessionDate = sd else: minSessionDate = sd if minSessionDate is not None: startTime = to_joda_datetime(minSessionDate, 'UTC') else: self.fail("Unable to find project sessions' start date") ctx = self.dsc.getContext() project = ctx.objectdWithURI(projectURI) eq_(project.getStartTime(), startTime)
def import_session(dsc, src, project, xnatSession, timezone='UTC'): if project == None: return try: attrs = xnatSession.attrs dtype = xnat_api(xnatSession.datatype) #print(dtype) _log.info(' Importing session: ' + dtype) dateString = xnat_api(attrs.get, dtype + '/date') timeString = xnat_api(attrs.get, dtype + '/time') purpose = xnat_api(attrs.get, dtype + '/note') if not purpose: #None or len 0 purpose = 'XNAT experiment ' + xnatSession._uri dateTimeString = dateString + ' ' + timeString if timeString: format = DATETIME_FORMAT if timeString.find( '.') > 0 else DATETIMESHORT_FORMAT else: format = DATE_FORMAT startTime = datetime.fromtimestamp( mktime(strptime(dateTimeString, format))) exp = project.insertExperiment(purpose, to_joda_datetime(startTime, timezone)) _import_entity_common(exp, xnatSession) scanIDs = xnat_api(xnatSession.scans().get) for scanID in scanIDs: import_scan(dsc, src, exp, xnat_api(xnatSession.scan, scanID), timezone=timezone) return exp except DatabaseError, e: _log.exception("Unable to import session: " + e.message) return None
_log.error("Unable to retrieve parameter " + k) except DatabaseError, e: _log.error("Unable to set parameters") parametersDict = {} parameters = dict2map(parametersDict) scannerParameters = parameters ovation = api.ovation_package() BYTE_DATATYPE = ovation.NumericDataType( ovation.NumericDataFormat.SignedFixedPointDataType, 1, ovation.NumericByteOrder.ByteOrderNeutral) for r in iterate_entity_collection(xnatScan.resources): _log.info(" Resource " + xnat_api(r.label)) # Insert one epoch group per resource group resourceGroup = g.insertEpochGroup(xnat_api(r.label), g.getStartTime()) for f in iterate_entity_collection(r.files): # Import one epoch per resource file e = resourceGroup.insertEpoch( g.getStartTime(), g.getEndTime(), #TODO scanType, parameters) url, uti = file_info(xnatScan._intf, f) _log.info(" File " + xnat_api(r.label)) e.insertURLResponse(
parametersDict[k] = xnat_api(xnatScan.attrs.get, k) except (DatabaseError,StopIteration): _log.error("Unable to retrieve parameter " + k) except DatabaseError, e: _log.error("Unable to set parameters") parametersDict = {} parameters = dict2map(parametersDict) scannerParameters = parameters ovation = api.ovation_package() BYTE_DATATYPE = ovation.NumericDataType(ovation.NumericDataFormat.SignedFixedPointDataType, 1, ovation.NumericByteOrder.ByteOrderNeutral) for r in iterate_entity_collection(xnatScan.resources): _log.info(" Resource " + xnat_api(r.label)) # Insert one epoch group per resource group resourceGroup = g.insertEpochGroup(xnat_api(r.label), g.getStartTime()) for f in iterate_entity_collection(r.files): # Import one epoch per resource file e = resourceGroup.insertEpoch( g.getStartTime(), g.getEndTime(), #TODO scanType, parameters ) url,uti = file_info(xnatScan._intf, f) _log.info(" File " + xnat_api(r.label))