def process(self, processingItem): processingItemObj = ProcessingItemObj(processingItem) matching_t1 = ADNI_T1_Helper().getMatchingT1(processingItemObj) if not matching_t1: PipelineLogger.log('root', 'error', 'PET cannot be processed no matching T1 found. - {0} - {1} - {2}.'.format(processingItemObj.subject_rid, processingItemObj.modality, processingItemObj.scan_date)) return 0 processed = ADNI_T1_Helper().checkProcessed(matching_t1) if not processed: PipelineLogger.log('root', 'error', 'PET cannot be processed due to matching T1 not being processed - {0}'.format(matching_t1)) return 0 else: PipelineLogger.log('root', 'INFO', '+++++++++ PET ready to be processed. Will check for xfm. - {0} - {1}'.format(processingItemObj.subject_rid, processingItemObj.scan_date)) if processingItemObj.manual_xfm == '': manualXFM = self.PETHelper.getManualXFM(processingItemObj, matching_t1) processingItemObj.manual_xfm = manualXFM elif processingItemObj.manual_xfm == 'Req_man_reg': coregDone = self.PETHelper.checkIfAlreadyDone(processingItemObj, matching_t1) if coregDone: manualXFM = coregDone setPPTableSQL = "UPDATE {0}_{1}_Pipeline SET MANUAL_XFM = '{2}' WHERE RECORD_ID = {3}".format(processingItemObj.study, processingItemObj.modality, manualXFM, processingItemObj.table_id) self.DBClient.executeNoResult(setPPTableSQL) else: self.PETHelper.requestCoreg(processingItemObj, matching_t1) PipelineLogger.log('root', 'INFO', 'Manual XFM was not found. Request to create one may have added. - {0} - {1}'.format(processingItemObj.subject_rid, processingItemObj.scan_date)) return 0 else: manualXFM = processingItemObj.manual_xfm if manualXFM: self.processPET(processingItemObj, processed) else: PipelineLogger.log('root', 'INFO', 'Manual XFM was not found. Request to create one may have added. - {0} - {1}'.format(processingItemObj.subject_rid, processingItemObj.scan_date)) return 0
def determineScanType(self, scanTypeRaw): try: return arc.scanTypeDict[scanTypeRaw] except KeyError: if 'FDG' in scanTypeRaw: PipelineLogger.log( 'root', 'error', 'Scan Type not defined : {0} -> Close match FDG...'.format( scanTypeRaw)) return 'FDG' if 'AV45' in scanTypeRaw: PipelineLogger.log( 'root', 'error', 'Scan Type not defined : {0} -> Close match AV45...'. format(scanTypeRaw)) return 'AV45' if 'AV1451' in scanTypeRaw or 'AV-1451' in scanTypeRaw or 'AV_1451' in scanTypeRaw: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match AV1451...'. format(scanTypeRaw)) return 'AV1451' if 'MPRAGE' in scanTypeRaw.upper(): PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match MPRAGE...'. format(scanTypeRaw)) return 'MPRAGE' else: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> No match...'.format( scanTypeRaw)) return 'unknown'
def checkMncFile(self, mncFile): cmd = Config.ConverterConfig.mincSource_exec + '; mincinfo ' + mncFile + ' | grep \"time\" ' p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if not out: # If no output, string empty PipelineLogger.log('converter', 'debug', mncFile + 'does not have a time axis!') os.remove(mncFile)
def checkNative(self, processingItemObj): orig_ScanType = self.getScanType(processingItemObj) converted_file = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format( processingItemObj.converted_folder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, orig_ScanType) nativeFolder = '{0}/native'.format(processingItemObj.root_folder) nativeFileName = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format( nativeFolder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, processingItemObj.modality.lower()) if not os.path.exists(nativeFileName): try: distutils.dir_util.mkpath(nativeFolder) shutil.copyfile(converted_file, nativeFileName) except Exception as e: PipelineLogger.log( 'manager', 'error', 'Error in creating folders or copying native file. \n {0}'. format(e)) PipelineLogger.log( 'manager', 'error', 'Setting to restart conversion. \n {0}'.format(e)) sql = "UPDATE Conversion SET CONVERTED = 0, SKIP = 0 WHERE S_IDENTIFIER = '{0}' AND I_IDENTIFIER = '{1}'".format( processingItemObj.s_identifier, processingItemObj.i_identifier) self.DBClient.executeNoResult(sql) return None return nativeFileName
def createNewScanSession(self,down_most_folder, filelist): # Return parts of the folder path, the ones of interest try: folder = down_most_folder.replace(self.root_folder,"") folder_parts = folder.split("/") # List containing each parts/folders of the full path filename_parts = filelist[0].split("_") # Takes the first filename and create a list of its parts rid = folder_parts[1][-4:] # Get the last 4 characters if re.search('[a-zA-Z]', rid) is not None: rid = filename_parts[3] if re.search('[a-zA-Z]', rid) is not None: PipelineLogger.log('root', 'error', 'File recurse error on Folder RID cannot be identified. - {0}, \n Filelist - {1}'.format(folder, filelist)) return None scan_type = self.determineScanType(folder_parts[-3]) scan_date = folder_parts[-2].split('_')[0] scan_time = folder_parts[-2].split('_', 1)[-1].replace("_", ":") s_identifier = filename_parts[-2] i_identifier = filename_parts[-1].split('.', 1)[0] file_type = self.determineExtension(filename_parts) download_folder = down_most_folder raw_folder = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/raw'.format(sc.studyDatabaseRootDict[self.study], 'ADNI', scan_type, rid, scan_date, s_identifier, i_identifier) newScanSession = ScanSession\ ('ADNI', rid, scan_type, scan_date, scan_time, s_identifier, i_identifier, download_folder, raw_folder, file_type) if scan_type == 'unknown': newScanSession.printObject() return None return newScanSession except Exception as e: return None
def convertRawData(self): def addTODB(result): if result['converted']: #### Add to correspoing table #self.conversionTable.insertFromConvertionObj(convertionObj, self.version) self.conversionTable.setConvertedTrue(result['obj']) else: PipelineLogger.log( 'manager', 'error', 'File conversion Error : {0} -> {1}. Moving to next...'. format(result['obj'].raw_folder, result['obj'].converted_folder)) self.conversionTable.setConvertedFailed(result['obj']) for study in self.studyList: totalToConv = len(self.toConvertObjListDict[study]) PipelineLogger.log( 'manager', 'info', 'Convertion started for study {0} - Total to be converted : {1}' .format(study, totalToConv)) results = [] for convObj in self.toConvertObjListDict[study]: convertedResult = self.pool.apply_async( self.raw2mincConverter.convert2minc, args=(convObj, ), callback=addTODB) results.append(convertedResult) for r in results: r.wait()
def restart_mongo(self): try: if self.old_pid > 0: os.kill(self.old_pid, 9) mongo_Cmd = '/data/data03/MongoDB/mongodb/bin/mongod --dbpath /data/data03/MongoDB/data/db/' Popen(mongo_Cmd, shell=True) except OSError as e: PipelineLogger.log('root', 'exception', 'MongoDB cannot be stopped or started.\n {0}\n'.format(e))
def restart_mongo(self): try: if self.old_pid > 0: os.kill(self.old_pid, 9) mongo_Cmd = '/data/data03/MongoDB/mongodb/bin/mongod --logpath Logs/MongoLog.log --dbpath /data/data03/MongoDB/data/db/' Popen(mongo_Cmd, shell=True) except OSError as e: PipelineLogger.log('root', 'exception', 'MongoDB cannot be stopped or started.\n {0}\n'.format(e))
def addTODB(result): if result['converted']: #### Add to correspoing table #self.conversionTable.insertFromConvertionObj(convertionObj, self.version) self.conversionTable.setConvertedTrue(result['obj']) else: PipelineLogger.log('manager', 'error', 'File conversion Error : {0} -> {1}. Moving to next...'.format(result['obj'].raw_folder, result['obj'].converted_folder)) self.conversionTable.setConvertedFailed(result['obj'])
def runNiak(self, processingItemObj): # Run Niak if not processingItemObj.skip: self.niak.process(processingItemObj) elif processingItemObj.skip: pass else: PipelineLogger.log('manager', 'error', 'Error handling obj for processing - {0}'.format(processingItemObj)) return 0
def convert_dicom(self, convertionObj): rawFolder = convertionObj.raw_folder outFile = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace('-', ''), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type) self.createNewFolder(convertionObj.converted_folder) # Create output folder tempFolder = convertionObj.converted_folder + '/../temp' # Generate path for temp folder self.createNewFolder(tempFolder) # Create temp folder # Move all the non-dicom stuff out of the original folder into tempFolder otherFiles = self.removeOtherFilesInFolder(rawFolder, '.dcm', tempFolder) # Run dcm2nii cmd = Config.ConverterConfig.dcmToNii_exec + ' -a N -e N -p N -g N -o ' + tempFolder + '/ -v Y ' + rawFolder PipelineLogger.log('converter', 'info', 'dcm2nii conversion starting for : {0} - {1} - {2} - {3}'.format(convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) PipelineLogger.log('converter', 'debug', 'Command : {0}'.format(cmd)) self.runShellCommand(cmd) # Move all the non-dicom stuff back into the original folder self.addBackOtherFiles(rawFolder, otherFiles, tempFolder) # Run nii2mnc fake_command = '{0} {1} {2}/../'.format(Config.ConverterConfig.niiToMnc_exec, rawFolder, convertionObj.converted_folder) PipelineLogger.log('converter', 'info', 'nii2mnc conversion starting for : {0} - {1} - {2} - {3}'.format(convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) PipelineLogger.log('converter', 'debug', 'Command : {0}'.format(fake_command)) iterator = 1 for niiFile in glob.glob(tempFolder + '/*.nii'): tempOutFile = outFile.replace('.mnc', '_run' + str(iterator) + '.mnc') cmd = '%s %s %s' % (Config.ConverterConfig.niiToMnc_exec, niiFile, tempOutFile) self.runShellCommand(cmd) self.checkMncFile(tempOutFile) # Check whether the fMRI files have a time component/axis iterator += 1 # Delete Temporary Folder self.deleteFolder(tempFolder) # Check how many mnc files were generated mncList = [] for root, dirnames, filenames in os.walk(convertionObj.converted_folder): for filename in fnmatch.filter(filenames, '*.mnc'): mncList.append(os.path.join(root, filename)) if len(mncList) == 0: PipelineLogger.log('converter', 'error', 'MINC Conversion unsuccessful : Check log for : {0} - {1} - {2} - {3}'.format( convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) return 0 else: return 1
def removeCommaIfThere(destFolder): PipelineLogger.log('manager', 'debug', 'Removing unsupported chars from file names...... :') for dpath, dnames, fnames in os.walk(destFolder): for f in fnames: os.chdir(dpath) if ',' in f: os.rename(f, f.replace(',', '')) PipelineLogger.log('manager', 'debug', 'Removing unsupported chars from file names done ...:')
def avgTime(self, inputMNC, outputMNC): avgCMD = "/opt/minc-toolkit/bin/mincaverage -short -avgdim time {0} {1}".format(inputMNC, outputMNC) p = subprocess.Popen(avgCMD, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() PipelineLogger.log("converter", "debug", "Averaging Time Output : \n{0}".format(out)) PipelineLogger.log("converter", "debug", "Averaging Time Err : \n{0}".format(err)) if os.path.exists(outputMNC): return 1 else: return 0
def runShellCommand(self, cmd): p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() PipelineLogger.log('converter', 'debug', 'Conversion Log Output : \n{0}'.format(out)) PipelineLogger.log('converter', 'debug', 'Conversion Log Err : \n{0}'.format(err))
def createNewScanSession(self, down_most_folder, filelist): # Return parts of the folder path, the ones of interest folder = down_most_folder.replace(self.root_folder, "") xmllist = [x for x in filelist if "xml" in x] filelist = [x for x in filelist if "xml" not in x] if filelist is None: # If no file in folder, ignore and skip return None try: folder_parts = folder.split("/") # List containing each parts/folders of the full path filename_parts = filelist[0].split("_") # Takes the first filename and create a list of its parts xmlFileS = open("{0}/{1}".format(down_most_folder, xmllist[0])).read() xmlDict = xmltodict.parse(xmlFileS) rid = xmlDict["metadata"]["subject"]["@id"].split("_")[-1] if re.search("[a-zA-Z]", rid) is not None: rid = filename_parts[3] if re.search("[a-zA-Z]", rid) is not None: PipelineLogger.log( "root", "error", "File recurse error on Folder RID cannot be identified. - {0}, \n Filelist - {1}".format( folder, filelist ), ) return None s_identifier = xmlDict["metadata"]["series"]["@uid"] i_identifier = xmlDict["metadata"]["image"]["@uid"] scan_type = self.determineScanType(folder_parts[-3], self.study, rid, s_identifier, i_identifier) scan_date = folder_parts[-2].split("_")[0] scan_time = folder_parts[-2].split("_", 1)[-1].replace("_", ":") file_type = self.determineExtension(filename_parts) download_folder = down_most_folder raw_folder = "{0}/{1}/{2}/{3}/{4}_{5}_{6}/raw".format( sc.studyDatabaseRootDict[self.study], self.study, scan_type, rid, scan_date, s_identifier, i_identifier ) except: PipelineLogger.log( "root", "exception", "File recurse error on Folder - {0}, \n Filelist - {1}".format(folder, filelist) ) return None newScanSession = ScanSession( self.study, rid, scan_type, scan_date, scan_time, s_identifier, i_identifier, download_folder, raw_folder, file_type, ) return newScanSession
def createNewScanSession(self, tup): down_most_folder, filelist = tup # Return parts of the folder path, the ones of interest folder = down_most_folder.replace(self.root_folder, "") filelist = [x for x in filelist if 'xml' not in x] if len(filelist) == 0: # If no file in folder, ignore and skip return None try: folder_parts = folder.split( "/") # List containing each parts/folders of the full path filename_parts = filelist[0].split( ".") # Takes the first filename and create a list of its parts rid = filename_parts[0] file_type = self.determineExtension(filename_parts) scan_date_str, scan_time_str = self.getScanDateTimeDCMHeader( '{0}/{1}'.format(down_most_folder, filelist[0])) scan_date = '{0}-{1}-{2}'.format(scan_date_str[:4], scan_date_str[4:6], scan_date_str[6:8]) scan_time = '{0}:{1}:{2}'.format(scan_time_str[:2], scan_time_str[2:4], scan_time_str[4:6]) visit = self.get_visit_info('{0}/{1}'.format( down_most_folder, filelist[0])) if not visit: return None scan_type = self.determineScanType('{0}/{1}'.format( down_most_folder, filelist[0])) if not scan_type or scan_type not in self.only_allowed_scan_types_to_move: return None i_identifier = '{0}{1}{2}{3}x{4}'.format( rid, scan_type, visit, scan_date_str, re.sub(r'[\W_]+', '', folder_parts[-2])) s_identifier = 'ySy' download_folder = down_most_folder raw_folder = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/raw'.format( sc.studyDatabaseRootDict[self.study], self.study, scan_type, rid, scan_date, s_identifier, i_identifier) except Exception as e: PipelineLogger.log( 'root', 'exception', 'File recurse error on Folder - {0}, \n Filelist - {1}'.format( folder, filelist)) PipelineLogger.log('root', 'exception', 'Exception - {0}'.format(e)) return None newScanSession = ScanSession(self.study, rid, scan_type, scan_date, scan_time, s_identifier, i_identifier, download_folder, raw_folder, file_type) return newScanSession
def removeCommaIfThere(destFolder): PipelineLogger.log( 'manager', 'debug', 'Removing unsupported chars from file names...... :') for dpath, dnames, fnames in os.walk(destFolder): for f in fnames: os.chdir(dpath) if ',' in f: os.rename(f, f.replace(',', '')) PipelineLogger.log( 'manager', 'debug', 'Removing unsupported chars from file names done ...:')
def addTODB(result): if result['converted']: #### Add to correspoing table #self.conversionTable.insertFromConvertionObj(convertionObj, self.version) self.conversionTable.setConvertedTrue(result['obj']) else: PipelineLogger.log( 'manager', 'error', 'File conversion Error : {0} -> {1}. Moving to next...'. format(result['obj'].raw_folder, result['obj'].converted_folder)) self.conversionTable.setConvertedFailed(result['obj'])
def findCorrespondingMRI(self, processingItemObj): # Find Matching T1 matching_t1 = ADNI_T1_Fmri_Helper().getMatchingT1(processingItemObj) if not matching_t1: return 0 # Find out whether T1 has been processed processed = ADNI_T1_Fmri_Helper().checkProcessed(matching_t1) if not processed: PipelineLogger.log('root', 'error', 'FMRI cannot be processed due to matching T1 not being processed.') return 0 else: return processed
def run(self): try: self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind((socket.gethostname(), 50500)) self.sock.settimeout(300) self.sock.listen(1000) PipelineLogger.log('manager', 'info', ' ++++++++ QSub Job Handler started.') PipelineLogger.log( 'manager', 'info', ' ++++++++ QSub Job Handler listening in Host : {0} at Port : {1}.' .format(socket.gethostname(), 50500)) while not self.QUIT and self.checkJobs(): try: conn = self.sock.accept()[0] thread = threading.Thread(target=self.doWork, args=(conn, )) thread.start() except socket.timeout: continue except Exception as e: PipelineLogger.log('manager', 'exception', e) PipelineLogger.log( 'manager', 'error', 'Cannot create QSubJobHandler... Will not listen to on jobs. ') del self.sock
def convertMinc(self, convertionObj): rawFile = '{0}/*.mnc'.format(convertionObj.raw_folder) outFile = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace('-', ''), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type) # Move files cmd = '{0} {1} {2}/../'.format('mv ' + rawFile + ' ' + outFile) PipelineLogger.log('converter', 'info', 'MINC transfer starting for : {0} - {1} - {2} - {3}'.format(convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) PipelineLogger.log('converter', 'debug', 'Command : {0}'.format(cmd)) self.runShellCommand(cmd)
def createNewScanSession(self, down_most_folder, filelist): # Return parts of the folder path, the ones of interest folder = down_most_folder.replace(self.root_folder, "") xmllist = [x for x in filelist if 'xml' in x] filelist = [x for x in filelist if 'xml' not in x] if len(filelist) == 0: # If no file in folder, ignore and skip return None try: folder_parts = folder.split( "/") # List containing each parts/folders of the full path filename_parts = filelist[0].split( "_") # Takes the first filename and create a list of its parts xmlFileS = open('{0}/{1}'.format(down_most_folder, xmllist[0])).read() xmlDict = xmltodict.parse(xmlFileS) rid = xmlDict['metadata']['subject']['@id'].split('_')[-1] if re.search('[a-zA-Z]', rid) is not None: rid = filename_parts[3] if re.search('[a-zA-Z]', rid) is not None: PipelineLogger.log( 'root', 'error', 'File recurse error on Folder RID cannot be identified. - {0}, \n Filelist - {1}' .format(folder, filelist)) return None s_identifier = xmlDict['metadata']['series']['@uid'] i_identifier = xmlDict['metadata']['image']['@uid'] scan_type = self.determineScanType(folder_parts[-3], self.study, rid, s_identifier, i_identifier) scan_date = folder_parts[-2].split('_')[0] scan_time = folder_parts[-2].split('_', 1)[-1].replace("_", ":") file_type = self.determineExtension(filename_parts) download_folder = down_most_folder raw_folder = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/raw'.format( sc.studyDatabaseRootDict[self.study], self.study, scan_type, rid, scan_date, s_identifier, i_identifier) except: PipelineLogger.log( 'root', 'exception', 'File recurse error on Folder - {0}, \n Filelist - {1}'.format( folder, filelist)) return None newScanSession = ScanSession\ (self.study, rid, scan_type, scan_date, scan_time, s_identifier, i_identifier, download_folder, raw_folder, file_type) return newScanSession
def findCorrespondingMRI(self, processingItemObj): # Find Matching T1 matching_t1 = ADNI_T1_Fmri_Helper().getMatchingT1(processingItemObj) if not matching_t1: return 0 # Find out whether T1 has been processed processed = ADNI_T1_Fmri_Helper().checkProcessed(matching_t1) if not processed: PipelineLogger.log( 'root', 'error', 'FMRI cannot be processed due to matching T1 not being processed.' ) return 0 else: return processed
def checkExternalJobs(self, study, modality): getExtJobSql = "SELECT * FROM externalWaitingJobs WHERE JOB_ID LIKE '{0}_{1}_%'".format(study, modality) extJobs = self.DBClient.executeAllResults(getExtJobSql) for job in extJobs: jobType = job[0].split('_')[-1] reportTable = job[1] tableID = job[0].split('_')[2] reportField = job[2] subjectScanID = job[0].split('_')[3] success = 0 if jobType == 'CIVETRUN': if glob.glob('{0}/{1}_{2}_*'.format(PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID)): getProccessRecSql = "SELECT * FROM Processing WHERE RECORD_ID IN (SELECT PROCESSING_TID FROM {0}_T1_Pipeline WHERE RECORD_ID = {1})".format(study, tableID) processingEntry = self.DBClient.executeAllResults(getProccessRecSql)[0] civetFolder = '{0}/civet'.format(processingEntry[8]) if os.path.exists(civetFolder): shutil.rmtree(civetFolder) try: PipelineLogger.log('manager', 'info', 'Copying - {0} -> {1}'.format(glob.glob('{0}/{1}_{2}_*'.format(PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID))[0], civetFolder)) dir_util.copy_tree(glob.glob('{0}/{1}_{2}_*'.format(PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID))[0], civetFolder) success = 1 except: success = 0 else: continue else: PipelineLogger.log('manager', 'error', 'Unknown external job type - {}'.format(jobType)) if success: updateSQL = "UPDATE {0} SET {1} = 1 WHERE RECORD_ID = {2}".format(reportTable, reportField, tableID) self.DBClient.executeNoResult(updateSQL) if jobType == 'CIVETRUN': finishSQL = "UPDATE {0} SET FINISHED = 1 WHERE RECORD_ID = {1}".format(reportTable, tableID) self.DBClient.executeNoResult(finishSQL) modal_table = reportTable modal_tableId = tableID qcField = 'QC' qctype = 'civet' qcFolder = civetFolder self.QCH.requestQC(study, modal_table, modal_tableId, qcField, qctype, qcFolder) rmSql = "DELETE FROM externalWaitingJobs WHERE JOB_ID LIKE '{0}_{1}_{2}_{3}_%'".format(study, modality, tableID, subjectScanID) self.DBClient.executeNoResult(rmSql)
def convertMinc(self, convertionObj): rawFile = '{0}/*.mnc'.format(convertionObj.raw_folder) outFile = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format( convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace('-', ''), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type) # Move files cmd = '{0} {1} {2}/../'.format('mv ' + rawFile + ' ' + outFile) PipelineLogger.log( 'converter', 'info', 'MINC transfer starting for : {0} - {1} - {2} - {3}'.format( convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) PipelineLogger.log('converter', 'debug', 'Command : {0}'.format(cmd)) self.runShellCommand(cmd)
def checkProcessed(self, t1Record): subject_id = t1Record[2] version = t1Record[11] s_id = t1Record[6] i_id = t1Record[7] checkProcessedSQL = "SELECT * FROM Processing WHERE RID = '{0}' AND VERSION = '{1}' AND S_IDENTIFIER = '{2}' AND I_IDENTIFIER = '{3}'".format(subject_id, version, s_id, i_id) result = self.DBClient.executeAllResults(checkProcessedSQL)[0] if len(result) < 1: PipelineLogger.log('root', 'error', 'Matched T1 is not added to the processing table. {0} - {1} - {2}'.format(subject_id, s_id, i_id)) return False else: if result[12] == 1 and result[13] == 1: return result[8] else: PipelineLogger.log('root', 'error', 'Matched T1 is not process or QC failed. {0} - {1} - {2}'.format(subject_id, s_id, i_id)) self.startProcessOFT1(result) return False
def process(self, processingItemObj): try: matlabScript, nativeFileName, niakFolder = self.readTemplateFile(processingItemObj) PipelineLogger.log('manager', 'info', 'NIAK starting for {0}'.format(nativeFileName)) except: return 0 # Delete PIPE.lock file, if is exists if os.path.isfile("%s/preprocessing/logs/PIPE.lock" % niakFolder): os.remove("%s/preprocessing/logs/PIPE.lock" % niakFolder) success = self.executeScript(processingItemObj, matlabScript, niakFolder) #### After, if Niak succeeded, concatenate all runs together using combiningRuns if False: if success: self.combiningRuns(processingItemObj) else: PipelineLogger.log()
def convertRawData(self): def addTODB(result): if result['converted']: #### Add to correspoing table #self.conversionTable.insertFromConvertionObj(convertionObj, self.version) self.conversionTable.setConvertedTrue(result['obj']) else: PipelineLogger.log('manager', 'error', 'File conversion Error : {0} -> {1}. Moving to next...'.format(result['obj'].raw_folder, result['obj'].converted_folder)) self.conversionTable.setConvertedFailed(result['obj']) for study in self.studyList: totalToConv = len(self.toConvertObjListDict[study]) PipelineLogger.log('manager', 'info', 'Convertion started for study {0} - Total to be converted : {1}'.format(study, totalToConv)) results = [] for convObj in self.toConvertObjListDict[study]: convertedResult = self.pool.apply_async(self.raw2mincConverter.convert2minc, args=(convObj,), callback=addTODB) results.append(convertedResult) for r in results: r.wait()
def copyFile(sourceFolder, destFolder): try: PipelineLogger.log('manager', 'debug', 'Raw Data Copying : {0} -> {1}'.format(sourceFolder, destFolder)) distutils.dir_util.copy_tree(sourceFolder, destFolder, update=True) PipelineLogger.log('manager', 'debug', 'Raw Data Copy Done...... : {0} -> {1}'.format(sourceFolder, destFolder)) removeCommaIfThere(destFolder) return 1 except Exception as exc: PipelineLogger.log('manager', 'error', 'Raw Data Move Error : {0} -> {1}'.format(sourceFolder, destFolder)) PipelineLogger.log('manager', 'exception', exc) return 0
def requestCoreg(self, processingItemObj, matchedT1entry): PipelineLogger.log('root', 'INFO', '$$$$$$$ Manual XFM not found. Requesting manual XFM. - {0} - {1}'.format(processingItemObj.subject_rid, processingItemObj.scan_date)) study = processingItemObj.study rid = processingItemObj.subject_rid pet_sid = processingItemObj.s_identifier pet_iid = processingItemObj.i_identifier t1_sid = matchedT1entry[6] t1_iid = matchedT1entry[7] pet_folder = processingItemObj.converted_folder pet_scanType = self.getScanType(processingItemObj) t1_civet_root = self.DBClient.executeAllResults("SELECT ROOT_FOLDER FROM Processing WHERE S_IDENTIFIER = '{0}' " "AND I_IDENTIFIER = '{1}' AND PROCESSED = 1".format(t1_sid, t1_iid)) if len(t1_civet_root)>0: t1_folder = t1_civet_root[0][0] t1_scanType = matchedT1entry[3] xfmFileName = '{0}_{1}_PET_{2}_{3}_T1_{4}_{5}'.format(study, rid, pet_sid, pet_iid, t1_sid, t1_iid) self.CoregHand.requestCoreg(study, rid, processingItemObj.modality, pet_folder, t1_folder, pet_scanType, t1_scanType, xfmFileName) else: PipelineLogger.log('root', 'error', 'T1 files not processed and cannot be added for manual coregistration - {0} - {1} - {2}'.format(processingItemObj.subject_rid, processingItemObj.scan_date, matchedT1entry[10]))
def determineScanType(self, scanTypeRaw): try: return arc.scanTypeDict[scanTypeRaw] except KeyError: if 'FDG' in scanTypeRaw: PipelineLogger.log('root', 'error', 'Scan Type not defined : {0} -> Close match FDG...'.format(scanTypeRaw)) return 'FDG' if 'AV45' in scanTypeRaw: PipelineLogger.log('root', 'error', 'Scan Type not defined : {0} -> Close match AV45...'.format(scanTypeRaw)) return 'AV45' if 'AV1451' in scanTypeRaw or 'AV-1451' in scanTypeRaw or 'AV_1451' in scanTypeRaw: PipelineLogger.log('root', 'error', 'Scan Type unidentified : {0} -> Close match AV1451...'.format(scanTypeRaw)) return 'AV1451' if 'MPRAGE' in scanTypeRaw.upper(): PipelineLogger.log('root', 'error', 'Scan Type unidentified : {0} -> Close match MPRAGE...'.format(scanTypeRaw)) return 'MPRAGE' else: PipelineLogger.log('root', 'error', 'Scan Type unidentified : {0} -> No match...'.format(scanTypeRaw)) return 'unknown'
def add_to_mongoDB(self, file): f = open(file, 'r').read() try: o = xmltodict.parse(f) except: print('XML cannot be read - {0}.'.format(f)) PipelineLogger.log('root', 'exception', 'XML cannot be read - {0}.'.format(f)) return 0 subId, SID, IID, valid = self.get_subid_sid_iid(file) if valid: _id = '{0}_{1}_{2}'.format(subId, SID, IID) o['_id'] = _id client = MongoClient('localhost', 27017) db = client.ADNI_Database collection = db.Scan_XML_Collection try: post_id = collection.insert_one(o).inserted_id except DuplicateKeyError: PipelineLogger.log('root', 'exception', 'XML already in DB - {0}.'.format(_id)) post_id = _id try: shutil.move(file, moncfg.XMLProcessedArchivePath) except shutil.Error: PipelineLogger.log( 'root', 'Info', 'XML already in processed path - {0}.'.format(_id)) os.remove(file) post_id = _id return post_id
def process(self, processingItem): processingItemObj = ProcessingItemObj(processingItem) if processingItemObj.beast_skip and processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'N') elif processingItemObj.manual_mask and not processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'M') elif processingItemObj.beast_mask == 0 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask: self.runBeast(processingItemObj) elif processingItemObj.beast_skip and not processingItemObj.manual_mask and not processingItemObj.manual_skip: PipelineLogger.log( 'manager', 'error', '$$$$$$$$$$$$$$$$$ Manual Mask Requested $$$$$$$$$$$$$$$$$$ - {0}' .format(processingItem)) pass elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 1 and not processingItemObj.manual_mask and not processingItemObj.civet: self.runCivet(processingItemObj, 'B') elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask and not processingItemObj.manual_skip: self.requestQC(processingItemObj, 'beast') elif processingItemObj.civet == 1 and processingItemObj.civet_qc == 0: self.requestQC(processingItemObj, 'civet') else: if processingItemObj.civet_qc == -1: PipelineLogger.log( 'manager', 'error', 'Civet QC failed. Skipping. - {0}'.format(processingItem)) PipelineLogger.log( 'manager', 'error', 'Error handling obj for processing - {0}'.format( processingItem)) return 0
def checkNative(self, processingItemObj): orig_ScanType = self.getScanType(processingItemObj) converted_file = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(processingItemObj.converted_folder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, orig_ScanType) nativeFolder = '{0}/native'.format(processingItemObj.root_folder) nativeFileName = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(nativeFolder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, processingItemObj.modality.lower()) if not os.path.exists(nativeFileName): try: distutils.dir_util.mkpath(nativeFolder) shutil.copyfile(converted_file, nativeFileName) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating folders or copying native file. \n {0}'.format(e)) PipelineLogger.log('manager', 'error', 'Setting to restart conversion. \n {0}'.format(e)) sql = "UPDATE Conversion SET CONVERTED = 0, SKIP = 0 WHERE S_IDENTIFIER = '{0}' AND I_IDENTIFIER = '{1}'".format(processingItemObj.s_identifier, processingItemObj.i_identifier) self.DBClient.executeNoResult(sql) return None return nativeFileName
def process(self, processingItemObj): try: matlabScript, nativeFileName, niakFolder = self.readTemplateFile( processingItemObj) PipelineLogger.log('manager', 'info', 'NIAK starting for {0}'.format(nativeFileName)) except: return 0 # Delete PIPE.lock file, if is exists if os.path.isfile("%s/preprocessing/logs/PIPE.lock" % niakFolder): os.remove("%s/preprocessing/logs/PIPE.lock" % niakFolder) success = self.executeScript(processingItemObj, matlabScript, niakFolder) #### After, if Niak succeeded, concatenate all runs together using combiningRuns if False: if success: self.combiningRuns(processingItemObj) else: PipelineLogger.log()
def createNewScanSession(self, down_most_folder, filelist): # Return parts of the folder path, the ones of interest try: folder = down_most_folder.replace(self.root_folder, "") folder_parts = folder.split( "/") # List containing each parts/folders of the full path filename_parts = filelist[0].split( "_") # Takes the first filename and create a list of its parts rid = folder_parts[1][-4:] # Get the last 4 characters if re.search('[a-zA-Z]', rid) is not None: rid = filename_parts[3] if re.search('[a-zA-Z]', rid) is not None: PipelineLogger.log( 'root', 'error', 'File recurse error on Folder RID cannot be identified. - {0}, \n Filelist - {1}' .format(folder, filelist)) return None scan_type = self.determineScanType(folder_parts[-3]) scan_date = folder_parts[-2].split('_')[0] scan_time = folder_parts[-2].split('_', 1)[-1].replace("_", ":") s_identifier = filename_parts[-2] i_identifier = filename_parts[-1].split('.', 1)[0] file_type = self.determineExtension(filename_parts) download_folder = down_most_folder raw_folder = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/raw'.format( sc.studyDatabaseRootDict[self.study], 'ADNI', scan_type, rid, scan_date, s_identifier, i_identifier) newScanSession = ScanSession\ ('ADNI', rid, scan_type, scan_date, scan_time, s_identifier, i_identifier, download_folder, raw_folder, file_type) if scan_type == 'unknown': newScanSession.printObject() return None return newScanSession except Exception as e: return None
def add_to_mongoDB(self, file): f = open(file, 'r').read() try: o = xmltodict.parse(f) except: print('XML cannot be read - {0}.'.format(f)) PipelineLogger.log('root', 'exception', 'XML cannot be read - {0}.'.format(f)) return 0 subId, SID, IID, valid = self.get_subid_sid_iid(file) if valid: _id = '{0}_{1}_{2}'.format(subId, SID, IID) o['_id'] = _id client = MongoClient('localhost', 27017) db = client.ADNI_Database collection = db.Scan_XML_Collection try: post_id = collection.insert_one(o).inserted_id shutil.move(file, moncfg.XMLProcessedArchivePath) except DuplicateKeyError: PipelineLogger.log('root', 'exception', 'XML already in DB - {0}.'.format(_id)) shutil.move(file, moncfg.XMLProcessedArchivePath) post_id = _id return post_id
def executeScript(self, processingItemObj, matlabScript, niakFolder): # Create a matlab file to be called later on matlabFile = self.createMatlabFile(matlabScript, niakFolder) # Prepare matlab command matlabCommand = '%s run %s;exit"' % (config.matlab_call, matlabFile) # Creating log folder logDir = '{0}/logs'.format(processingItemObj.root_folder) try: distutils.dir_util.mkpath(logDir) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating log folder \n {0}'.format(e)) return 0 # Create list of files that should be present fmri_file = niakFolder + '/fmri/fmri_subject1_session1_run1.mnc' anat_ln_file = niakFolder + '/anat/anat_subject1_nuc_stereolin.mnc' anat_nl_file = niakFolder + '/anat/anat_subject1_nuc_stereonl.mnc' fmri_mean_file = niakFolder + '/anat/func_subject1_mean_stereonl.mnc' func_coregister = niakFolder + '/quality_control/group_coregistration/func_tab_qc_coregister_stereonl.csv' anat_ln_coregister = niakFolder + '/quality_control/group_coregistration/anat_tab_qc_coregister_stereolin.csv' anat_nl_coregister = niakFolder + '/quality_control/group_coregistration/anat_tab_qc_coregister_stereonl.csv' func_motion = niakFolder + '/quality_control/group_motion/qc_scrubbing_group.csv' outputFiles = ' '.join([fmri_file, anat_ln_file, anat_nl_file, fmri_mean_file, func_coregister, anat_ln_coregister, anat_nl_coregister, func_motion]) # Prepare bash command id = '{0}{1}{2}{3}'.format(processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) command = '%s; Pipelines/ADNI_Fmri/MatlabScripts/startMatlabScript.sh %s %s %s %s %s %s %s' % \ (config.sourcing, id, matlabCommand, niakFolder, logDir, socket.gethostname(), '50500', outputFiles) # Create NIAK folder if not os.path.exists(niakFolder): os.makedirs(niakFolder) # Run converter command PipelineLogger.log('converter', 'debug', 'Command : {0}'.format(command)) p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log('converter', 'debug', 'Conversion Log Output : \n{0}'.format(out)) PipelineLogger.log('converter', 'debug', 'Conversion Log Err : \n{0}'.format(err)) QSubJobHandler.submittedJobs[id] = QSubJob(id, '01:00:00', processingItemObj, 'niak') return 1
def convert_nii(self, convertionObj): # Run nii2mnc cmd = '{0} {1} {2}/../'.format(Config.ConverterConfig.niiToMnc_exec, convertionObj.raw_folder, convertionObj.converted_folder) PipelineLogger.log( 'converter', 'info', 'nii2mnc conversion starting for : {0} - {1} - {2} - {3}'.format( convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) PipelineLogger.log('converter', 'debug', 'Command : {0}'.format(cmd)) outFile = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format( convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace('-', ''), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type) iterator = 1 for niiFile in glob.glob(convertionObj.raw_folder + '/*.nii*'): if niiFile.endswith('.gz'): subprocess.Popen('gzip -d ' + niiFile, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) niiFile = niiFile.replace('.gz', '') tempOutFile = outFile.replace('.mnc', '_run' + str(iterator) + '.mnc') cmd = '%s %s %s' % (Config.ConverterConfig.niiToMnc_exec, niiFile, tempOutFile) self.runShellCommand(cmd) self.checkMncFile( tempOutFile ) # Check whether the fMRI files have a time component/axis iterator += 1 # Check how many mnc files were generated mncList = [] for root, dirnames, filenames in os.walk( convertionObj.converted_folder): for filename in fnmatch.filter(filenames, '*.mnc'): mncList.append(os.path.join(root, filename)) if len(mncList) == 0: PipelineLogger.log( 'converter', 'error', 'MINC Conversion unsuccessful : Check log for : {0} - {1} - {2} - {3}' .format(convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) return 0 else: return 1
def run(self): try: self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind((socket.gethostname(), 50500)) self.sock.settimeout(300) self.sock.listen(1000) PipelineLogger.log('manager', 'info',' ++++++++ QSub Job Handler started.') PipelineLogger.log('manager', 'info',' ++++++++ QSub Job Handler listening in Host : {0} at Port : {1}.'.format(socket.gethostname(), 50500)) while not self.QUIT and self.checkJobs(): try: conn = self.sock.accept()[0] thread = threading.Thread(target=self.doWork, args=(conn, )) thread.start() except socket.timeout: continue except Exception as e: PipelineLogger.log('manager', 'exception',e) PipelineLogger.log('manager', 'error','Cannot create QSubJobHandler... Will not listen to on jobs. ') del self.sock
def copyFile(sourceFolder, destFolder): try: PipelineLogger.log( 'manager', 'debug', 'Raw Data Copying : {0} -> {1}'.format( sourceFolder, destFolder)) distutils.dir_util.copy_tree(sourceFolder, destFolder, update=True) PipelineLogger.log( 'manager', 'debug', 'Raw Data Copy Done...... : {0} -> {1}'.format( sourceFolder, destFolder)) removeCommaIfThere(destFolder) return 1 except Exception as exc: PipelineLogger.log( 'manager', 'error', 'Raw Data Move Error : {0} -> {1}'.format( sourceFolder, destFolder)) PipelineLogger.log('manager', 'exception', exc) return 0
def combiningRuns(self, processingItemObj): #### Needs to improve it a lot more command = "%s combiningRuns('%s', '%s', %s, %s, %s)" %\ (config.matlab_call, config.fmristat_location, config.emma_tools_location, processingItemObj.root_folder, processingItemObj.subject_rid, '1') # Run matlab command PipelineLogger.log('processing', 'debug', 'Command : {0}'.format(command)) p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log('processing', 'debug', 'combiningRuns Log Output : \n{0}'.format(out)) PipelineLogger.log('processing', 'debug', 'combiningRuns Log Err : \n{0}'.format(err)) return out
def runCivet(self, processingItemObj, maskStatus): nativeFileName = self.checkNative(processingItemObj) if not nativeFileName: return 0 copyFolder = pc.T1TempDirForCIVETProcessing subjectFileName_base = '{0}_{1}{2}{3}{4}_{5}'.format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, processingItemObj.modality.lower()) jobId = '{0}_{1}_{2}_{3}{4}{5}{6}_CIVETRUN'.format(processingItemObj.study, processingItemObj.modality, processingItemObj.table_id, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) checkJobPresentSql = "SELECT * FROM externalWaitingJobs WHERE JOB_ID = '{0}'".format(jobId) if len(self.DBClient.executeAllResults(checkJobPresentSql)) is 0: beastFileName = '{0}/beast/mask/{1}_skull_mask_native.mnc'.format(processingItemObj.root_folder, subjectFileName_base) beastMaskName_base = '{0}_{1}{2}{3}{4}_mask'.format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) beastMaskName = '{0}/{1}.mnc'.format(copyFolder, beastMaskName_base) manualFileName = '{0}/manual/mask/{1}_skull_mask_native.mnc'.format(processingItemObj.root_folder, subjectFileName_base) manualMaskName_base = '{0}_{1}{2}{3}{4}_mask'.format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) manualMaskName = '{0}/{1}.mnc'.format(copyFolder, manualMaskName_base) try: distutils.file_util.copy_file(nativeFileName, copyFolder) if maskStatus == 'B': distutils.file_util.copy_file(beastFileName, beastMaskName) elif maskStatus == 'M': distutils.file_util.copy_file(manualFileName, manualMaskName) elif maskStatus == 'N': pass else: PipelineLogger.log('manager', 'error', 'Unknown mask status - {0} Entry : Processing ID - {1}, Table ID - {3}'.format(maskStatus, processingItemObj.processing_rid, processingItemObj.table_id)) addExternalJobSQL = "INSERT INTO externalWaitingJobs VALUES ('{0}', '{1}', '{2}', NULL, NULL, NULL)".format(jobId, '{0}_{1}_Pipeline'.format(processingItemObj.study, processingItemObj.modality), 'CIVET') self.DBClient.executeNoResult(addExternalJobSQL) except Exception as e: PipelineLogger.log('manager', 'error', 'Error copying for CIVET input. Rolling back... - Processing Table ID -> {0} Table ID -> {1}'.format( processingItemObj.processing_rid, processingItemObj.table_id)) PipelineLogger.log('manager', 'exception', e) nativeFileOnCopyFolder = '{0}/{1}'.format(copyFolder, os.path.basename(nativeFileName)) os.remove(nativeFileOnCopyFolder) if os.path.exists(nativeFileOnCopyFolder) else None os.remove(beastMaskName) if os.path.exists(beastMaskName) else None os.remove(manualMaskName) if os.path.exists(manualMaskName) else None
def getManualXFM(self, processingItemObj, matchedT1entry): study = processingItemObj.study rid = processingItemObj.subject_rid pet_sid = processingItemObj.s_identifier pet_iid = processingItemObj.i_identifier t1_sid = matchedT1entry[6] t1_iid = matchedT1entry[7] xfmUID = 'PET_{0}_{1}_T1_{2}_{3}'.format(pet_sid, pet_iid, t1_sid, t1_iid) getXFMSQL = "SELECT * FROM MANUAL_XFM WHERE STUDY = '{0}' AND RID = '{1}' AND XFM_UNIQUEID = '{2}'".format(study, rid, xfmUID) res = self.DBClient.executeAllResults(getXFMSQL) if len(res) > 0: PipelineLogger.log('root', 'INFO', '++ Manual XFM found. - {0} - {1}'.format(processingItemObj.subject_rid, processingItemObj.scan_date)) manXFM = res[0][4] updateSQL = "UPDATE {0}_{1}_Pipeline SET MANUAL_XFM = '{2}' WHERE PROCESSING_TID = {3}".format(study, processingItemObj.modality, manXFM, processingItemObj.processing_rid) self.DBClient.executeNoResult(updateSQL) return manXFM else: PipelineLogger.log('root', 'INFO', '$$$$$$$ Manual XFM not found. Trying to find using uncorrected T1s. - {0} - {1}'.format(processingItemObj.subject_rid, processingItemObj.scan_date)) mustMatchT1SID = t1_sid mustMatchT1IID = t1_iid xfmApproximation = 'PET_{0}_{1}_T1_%_%'.format(pet_sid, pet_iid) getAllT1s = "SELECT * FROM MANUAL_XFM WHERE STUDY = '{0}' AND RID = '{1}' AND XFM_UNIQUEID LIKE '{2}'".format(study, rid, xfmApproximation) approxRes = self.DBClient.executeAllResults(getAllT1s) getFromProcessingSQL = "SELECT * FROM Processing WHERE (STUDY, RID, SCAN_DATE, SCAN_TIME) = (SELECT `STUDY`, `RID`, `SCAN_DATE`, `SCAN_TIME` FROM `Processing` WHERE MODALITY = 'T1' AND `S_IDENTIFIER` = '{0}' AND `I_IDENTIFIER` = '{1}')".format(mustMatchT1SID, mustMatchT1IID) allT1s = self.DBClient.executeAllResults(getFromProcessingSQL) for t1 in allT1s: t1sid = t1[6] t1iid = t1[7] for appRes in approxRes: approxResSID = appRes[3].split('_')[4] approxResIID = appRes[3].split('_')[5] if t1sid == approxResSID and t1iid == approxResIID: PipelineLogger.log('root', 'INFO', '++ Manual XFM found from approximate matching. - {0} - {1}'.format(processingItemObj.subject_rid, processingItemObj.scan_date)) manXFM = appRes[4] updateSQL = "UPDATE {0}_{1}_Pipeline SET MANUAL_XFM = '{2}' WHERE PROCESSING_TID = {3}".format(study, processingItemObj.modality, manXFM, processingItemObj.processing_rid) self.DBClient.executeNoResult(updateSQL) return manXFM self.requestCoreg(processingItemObj, matchedT1entry) return None
def combiningRuns(self, processingItemObj): #### Needs to improve it a lot more command = "%s combiningRuns('%s', '%s', %s, %s, %s)" % \ (config.matlab_call, config.fmristat_location, config.emma_tools_location, processingItemObj.root_folder, processingItemObj.subject_rid, '1') # Run matlab command PipelineLogger.log('processing', 'debug', 'Command : {0}'.format(command)) p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log('processing', 'debug', 'combiningRuns Log Output : \n{0}'.format(out)) PipelineLogger.log('processing', 'debug', 'combiningRuns Log Err : \n{0}'.format(err)) return out
def convert_nii(self, convertionObj): # Run nii2mnc cmd = '{0} {1} {2}/../'.format(Config.ConverterConfig.niiToMnc_exec, convertionObj.raw_folder, convertionObj.converted_folder) PipelineLogger.log('converter', 'info', 'nii2mnc conversion starting for : {0} - {1} - {2} - {3}'.format(convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) PipelineLogger.log('converter', 'debug', 'Command : {0}'.format(cmd)) outFile = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace('-', ''), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type) iterator = 1 for niiFile in glob.glob(convertionObj.raw_folder + '/*.nii*'): if niiFile.endswith('.gz'): subprocess.Popen('gzip -d ' + niiFile, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) niiFile = niiFile.replace('.gz', '') tempOutFile = outFile.replace('.mnc', '_run' + str(iterator) + '.mnc') cmd = '%s %s %s' % (Config.ConverterConfig.niiToMnc_exec, niiFile, tempOutFile) self.runShellCommand(cmd) self.checkMncFile(tempOutFile) # Check whether the fMRI files have a time component/axis iterator += 1 # Check how many mnc files were generated mncList = [] for root, dirnames, filenames in os.walk(convertionObj.converted_folder): for filename in fnmatch.filter(filenames, '*.mnc'): mncList.append(os.path.join(root, filename)) if len(mncList) == 0: PipelineLogger.log('converter', 'error', 'MINC Conversion unsuccessful : Check log for : {0} - {1} - {2} - {3}'.format( convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type)) return 0 else: return 1
def setStatus(self, job, status): if job.jobType == 'beast': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = 'UPDATE {0} SET BEAST_MASK = 1 WHERE RECORD_ID = {1}'.format(table, table_id) elif status == 'Fail': setSql = 'UPDATE {0} SET BEAST_MASK = -1, BEAST_SKIP = 1 WHERE RECORD_ID = {1}'.format(table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log('manager', 'error','QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}'.format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'av45': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format(table, table_id) self.requestQC(nestedJob, 'av45') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format(table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log('manager', 'error','QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}'.format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'fdg': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format(table, table_id) self.requestQC(nestedJob, 'fdg') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format(table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log('manager', 'error','QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}'.format(job.jobType, nestedJob.processing_rid, nestedJob.table_id))
def moveRawData(self): def removeCommaIfThere(destFolder): PipelineLogger.log('manager', 'debug', 'Removing unsupported chars from file names...... :') for dpath, dnames, fnames in os.walk(destFolder): for f in fnames: os.chdir(dpath) if ',' in f: os.rename(f, f.replace(',', '')) PipelineLogger.log('manager', 'debug', 'Removing unsupported chars from file names done ...:') def copyFile(sourceFolder, destFolder): try: PipelineLogger.log('manager', 'debug', 'Raw Data Copying : {0} -> {1}'.format(sourceFolder, destFolder)) distutils.dir_util.copy_tree(sourceFolder, destFolder, update=True) PipelineLogger.log('manager', 'debug', 'Raw Data Copy Done...... : {0} -> {1}'.format(sourceFolder, destFolder)) removeCommaIfThere(destFolder) return 1 except Exception as exc: PipelineLogger.log('manager', 'error', 'Raw Data Move Error : {0} -> {1}'.format(sourceFolder, destFolder)) PipelineLogger.log('manager', 'exception', exc) return 0 for study in self.studyList: totalToMove = len(self.moveSortingObjListDict[study]) PipelineLogger.log('manager', 'info', 'Moving started for study {0} - Total to be moved : {1}'.format(study, totalToMove)) count = 1 for sortingObj in self.moveSortingObjListDict[study]: PipelineLogger.log('manager', 'info', 'Moving {0}/{1} - {2}'.format(count, totalToMove, sortingObj.download_folder)) copied = copyFile(sortingObj.download_folder, sortingObj.raw_folder) count += 1 if copied: self.conversionTable.insertFromSortingObj(sortingObj, self.version) self.sortingTable.setMovedTrue(sortingObj) else: PipelineLogger.log('manager', 'error', 'File Move Error : {0} -> {1}. Moving to next...'.format(sortingObj.download_folder, sortingObj.raw_folder))
def process(self, processingItem): processingItemObj = ProcessingItemObj(processingItem) if processingItemObj.beast_skip and processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'N') elif processingItemObj.manual_mask and not processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'M') elif processingItemObj.beast_mask == 0 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask: self.runBeast(processingItemObj) elif processingItemObj.beast_skip and not processingItemObj.manual_mask and not processingItemObj.manual_skip: PipelineLogger.log('manager', 'error', '$$$$$$$$$$$$$$$$$ Manual Mask Requested $$$$$$$$$$$$$$$$$$ - {0}'.format(processingItem)) pass elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 1 and not processingItemObj.manual_mask and not processingItemObj.civet: self.runCivet(processingItemObj, 'B') elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask and not processingItemObj.manual_skip: self.requestQC(processingItemObj, 'beast') elif processingItemObj.civet == 1 and processingItemObj.civet_qc == 0: self.requestQC(processingItemObj, 'civet') else: if processingItemObj.civet_qc == -1: PipelineLogger.log('manager', 'error', 'Civet QC failed. Skipping. - {0}'.format(processingItem)) PipelineLogger.log('manager', 'error', 'Error handling obj for processing - {0}'.format(processingItem)) return 0
def processPET(self, processingItemObj, matchT1Path): petFileName = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(processingItemObj.converted_folder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, self.getScanType(processingItemObj)) processedFolder = '{0}/processed'.format(processingItemObj.root_folder) logDir = '{0}/logs'.format(processingItemObj.root_folder) PipelineLogger.log('manager', 'info', 'PET processing starting for {0}'.format(petFileName)) try: distutils.dir_util.mkpath(logDir) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating log folder \n {0}'.format(e)) return 0 id = '{0}{1}{2}{3}'.format(processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) paramStrd = ast.literal_eval(processingItemObj.parameters) paramStrt = ' '.join(['[\"{0}\"]=\"{1}\"'.format(k, v) for k,v in paramStrd.items()]) paramStr = '({0})'.format(paramStrt) petCMD = "source /opt/minc-toolkit/minc-toolkit-config.sh; Pipelines/ADNI_AV45/ADNI_V2_AV45_Process {0} {1} {2} {3} {4} {5} '{6}' {7} {8}".format(id, petFileName, processedFolder, matchT1Path, 'auto' if processingItemObj.manual_xfm == '' else processingItemObj.manual_xfm, logDir, paramStr,socket.gethostname(), 50500) try: processedFolder_del = '{0}/processed_del'.format(processingItemObj.root_folder) os.rename(processedFolder, processedFolder_del) shutil.rmtree(processedFolder_del) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in deleting old processing folder. \n {0}'.format(e)) try: distutils.dir_util.mkpath(processedFolder) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating processing folder. \n {0}'.format(e)) return 0 PipelineLogger.log('manager', 'debug', 'Command : {0}'.format(petCMD)) p = subprocess.Popen(petCMD, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log('manager', 'debug', 'Process Log Output : \n{0}'.format(out)) PipelineLogger.log('manager', 'debug', 'Process Log Err : \n{0}'.format(err)) QSubJobHandler.submittedJobs[id] = QSubJob(id, '02:00:00', processingItemObj, 'av45') return 1
def convert_v(self, convertionObj): rawFile = "{0}/*.v".format(convertionObj.raw_folder) outFile = "{0}/{1}_{2}{3}{4}{5}_{6}.mnc".format( convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace("-", ""), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type, ) outDynFile = "{0}/{1}_{2}{3}{4}{5}_{6}_Dyn.mnc".format( convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace("-", ""), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type, ) outTempFile = "{0}/{1}_{2}{3}{4}{5}_{6}_temp.mnc".format( convertionObj.converted_folder, convertionObj.study, convertionObj.rid, convertionObj.scan_date.replace("-", ""), convertionObj.s_identifier, convertionObj.i_identifier, convertionObj.scan_type, ) cmd = "/opt/minc/bin/ecattominc {0} {1}".format(rawFile, outTempFile) cmdRes = "mincresample -short {0} {1}".format(outTempFile, outDynFile) PipelineLogger.log( "converter", "info", "MINC conversion starting for : {0} - {1} - {2} - {3}".format( convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type ), ) PipelineLogger.log("converter", "debug", "Command : {0}".format(cmd)) try: os.remove(outDynFile) except: pass if os.path.exists(convertionObj.converted_folder): try: convertedFolder_del = "{0}_del".format(convertionObj.converted_folder) os.rename(convertionObj.converted_folder, convertedFolder_del) shutil.rmtree(convertedFolder_del) except Exception as e: PipelineLogger.log("manager", "error", "Error in deleting old converted folder. \n {0}".format(e)) try: distutils.dir_util.mkpath(convertionObj.converted_folder) except: pass p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() PipelineLogger.log("converter", "debug", "Conversion Log Output : \n{0}".format(out)) PipelineLogger.log("converter", "debug", "Conversion Log Err : \n{0}".format(err)) subprocess.Popen(cmdRes, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() if self.avgTime(outDynFile, outFile): PipelineLogger.log( "converter", "info", "MINC Conversion success : {0} - {1} - {2} - {3}".format( convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type ), ) return 1 else: PipelineLogger.log( "converter", "error", "MINC Conversion unsuccessful : Check log for : {0} - {1} - {2} - {3}".format( convertionObj.study, convertionObj.rid, convertionObj.scan_date, convertionObj.scan_type ), ) return 0
def processPET(self, processingItemObj, matchT1Path): petFileName = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(processingItemObj.converted_folder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, self.getScanType(processingItemObj)) processedFolder = '{0}/processed'.format(processingItemObj.root_folder) logDir = '{0}/logs'.format(processingItemObj.root_folder) PipelineLogger.log('manager', 'info', 'PET processing starting for {0}'.format(petFileName)) try: distutils.dir_util.mkpath(logDir) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating log folder \n {0}'.format(e)) return 0 id = '{0}{1}{2}{3}'.format(processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) paramStrd = ast.literal_eval(processingItemObj.parameters) paramStrt = ' '.join(['[\"{0}\"]=\"{1}\"'.format(k, v) for k,v in paramStrd.items()]) paramStr = '({0})'.format(paramStrt) petCMD = "source /opt/minc-1.9.15/minc-toolkit-config.sh; Pipelines/ADNI_AV1451/ADNI_V1_AV1451_Process {0} {1} {2} {3} {4} {5} '{6}' {7} {8}".format(id, petFileName, processedFolder, matchT1Path, processingItemObj.manual_xfm, logDir, paramStr,socket.gethostname(), 50500) try: processedFolder_del = '{0}/processed_del'.format(processingItemObj.root_folder) os.rename(processedFolder, processedFolder_del) shutil.rmtree(processedFolder_del) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in deleting old processing folder. \n {0}'.format(e)) try: distutils.dir_util.mkpath(processedFolder) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating processing folder. \n {0}'.format(e)) return 0 ### This section is new for ADNI Pre processing - Per scanner type blurring. Only required if ### the images are aquired from different scanners and need to get to same PSF. blur_x, blur_y, blur_z = self.PETHelper.getBlurringParams(processingItemObj) ### End pre processing. PipelineLogger.log('manager', 'debug', 'Command : {0}'.format(petCMD)) p = subprocess.Popen(petCMD, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log('manager', 'debug', 'Process Log Output : \n{0}'.format(out)) PipelineLogger.log('manager', 'debug', 'Process Log Err : \n{0}'.format(err)) QSubJobHandler.submittedJobs[id] = QSubJob(id, '02:00:00', processingItemObj, 'av1451') return 1
def processPET(self, processingItemObj, matchT1Path): petFileName = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format( processingItemObj.converted_folder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, self.getScanType(processingItemObj)) processedFolder = '{0}/processed'.format(processingItemObj.root_folder) logDir = '{0}/logs'.format(processingItemObj.root_folder) PipelineLogger.log( 'manager', 'info', 'PET processing starting for {0}'.format(petFileName)) try: distutils.dir_util.mkpath(logDir) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating log folder \n {0}'.format(e)) return 0 id = '{0}{1}{2}{3}'.format( processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) paramStrd = ast.literal_eval(processingItemObj.parameters) paramStrt = ' '.join( ['[\"{0}\"]=\"{1}\"'.format(k, v) for k, v in paramStrd.items()]) paramStr = '({0})'.format(paramStrt) petCMD = "source /opt/minc-1.9.15/minc-toolkit-config.sh; Pipelines/ADNI_FDG/ADNI_V2_FDG_Process {0} {1} {2} {3} {4} {5} '{6}' {7} {8}".format( id, petFileName, processedFolder, matchT1Path, 'auto' if processingItemObj.manual_xfm == '' else processingItemObj.manual_xfm, logDir, paramStr, socket.gethostname(), 50500) try: processedFolder_del = '{0}/processed_del'.format( processingItemObj.root_folder) os.rename(processedFolder, processedFolder_del) shutil.rmtree(processedFolder_del) except Exception as e: PipelineLogger.log( 'manager', 'error', 'Error in deleting old processing folder. \n {0}'.format(e)) try: distutils.dir_util.mkpath(processedFolder) except Exception as e: PipelineLogger.log( 'manager', 'error', 'Error in creating processing folder. \n {0}'.format(e)) return 0 PipelineLogger.log('manager', 'debug', 'Command : {0}'.format(petCMD)) p = subprocess.Popen(petCMD, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log('manager', 'debug', 'Process Log Output : \n{0}'.format(out)) PipelineLogger.log('manager', 'debug', 'Process Log Err : \n{0}'.format(err)) QSubJobHandler.submittedJobs[id] = QSubJob(id, '23:00:00', processingItemObj, 'fdg') return 1
__author__ = 'Sulantha' import logging.config from Utils.PipelineLogger import PipelineLogger from Test.logTestSubModule import testSubModuleLogging logging.config.fileConfig('../Config/LoggingConfig.conf') PipelineLogger.log('root', 'info', 'Main module log') lg = testSubModuleLogging() lg.log()
def getMatchingT1(self, processingItemObj): modalityID = '{0}{1}{2}{3}{4}{5}{6}'.format( processingItemObj.study, processingItemObj.version, processingItemObj.subject_rid, processingItemObj.modality, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) getFromMatchTableSQL = "SELECT * FROM MatchT1 WHERE MODALITY_ID = '{0}'".format( modalityID) existingMatchedRec = self.DBClient.executeAllResults( getFromMatchTableSQL) if len(existingMatchedRec) == 1: getConvSQL = "SELECT * FROM Conversion WHERE RECORD_ID = '{0}'".format( existingMatchedRec[0][3]) return self.DBClient.executeAllResults(getConvSQL)[0] else: if processingItemObj.modality == 'FMRI': PipelineLogger.log( 'root', 'error', 'FMRI T1 Matching not implemented. {0} - {1} - {2}'.format( processingItemObj.subject_rid, processingItemObj.s_identifier.replace('S', ''), processingItemObj.i_identifier.replace('I', ''))) return None else: # By Default, for PET images date_str = processingItemObj.scan_date.replace('-', '') name_and_Mod = '{0}{1}'.format(processingItemObj.subject_rid, processingItemObj.modality) visit = processingItemObj.i_identifier.split('x')[0].replace( date_str, '').replace(name_and_Mod, '') pet_label = '{0}_{1}_{2}'.format( processingItemObj.subject_rid, visit, processingItemObj.modality.lower()) getRecordSQL = "SELECT * FROM PET_MRI_Proc_Match WHERE Label LIKE '{0}'".format( pet_label) petrecord = self.MatchDBClient.executeAllResults(getRecordSQL) if not petrecord: PipelineLogger.log( 'root', 'error', 'Cannot find PET record : {0} - {1} - {2}'.format( processingItemObj.subject_rid, processingItemObj.s_identifier.replace('S', ''), processingItemObj.i_identifier.replace('I', ''))) return None mr_name = petrecord[0][5] if mr_name == '': ### Processed with MR entry not found. Have to switch to date based matching. PipelineLogger.log( 'root', 'error', 'Processed with MR entry not found. : {0} - {1} - {2} - Searching based on scan date. +/- 60 days from PET date' .format(processingItemObj.subject_rid, processingItemObj.modality, visit)) return None mr_fid = petrecord[0][6] mr_visit = mr_name.split('_')[1] matchedT1withScanDescriptions = [] for t1_type in ['MPRAGE', 'IRFSPGR', 'MPR', 'FSPGR']: mr_DB_iid = '{0}{3}{1}%x{2}'.format( processingItemObj.subject_rid, mr_visit, mr_fid, t1_type) getScanFromConversionSQL = "SELECT * FROM Conversion WHERE STUDY = '{0}' AND I_IDENTIFIER LIKE '{1}' AND SKIP = 0".format( processingItemObj.study, mr_DB_iid) t1_conversion = self.DBClient.executeAllResults( getScanFromConversionSQL) if len(t1_conversion) > 0: matchedT1withScanDescriptions.append(t1_conversion[0]) if len(matchedT1withScanDescriptions) < 1: PipelineLogger.log( 'root', 'error', 'Matched T1s are not in the database. : Subject, visit and FID - {0} {1} {2}' .format(processingItemObj.subject_rid, mr_visit, mr_fid)) return None else: if len(matchedT1withScanDescriptions) == 1: ## ONLY ONE MATCHED T1. GOOD> CHECK IF THE T1 is a good scan type and not a bluff !!! self.addToMatchT1Table(processingItemObj, modalityID, matchedT1withScanDescriptions[0]) return matchedT1withScanDescriptions[0] else: #### MORE THAN ONE FOUND. Very weird fro DIAN. PipelineLogger.log( 'root', 'error', 'MORE THAN ONE T1 Match FOUND. Very weird fro DIAN. : Subject and visit - {0} {1}' .format(processingItemObj.subject_rid, mr_visit)) return None
def getMatchingT1(self, processingItemObj): modalityID = '{0}{1}{2}{3}{4}{5}{6}'.format( processingItemObj.study, processingItemObj.version, processingItemObj.subject_rid, processingItemObj.modality, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) getFromMatchTableSQL = "SELECT * FROM MatchT1 WHERE MODALITY_ID = '{0}'".format( modalityID) # Find matching record in matching T1 table existingMatchedRec = self.DBClient.executeAllResults( getFromMatchTableSQL) if len(existingMatchedRec) == 1: getConvSQL = "SELECT * FROM Conversion WHERE RECORD_ID = '{0}'".format( existingMatchedRec[0][3]) return self.DBClient.executeAllResults(getConvSQL)[0] else: # If can't find them, look into MRIList to find an equivalent getFmriRecordSQL = "SELECT * FROM MRILIST WHERE subject LIKE '%_%_{0}' AND seriesid = {1} AND imageuid = {2}".format( processingItemObj.subject_rid, processingItemObj.s_identifier.replace('S', ''), processingItemObj.i_identifier.replace('I', '')) FmriRecord = self.MatchDBClient.executeAllResults(getFmriRecordSQL) if not FmriRecord: PipelineLogger.log( 'root', 'error', 'Cannot find Fmri record : {0} - {1} - {2}'.format( processingItemObj.subject_rid, processingItemObj.s_identifier.replace('S', ''), processingItemObj.i_identifier.replace('I', ''))) return None visit_code = pc.ADNI_visitCode_Dict[FmriRecord[0][2]] getMRIRecordsSQL = "SELECT * FROM MPRAGEMETA WHERE subjectid LIKE '%_%_{0}'".format( processingItemObj.subject_rid) mrirecords = self.MatchDBClient.executeAllResults(getMRIRecordsSQL) if not mrirecords: PipelineLogger.log( 'root', 'error', '################################ - Error !!!!! Cannot find any MRI records : {0} - Please check ADNI recs. ################################' .format(processingItemObj.subject_rid)) return None # getMRISecondarySQL = "SELECT * FROM MRILIST WHERE subject LIKE '%_%_{0}'".format(processingItemObj.subject_rid) # mriSecondaryRecords = self.MatchDBClient.executeAllResults(getMRISecondarySQL) # t_mrirecords = mrirecords # for record in mriSecondaryRecords: # distint = 1 # for i in t_mrirecords: # if record[7] == i[7] and record[8] == i[8]: # distint = 0 # if distint: # mrirecords.append(record) matchedT1Recs = [] for rec in mrirecords: if pc.ADNI_visitCode_Dict[rec[2]] == visit_code: matchedT1Recs.append(rec) if len(matchedT1Recs) == 0: PipelineLogger.log( 'root', 'error', 'Cannot match visit codes for : {0} - {1} - {2} - Searching based on scan date. +/- 60 days from PET date' .format(processingItemObj.subject_rid, processingItemObj.modality, visit_code)) pet_date = datetime.strptime(processingItemObj.scan_date, '%Y-%m-%d') sortedRecs = sorted( mrirecords, key=lambda x: abs( datetime.strptime(x[5], '%Y-%m-%d') - pet_date)) closestDate = [ k for k, g in itertools.groupby( sortedRecs, key=lambda x: abs( datetime.strptime(x[5], '%Y-%m-%d') - pet_date)) ][0] PipelineLogger.log( 'root', 'error', 'PET MRI Matching based on dates - match visit codes for : {0} - {1} - {2} - Distance between MRI/PET : {3} days.' .format(processingItemObj.subject_rid, processingItemObj.modality, visit_code, closestDate)) closestMatchedRecs = [ list(g) for k, g in itertools.groupby( sortedRecs, key=lambda x: abs( datetime.strptime(x[5], '%Y-%m-%d') - pet_date)) ][0] matchedT1Recs = closestMatchedRecs if len(matchedT1Recs) == 0: PipelineLogger.log( 'root', 'error', 'Cannot match visit codes for : {0} - {1} - {2}'.format( processingItemObj.subject_rid, processingItemObj.modality, visit_code)) return None matchedT1withScanDescriptions = [] for rec in matchedT1Recs: getScanFromConversionSQL = "SELECT * FROM Conversion WHERE STUDY = '{0}' AND S_IDENTIFIER = '{1}' AND I_IDENTIFIER = '{2}' AND SKIP = 0".format( processingItemObj.study, 'S{0}'.format(rec[7]), 'I{0}'.format(rec[8])) t1_conversion = self.DBClient.executeAllResults( getScanFromConversionSQL) if len(t1_conversion) > 0: matchedT1withScanDescriptions.append(t1_conversion[0]) else: PipelineLogger.log( 'root', 'error', 'Correspoding MRI was not found in the system : {0} - {1} - {2}' .format(processingItemObj.subject_rid, 'S{0}'.format(rec[7]), 'I{0}'.format(rec[8]))) continue if len(matchedT1withScanDescriptions) < 1: PipelineLogger.log( 'root', 'error', 'Matched T1s are not in the database. : Matched T1 s - \n {0}' .format(matchedT1Recs)) return None else: if len(matchedT1withScanDescriptions) == 1: ## ONLY ONE MATCHED T1. GOOD> CHECK IF THE T1 is a good scan type and not a bluff !!! if matchedT1withScanDescriptions[0][ 3] in pc.ADNI_T1_match_accepted_scantypes: self.addToMatchT1Table( processingItemObj, modalityID, matchedT1withScanDescriptions[0]) return matchedT1withScanDescriptions[0] else: PipelineLogger.log( 'root', 'error', 'Matched T1s is not accepted scan type. : Matched T1 s - \n {0}' .format(matchedT1withScanDescriptions[0])) return None else: #### MORE THAN ONE FOUND. SELECT ONE BASED ON SCAN TYPE PRIORITY sortedList = sorted(matchedT1withScanDescriptions, key=lambda x: (pc.ADNI_T1_match_scantype_priorityList .index(x[3]), -x[5])) self.addToMatchT1Table(processingItemObj, modalityID, sortedList[0]) return sortedList[0]
def determineScanType(self, scanTypeRaw, study, rid, sid, iid): try: return arc.scanTypeDict[scanTypeRaw] except KeyError: if 'FDG' in scanTypeRaw: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match FDG...'. format(scanTypeRaw)) return 'FDG' if 'PIB' in scanTypeRaw: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match PIB...'. format(scanTypeRaw)) return 'PIB' if 'AV45' in scanTypeRaw or 'AV-45' in scanTypeRaw or 'AV_45' in scanTypeRaw: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match AV45...'. format(scanTypeRaw)) return 'AV45' if 'AV1451' in scanTypeRaw or 'AV-1451' in scanTypeRaw or 'AV_1451' in scanTypeRaw or 'tau' in scanTypeRaw.lower( ): PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match AV1451...'. format(scanTypeRaw)) return 'AV1451' if 'MPRAGE' in scanTypeRaw.upper(): PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match MPRAGE...'. format(scanTypeRaw)) return 'MPRAGE' else: typeFromXML = self.getScanTypeFromXMLs(sc.xmlPath, study, rid, sid, iid) if typeFromXML: if 'FDG' in typeFromXML: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match from XML - FDG...' .format(typeFromXML)) return 'FDG' if 'PIB' in typeFromXML: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match from XML - PIB...' .format(typeFromXML)) return 'PIB' if 'AV45' in typeFromXML or 'AV-45' in typeFromXML or 'AV_45' in typeFromXML: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match from XML - AV45...' .format(typeFromXML)) return 'AV45' if 'AV1451' in scanTypeRaw or 'AV-1451' in scanTypeRaw or 'AV_1451' in scanTypeRaw or 'tau' in scanTypeRaw.lower( ): PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> Close match AV1451...' .format(scanTypeRaw)) return 'AV1451' else: PipelineLogger.log( 'root', 'error', 'Scan Type unidentified : {0} -> No match...'.format( scanTypeRaw)) return 'unknown'