class QSubJobStatusReporter: def __init__(self): self.DBClient = DbUtils() self.QCHandler = QCHandler() def setStatus(self, job, status): if job.jobType == 'beast': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = 'UPDATE {0} SET BEAST_MASK = 1 WHERE RECORD_ID = {1}'.format(table, table_id) elif status == 'Fail': setSql = 'UPDATE {0} SET BEAST_MASK = -1, BEAST_SKIP = 1 WHERE RECORD_ID = {1}'.format(table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log('manager', 'error','QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}'.format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'av45': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format(table, table_id) self.requestQC(nestedJob, 'av45') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format(table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log('manager', 'error','QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}'.format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'fdg': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format(table, table_id) self.requestQC(nestedJob, 'fdg') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format(table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log('manager', 'error','QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}'.format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) def requestQC(self, processingItemObj, qctype): qcFieldDict = dict(civet='QC', beast='BEAST_QC', av45='QC', fdg='QC') qcFolderDict = { 'civet' : '{0}/civet'.format(processingItemObj.root_folder), 'beast' : '{0}/beast'.format(processingItemObj.root_folder), 'av45' : '{0}/processed'.format(processingItemObj.root_folder), 'fdg' : '{0}/processed'.format(processingItemObj.root_folder)} self.QCHandler.requestQC(processingItemObj.study, '{0}_{1}_Pipeline'.format(processingItemObj.study, processingItemObj.modality), processingItemObj.table_id, qcFieldDict[qctype], qctype, qcFolderDict[qctype])
class PipelineHandler: def __init__(self): self.processingPPDict = {'ADNI':{'V1':{'T1':ADNI_V1_T1(), 'FMRI':ADNI_V1_FMRI(), 'AV45':ADNI_V1_AV45(), 'FDG':ADNI_V1_FDG(), 'AV1451': ADNI_V1_AV1451()}, 'V2':{'T1':ADNI_V1_T1(), 'FMRI':ADNI_V1_FMRI(), 'AV45':ADNI_V2_AV45(), 'FDG':ADNI_V2_FDG(), 'AV1451': ADNI_V2_AV1451()}}} self.DBClient = DbUtils() self.QCH = QCHandler() def checkExternalJobs(self, study, modality): getExtJobSql = "SELECT * FROM externalWaitingJobs WHERE JOB_ID LIKE '{0}_{1}_%'".format(study, modality) extJobs = self.DBClient.executeAllResults(getExtJobSql) for job in extJobs: jobType = job[0].split('_')[-1] reportTable = job[1] tableID = job[0].split('_')[2] reportField = job[2] subjectScanID = job[0].split('_')[3] success = 0 if jobType == 'CIVETRUN': if glob.glob('{0}/{1}_{2}_*'.format(PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID)): getProccessRecSql = "SELECT * FROM Processing WHERE RECORD_ID IN (SELECT PROCESSING_TID FROM {0}_T1_Pipeline WHERE RECORD_ID = {1})".format(study, tableID) processingEntry = self.DBClient.executeAllResults(getProccessRecSql)[0] civetFolder = '{0}/civet'.format(processingEntry[8]) if os.path.exists(civetFolder): shutil.rmtree(civetFolder) try: PipelineLogger.log('manager', 'info', 'Copying - {0} -> {1}'.format(glob.glob('{0}/{1}_{2}_*'.format(PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID))[0], civetFolder)) dir_util.copy_tree(glob.glob('{0}/{1}_{2}_*'.format(PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID))[0], civetFolder) success = 1 except: success = 0 else: continue else: PipelineLogger.log('manager', 'error', 'Unknown external job type - {}'.format(jobType)) if success: updateSQL = "UPDATE {0} SET {1} = 1 WHERE RECORD_ID = {2}".format(reportTable, reportField, tableID) self.DBClient.executeNoResult(updateSQL) if jobType == 'CIVETRUN': finishSQL = "UPDATE {0} SET FINISHED = 1 WHERE RECORD_ID = {1}".format(reportTable, tableID) self.DBClient.executeNoResult(finishSQL) modal_table = reportTable modal_tableId = tableID qcField = 'QC' qctype = 'civet' qcFolder = civetFolder self.QCH.requestQC(study, modal_table, modal_tableId, qcField, qctype, qcFolder) rmSql = "DELETE FROM externalWaitingJobs WHERE JOB_ID LIKE '{0}_{1}_{2}_{3}_%'".format(study, modality, tableID, subjectScanID) self.DBClient.executeNoResult(rmSql) def process(self, study, modality): os.environ['PATH'] = ':'.join(libpath.PATH) os.environ['LD_LIBRARY_PATH'] = ':'.join(libpath.LD_LIBRARY_PATH) os.environ['LD_LIBRARYN32_PATH'] = ':'.join(libpath.LD_LIBRARYN32_PATH) os.environ['PERL5LIB'] = ':'.join(libpath.PERL5LIB) os.environ['MNI_DATAPATH'] = ':'.join(libpath.MNI_DATAPATH) os.environ['ROOT'] = ';'.join(libpath.ROOT) os.environ['MINC_TOOLKIT_VERSION'] = libpath.MINC_TOOLKIT_VERSION os.environ['MINC_COMPRESS'] = libpath.MINC_COMPRESS os.environ['MINC_FORCE_V2'] = libpath.MINC_FORCE_V2 toProcessinModalityPerStudy = self.DBClient.executeAllResults("SELECT * FROM Processing INNER JOIN (SELECT * FROM {0}_{1}_Pipeline WHERE NOT (FINISHED OR SKIP)) as TMP ON Processing.RECORD_ID=TMP.PROCESSING_TID".format(study, modality)) for processingItem in toProcessinModalityPerStudy: version = processingItem[10] # Calling on the process .section of given studies and modalities self.processingPPDict[study][version][modality].process(processingItem) return 0 def addToPipelineTable(self, processingObj): study = processingObj.study version = processingObj.version modality = processingObj.modality r_id = processingObj.record_id addToTableDict = dict(T1="INSERT IGNORE INTO {0}_T1_Pipeline VALUES (NULL, {1}, \"{2}\", 0, 0, 0, 0, 0, 0, 0, 0, 0, NULL, NULL)".format(study, r_id, PipelineConfig.defaultT1config), AV45="INSERT IGNORE INTO {0}_AV45_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)".format(study, r_id, PipelineConfig.defaultAV45config, ''), AV1451="INSERT IGNORE INTO {0}_AV1451_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)".format(study, r_id, PipelineConfig.defaultAV1451config, ''), FDG="INSERT IGNORE INTO {0}_FDG_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)".format(study, r_id, PipelineConfig.defaultFDGconfig, ''), FMRI="INSERT IGNORE INTO {0}_FMRI_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)".format(study, r_id, PipelineConfig.defaultFMRIconfig, 'NIAK_STH_COMESHERE')) self.DBClient.executeNoResult(addToTableDict[modality])
class ADNI_V1_T1: def __init__(self): self.DBClient = DbUtils() self.QCHandler = QCHandler() def process(self, processingItem): processingItemObj = ProcessingItemObj(processingItem) if processingItemObj.beast_skip and processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'N') elif processingItemObj.manual_mask and not processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'M') elif processingItemObj.beast_mask == 0 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask: self.runBeast(processingItemObj) elif processingItemObj.beast_skip and not processingItemObj.manual_mask and not processingItemObj.manual_skip: PipelineLogger.log( 'manager', 'error', '$$$$$$$$$$$$$$$$$ Manual Mask Requested $$$$$$$$$$$$$$$$$$ - {0}' .format(processingItem)) pass elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 1 and not processingItemObj.manual_mask and not processingItemObj.civet: self.runCivet(processingItemObj, 'B') elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask and not processingItemObj.manual_skip: self.requestQC(processingItemObj, 'beast') elif processingItemObj.civet == 1 and processingItemObj.civet_qc == 0: self.requestQC(processingItemObj, 'civet') else: if processingItemObj.civet_qc == -1: PipelineLogger.log( 'manager', 'error', 'Civet QC failed. Skipping. - {0}'.format(processingItem)) PipelineLogger.log( 'manager', 'error', 'Error handling obj for processing - {0}'.format( processingItem)) return 0 def getScanType(self, processingItemObj): r = self.DBClient.executeAllResults( "SELECT SCAN_TYPE FROM Conversion WHERE STUDY = '{0}' AND RID = '{1}' " "AND SCAN_DATE = '{2}' AND S_IDENTIFIER = '{3}' " "AND I_IDENTIFIER = '{4}'".format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date, processingItemObj.s_identifier, processingItemObj.i_identifier)) return r[0][0] def checkNative(self, processingItemObj): orig_ScanType = self.getScanType(processingItemObj) converted_file = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format( processingItemObj.converted_folder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, orig_ScanType) nativeFolder = '{0}/native'.format(processingItemObj.root_folder) nativeFileName = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format( nativeFolder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, processingItemObj.modality.lower()) if not os.path.exists(nativeFileName): try: distutils.dir_util.mkpath(nativeFolder) shutil.copyfile(converted_file, nativeFileName) except Exception as e: PipelineLogger.log( 'manager', 'error', 'Error in creating folders or copying native file. \n {0}'. format(e)) PipelineLogger.log( 'manager', 'error', 'Setting to restart conversion. \n {0}'.format(e)) sql = "UPDATE Conversion SET CONVERTED = 0, SKIP = 0 WHERE S_IDENTIFIER = '{0}' AND I_IDENTIFIER = '{1}'".format( processingItemObj.s_identifier, processingItemObj.i_identifier) self.DBClient.executeNoResult(sql) return None return nativeFileName def runBeast(self, processingItemObj): nativeFileName = self.checkNative(processingItemObj) if not nativeFileName: return 0 beastFolder = '{0}/beast'.format(processingItemObj.root_folder) logDir = '{0}/logs'.format(processingItemObj.root_folder) PipelineLogger.log('manager', 'info', 'BeAST starting for {0}'.format(nativeFileName)) PipelineLogger.log('manager', 'info', 'Current working folder : {0}'.format(os.getcwd())) try: distutils.dir_util.mkpath(logDir) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating log folder \n {0}'.format(e)) return 0 id = '{0}{1}{2}{3}'.format( processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) beastCMD = 'source /opt/minc-1.9.15/minc-toolkit-config.sh; Pipelines/ADNI_T1/ADNI_V1_T1_BeAST {0} {1} {2} {3} {4} {5}'.format( id, nativeFileName, beastFolder, logDir, socket.gethostname(), 50500) try: shutil.rmtree(beastFolder) except: pass try: distutils.dir_util.mkpath(beastFolder) except Exception as e: PipelineLogger.log( 'manager', 'error', 'Error in creating BeAST folder. \n {0}'.format(e)) return 0 PipelineLogger.log('manager', 'debug', 'Command : {0}'.format(beastCMD)) os.chdir(pc.SourcePath) p = subprocess.Popen(beastCMD, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log( 'manager', 'debug', 'Beast Log Output : \n{0}'.format(out.decode("utf-8"))) PipelineLogger.log('manager', 'debug', 'Beast Log Err : \n{0}'.format(err.decode("utf-8"))) QSubJobHandler.submittedJobs[id] = QSubJob(id, '02:00:00', processingItemObj, 'beast') return 1 def runCivet(self, processingItemObj, maskStatus): nativeFileName = self.checkNative(processingItemObj) if not nativeFileName: return 0 copyFolder = pc.T1TempDirForCIVETProcessing subjectFileName_base = '{0}_{1}{2}{3}{4}_{5}'.format( processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, processingItemObj.modality.lower()) jobId = '{0}_{1}_{2}_{3}{4}{5}{6}_CIVETRUN'.format( processingItemObj.study, processingItemObj.modality, processingItemObj.table_id, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) checkJobPresentSql = "SELECT * FROM externalWaitingJobs WHERE JOB_ID = '{0}'".format( jobId) if len(self.DBClient.executeAllResults(checkJobPresentSql)) is 0: beastFileName = '{0}/beast/mask/{1}_skull_mask_native.mnc'.format( processingItemObj.root_folder, subjectFileName_base) beastMaskName_base = '{0}_{1}{2}{3}{4}_mask'.format( processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) beastMaskName = '{0}/{1}.mnc'.format(copyFolder, beastMaskName_base) manualFileName = '{0}/manual/mask/{1}_skull_mask_native.mnc'.format( processingItemObj.root_folder, subjectFileName_base) manualMaskName_base = '{0}_{1}{2}{3}{4}_mask'.format( processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) manualMaskName = '{0}/{1}.mnc'.format(copyFolder, manualMaskName_base) try: distutils.file_util.copy_file(nativeFileName, copyFolder) if maskStatus == 'B': distutils.file_util.copy_file(beastFileName, beastMaskName) elif maskStatus == 'M': distutils.file_util.copy_file(manualFileName, manualMaskName) elif maskStatus == 'N': pass else: PipelineLogger.log( 'manager', 'error', 'Unknown mask status - {0} Entry : Processing ID - {1}, Table ID - {3}' .format(maskStatus, processingItemObj.processing_rid, processingItemObj.table_id)) addExternalJobSQL = "INSERT INTO externalWaitingJobs VALUES ('{0}', '{1}', '{2}', NULL, NULL, NULL)".format( jobId, '{0}_{1}_Pipeline'.format(processingItemObj.study, processingItemObj.modality), 'CIVET') self.DBClient.executeNoResult(addExternalJobSQL) except Exception as e: PipelineLogger.log( 'manager', 'error', 'Error copying for CIVET input. Rolling back... - Processing Table ID -> {0} Table ID -> {1}' .format(processingItemObj.processing_rid, processingItemObj.table_id)) PipelineLogger.log('manager', 'exception', e) nativeFileOnCopyFolder = '{0}/{1}'.format( copyFolder, os.path.basename(nativeFileName)) os.remove(nativeFileOnCopyFolder) if os.path.exists( nativeFileOnCopyFolder) else None os.remove(beastMaskName) if os.path.exists( beastMaskName) else None os.remove(manualMaskName) if os.path.exists( manualMaskName) else None def requestQC(self, processingItemObj, qctype): qcFieldDict = dict(civet='QC', beast='BEAST_QC') qcFolderDict = { 'civet': '{0}/civet'.format(processingItemObj.root_folder), 'beast': '{0}/beast'.format(processingItemObj.root_folder) } tablename = '{0}_{1}_Pipeline'.format(processingItemObj.study, processingItemObj.modality) self.QCHandler.requestQC(processingItemObj.study, tablename, processingItemObj.table_id, qcFieldDict[qctype], qctype, qcFolderDict[qctype])
class ADNI_V1_T1: def __init__(self): self.DBClient = DbUtils() self.QCHandler = QCHandler() def process(self, processingItem): processingItemObj = ProcessingItemObj(processingItem) if processingItemObj.beast_skip and processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'N') elif processingItemObj.manual_mask and not processingItemObj.manual_skip and not processingItemObj.civet: self.runCivet(processingItemObj, 'M') elif processingItemObj.beast_mask == 0 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask: self.runBeast(processingItemObj) elif processingItemObj.beast_skip and not processingItemObj.manual_mask and not processingItemObj.manual_skip: PipelineLogger.log('manager', 'error', '$$$$$$$$$$$$$$$$$ Manual Mask Requested $$$$$$$$$$$$$$$$$$ - {0}'.format(processingItem)) pass elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 1 and not processingItemObj.manual_mask and not processingItemObj.civet: self.runCivet(processingItemObj, 'B') elif processingItemObj.beast_mask == 1 and not processingItemObj.beast_skip and processingItemObj.beast_qc == 0 and not processingItemObj.manual_mask and not processingItemObj.manual_skip: self.requestQC(processingItemObj, 'beast') elif processingItemObj.civet == 1 and processingItemObj.civet_qc == 0: self.requestQC(processingItemObj, 'civet') else: if processingItemObj.civet_qc == -1: PipelineLogger.log('manager', 'error', 'Civet QC failed. Skipping. - {0}'.format(processingItem)) PipelineLogger.log('manager', 'error', 'Error handling obj for processing - {0}'.format(processingItem)) return 0 def getScanType(self, processingItemObj): r = self.DBClient.executeAllResults("SELECT SCAN_TYPE FROM Conversion WHERE STUDY = '{0}' AND RID = '{1}' " "AND SCAN_DATE = '{2}' AND S_IDENTIFIER = '{3}' " "AND I_IDENTIFIER = '{4}'".format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date, processingItemObj.s_identifier, processingItemObj.i_identifier)) return r[0][0] def checkNative(self, processingItemObj): orig_ScanType = self.getScanType(processingItemObj) converted_file = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(processingItemObj.converted_folder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, orig_ScanType) nativeFolder = '{0}/native'.format(processingItemObj.root_folder) nativeFileName = '{0}/{1}_{2}{3}{4}{5}_{6}.mnc'.format(nativeFolder, processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, processingItemObj.modality.lower()) if not os.path.exists(nativeFileName): try: distutils.dir_util.mkpath(nativeFolder) shutil.copyfile(converted_file, nativeFileName) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating folders or copying native file. \n {0}'.format(e)) PipelineLogger.log('manager', 'error', 'Setting to restart conversion. \n {0}'.format(e)) sql = "UPDATE Conversion SET CONVERTED = 0, SKIP = 0 WHERE S_IDENTIFIER = '{0}' AND I_IDENTIFIER = '{1}'".format(processingItemObj.s_identifier, processingItemObj.i_identifier) self.DBClient.executeNoResult(sql) return None return nativeFileName def runBeast(self, processingItemObj): nativeFileName = self.checkNative(processingItemObj) if not nativeFileName: return 0 beastFolder = '{0}/beast'.format(processingItemObj.root_folder) logDir = '{0}/logs'.format(processingItemObj.root_folder) PipelineLogger.log('manager', 'info', 'BeAST starting for {0}'.format(nativeFileName)) PipelineLogger.log('manager', 'info', 'Current working folder : {0}'.format(os.getcwd())) try: distutils.dir_util.mkpath(logDir) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating log folder \n {0}'.format(e)) return 0 id = '{0}{1}{2}{3}'.format(processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) beastCMD = 'source /opt/minc-toolkit/minc-toolkit-config.sh; Pipelines/ADNI_T1/ADNI_V1_T1_BeAST {0} {1} {2} {3} {4} {5}'.format(id, nativeFileName, beastFolder, logDir, socket.gethostname(), 50500) try: shutil.rmtree(beastFolder) except: pass try: distutils.dir_util.mkpath(beastFolder) except Exception as e: PipelineLogger.log('manager', 'error', 'Error in creating BeAST folder. \n {0}'.format(e)) return 0 PipelineLogger.log('manager', 'debug', 'Command : {0}'.format(beastCMD)) os.chdir(pc.SourcePath) p = subprocess.Popen(beastCMD, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') out, err = p.communicate() PipelineLogger.log('manager', 'debug', 'Beast Log Output : \n{0}'.format(out.decode("utf-8"))) PipelineLogger.log('manager', 'debug', 'Beast Log Err : \n{0}'.format(err.decode("utf-8"))) QSubJobHandler.submittedJobs[id] = QSubJob(id, '02:00:00', processingItemObj, 'beast') return 1 def runCivet(self, processingItemObj, maskStatus): nativeFileName = self.checkNative(processingItemObj) if not nativeFileName: return 0 copyFolder = pc.T1TempDirForCIVETProcessing subjectFileName_base = '{0}_{1}{2}{3}{4}_{5}'.format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier, processingItemObj.modality.lower()) jobId = '{0}_{1}_{2}_{3}{4}{5}{6}_CIVETRUN'.format(processingItemObj.study, processingItemObj.modality, processingItemObj.table_id, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) checkJobPresentSql = "SELECT * FROM externalWaitingJobs WHERE JOB_ID = '{0}'".format(jobId) if len(self.DBClient.executeAllResults(checkJobPresentSql)) is 0: beastFileName = '{0}/beast/mask/{1}_skull_mask_native.mnc'.format(processingItemObj.root_folder, subjectFileName_base) beastMaskName_base = '{0}_{1}{2}{3}{4}_mask'.format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) beastMaskName = '{0}/{1}.mnc'.format(copyFolder, beastMaskName_base) manualFileName = '{0}/manual/mask/{1}_skull_mask_native.mnc'.format(processingItemObj.root_folder, subjectFileName_base) manualMaskName_base = '{0}_{1}{2}{3}{4}_mask'.format(processingItemObj.study, processingItemObj.subject_rid, processingItemObj.scan_date.replace('-', ''), processingItemObj.s_identifier, processingItemObj.i_identifier) manualMaskName = '{0}/{1}.mnc'.format(copyFolder, manualMaskName_base) try: distutils.file_util.copy_file(nativeFileName, copyFolder) if maskStatus == 'B': distutils.file_util.copy_file(beastFileName, beastMaskName) elif maskStatus == 'M': distutils.file_util.copy_file(manualFileName, manualMaskName) elif maskStatus == 'N': pass else: PipelineLogger.log('manager', 'error', 'Unknown mask status - {0} Entry : Processing ID - {1}, Table ID - {3}'.format(maskStatus, processingItemObj.processing_rid, processingItemObj.table_id)) addExternalJobSQL = "INSERT INTO externalWaitingJobs VALUES ('{0}', '{1}', '{2}', NULL, NULL, NULL)".format(jobId, '{0}_{1}_Pipeline'.format(processingItemObj.study, processingItemObj.modality), 'CIVET') self.DBClient.executeNoResult(addExternalJobSQL) except Exception as e: PipelineLogger.log('manager', 'error', 'Error copying for CIVET input. Rolling back... - Processing Table ID -> {0} Table ID -> {1}'.format( processingItemObj.processing_rid, processingItemObj.table_id)) PipelineLogger.log('manager', 'exception', e) nativeFileOnCopyFolder = '{0}/{1}'.format(copyFolder, os.path.basename(nativeFileName)) os.remove(nativeFileOnCopyFolder) if os.path.exists(nativeFileOnCopyFolder) else None os.remove(beastMaskName) if os.path.exists(beastMaskName) else None os.remove(manualMaskName) if os.path.exists(manualMaskName) else None def requestQC(self, processingItemObj, qctype): qcFieldDict = dict(civet='QC', beast='BEAST_QC') qcFolderDict = { 'civet' : '{0}/civet'.format(processingItemObj.root_folder), 'beast' : '{0}/beast'.format(processingItemObj.root_folder)} tablename = '{0}_{1}_Pipeline'.format(processingItemObj.study, processingItemObj.modality) self.QCHandler.requestQC(processingItemObj.study, tablename, processingItemObj.table_id, qcFieldDict[qctype], qctype, qcFolderDict[qctype])
class PipelineHandler: def __init__(self): self.processingPPDict = { 'ADNI': { 'V1': { 'T1': ADNI_V1_T1(), 'FMRI': ADNI_V1_FMRI(), 'AV45': ADNI_V1_AV45(), 'FDG': ADNI_V1_FDG(), 'AV1451': ADNI_V1_AV1451() }, 'V2': { 'T1': ADNI_V1_T1(), 'FMRI': ADNI_V1_FMRI(), 'AV45': ADNI_V2_AV45(), 'FDG': ADNI_V2_FDG(), 'AV1451': ADNI_V2_AV1451() } }, 'DIAN': { 'V1': { 'T1': DIAN_V1_T1(), 'FDG': DIAN_V1_FDG(), 'PIB': DIAN_V1_PIB() } } } self.DBClient = DbUtils() self.QCH = QCHandler() def checkExternalJobs(self, study, modality): getExtJobSql = "SELECT * FROM externalWaitingJobs WHERE JOB_ID LIKE '{0}_{1}_%'".format( study, modality) extJobs = self.DBClient.executeAllResults(getExtJobSql) for job in extJobs: jobType = job[0].split('_')[-1] reportTable = job[1] tableID = job[0].split('_')[2] reportField = job[2] subjectScanID = job[0].split('_')[3] success = 0 if jobType == 'CIVETRUN': if glob.glob('{0}/{1}_{2}_*'.format( PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID)): getProccessRecSql = "SELECT * FROM Processing WHERE RECORD_ID IN (SELECT PROCESSING_TID FROM {0}_T1_Pipeline WHERE RECORD_ID = {1})".format( study, tableID) processingEntry = self.DBClient.executeAllResults( getProccessRecSql)[0] civetFolder = '{0}/civet'.format(processingEntry[8]) if os.path.exists(civetFolder): shutil.rmtree(civetFolder) try: PipelineLogger.log( 'manager', 'info', 'Copying - {0} -> {1}'.format( glob.glob('{0}/{1}_{2}_*'.format( PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID))[0], civetFolder)) dir_util.copy_tree( glob.glob('{0}/{1}_{2}_*'.format( PipelineConfig.T1TempDirForCIVETDownload, study, subjectScanID))[0], civetFolder) success = 1 except: success = 0 else: continue else: PipelineLogger.log( 'manager', 'error', 'Unknown external job type - {}'.format(jobType)) if success: updateSQL = "UPDATE {0} SET {1} = 1 WHERE RECORD_ID = {2}".format( reportTable, reportField, tableID) self.DBClient.executeNoResult(updateSQL) if jobType == 'CIVETRUN': finishSQL = "UPDATE {0} SET FINISHED = 1 WHERE RECORD_ID = {1}".format( reportTable, tableID) self.DBClient.executeNoResult(finishSQL) modal_table = reportTable modal_tableId = tableID qcField = 'QC' qctype = 'civet' qcFolder = civetFolder self.QCH.requestQC(study, modal_table, modal_tableId, qcField, qctype, qcFolder) rmSql = "DELETE FROM externalWaitingJobs WHERE JOB_ID LIKE '{0}_{1}_{2}_{3}_%'".format( study, modality, tableID, subjectScanID) self.DBClient.executeNoResult(rmSql) def process(self, study, modality): os.environ['PATH'] = ':'.join(libpath.PATH) os.environ['LD_LIBRARY_PATH'] = ':'.join(libpath.LD_LIBRARY_PATH) os.environ['LD_LIBRARYN32_PATH'] = ':'.join(libpath.LD_LIBRARYN32_PATH) os.environ['PERL5LIB'] = ':'.join(libpath.PERL5LIB) os.environ['MNI_DATAPATH'] = ':'.join(libpath.MNI_DATAPATH) os.environ['ROOT'] = ';'.join(libpath.ROOT) os.environ['MINC_TOOLKIT_VERSION'] = libpath.MINC_TOOLKIT_VERSION os.environ['MINC_COMPRESS'] = libpath.MINC_COMPRESS os.environ['MINC_FORCE_V2'] = libpath.MINC_FORCE_V2 toProcessinModalityPerStudy = self.DBClient.executeAllResults( "SELECT * FROM Processing INNER JOIN (SELECT * FROM {0}_{1}_Pipeline WHERE NOT (FINISHED OR SKIP)) as TMP ON Processing.RECORD_ID=TMP.PROCESSING_TID" .format(study, modality)) for processingItem in toProcessinModalityPerStudy: version = processingItem[10] # Calling on the process .section of given studies and modalities self.processingPPDict[study][version][modality].process( processingItem) return 0 def addToPipelineTable(self, processingObj): study = processingObj.study version = processingObj.version modality = processingObj.modality r_id = processingObj.record_id addToTableDict = dict( T1= "INSERT IGNORE INTO {0}_T1_Pipeline VALUES (NULL, {1}, \"{2}\", 0, 0, 0, 0, 0, 0, 0, 0, 0, NULL, NULL)" .format(study, r_id, PipelineConfig.defaultT1config), AV45= "INSERT IGNORE INTO {0}_AV45_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)" .format(study, r_id, PipelineConfig.defaultAV45config, ''), AV1451= "INSERT IGNORE INTO {0}_AV1451_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)" .format(study, r_id, PipelineConfig.defaultAV1451config, ''), FDG= "INSERT IGNORE INTO {0}_FDG_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)" .format(study, r_id, PipelineConfig.defaultFDGconfig, ''), PIB= "INSERT IGNORE INTO {0}_PIB_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)" .format(study, r_id, PipelineConfig.defaultFDGconfig, ''), FMRI= "INSERT IGNORE INTO {0}_FMRI_Pipeline VALUES (NULL, {1}, \"{2}\", '{3}', 0, 0, 0, NULL, NULL)" .format(study, r_id, PipelineConfig.defaultFMRIconfig, 'NIAK_STH_COMESHERE')) self.DBClient.executeNoResult(addToTableDict[modality])
class QSubJobStatusReporter: def __init__(self): self.DBClient = DbUtils() self.QCHandler = QCHandler() def setStatus(self, job, status): if job.jobType == 'beast': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = 'UPDATE {0} SET BEAST_MASK = 1 WHERE RECORD_ID = {1}'.format( table, table_id) elif status == 'Fail': setSql = 'UPDATE {0} SET BEAST_MASK = -1, BEAST_SKIP = 1 WHERE RECORD_ID = {1}'.format( table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log( 'manager', 'error', 'QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}' .format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'av45': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format( table, table_id) self.requestQC(nestedJob, 'av45') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format( table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log( 'manager', 'error', 'QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}' .format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'av1451': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format( table, table_id) self.requestQC(nestedJob, 'av1451') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format( table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log( 'manager', 'error', 'QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}' .format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'fdg': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format( table, table_id) self.requestQC(nestedJob, 'fdg') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format( table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log( 'manager', 'error', 'QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}' .format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) if job.jobType == 'pib': nestedJob = job.job table = '{0}_{1}_Pipeline'.format(nestedJob.study, nestedJob.modality) table_id = nestedJob.table_id if status == 'Success': setSql = "UPDATE {0} SET FINISHED = 1, PROC_Failed = Null WHERE RECORD_ID = {1}".format( table, table_id) self.requestQC(nestedJob, 'pib') elif status == 'Fail': setSql = "UPDATE {0} SET PROC_Failed = 'Failed' , SKIP = 1 WHERE RECORD_ID = {1}".format( table, table_id) self.DBClient.executeNoResult(setSql) if status == 'Fail': PipelineLogger.log( 'manager', 'error', 'QSUB job Status Failed: - {0} - Processing Table ID : {1} - Modality Table ID : {2}' .format(job.jobType, nestedJob.processing_rid, nestedJob.table_id)) def requestQC(self, processingItemObj, qctype): qcFieldDict = dict(civet='QC', beast='BEAST_QC', av45='QC', fdg='QC', av1451='QC', pib='QC') qcFolderDict = { 'civet': '{0}/civet'.format(processingItemObj.root_folder), 'beast': '{0}/beast'.format(processingItemObj.root_folder), 'av45': '{0}/processed'.format(processingItemObj.root_folder), 'av1451': '{0}/processed'.format(processingItemObj.root_folder), 'fdg': '{0}/processed'.format(processingItemObj.root_folder), 'pib': '{0}/processed'.format(processingItemObj.root_folder) } self.QCHandler.requestQC( processingItemObj.study, '{0}_{1}_Pipeline'.format(processingItemObj.study, processingItemObj.modality), processingItemObj.table_id, qcFieldDict[qctype], qctype, qcFolderDict[qctype])