__author__ = 'sulantha'
import Config.EmailConfig as ec
from Utils.EmailClient import EmailClient
emailClient = EmailClient()
emailClient.send_email(ec.EmailRecList_admin, 'First test email subject',
'Body of text. ')
__author__ = 'sulantha' import Config.EmailConfig as ec from Utils.EmailClient import EmailClient emailClient = EmailClient() emailClient.send_email(ec.EmailRecList_admin, 'First test email subject', 'Body of text. ')
def main():
# Start an email client and logging capacity
emailClient = EmailClient()
logging.config.fileConfig('Config/LoggingConfig.conf')
studyList = None
## Added ability to run from command line.
try:
## Open up for arguments
parser = argparse.ArgumentParser()
parser.add_argument('--studyList', required=True, nargs='+', choices=StudyConfig.AllowedStudyList, help='Space seperated study list.')
parser.add_argument('--steps', required=False, nargs='+', choices=StudyConfig.AllowedStepsList, help='Space seperated steps list.')
parser.add_argument('--modalities', required=False, nargs='+', choices=StudyConfig.AllowedModalityList, help='Space seperated modality list.')
parser.add_argument('--pipe_v', required=False, nargs='+', choices=StudyConfig.AllowedVersions, help='Version of pipeline need to run.')
args = parser.parse_args()
studyList = args.studyList
steps = args.steps
modalities = args.modalities
version = args.pipe_v
if not validateStepSequence(steps):
sys.exit(2)
steps = ['ALL'] if not steps else steps
modalities = StudyConfig.AllowedModalityList if not modalities else modalities
if version and len(version) > 1 and len(studyList) > 1:
PipelineLogger.log('root', 'info', 'Versioning with multiple studies is not supported. ')
sys.exit(2)
version = StudyConfig.defaultVersioningForStudy[studyList[0]] if not version else dict(zip(modalities, version))
PipelineLogger.log('root', 'info', '##################Pipeline Started.#################')
PipelineLogger.log('root', 'info', 'StudyIds = %s' %', '.join(map(str, studyList)))
PipelineLogger.log('root', 'info', 'Steps = %s' %', '.join(map(str, steps)))
PipelineLogger.log('root', 'info', 'Version = %s' %version)
pipeline = PipelineManager(studyList, version)
####ToDo: Process steps sequence.
## Recurse for new data
PipelineLogger.log('root', 'info', 'Recursing for new data started ...')
pipeline.recurseForNewData()
PipelineLogger.log('root', 'info', 'Recursing for new data done ...############')
## Add data to Sorting table.
pipeline.addNewDatatoDB()
##Get Unmoved Raw File List
pipeline.getUnmovedRawDataList()
PipelineLogger.log('root', 'info', 'Moving new data started ...')
pipeline.moveRawData()
PipelineLogger.log('root', 'info', 'Moving new data done ...############')
pipeline.getConversionList()
PipelineLogger.log('root', 'info', 'Converting to MINC started ...')
pipeline.convertRawData()
PipelineLogger.log('root', 'info', 'Converting to MINC done ...############')
PipelineLogger.log('root', 'info', 'Modifying processing pipeline table. This may take a while. Please wait....############')
pipeline.getConvertedList() # Get all files that have been converted, from Conversion table
pipeline.refreshModalityTables() # Get the correct modality, insert or ignore information in Processing table, takes quite some time
pipeline.getProcessList() # Returning all rows from the Processing table that has not been processed or skipped, takes time too
pipeline.fillPipelineTables() # Creating jobs queue in {study}_{modality}_Pipeline table, takes time too
for modality in modalities:
if modality == 'BLUFF':
break
pipeline.checkExternalJobs(modality) # Update list of external waiting jobs in externalWaitingJobs table
for modality in modalities:
if modality == 'BLUFF':
break
pipeline.checkOnQCJobs(modality) # Update jobs with success QC in Processing table
for modality in modalities:
if modality == 'BLUFF':
break
pipeline.processModality(modality) # Actually processing jobs
pipeline.qsubJobHandler.submittedJobs['xxxx'].Fin = True
PipelineLogger.log('root', 'info', 'Pipeline reached end. Waiting on submitted jobs. ')
#### End
if not pipeline.qsubJobHandler.submittedJobs:
PipelineLogger.log('root', 'info', 'No QSUB Jobs in waiting ...############')
PipelineLogger.log('root', 'info', 'Pipeline exiting ...############')
PipelineLogger.log('root', 'info', '##################Pipeline Done.#################')
pipeline.qsubJobHandler.QUIT = 1
except:
PipelineLogger.log('root', 'exception', 'Pipeline crashed with exception. ')
emailClient.send_email(ec.EmailRecList_admin, 'Pipeline crashed with exception. ', ' {0} '.format(traceback.format_exc()))
def main():
# Start an email client and logging capacity
emailClient = EmailClient()
logging.config.fileConfig("Config/LoggingConfig.conf")
studyList = None
## Added ability to run from command line.
try:
## Open up for arguments
parser = argparse.ArgumentParser()
parser.add_argument(
"--studyList",
required=True,
nargs="+",
choices=StudyConfig.AllowedStudyList,
help="Space seperated study list.",
)
parser.add_argument(
"--steps",
required=False,
nargs="+",
choices=StudyConfig.AllowedStepsList,
help="Space seperated steps list.",
)
parser.add_argument(
"--modalities",
required=False,
nargs="+",
choices=StudyConfig.AllowedModalityList,
help="Space seperated modality list.",
)
parser.add_argument(
"--pipe_v",
required=False,
nargs="+",
choices=StudyConfig.AllowedVersions,
help="Version of pipeline need to run.",
)
args = parser.parse_args()
studyList = args.studyList
steps = args.steps
modalities = args.modalities
version = args.pipe_v
if not validateStepSequence(steps):
sys.exit(2)
steps = ["ALL"] if not steps else steps
modalities = StudyConfig.AllowedModalityList if not modalities else modalities
if version and len(version) > 1 and len(studyList) > 1:
PipelineLogger.log("root", "info", "Versioning with multiple studies is not supported. ")
sys.exit(2)
version = StudyConfig.defaultVersioningForStudy[studyList[0]] if not version else dict(zip(modalities, version))
PipelineLogger.log("root", "info", "##################Pipeline Started.#################")
PipelineLogger.log("root", "info", "StudyIds = %s" % ", ".join(map(str, studyList)))
PipelineLogger.log("root", "info", "Steps = %s" % ", ".join(map(str, steps)))
PipelineLogger.log("root", "info", "Version = %s" % version)
pipeline = PipelineManager(studyList, version)
####ToDo: Process steps sequence.
## Recurse for new data
PipelineLogger.log("root", "info", "Recursing for new data started ...")
pipeline.recurseForNewData()
PipelineLogger.log("root", "info", "Recursing for new data done ...############")
## Add data to Sorting table.
pipeline.addNewDatatoDB()
##Get Unmoved Raw File List
pipeline.getUnmovedRawDataList()
PipelineLogger.log("root", "info", "Moving new data started ...")
pipeline.moveRawData()
PipelineLogger.log("root", "info", "Moving new data done ...############")
pipeline.getConversionList()
PipelineLogger.log("root", "info", "Converting to MINC started ...")
pipeline.convertRawData()
PipelineLogger.log("root", "info", "Converting to MINC done ...############")
PipelineLogger.log(
"root", "info", "Modifying processing pipeline table. This may take a while. Please wait....############"
)
pipeline.getConvertedList() # Get all files that have been converted, from Conversion table
pipeline.refreshModalityTables() # Get the correct modality, insert or ignore information in Processing table, takes quite some time
pipeline.getProcessList() # Returning all rows from the Processing table that has not been processed or skipped, takes time too
pipeline.fillPipelineTables() # Creating jobs queue in {study}_{modality}_Pipeline table, takes time too
for modality in modalities:
if modality == "BLUFF":
break
pipeline.checkExternalJobs(modality) # Update list of external waiting jobs in externalWaitingJobs table
for modality in modalities:
if modality == "BLUFF":
break
pipeline.checkOnQCJobs(modality) # Update jobs with success QC in Processing table
for modality in modalities:
if modality == "BLUFF":
break
pipeline.processModality(modality) # Actually processing jobs
QSubJobHandler.submittedJobs["xxxx"].Fin = True
PipelineLogger.log("root", "info", "Pipeline reached end. Waiting on submitted jobs. ")
#### End
if not QSubJobHandler.submittedJobs:
PipelineLogger.log("root", "info", "No QSUB Jobs in waiting ...############")
PipelineLogger.log("root", "info", "Pipeline exiting ...############")
PipelineLogger.log("root", "info", "##################Pipeline Done.#################")
pipeline.qsubJobHandler.QUIT = 1
except:
PipelineLogger.log("root", "exception", "Pipeline crashed with exception. ")
emailClient.send_email(
ec.EmailRecList_admin, "Pipeline crashed with exception. ", " {0} ".format(traceback.format_exc())
)
