def run(options):
logging.logPrint('Starting')
batchConfig = config.configFromStream(open(options.configFile), lazy=True)
machineConf = config.configFromStream(open('/tmp/machine.conf'))
state = State(
options.workflowConfig, options.batchStatesFile,
_validateWrapper(
batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'),
pipeline_misc.determineWrapper(
machineConf,
batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'))),
_interpretBatchFile(options.batchFile),
_extractInnerPipelineConfig(batchConfig),
batchConfig('pipeline.PIPELINE_WRAPPER_NAME'),
int(batchConfig('batch.options.CONCURRENT_PRERUN')),
int(batchConfig('batch.options.CONCURRENT_PIPELINES')),
int(batchConfig('batch.options.CONCURRENT_POSTRUN')))
logging.logPrint('Queuing any incomplete work')
queueCount = _queueIncompleteWork(state)
logging.logPrint('Queued: %d' % queueCount)
if state.pipelinesQueue.hasWork():
yield defer_work_queue.waitForCompletion(state.pipelinesQueue)
for batchState in state.batchStates.values():
if 'state' not in batchState or batchState['state'] == 'failed':
raise JobFailed()
def run(options):
logging.logPrint('Starting')
batchConfig = config.configFromStream(open(options.configFile), lazy=True)
machineConf = config.configFromStream(open('/tmp/machine.conf'))
state = State(options.workflowConfig,
options.batchStatesFile,
_validateWrapper(batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'),
pipeline_misc.determineWrapper(machineConf,
batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'))),
_interpretBatchFile(options.batchFile),
_extractInnerPipelineConfig(batchConfig),
batchConfig('pipeline.PIPELINE_WRAPPER_NAME'),
int(batchConfig('batch.options.CONCURRENT_PRERUN')),
int(batchConfig('batch.options.CONCURRENT_PIPELINES')),
int(batchConfig('batch.options.CONCURRENT_POSTRUN')))
logging.logPrint('Queuing any incomplete work')
queueCount = _queueIncompleteWork(state)
logging.logPrint('Queued: %d' % queueCount)
if state.pipelinesQueue.hasWork():
yield defer_work_queue.waitForCompletion(state.pipelinesQueue)
for batchState in state.batchStates.values():
if 'state' not in batchState or batchState['state'] == 'failed':
raise JobFailed()
def _determineProtocol(request):
if request.body['bare_run']:
return request.body['config']['pipeline.PIPELINE_TEMPLATE']
else:
return pipeline_misc.determineWrapper(
request.state.machineconf,
request.body['config']['pipeline.PIPELINE_TEMPLATE'])
def pipelineToDict(self, pipeline):
protocolConf = protocol_format.load(self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE"))
inputTagsList = [
pipeline.config(k).split(",")
for k, v in protocolConf
if v.get("type").split()[0] in ["dataset", "blastdb_dataset", "paired_dataset", "singleton_dataset"]
and pipeline.config(k)
]
inputTags = []
for i in inputTagsList:
inputTags.extend(i)
possibleOutputTags = set(
[
pipeline.pipelineName + "_" + t.strip()
for t in pipeline.config("output.TAGS_TO_DOWNLOAD", default="").split(",")
]
)
query = [{"tag_name": t} for t in possibleOutputTags]
tags = yield www_tags.loadTagsBy("localhost", "local", pipeline.userName, {"$or": query}, False)
tags = set([t["tag_name"] for t in tags])
outputTags = list(tags & possibleOutputTags)
pipelineTask = yield tasks_tx.loadTask(pipeline.taskName)
pipelineWrapper = pipeline_misc.determineWrapper(
self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE")
)
pipelineDict = {
"pipeline_id": pipeline.pipelineId,
"pipeline_name": pipeline.pipelineName,
"user_name": pipeline.userName,
"wrapper": pipeline.protocol == pipelineWrapper,
"protocol": pipeline.config("pipeline.PIPELINE_TEMPLATE"),
"checksum": pipeline.checksum,
"task_name": pipeline.taskName,
"queue": pipeline.queue,
"children": pipeline.children,
"state": pipelineTask.state,
"num_steps": pipelineTask.numTasks,
"num_complete": pipelineTask.completedTasks,
"input_tags": inputTags,
"output_tags": outputTags,
"pipeline_desc": pipeline.config("pipeline.PIPELINE_DESC", default=""),
"config": config.configToDict(pipeline.config, lazy=True),
}
defer.returnValue(pipelineDict)
def pipelineToDict(self, pipeline):
protocolConf = protocol_format.load(self.machineConf, pipeline.config('pipeline.PIPELINE_TEMPLATE'))
inputTagsList = [pipeline.config(k).split(',')
for k, v in protocolConf
if v.get('type').split()[0] in ['dataset',
'blastdb_dataset',
'paired_dataset',
'singleton_dataset'] and pipeline.config(k)]
inputTags = []
for i in inputTagsList:
inputTags.extend(i)
possibleOutputTags = set([pipeline.pipelineName + '_' + t.strip()
for t in pipeline.config('output.TAGS_TO_DOWNLOAD', default='').split(',')])
query = [{'tag_name': t} for t in possibleOutputTags]
tags = yield www_tags.loadTagsBy('localhost', 'local', pipeline.userName, {'$or': query}, False)
tags = set([t['tag_name'] for t in tags])
outputTags = list(tags & possibleOutputTags)
pipelineTask = yield tasks_tx.loadTask(pipeline.taskName)
pipelineWrapper = pipeline_misc.determineWrapper(self.machineConf,
pipeline.config('pipeline.PIPELINE_TEMPLATE'))
pipelineDict = {'pipeline_id': pipeline.pipelineId,
'pipeline_name': pipeline.pipelineName,
'user_name': pipeline.userName,
'wrapper': pipeline.protocol == pipelineWrapper,
'protocol': pipeline.config('pipeline.PIPELINE_TEMPLATE'),
'checksum': pipeline.checksum,
'task_name': pipeline.taskName,
'queue': pipeline.queue,
'children': pipeline.children,
'state': pipelineTask.state,
'num_steps': pipelineTask.numTasks,
'num_complete': pipelineTask.completedTasks,
'input_tags': inputTags,
'output_tags': outputTags,
'pipeline_desc': pipeline.config('pipeline.PIPELINE_DESC', default=''),
'config': config.configToDict(pipeline.config, lazy=True),
}
defer.returnValue(pipelineDict)
def _determineProtocol(request):
if request.body['bare_run']:
return request.body['config']['pipeline.PIPELINE_TEMPLATE']
else:
return pipeline_misc.determineWrapper(request.state.machineconf,
request.body['config']['pipeline.PIPELINE_TEMPLATE'])
def handleWWWListProtocols(request):
"""
Input:
{ cluster: string
?criteria: { key/value }
?detail: boolean
?batch_mode: boolean
}
Output:
[{ protocol: string, config: [ { key/value } ]}]
"""
def _removeAlwaysHidden(protocolConfig):
return [
pc for pc in protocolConfig
if pc[1].get('visibility') != 'always_hidden'
]
# Create a criterial filter called protocolF
if 'criteria' in request.body:
if '$or' in request.body['criteria']:
protocolNames = [
p['protocol'] for p in request.body['criteria']['$or']
]
protocolF = lambda p: p in protocolNames
elif 'protocol' in request.body['criteria']:
protocolF = lambda p: p == request.body['criteria']['protocol']
else:
protocolF = lambda _: True
else:
protocolF = lambda _: True
# Get all protocols and filter out the ones we don't need
protocols = [
p for p in protocol_format.protocols(request.state.machineconf)
if protocolF(p)
]
if request.body.get('batch_mode'):
batchConfig = [
func.updateDict(c[1], {'name': c[0]}) for c in _removeAlwaysHidden(
protocol_format.load(request.state.machineconf,
'clovr_batch_wrapper'))
]
else:
batchConfig = []
protocolConfs = []
for p in protocols:
protocolConfig = protocol_format.load(request.state.machineconf, p)
wrapperName = pipeline_misc.determineWrapper(request.state.machineconf,
p)
if wrapperName != p:
protocolKeys = set([k for k, _ in protocolConfig])
wrapperConfig = [(k, v) for (k, v) in protocol_format.load(
request.state.machineconf, wrapperName)
if k not in protocolKeys]
protocolConfig += wrapperConfig
if request.body.get('detail', False):
conf = [
func.updateDict(c[1], {'name': c[0]})
for c in _removeAlwaysHidden(protocolConfig)
]
if request.body.get('batch_mode', False):
conf = batchConfig + [
func.updateDict(c, {'name': 'batch_pipeline.' + c['name']})
for c in conf
]
else:
conf = []
protocolConfs.append({'protocol': p, 'config': conf})
return defer_pipe.ret(request.update(response=protocolConfs))
def handleWWWListProtocols(request):
"""
Input:
{ cluster: string
?criteria: { key/value }
?detail: boolean
?batch_mode: boolean
}
Output:
[{ protocol: string, config: [ { key/value } ]}]
"""
def _removeAlwaysHidden(protocolConfig):
return [pc
for pc in protocolConfig
if pc[1].get('visibility') != 'always_hidden']
# Create a criterial filter called protocolF
if 'criteria' in request.body:
if '$or' in request.body['criteria']:
protocolNames = [p['protocol'] for p in request.body['criteria']['$or']]
protocolF = lambda p : p in protocolNames
elif 'protocol' in request.body['criteria']:
protocolF = lambda p : p == request.body['criteria']['protocol']
else:
protocolF = lambda _ : True
else:
protocolF = lambda _ : True
# Get all protocols and filter out the ones we don't need
protocols = [p
for p in protocol_format.protocols(request.state.machineconf)
if protocolF(p)]
if request.body.get('batch_mode'):
batchConfig = [func.updateDict(c[1], {'name': c[0]})
for c in _removeAlwaysHidden(protocol_format.load(request.state.machineconf,
'clovr_batch_wrapper'))]
else:
batchConfig = []
protocolConfs = []
for p in protocols:
protocolConfig = protocol_format.load(request.state.machineconf, p)
wrapperName = pipeline_misc.determineWrapper(request.state.machineconf, p)
if wrapperName != p:
protocolKeys = set([k for k, _ in protocolConfig])
wrapperConfig = [(k, v)
for (k,v) in protocol_format.load(request.state.machineconf,
wrapperName)
if k not in protocolKeys]
protocolConfig += wrapperConfig
if request.body.get('detail', False):
conf = [func.updateDict(c[1], {'name': c[0]})
for c in _removeAlwaysHidden(protocolConfig)]
if request.body.get('batch_mode', False):
conf = batchConfig + [func.updateDict(c, {'name': 'batch_pipeline.' + c['name']})
for c in conf]
else:
conf = []
protocolConfs.append({'protocol': p, 'config': conf})
return defer_pipe.ret(request.update(response=protocolConfs))