def validatePipelineConfig(request): validateState = func.Record(conf=request.state.conf, machineconf=request.state.machineconf, mq=request.mq) protocolConf = protocol_format.load(request.state.machineconf, request.body['config']['pipeline.PIPELINE_TEMPLATE']) pipelineWrapper = determineWrapper(request.state.machineconf, request.body['config']['pipeline.PIPELINE_TEMPLATE']) if not request.body['bare_run']: protocolConf += protocol_format.load(request.state.machineconf, pipelineWrapper) protocol_format.applyProtocol(protocolConf, request.body['config']) return pipeline_validate.validate(validateState, protocolConf, request.body['config'])
def pipelineToDict(self, pipeline): protocolConf = protocol_format.load(self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE")) inputTagsList = [ pipeline.config(k).split(",") for k, v in protocolConf if v.get("type").split()[0] in ["dataset", "blastdb_dataset", "paired_dataset", "singleton_dataset"] and pipeline.config(k) ] inputTags = [] for i in inputTagsList: inputTags.extend(i) possibleOutputTags = set( [ pipeline.pipelineName + "_" + t.strip() for t in pipeline.config("output.TAGS_TO_DOWNLOAD", default="").split(",") ] ) query = [{"tag_name": t} for t in possibleOutputTags] tags = yield www_tags.loadTagsBy("localhost", "local", pipeline.userName, {"$or": query}, False) tags = set([t["tag_name"] for t in tags]) outputTags = list(tags & possibleOutputTags) pipelineTask = yield tasks_tx.loadTask(pipeline.taskName) pipelineWrapper = pipeline_misc.determineWrapper( self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE") ) pipelineDict = { "pipeline_id": pipeline.pipelineId, "pipeline_name": pipeline.pipelineName, "user_name": pipeline.userName, "wrapper": pipeline.protocol == pipelineWrapper, "protocol": pipeline.config("pipeline.PIPELINE_TEMPLATE"), "checksum": pipeline.checksum, "task_name": pipeline.taskName, "queue": pipeline.queue, "children": pipeline.children, "state": pipelineTask.state, "num_steps": pipelineTask.numTasks, "num_complete": pipelineTask.completedTasks, "input_tags": inputTags, "output_tags": outputTags, "pipeline_desc": pipeline.config("pipeline.PIPELINE_DESC", default=""), "config": config.configToDict(pipeline.config, lazy=True), } defer.returnValue(pipelineDict)
def determineWrapper(machineconf, pipelineTemplate): protocolConf = protocol_format.load(machineconf, pipelineTemplate) wrapper = [v for k, v in protocolConf if k == 'pipeline.PIPELINE_WRAPPER'] if wrapper and wrapper[0]['default'] is None: return pipelineTemplate elif wrapper: return wrapper[0]['default'] else: return 'clovr_wrapper'
def pipelineToDict(self, pipeline): protocolConf = protocol_format.load(self.machineConf, pipeline.config('pipeline.PIPELINE_TEMPLATE')) inputTagsList = [pipeline.config(k).split(',') for k, v in protocolConf if v.get('type').split()[0] in ['dataset', 'blastdb_dataset', 'paired_dataset', 'singleton_dataset'] and pipeline.config(k)] inputTags = [] for i in inputTagsList: inputTags.extend(i) possibleOutputTags = set([pipeline.pipelineName + '_' + t.strip() for t in pipeline.config('output.TAGS_TO_DOWNLOAD', default='').split(',')]) query = [{'tag_name': t} for t in possibleOutputTags] tags = yield www_tags.loadTagsBy('localhost', 'local', pipeline.userName, {'$or': query}, False) tags = set([t['tag_name'] for t in tags]) outputTags = list(tags & possibleOutputTags) pipelineTask = yield tasks_tx.loadTask(pipeline.taskName) pipelineWrapper = pipeline_misc.determineWrapper(self.machineConf, pipeline.config('pipeline.PIPELINE_TEMPLATE')) pipelineDict = {'pipeline_id': pipeline.pipelineId, 'pipeline_name': pipeline.pipelineName, 'user_name': pipeline.userName, 'wrapper': pipeline.protocol == pipelineWrapper, 'protocol': pipeline.config('pipeline.PIPELINE_TEMPLATE'), 'checksum': pipeline.checksum, 'task_name': pipeline.taskName, 'queue': pipeline.queue, 'children': pipeline.children, 'state': pipelineTask.state, 'num_steps': pipelineTask.numTasks, 'num_complete': pipelineTask.completedTasks, 'input_tags': inputTags, 'output_tags': outputTags, 'pipeline_desc': pipeline.config('pipeline.PIPELINE_DESC', default=''), 'config': config.configToDict(pipeline.config, lazy=True), } defer.returnValue(pipelineDict)
def handleWWWListProtocols(request): """ Input: { cluster: string ?criteria: { key/value } ?detail: boolean ?batch_mode: boolean } Output: [{ protocol: string, config: [ { key/value } ]}] """ def _removeAlwaysHidden(protocolConfig): return [ pc for pc in protocolConfig if pc[1].get('visibility') != 'always_hidden' ] # Create a criterial filter called protocolF if 'criteria' in request.body: if '$or' in request.body['criteria']: protocolNames = [ p['protocol'] for p in request.body['criteria']['$or'] ] protocolF = lambda p: p in protocolNames elif 'protocol' in request.body['criteria']: protocolF = lambda p: p == request.body['criteria']['protocol'] else: protocolF = lambda _: True else: protocolF = lambda _: True # Get all protocols and filter out the ones we don't need protocols = [ p for p in protocol_format.protocols(request.state.machineconf) if protocolF(p) ] if request.body.get('batch_mode'): batchConfig = [ func.updateDict(c[1], {'name': c[0]}) for c in _removeAlwaysHidden( protocol_format.load(request.state.machineconf, 'clovr_batch_wrapper')) ] else: batchConfig = [] protocolConfs = [] for p in protocols: protocolConfig = protocol_format.load(request.state.machineconf, p) wrapperName = pipeline_misc.determineWrapper(request.state.machineconf, p) if wrapperName != p: protocolKeys = set([k for k, _ in protocolConfig]) wrapperConfig = [(k, v) for (k, v) in protocol_format.load( request.state.machineconf, wrapperName) if k not in protocolKeys] protocolConfig += wrapperConfig if request.body.get('detail', False): conf = [ func.updateDict(c[1], {'name': c[0]}) for c in _removeAlwaysHidden(protocolConfig) ] if request.body.get('batch_mode', False): conf = batchConfig + [ func.updateDict(c, {'name': 'batch_pipeline.' + c['name']}) for c in conf ] else: conf = [] protocolConfs.append({'protocol': p, 'config': conf}) return defer_pipe.ret(request.update(response=protocolConfs))
def handleWWWListProtocols(request): """ Input: { cluster: string ?criteria: { key/value } ?detail: boolean ?batch_mode: boolean } Output: [{ protocol: string, config: [ { key/value } ]}] """ def _removeAlwaysHidden(protocolConfig): return [pc for pc in protocolConfig if pc[1].get('visibility') != 'always_hidden'] # Create a criterial filter called protocolF if 'criteria' in request.body: if '$or' in request.body['criteria']: protocolNames = [p['protocol'] for p in request.body['criteria']['$or']] protocolF = lambda p : p in protocolNames elif 'protocol' in request.body['criteria']: protocolF = lambda p : p == request.body['criteria']['protocol'] else: protocolF = lambda _ : True else: protocolF = lambda _ : True # Get all protocols and filter out the ones we don't need protocols = [p for p in protocol_format.protocols(request.state.machineconf) if protocolF(p)] if request.body.get('batch_mode'): batchConfig = [func.updateDict(c[1], {'name': c[0]}) for c in _removeAlwaysHidden(protocol_format.load(request.state.machineconf, 'clovr_batch_wrapper'))] else: batchConfig = [] protocolConfs = [] for p in protocols: protocolConfig = protocol_format.load(request.state.machineconf, p) wrapperName = pipeline_misc.determineWrapper(request.state.machineconf, p) if wrapperName != p: protocolKeys = set([k for k, _ in protocolConfig]) wrapperConfig = [(k, v) for (k,v) in protocol_format.load(request.state.machineconf, wrapperName) if k not in protocolKeys] protocolConfig += wrapperConfig if request.body.get('detail', False): conf = [func.updateDict(c[1], {'name': c[0]}) for c in _removeAlwaysHidden(protocolConfig)] if request.body.get('batch_mode', False): conf = batchConfig + [func.updateDict(c, {'name': 'batch_pipeline.' + c['name']}) for c in conf] else: conf = [] protocolConfs.append({'protocol': p, 'config': conf}) return defer_pipe.ret(request.update(response=protocolConfs))
def validateBatchPipelineConfig(request): """ For batch configs we unfortunately cannot do as complete of validation as we would like for the inner pipeline. In this case we are going to create a validation scheme on the fly. First we are going to pull out the inner pipeline config and apply its protocol configuration to it. Then we are going to modify the protocol configuration to match config names in the request for the inner pipeline (prepending 'batch_pipeline.'). After that we are going to load the clovr_batch_wrapper protocol conf and add it to the protocol conf we are building. Then we need to modify the protocol conf to not try to validate things we know will not validate. """ def _makeNoop(params): # Turn off any validation if we are going to be replacing a variable # in the variable v['type'] = 'string' v['require_value'] = False # Remove any transform name so it doesn't accidently stomp on a variable # we actually care about if 'type_params' in v and 'transform_name' in v['type_params']: v['type_params'].pop('transform_name') # First things first, rip out the inner pipeline config: innerConfig = dict([(k.split('.', 1)[-1], v) for k, v in request.body['config'].iteritems() if k.startswith('batch_pipeline.')]) protocolConf = protocol_format.load(request.state.machineconf, innerConfig['pipeline.PIPELINE_TEMPLATE']) protocol_format.applyProtocol(protocolConf, innerConfig) # Push the applied protocol back into the request config for k, v in innerConfig.iteritems(): request.body['config']['batch_pipeline.' + k] = v additionalConf = [] for k, v in protocolConf: if v['type'] in ['dataset', 'paired_dataset', 'singleton_dataset']: if ('batch.tag_list.' + k) in request.body['config']: params = dict(v) params['type_params'] = dict(params.get('type_params', {})) typeParams = params['type_params'] typeParams['transform_name'] = 'batch.tag_list.file_list.' + k + '_throwaway' additionalConf.append(('batch.tag_list.' + k, params)) _makeNoop(v) elif ('batch.param_list.' + k) in request.body['config']: params = dict(v) params['type'] = params['type'] + ' list' params['type_params'] = dict(params.get('type_params', {})) typeParams = params['type_params'] typeParams['transform_name'] = 'batch.param_list.file_list.' + k + '_throwaway' additionalConf.append(('batch.param_list.' + k, params)) _makeNoop(v) elif '${BATCH_NUM}' in request.body['config'].get(k, ''): _makeNoop(v) elif ('batch.param_list.' + k) in request.body['config']: params = dict(v) params['type'] = params['type'] + ' list' params['type_params'] = dict(params.get('type_params', {})) typeParams = params['type_params'] typeParams['transform_name'] = 'batch.param_list.transformed.' + k + '_throwaway' elif '${BATCH_NUM}' in request.body['config'].get(k, ''): _makeNoop(v) else: if 'type_params' in v and 'transform_name' in v['type_params']: v['type_params']['transform_name'] = 'batch_pipeline.' + v['type_params']['transform_name'] protocolConf = [('batch_pipeline.' + k, v) for k, v in protocolConf] batchWrapperConf = protocol_format.load(request.state.machineconf, 'clovr_batch_wrapper') protocol_format.applyProtocol(batchWrapperConf, request.body['config']) protocolConf += batchWrapperConf protocolConf += additionalConf validateState = func.Record(conf=request.state.conf, machineconf=request.state.machineconf, mq=request.mq) return pipeline_validate.validate(validateState, protocolConf, request.body['config'])