Ejemplo n.º 1
0
def validatePipelineConfig(request):
    validateState = func.Record(conf=request.state.conf,
                                machineconf=request.state.machineconf,
                                mq=request.mq)
    protocolConf = protocol_format.load(request.state.machineconf,
                                        request.body['config']['pipeline.PIPELINE_TEMPLATE'])

    pipelineWrapper = determineWrapper(request.state.machineconf,
                                       request.body['config']['pipeline.PIPELINE_TEMPLATE'])
    
    if not request.body['bare_run']:
        protocolConf += protocol_format.load(request.state.machineconf,
                                             pipelineWrapper)
        
    protocol_format.applyProtocol(protocolConf, request.body['config'])
    return pipeline_validate.validate(validateState, protocolConf, request.body['config'])    
Ejemplo n.º 2
0
    def pipelineToDict(self, pipeline):
        protocolConf = protocol_format.load(self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE"))

        inputTagsList = [
            pipeline.config(k).split(",")
            for k, v in protocolConf
            if v.get("type").split()[0] in ["dataset", "blastdb_dataset", "paired_dataset", "singleton_dataset"]
            and pipeline.config(k)
        ]
        inputTags = []
        for i in inputTagsList:
            inputTags.extend(i)

        possibleOutputTags = set(
            [
                pipeline.pipelineName + "_" + t.strip()
                for t in pipeline.config("output.TAGS_TO_DOWNLOAD", default="").split(",")
            ]
        )

        query = [{"tag_name": t} for t in possibleOutputTags]

        tags = yield www_tags.loadTagsBy("localhost", "local", pipeline.userName, {"$or": query}, False)

        tags = set([t["tag_name"] for t in tags])

        outputTags = list(tags & possibleOutputTags)

        pipelineTask = yield tasks_tx.loadTask(pipeline.taskName)

        pipelineWrapper = pipeline_misc.determineWrapper(
            self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE")
        )

        pipelineDict = {
            "pipeline_id": pipeline.pipelineId,
            "pipeline_name": pipeline.pipelineName,
            "user_name": pipeline.userName,
            "wrapper": pipeline.protocol == pipelineWrapper,
            "protocol": pipeline.config("pipeline.PIPELINE_TEMPLATE"),
            "checksum": pipeline.checksum,
            "task_name": pipeline.taskName,
            "queue": pipeline.queue,
            "children": pipeline.children,
            "state": pipelineTask.state,
            "num_steps": pipelineTask.numTasks,
            "num_complete": pipelineTask.completedTasks,
            "input_tags": inputTags,
            "output_tags": outputTags,
            "pipeline_desc": pipeline.config("pipeline.PIPELINE_DESC", default=""),
            "config": config.configToDict(pipeline.config, lazy=True),
        }

        defer.returnValue(pipelineDict)
Ejemplo n.º 3
0
def determineWrapper(machineconf, pipelineTemplate):
    protocolConf = protocol_format.load(machineconf,
                                        pipelineTemplate)
    wrapper = [v for k, v in protocolConf if k == 'pipeline.PIPELINE_WRAPPER']

    if wrapper and wrapper[0]['default'] is None:
        return pipelineTemplate
    elif wrapper:
        return wrapper[0]['default']
    else:
        return 'clovr_wrapper'
Ejemplo n.º 4
0
    def pipelineToDict(self, pipeline):
        protocolConf = protocol_format.load(self.machineConf, pipeline.config('pipeline.PIPELINE_TEMPLATE'))

        inputTagsList = [pipeline.config(k).split(',')
                         for k, v in protocolConf
                         if v.get('type').split()[0] in ['dataset',
                                                         'blastdb_dataset',
                                                         'paired_dataset',
                                                         'singleton_dataset'] and pipeline.config(k)]
        inputTags = []
        for i in inputTagsList:
            inputTags.extend(i)


        possibleOutputTags = set([pipeline.pipelineName + '_' + t.strip()
                                  for t in pipeline.config('output.TAGS_TO_DOWNLOAD', default='').split(',')])

        query = [{'tag_name': t} for t in possibleOutputTags]

        tags = yield www_tags.loadTagsBy('localhost', 'local', pipeline.userName, {'$or': query}, False)

        tags = set([t['tag_name'] for t in tags])

        outputTags = list(tags & possibleOutputTags)

        pipelineTask = yield tasks_tx.loadTask(pipeline.taskName)

        pipelineWrapper = pipeline_misc.determineWrapper(self.machineConf,
                                                         pipeline.config('pipeline.PIPELINE_TEMPLATE'))

        pipelineDict = {'pipeline_id': pipeline.pipelineId,
                        'pipeline_name': pipeline.pipelineName,
                        'user_name': pipeline.userName,
                        'wrapper': pipeline.protocol == pipelineWrapper,
                        'protocol': pipeline.config('pipeline.PIPELINE_TEMPLATE'),
                        'checksum': pipeline.checksum,
                        'task_name': pipeline.taskName,
                        'queue': pipeline.queue,
                        'children': pipeline.children,
                        'state': pipelineTask.state,
                        'num_steps': pipelineTask.numTasks,
                        'num_complete': pipelineTask.completedTasks,
                        'input_tags': inputTags,
                        'output_tags': outputTags,
                        'pipeline_desc': pipeline.config('pipeline.PIPELINE_DESC', default=''),
                        'config': config.configToDict(pipeline.config, lazy=True),
                        }

        defer.returnValue(pipelineDict)
Ejemplo n.º 5
0
def handleWWWListProtocols(request):
    """
    Input:
    { cluster: string
      ?criteria: { key/value }
      ?detail: boolean
      ?batch_mode: boolean
    }
    Output:
    [{ protocol: string, config: [ { key/value } ]}]
    """
    def _removeAlwaysHidden(protocolConfig):
        return [
            pc for pc in protocolConfig
            if pc[1].get('visibility') != 'always_hidden'
        ]

    # Create a criterial filter called protocolF
    if 'criteria' in request.body:
        if '$or' in request.body['criteria']:
            protocolNames = [
                p['protocol'] for p in request.body['criteria']['$or']
            ]
            protocolF = lambda p: p in protocolNames
        elif 'protocol' in request.body['criteria']:
            protocolF = lambda p: p == request.body['criteria']['protocol']
        else:
            protocolF = lambda _: True
    else:
        protocolF = lambda _: True

    # Get all protocols and filter out the ones we don't need
    protocols = [
        p for p in protocol_format.protocols(request.state.machineconf)
        if protocolF(p)
    ]

    if request.body.get('batch_mode'):
        batchConfig = [
            func.updateDict(c[1], {'name': c[0]}) for c in _removeAlwaysHidden(
                protocol_format.load(request.state.machineconf,
                                     'clovr_batch_wrapper'))
        ]
    else:
        batchConfig = []

    protocolConfs = []
    for p in protocols:
        protocolConfig = protocol_format.load(request.state.machineconf, p)
        wrapperName = pipeline_misc.determineWrapper(request.state.machineconf,
                                                     p)
        if wrapperName != p:
            protocolKeys = set([k for k, _ in protocolConfig])
            wrapperConfig = [(k, v) for (k, v) in protocol_format.load(
                request.state.machineconf, wrapperName)
                             if k not in protocolKeys]

            protocolConfig += wrapperConfig

        if request.body.get('detail', False):
            conf = [
                func.updateDict(c[1], {'name': c[0]})
                for c in _removeAlwaysHidden(protocolConfig)
            ]
            if request.body.get('batch_mode', False):
                conf = batchConfig + [
                    func.updateDict(c, {'name': 'batch_pipeline.' + c['name']})
                    for c in conf
                ]
        else:
            conf = []

        protocolConfs.append({'protocol': p, 'config': conf})

    return defer_pipe.ret(request.update(response=protocolConfs))
Ejemplo n.º 6
0
def handleWWWListProtocols(request):
    """
    Input:
    { cluster: string
      ?criteria: { key/value }
      ?detail: boolean
      ?batch_mode: boolean
    }
    Output:
    [{ protocol: string, config: [ { key/value } ]}]
    """
    def _removeAlwaysHidden(protocolConfig):
        return [pc
                for pc in protocolConfig
                if pc[1].get('visibility') != 'always_hidden']

    # Create a criterial filter called protocolF
    if 'criteria' in request.body:
        if '$or' in request.body['criteria']:
            protocolNames = [p['protocol'] for p in request.body['criteria']['$or']]
            protocolF = lambda p : p in protocolNames
        elif 'protocol' in request.body['criteria']:
            protocolF = lambda p : p == request.body['criteria']['protocol']
        else:
            protocolF = lambda _ : True
    else:
        protocolF = lambda _ : True

    # Get all protocols and filter out the ones we don't need
    protocols = [p
                 for p in protocol_format.protocols(request.state.machineconf)
                 if protocolF(p)]

    if request.body.get('batch_mode'):
        batchConfig = [func.updateDict(c[1], {'name': c[0]})
                       for c in _removeAlwaysHidden(protocol_format.load(request.state.machineconf,
                                                                         'clovr_batch_wrapper'))]
    else:
        batchConfig = []
    
    protocolConfs = []
    for p in protocols:
        protocolConfig = protocol_format.load(request.state.machineconf, p)
        wrapperName = pipeline_misc.determineWrapper(request.state.machineconf, p)
        if wrapperName != p:
            protocolKeys = set([k for k, _ in protocolConfig])
            wrapperConfig = [(k, v) 
                             for (k,v) in protocol_format.load(request.state.machineconf, 
                                                               wrapperName) 
                             if k not in protocolKeys]

            protocolConfig += wrapperConfig

        if request.body.get('detail', False):
            conf = [func.updateDict(c[1], {'name': c[0]})
                    for c in _removeAlwaysHidden(protocolConfig)]
            if request.body.get('batch_mode', False):
                conf = batchConfig + [func.updateDict(c, {'name': 'batch_pipeline.' + c['name']})
                                      for c in conf]
        else:
            conf = []
            
        protocolConfs.append({'protocol': p, 'config': conf})
    
    return defer_pipe.ret(request.update(response=protocolConfs))
Ejemplo n.º 7
0
def validateBatchPipelineConfig(request):
    """
    For batch configs we unfortunately cannot do as complete of
    validation as we would like for the inner pipeline.  In this case
    we are going to create a validation scheme on the fly.

    First we are going to pull out the inner pipeline config and
    apply its protocol configuration to it.  Then we are going to
    modify the protocol configuration to match config names in
    the request for the inner pipeline (prepending 'batch_pipeline.').
    After that we are going to load the clovr_batch_wrapper protocol conf
    and add it to the protocol conf we are building.

    Then we need to modify the protocol conf to not try to validate things we
    know will not validate.
    """
    def _makeNoop(params):
        # Turn off any validation if we are going to be replacing a variable
        # in the variable
        v['type'] = 'string'
        v['require_value'] = False
        # Remove any transform name so it doesn't accidently stomp on a variable
        # we actually care about
        if 'type_params' in v and 'transform_name' in v['type_params']:
            v['type_params'].pop('transform_name')

    
    # First things first, rip out the inner pipeline config:
    innerConfig = dict([(k.split('.', 1)[-1], v)
                        for k, v in request.body['config'].iteritems()
                        if k.startswith('batch_pipeline.')])

    protocolConf = protocol_format.load(request.state.machineconf,
                                        innerConfig['pipeline.PIPELINE_TEMPLATE'])
    protocol_format.applyProtocol(protocolConf, innerConfig)
    
    # Push the applied protocol back into the request config
    for k, v in innerConfig.iteritems():
        request.body['config']['batch_pipeline.' + k] = v

    additionalConf = []
    for k, v in protocolConf:
        if v['type'] in ['dataset', 'paired_dataset', 'singleton_dataset']:
            if ('batch.tag_list.' + k) in request.body['config']:
                params = dict(v)
                params['type_params'] = dict(params.get('type_params', {}))
                typeParams = params['type_params']
                typeParams['transform_name'] = 'batch.tag_list.file_list.' + k + '_throwaway'
                additionalConf.append(('batch.tag_list.' + k, params))
                _makeNoop(v)
            elif ('batch.param_list.' + k) in request.body['config']:
                params = dict(v)
                params['type'] = params['type'] + ' list'
                params['type_params'] = dict(params.get('type_params', {}))
                typeParams = params['type_params']
                typeParams['transform_name'] = 'batch.param_list.file_list.' + k + '_throwaway'
                additionalConf.append(('batch.param_list.' + k, params))
                _makeNoop(v)
            elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
                _makeNoop(v)
        elif ('batch.param_list.' + k) in request.body['config']:
            params = dict(v)
            params['type'] = params['type'] + ' list'
            params['type_params'] = dict(params.get('type_params', {}))
            typeParams = params['type_params']
            typeParams['transform_name'] = 'batch.param_list.transformed.' + k + '_throwaway'
        elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
            _makeNoop(v)
        else:
            if 'type_params' in v and 'transform_name' in v['type_params']:
                v['type_params']['transform_name'] = 'batch_pipeline.' + v['type_params']['transform_name']

    protocolConf = [('batch_pipeline.' + k, v)
                    for k, v in protocolConf]

    batchWrapperConf = protocol_format.load(request.state.machineconf,
                                            'clovr_batch_wrapper')
    protocol_format.applyProtocol(batchWrapperConf, request.body['config'])
    
    protocolConf += batchWrapperConf

    protocolConf += additionalConf

    validateState = func.Record(conf=request.state.conf,
                                machineconf=request.state.machineconf,
                                mq=request.mq)
    return pipeline_validate.validate(validateState, protocolConf, request.body['config'])