示例#1
0
def validatePipelineConfig(request):
    validateState = func.Record(conf=request.state.conf,
                                machineconf=request.state.machineconf,
                                mq=request.mq)
    protocolConf = protocol_format.load(request.state.machineconf,
                                        request.body['config']['pipeline.PIPELINE_TEMPLATE'])

    pipelineWrapper = determineWrapper(request.state.machineconf,
                                       request.body['config']['pipeline.PIPELINE_TEMPLATE'])
    
    if not request.body['bare_run']:
        protocolConf += protocol_format.load(request.state.machineconf,
                                             pipelineWrapper)
        
    protocol_format.applyProtocol(protocolConf, request.body['config'])
    return pipeline_validate.validate(validateState, protocolConf, request.body['config'])    
示例#2
0
def validateBatchPipelineConfig(request):
    """
    For batch configs we unfortunately cannot do as complete of
    validation as we would like for the inner pipeline.  In this case
    we are going to create a validation scheme on the fly.

    First we are going to pull out the inner pipeline config and
    apply its protocol configuration to it.  Then we are going to
    modify the protocol configuration to match config names in
    the request for the inner pipeline (prepending 'batch_pipeline.').
    After that we are going to load the clovr_batch_wrapper protocol conf
    and add it to the protocol conf we are building.

    Then we need to modify the protocol conf to not try to validate things we
    know will not validate.
    """
    def _makeNoop(params):
        # Turn off any validation if we are going to be replacing a variable
        # in the variable
        v['type'] = 'string'
        v['require_value'] = False
        # Remove any transform name so it doesn't accidently stomp on a variable
        # we actually care about
        if 'type_params' in v and 'transform_name' in v['type_params']:
            v['type_params'].pop('transform_name')

    
    # First things first, rip out the inner pipeline config:
    innerConfig = dict([(k.split('.', 1)[-1], v)
                        for k, v in request.body['config'].iteritems()
                        if k.startswith('batch_pipeline.')])

    protocolConf = protocol_format.load(request.state.machineconf,
                                        innerConfig['pipeline.PIPELINE_TEMPLATE'])
    protocol_format.applyProtocol(protocolConf, innerConfig)
    
    # Push the applied protocol back into the request config
    for k, v in innerConfig.iteritems():
        request.body['config']['batch_pipeline.' + k] = v

    additionalConf = []
    for k, v in protocolConf:
        if v['type'] in ['dataset', 'paired_dataset', 'singleton_dataset']:
            if ('batch.tag_list.' + k) in request.body['config']:
                params = dict(v)
                params['type_params'] = dict(params.get('type_params', {}))
                typeParams = params['type_params']
                typeParams['transform_name'] = 'batch.tag_list.file_list.' + k + '_throwaway'
                additionalConf.append(('batch.tag_list.' + k, params))
                _makeNoop(v)
            elif ('batch.param_list.' + k) in request.body['config']:
                params = dict(v)
                params['type'] = params['type'] + ' list'
                params['type_params'] = dict(params.get('type_params', {}))
                typeParams = params['type_params']
                typeParams['transform_name'] = 'batch.param_list.file_list.' + k + '_throwaway'
                additionalConf.append(('batch.param_list.' + k, params))
                _makeNoop(v)
            elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
                _makeNoop(v)
        elif ('batch.param_list.' + k) in request.body['config']:
            params = dict(v)
            params['type'] = params['type'] + ' list'
            params['type_params'] = dict(params.get('type_params', {}))
            typeParams = params['type_params']
            typeParams['transform_name'] = 'batch.param_list.transformed.' + k + '_throwaway'
        elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
            _makeNoop(v)
        else:
            if 'type_params' in v and 'transform_name' in v['type_params']:
                v['type_params']['transform_name'] = 'batch_pipeline.' + v['type_params']['transform_name']

    protocolConf = [('batch_pipeline.' + k, v)
                    for k, v in protocolConf]

    batchWrapperConf = protocol_format.load(request.state.machineconf,
                                            'clovr_batch_wrapper')
    protocol_format.applyProtocol(batchWrapperConf, request.body['config'])
    
    protocolConf += batchWrapperConf

    protocolConf += additionalConf

    validateState = func.Record(conf=request.state.conf,
                                machineconf=request.state.machineconf,
                                mq=request.mq)
    return pipeline_validate.validate(validateState, protocolConf, request.body['config'])