def validatePipelineConfig(request):
validateState = func.Record(conf=request.state.conf,
machineconf=request.state.machineconf,
mq=request.mq)
protocolConf = protocol_format.load(request.state.machineconf,
request.body['config']['pipeline.PIPELINE_TEMPLATE'])
pipelineWrapper = determineWrapper(request.state.machineconf,
request.body['config']['pipeline.PIPELINE_TEMPLATE'])
if not request.body['bare_run']:
protocolConf += protocol_format.load(request.state.machineconf,
pipelineWrapper)
protocol_format.applyProtocol(protocolConf, request.body['config'])
return pipeline_validate.validate(validateState, protocolConf, request.body['config'])
def validateBatchPipelineConfig(request):
"""
For batch configs we unfortunately cannot do as complete of
validation as we would like for the inner pipeline. In this case
we are going to create a validation scheme on the fly.
First we are going to pull out the inner pipeline config and
apply its protocol configuration to it. Then we are going to
modify the protocol configuration to match config names in
the request for the inner pipeline (prepending 'batch_pipeline.').
After that we are going to load the clovr_batch_wrapper protocol conf
and add it to the protocol conf we are building.
Then we need to modify the protocol conf to not try to validate things we
know will not validate.
"""
def _makeNoop(params):
# Turn off any validation if we are going to be replacing a variable
# in the variable
v['type'] = 'string'
v['require_value'] = False
# Remove any transform name so it doesn't accidently stomp on a variable
# we actually care about
if 'type_params' in v and 'transform_name' in v['type_params']:
v['type_params'].pop('transform_name')
# First things first, rip out the inner pipeline config:
innerConfig = dict([(k.split('.', 1)[-1], v)
for k, v in request.body['config'].iteritems()
if k.startswith('batch_pipeline.')])
protocolConf = protocol_format.load(request.state.machineconf,
innerConfig['pipeline.PIPELINE_TEMPLATE'])
protocol_format.applyProtocol(protocolConf, innerConfig)
# Push the applied protocol back into the request config
for k, v in innerConfig.iteritems():
request.body['config']['batch_pipeline.' + k] = v
additionalConf = []
for k, v in protocolConf:
if v['type'] in ['dataset', 'paired_dataset', 'singleton_dataset']:
if ('batch.tag_list.' + k) in request.body['config']:
params = dict(v)
params['type_params'] = dict(params.get('type_params', {}))
typeParams = params['type_params']
typeParams['transform_name'] = 'batch.tag_list.file_list.' + k + '_throwaway'
additionalConf.append(('batch.tag_list.' + k, params))
_makeNoop(v)
elif ('batch.param_list.' + k) in request.body['config']:
params = dict(v)
params['type'] = params['type'] + ' list'
params['type_params'] = dict(params.get('type_params', {}))
typeParams = params['type_params']
typeParams['transform_name'] = 'batch.param_list.file_list.' + k + '_throwaway'
additionalConf.append(('batch.param_list.' + k, params))
_makeNoop(v)
elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
_makeNoop(v)
elif ('batch.param_list.' + k) in request.body['config']:
params = dict(v)
params['type'] = params['type'] + ' list'
params['type_params'] = dict(params.get('type_params', {}))
typeParams = params['type_params']
typeParams['transform_name'] = 'batch.param_list.transformed.' + k + '_throwaway'
elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
_makeNoop(v)
else:
if 'type_params' in v and 'transform_name' in v['type_params']:
v['type_params']['transform_name'] = 'batch_pipeline.' + v['type_params']['transform_name']
protocolConf = [('batch_pipeline.' + k, v)
for k, v in protocolConf]
batchWrapperConf = protocol_format.load(request.state.machineconf,
'clovr_batch_wrapper')
protocol_format.applyProtocol(batchWrapperConf, request.body['config'])
protocolConf += batchWrapperConf
protocolConf += additionalConf
validateState = func.Record(conf=request.state.conf,
machineconf=request.state.machineconf,
mq=request.mq)
return pipeline_validate.validate(validateState, protocolConf, request.body['config'])