def create(self, publish=True, wfString='', requestID=0, reqUsed=0):
""" Will create the production and subsequently publish to the BK.
Production parameters are also added at this point.
publish = True - will add production to the production management system
False - does not publish the production
The workflow XML is created regardless of the flags.
"""
if wfString:
self.LHCbJob.workflow = fromXMLString(wfString)
# self.name = self.LHCbJob.workflow.getName()
bkConditions = self.LHCbJob.workflow.findParameter(
'conditions').getValue()
bkSteps = self.LHCbJob.workflow.findParameter(
'BKProcessingPass').getValue()
bkDictStep = {}
# Add the BK conditions metadata / name
simConds = self.bkkClient.getSimConditions()
if not simConds['OK']:
gLogger.error('Could not retrieve conditions data from BK:\n%s' %
simConds)
return simConds
simulationDescriptions = []
for record in simConds['Value']:
simulationDescriptions.append(str(record[1]))
if bkConditions not in simulationDescriptions:
gLogger.verbose(
'Assuming BK conditions %s are DataTakingConditions' %
bkConditions)
bkDictStep['DataTakingConditions'] = bkConditions
else:
gLogger.verbose('Found simulation conditions for %s' %
bkConditions)
bkDictStep['SimulationConditions'] = bkConditions
descShort = self.LHCbJob.workflow.getDescrShort()
descLong = self.LHCbJob.workflow.getDescription()
prodID = 0
if publish:
self.setParameter('ProcessingType', 'JDL', str(self.prodGroup),
'ProductionGroupOrType')
self.setParameter('Priority', 'JDL', str(self.priority),
'UserPriority')
try:
fileName = self.__createWorkflow()
except Exception as x: # pylint: disable=broad-except
gLogger.error(x)
return S_ERROR('Could not create workflow')
gLogger.verbose('Workflow XML file name is: %s' % fileName)
workflowBody = ''
if os.path.exists(fileName):
with open(fileName, 'r') as fopen:
workflowBody = fopen.read()
else:
return S_ERROR('Could not get workflow body')
# Standard parameters
self.transformation.setTransformationName(fileName)
self.transformation.setTransformationGroup(self.prodGroup)
self.transformation.setDescription(descShort)
self.transformation.setLongDescription(descLong)
self.transformation.setType(self.LHCbJob.type)
self.transformation.setBody(workflowBody)
self.transformation.setPlugin(self.plugin)
self.transformation.setBkQuery(self.inputBKSelection)
self.transformation.setTransformationFamily(
self.transformationFamily)
self.transformation.setFileMask(self.inputFileMask)
self.transformation.setGroupSize(int(self.jobFileGroupSize))
self.transformation.setInheritedFrom(int(self.ancestorProduction))
result = self.transformation.addTransformation()
if not result['OK']:
gLogger.error('Problem creating production:\n%s' % result)
return result
prodID = result['Value']
gLogger.info('Production %s successfully created' % prodID)
# All other parameters
groupDesc = self.LHCbJob.workflow.findParameter(
'groupDescription').getValue()
paramsDict = self.__getProductionParameters(
prodID=prodID,
prodXMLFile=fileName,
groupDescription=groupDesc,
bkPassInfo=bkSteps,
reqID=requestID,
derivedProd=self.ancestorProduction)
for parName, parValue in paramsDict.iteritems():
result = getattr(self.transformation,
'set' + parName)(parValue)
else:
gLogger.verbose(
'Publish flag is disabled, using default production ID')
bkDictStep['Production'] = int(prodID)
if self.inputBKSelection:
queryProdID = int(self.inputBKSelection.get('ProductionID', 0))
queryProcPass = self.inputBKSelection.get(
'ProcessingPass', '') if self.inputBKSelection.get(
'ProcessingPass', '') != 'All' else ''
if queryProdID:
inputPass = self.bkkClient.getProductionProcessingPass(
queryProdID)
if not inputPass['OK']:
gLogger.error(inputPass)
gLogger.error(
'Production %s was created but BK processing pass for %d was not found'
% (prodID, queryProdID))
return inputPass
inputPass = inputPass['Value']
gLogger.info(
'Setting %d as BK input production for %s with processing pass %s'
% (queryProdID, prodID, inputPass))
bkDictStep['InputProductionTotalProcessingPass'] = inputPass
elif queryProcPass:
gLogger.info(
'Adding input BK processing pass for production %s from input data query: %s'
% (prodID, queryProcPass))
bkDictStep[
'InputProductionTotalProcessingPass'] = queryProcPass
else:
# has to account for an input processing pass anyway, if there is one,
# or output files may not have the output processing pass correctly computed
try:
bkDictStep[
'InputProductionTotalProcessingPass'] = self.LHCbJob.workflow.findParameter(
'processingPass').getValue()
except AttributeError:
pass
stepList = []
stepKeys = sorted(bkSteps.keys())
# The BK needs an ordered list of steps
for step in stepKeys:
stepID = bkSteps[step]['BKStepID']
if stepID:
stepName = bkSteps[step]['StepName']
stepVisible = bkSteps[step]['StepVisible']
outputFileTypes = bkSteps[step]['OutputFileTypes']
stepList.append({
'StepId': int(stepID),
'OutputFileTypes': outputFileTypes,
'StepName': stepName,
'Visible': stepVisible
})
# This is the last component necessary for the BK publishing (post reorganisation)
bkDictStep['Steps'] = stepList
bkDictStep['EventType'] = paramsDict['eventType']
bkDictStep['ConfigName'] = self.LHCbJob.workflow.findParameter(
'configName').getValue()
bkDictStep['ConfigVersion'] = self.LHCbJob.workflow.findParameter(
'configVersion').getValue()
if publish:
gLogger.verbose('Attempting to publish production %s to the BK' %
(prodID))
result = self.bkkClient.addProduction(bkDictStep)
if not result['OK']:
gLogger.error(result)
return result
if requestID and publish:
from DIRAC.Core.DISET.RPCClient import RPCClient
reqClient = RPCClient('ProductionManagement/ProductionRequest',
timeout=120)
reqDict = {
'ProductionID': long(prodID),
'RequestID': requestID,
'Used': reqUsed,
'BkEvents': 0
}
result = reqClient.addProductionToRequest(reqDict)
if not result['OK']:
gLogger.error(
'Attempt to add production %s to request %s failed: %s ' %
(prodID, requestID, result['Message']))
gLogger.error('Dictionary below:\n%s' % reqDict)
else:
gLogger.info(
'Successfully added production %s to request %s with flag set to %s'
% (prodID, requestID, reqUsed))
return S_OK(prodID)
