def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
transName = 'testProduction_' + str(int(time.time())) desc = 'just test' prodJob = Job() prodJob._addParameter(prodJob.workflow, 'PRODUCTION_ID', 'string', '00012345', 'ProductionID') prodJob._addParameter(prodJob.workflow, 'JOB_ID', 'string', '00006789', 'ProductionJobID') prodJob._addParameter(prodJob.workflow, 'eventType', 'string', 'TestEventType', 'Event Type of the prodJobuction') prodJob._addParameter(prodJob.workflow, 'numberOfEvents', 'string', '-1', 'Number of events requested') prodJob._addParameter(prodJob.workflow, 'ProcessingType', 'JDL', str('Test'), 'ProductionGroupOrType') prodJob._addParameter(prodJob.workflow, 'Priority', 'JDL', str(9), 'UserPriority') prodJob.setType(prodJobType) prodJob.workflow.setName(transName) prodJob.workflow.setDescrShort(desc) prodJob.workflow.setDescription(desc) prodJob.setCPUTime(86400) prodJob.setInputDataPolicy('Download') prodJob.setExecutable('/bin/ls', '-l') # Let's submit the prodJobuction now #result = prodJob.create() name = prodJob.workflow.getName() name = name.replace('/', '').replace('\\', '') prodJob.workflow.toXMLFile(name) print 'Workflow XML file name is: %s' % name workflowBody = '' if os.path.exists(name): with open(name, 'r') as fopen: workflowBody = fopen.read()
transName = 'testProduction_' + str(int(time.time())) desc = 'just test' prodJob = Job() prodJob._addParameter( prodJob.workflow, 'PRODUCTION_ID', 'string', '00012345', 'ProductionID' ) prodJob._addParameter( prodJob.workflow, 'JOB_ID', 'string', '00006789', 'ProductionJobID' ) prodJob._addParameter( prodJob.workflow, 'eventType', 'string', 'TestEventType', 'Event Type of the prodJobuction' ) prodJob._addParameter( prodJob.workflow, 'numberOfEvents', 'string', '-1', 'Number of events requested' ) prodJob._addParameter( prodJob.workflow, 'ProcessingType', 'JDL', str( 'Test' ), 'ProductionGroupOrType' ) prodJob._addParameter( prodJob.workflow, 'Priority', 'JDL', str( 9 ), 'UserPriority' ) prodJob.setType( prodJobType ) prodJob.workflow.setName(transName) prodJob.workflow.setDescrShort( desc ) prodJob.workflow.setDescription( desc ) prodJob.setCPUTime( 86400 ) prodJob.setInputDataPolicy( 'Download' ) prodJob.setExecutable('/bin/ls', '-l') #Let's submit the prodJobuction now #result = prodJob.create() name = prodJob.workflow.getName() name = name.replace( '/', '' ).replace( '\\', '' ) prodJob.workflow.toXMLFile( name ) print 'Workflow XML file name is: %s' % name workflowBody = '' if os.path.exists( name ): with open( name, 'r' ) as fopen: workflowBody = fopen.read()
transName = "testProduction_" + str(int(time.time())) desc = "just test" prodJob = Job() prodJob._addParameter(prodJob.workflow, "PRODUCTION_ID", "string", "00012345", "ProductionID") prodJob._addParameter(prodJob.workflow, "JOB_ID", "string", "00006789", "ProductionJobID") prodJob._addParameter(prodJob.workflow, "eventType", "string", "TestEventType", "Event Type of the prodJobuction") prodJob._addParameter(prodJob.workflow, "numberOfEvents", "string", "-1", "Number of events requested") prodJob._addParameter(prodJob.workflow, "ProcessingType", "JDL", str("Test"), "ProductionGroupOrType") prodJob._addParameter(prodJob.workflow, "Priority", "JDL", str(9), "UserPriority") prodJob.setType(prodJobType) prodJob.workflow.setName(transName) prodJob.workflow.setDescrShort(desc) prodJob.workflow.setDescription(desc) prodJob.setCPUTime(86400) prodJob.setInputDataPolicy("Download") prodJob.setExecutable("/bin/ls", "-l", modulesList=["Script", "FailoverRequest"]) # Let's submit the prodJobuction now # result = prodJob.create() name = prodJob.workflow.getName() name = name.replace("/", "").replace("\\", "") prodJob.workflow.toXMLFile(name) print("Workflow XML file name is: %s" % name) workflowBody = "" if os.path.exists(name): with open(name, "r") as fopen: workflowBody = fopen.read()