def submitJob(self, jdl): """ Submit one job specified by its JDL to WMS """ if os.path.exists(jdl): fic = open(jdl, "r") jdlString = fic.read() fic.close() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl # Check the validity of the input JDL jdlString = jdlString.strip() if jdlString.find("[") != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd(jdlString) if not classAdJob.isOK(): return S_ERROR('Invalid job JDL') # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox(classAdJob) if not result['OK']: return result # Submit the job now and get the new job ID if not self.jobManager: self.jobManager = RPCClient('WorkloadManagement/JobManager', useCertificates=self.useCertificates, timeout=self.timeout) result = self.jobManager.submitJob(classAdJob.asJDL()) if 'requireProxyUpload' in result and result['requireProxyUpload']: gLogger.warn("Need to upload the proxy") return result
def getJobDefinition(self, job, jobDef=False): """ Retrieve JDL of the Job and return jobDef dictionary """ if not jobDef: jobDef = {} # If not jdl in jobinfo load it if 'jdl' not in jobDef: if self.requiredJobInfo == 'jdlOriginal': result = self.jobDB.getJobJDL(job, original=True) if not result['OK']: self.log.error("No JDL for job", "%s" % job) return S_ERROR("No JDL for job") jobDef['jdl'] = result['Value'] if self.requiredJobInfo == 'jdl': result = self.jobDB.getJobJDL(job) if not result['OK']: self.log.error("No JDL for job", "%s" % job) return S_ERROR("No JDL for job") jobDef['jdl'] = result['Value'] # Load the classad if needed if 'jdl' in jobDef and 'classad' not in jobDef: try: classad = ClassAd(jobDef['jdl']) except BaseException: self.log.debug("Cannot load JDL") return S_ERROR('Illegal Job JDL') if not classad.isOK(): self.log.debug( "Warning: illegal JDL for job %s, will be marked problematic" % (job)) return S_ERROR('Illegal Job JDL') jobDef['classad'] = classad return S_OK(jobDef)
def insertJobInQueue(self, job, classAdJob): """ Check individual job and add to the Task Queue eventually. """ jobReq = classAdJob.get_expression("JobRequirements") classAdJobReq = ClassAd(jobReq) jobReqDict = {} for name in self.taskQueueDB.getSingleValueTQDefFields(): if classAdJobReq.lookupAttribute(name): if name == 'CPUTime': jobReqDict[name] = classAdJobReq.getAttributeInt(name) else: jobReqDict[name] = classAdJobReq.getAttributeString(name) for name in self.taskQueueDB.getMultiValueTQDefFields(): if classAdJobReq.lookupAttribute(name): jobReqDict[name] = classAdJobReq.getListFromExpression(name) jobPriority = classAdJobReq.getAttributeInt('UserPriority') result = self.taskQueueDB.insertJob(job, jobReqDict, jobPriority) if not result['OK']: self.log.error("Cannot insert job %s in task queue: %s" % (job, result['Message'])) # Force removing the job from the TQ if it was actually inserted result = self.taskQueueDB.deleteJob(job) if result['OK']: if result['Value']: self.log.info("Job %s removed from the TQ" % job) return S_ERROR("Cannot insert in task queue") return S_OK()
def __getJDLParameters(self, jdl): """Returns a dictionary of JDL parameters. """ try: parameters = {} # print jdl if not re.search('\[', jdl): jdl = '[' + jdl + ']' classAdJob = ClassAd(jdl) paramsDict = classAdJob.contents for param, value in paramsDict.items(): if value.strip().startswith('{'): self.log.debug('Found list type parameter %s' % (param)) rawValues = value.replace('{', '').replace('}', '').replace( '"', '').split() valueList = [] for val in rawValues: if re.search(',$', val): valueList.append(val[:-1]) else: valueList.append(val) parameters[param] = valueList else: parameters[param] = value.replace('"', '').replace( '{', '"{').replace('}', '}"') self.log.debug('Found standard parameter %s: %s' % (param, parameters[param])) return S_OK(parameters) except Exception, x: self.log.exception(lException=x) return S_ERROR('Exception while extracting JDL parameters for job')
def test_SimpleParametricJob(): job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") parList = [1, 2, 3] job.setParameterSequence("JOB_ID", parList, addToWorkflow=True) inputDataList = [ ["/lhcb/data/data1", "/lhcb/data/data2"], ["/lhcb/data/data3", "/lhcb/data/data4"], ["/lhcb/data/data5", "/lhcb/data/data6"], ] job.setParameterSequence("InputData", inputDataList, addToWorkflow=True) jdl = job._toJDL() with open(join(dirname(__file__), "testWF.jdl")) as fd: expected = fd.read() assert jdl == expected clad = ClassAd("[" + jdl + "]") arguments = clad.getAttributeString("Arguments") job_id = clad.getAttributeString("JOB_ID") inputData = clad.getAttributeString("InputData") assert job_id == "%(JOB_ID)s" assert inputData == "%(InputData)s" assert "jobDescription.xml" in arguments assert "-o LogLevel=DEBUG" in arguments assert "-p JOB_ID=%(JOB_ID)s" in arguments assert "-p InputData=%(InputData)s" in arguments
def test_NoParameters(self): clad = ClassAd(TEST_JDL_NO_PARAMETERS) result = getParameterVectorLength(clad) self.assertTrue(result['OK']) nParam = result['Value'] self.assertTrue(nParam is None)
def test_SimpleParametricJob(self): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'], ['/lhcb/data/data3', '/lhcb/data/data4'], ['/lhcb/data/data5', '/lhcb/data/data6']] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() print jdl clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') print "arguments", arguments self.assertEqual(job_id, '%(JOB_ID)s') self.assertEqual(inputData, '%(InputData)s') self.assertIn('jobDescription.xml', arguments) self.assertIn('-o LogLevel=DEBUG', arguments) self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments) self.assertIn('-p InputData=%(InputData)s', arguments)
def getJobDefinition(self, job, jobDef=False): """Retrieve JDL of the Job and return jobDef dictionary""" if not jobDef: jobDef = {} # If not jdl in jobinfo load it if "jdl" not in jobDef: if self.requiredJobInfo == "jdlOriginal": result = self.jobDB.getJobJDL(job, original=True) if not result["OK"]: self.log.error("No JDL for job", "%s" % job) return S_ERROR("No JDL for job") jobDef["jdl"] = result["Value"] if self.requiredJobInfo == "jdl": result = self.jobDB.getJobJDL(job) if not result["OK"]: self.log.error("No JDL for job", "%s" % job) return S_ERROR("No JDL for job") jobDef["jdl"] = result["Value"] # Load the classad if needed if "jdl" in jobDef and "classad" not in jobDef: try: classad = ClassAd(jobDef["jdl"]) except Exception: self.log.debug("Cannot load JDL") return S_ERROR(JobMinorStatus.ILLEGAL_JOB_JDL) if not classad.isOK(): self.log.debug( "Warning: illegal JDL for job %s, will be marked problematic" % (job)) return S_ERROR(JobMinorStatus.ILLEGAL_JOB_JDL) jobDef["classad"] = classad return S_OK(jobDef)
def _getJDLParameters(self, jdl): """Returns a dictionary of JDL parameters.""" try: parameters = {} # print jdl if not re.search(r"\[", jdl): jdl = "[" + jdl + "]" classAdJob = ClassAd(jdl) paramsDict = classAdJob.contents for param, value in paramsDict.items(): if value.strip().startswith("{"): self.log.debug("Found list type parameter %s" % (param)) rawValues = value.replace("{", "").replace("}", "").replace( '"', "").split() valueList = [] for val in rawValues: if re.search(",$", val): valueList.append(val[:-1]) else: valueList.append(val) parameters[param] = valueList else: parameters[param] = value.replace('"', "").replace( "{", '"{').replace("}", '}"') self.log.debug("Found standard parameter %s: %s" % (param, parameters[param])) return S_OK(parameters) except Exception as x: self.log.exception(lException=x) return S_ERROR("Exception while extracting JDL parameters for job")
def test_SimpleParametricJob(): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'], ['/lhcb/data/data3', '/lhcb/data/data4'], ['/lhcb/data/data5', '/lhcb/data/data6']] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() try: with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() assert jdl == expected clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') assert job_id == '%(JOB_ID)s' assert inputData == '%(InputData)s' assert 'jobDescription.xml' in arguments assert '-o LogLevel=DEBUG' in arguments assert '-p JOB_ID=%(JOB_ID)s' in arguments assert '-p InputData=%(InputData)s' in arguments
def test_SimpleProgression(self): clad = ClassAd(TEST_JDL_SIMPLE_PROGRESSION) nParam = getNumberOfParameters(clad) self.assertEqual(nParam, 3) result = generateParametricJobs(clad) self.assert_(result['OK']) jobDescList = result['Value'] self.assertEqual(nParam, len(jobDescList)) # Check the definition of the 2nd job jobClassAd = ClassAd(jobDescList[1]) self.assertEqual(jobClassAd.getAttributeString('Arguments'), '3') self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
def _saveJobJDLRequest(self, jobID, jobJDL): """Save job JDL local to JobAgent.""" classAdJob = ClassAd(jobJDL) classAdJob.insertAttributeString("LocalCE", self.ceName) jdlFileName = jobID + ".jdl" jdlFile = open(jdlFileName, "w") jdl = classAdJob.asJDL() jdlFile.write(jdl) jdlFile.close()
def test_Simple(self): clad = ClassAd(TEST_JDL_SIMPLE) result = getParameterVectorLength(clad) self.assertTrue(result['OK']) nParam = result['Value'] self.assertEqual(nParam, 3) result = generateParametricJobs(clad) self.assertTrue(result['OK']) jobDescList = result['Value'] self.assertEqual(nParam, len(jobDescList)) # Check the definition of the 2nd job jobClassAd = ClassAd(jobDescList[1]) self.assertEqual(jobClassAd.getAttributeString('Arguments'), 'b') self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
def test_SimpleBunch(self): clad = ClassAd(TEST_JDL_SIMPLE_BUNCH) result = getParameterVectorLength(clad) self.assertTrue(result["OK"]) nParam = result["Value"] self.assertEqual(nParam, 3) result = generateParametricJobs(clad) self.assertTrue(result["OK"]) jobDescList = result["Value"] self.assertEqual(nParam, len(jobDescList)) # Check the definition of the 2nd job jobClassAd = ClassAd(jobDescList[1]) self.assertEqual(jobClassAd.getAttributeString("Arguments"), "5") self.assertEqual(jobClassAd.getAttributeString("JobName"), "Test_1")
def _getProcessingType(self, jobID): """Get the Processing Type from the JDL, until it is promoted to a real Attribute""" processingType = "unknown" result = self.jobDB.getJobJDL(jobID, original=True) if not result["OK"]: return processingType classAdJob = ClassAd(result["Value"]) if classAdJob.lookupAttribute("ProcessingType"): processingType = classAdJob.getAttributeString("ProcessingType") return processingType
def __processResourceDescription(self, resourceDescription): # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR('Illegal Resource JDL') gLogger.verbose(classAdAgent.asJDL()) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): if name == 'SubmitPool': resourceDict[ name] = classAdAgent.getListFromExpression(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization'): if classAdAgent.lookupAttribute(k): resourceDict[k] = classAdAgent.getAttributeString(k) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key('JobID'): resourceDict['JobID'] = resourceDescription['JobID'] for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotInfoReportedFlag', 'PilotBenchmark', 'LHCbPlatform'): if k in resourceDescription: resourceDict[k] = resourceDescription[k] return resourceDict
def __processResourceDescription(self, resourceDescription): # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR('Illegal Resource JDL') gLogger.verbose(classAdAgent.asJDL()) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): resourceDict[name] = classAdAgent.getAttributeString(name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') if classAdAgent.lookupAttribute('DIRACVersion'): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion') if classAdAgent.lookupAttribute('VirtualOrganization'): resourceDict[ 'VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization') else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key('JobID'): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key('DIRACVersion'): resourceDict['DIRACVersion'] = resourceDescription[ 'DIRACVersion'] if resourceDescription.has_key('VirtualOrganization'): resourceDict['VirtualOrganization'] = resourceDescription[ 'VirtualOrganization'] return resourceDict
def __saveJobJDLRequest(self, jobID, jobJDL): """Save job JDL local to JobAgent. """ classAdJob = ClassAd(jobJDL) classAdJob.insertAttributeString('LocalCE', self.ceName) jdlFileName = jobID + '.jdl' jdlFile = open(jdlFileName, 'w') jdl = classAdJob.asJDL() jdlFile.write(jdl) jdlFile.close() return S_OK(jdlFileName)
def __getProcessingType(self, jobID): """ Get the Processing Type from the JDL, until it is promoted to a real Attribute """ processingType = 'unknown' result = self.jobDB.getJobJDL(jobID, original=True) if not result['OK']: return processingType classAdJob = ClassAd(result['Value']) if classAdJob.lookupAttribute('ProcessingType'): processingType = classAdJob.getAttributeString('ProcessingType') return processingType
def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") job.setNumberOfProcessors(proc, minProc, maxProc) jdl = job._toJDL() clad = ClassAd("[" + jdl + "]") processors = clad.getAttributeInt("NumberOfProcessors") minProcessors = clad.getAttributeInt("MinNumberOfProcessors") maxProcessors = clad.getAttributeInt("MaxNumberOfProcessors") assert processors == expectedProc assert minProcessors == expectedMinProc assert maxProcessors == expectedMaxProc
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') job.setNumberOfProcessors(proc, minProc, maxProc) jdl = job._toJDL() clad = ClassAd('[' + jdl + ']') processors = clad.getAttributeInt('NumberOfProcessors') minProcessors = clad.getAttributeInt('MinNumberOfProcessors') maxProcessors = clad.getAttributeInt('MaxNumberOfProcessors') assert processors == expectedProc assert minProcessors == expectedMinProc assert maxProcessors == expectedMaxProc
def submitJob(self, jdl, jobDescriptionObject=None): """ Submit one job specified by its JDL to WMS """ if os.path.exists(jdl): fic = open(jdl, "r") jdlString = fic.read() fic.close() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl jdlString = jdlString.strip() # Strip of comments in the jdl string newJdlList = [] for line in jdlString.split('\n'): if not line.strip().startswith('#'): newJdlList.append(line) jdlString = '\n'.join(newJdlList) # Check the validity of the input JDL if jdlString.find("[") != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd(jdlString) if not classAdJob.isOK(): return S_ERROR('Invalid job JDL') # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject) if not result['OK']: return result # Submit the job now and get the new job ID if not self.jobManager: self.jobManager = RPCClient('WorkloadManagement/JobManager', useCertificates=self.useCertificates, timeout=self.timeout) result = self.jobManager.submitJob(classAdJob.asJDL()) if 'requireProxyUpload' in result and result['requireProxyUpload']: gLogger.warn("Need to upload the proxy") return result
def export_submitJob(self, jobDesc): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR("Can't submit using a limited proxy! (bad boy!)") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR('Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR('Job submission not authorized') #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parametric one jobClassAd = ClassAd(jobDesc) nParameters = getNumberOfParameters(jobClassAd) parametricJob = False if nParameters > 0: parametricJob = True result = generateParametricJobs(jobClassAd) if not result['OK']: return result jobDescList = result['Value'] else: jobDescList = [jobDesc] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) #Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() self.__sendJobsToOptimizationMind(jobIDList) return result
def export_submitJob(self, jobDesc): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR("Can't submit using a limited proxy! (bad boy!)") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR('Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR('Job submission not authorized') #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parameteric one jobClassAd = ClassAd(jobDesc) parametricJob = False if jobClassAd.lookupAttribute('Parameters'): parametricJob = True if jobClassAd.isAttributeList('Parameters'): parameterList = jobClassAd.getListFromExpression('Parameters') else: pStep = 0 pFactor = 1 pStart = 1 nParameters = jobClassAd.getAttributeInt('Parameters') if not nParameters: value = jobClassAd.get_expression('Parameters') return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterStart'): value = jobClassAd.get_expression( 'ParameterStart').replace('"', '') try: pStart = int(value) except: try: pStart = float(value) except: return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterStep'): pStep = jobClassAd.getAttributeInt('ParameterStep') if not pStep: pStep = jobClassAd.getAttributeFloat('ParameterStep') if not pStep: value = jobClassAd.get_expression('ParameterStep') return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterFactor'): pFactor = jobClassAd.getAttributeInt('ParameterFactor') if not pFactor: pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor') if not pFactor: value = jobClassAd.get_expression( 'ParameterFactor') return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value) parameterList = list() parameterList.append(pStart) for i in range(nParameters - 1): parameterList.append(parameterList[i] * pFactor + pStep) if len(parameterList) > self.maxParametricJobs: return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs) jobDescList = [] nParam = len(parameterList) - 1 for n, p in enumerate(parameterList): newJobDesc = jobDesc.replace('%s', str(p)).replace( '%n', str(n).zfill(len(str(nParam)))) newClassAd = ClassAd(newJobDesc) for attr in ['Parameters', 'ParameterStep', 'ParameterFactor']: newClassAd.deleteAttribute(attr) if type(p) == type(' ') and p.startswith('{'): newClassAd.insertAttributeInt('Parameter', str(p)) else: newClassAd.insertAttributeString('Parameter', str(p)) newClassAd.insertAttributeInt('ParameterNumber', n) newJDL = newClassAd.asJDL() jobDescList.append(newJDL) else: jobDescList = [jobDesc] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) #Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() self.__sendNewJobsToMind(jobIDList) return result
def submitJob(self, jdl, jobDescriptionObject=None): """Submit one job specified by its JDL to WMS. The JDL may actually be the desciption of a parametric job, resulting in multiple DIRAC jobs submitted to the DIRAC WMS """ if os.path.exists(jdl): with open(jdl, "r") as fic: jdlString = fic.read() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl jdlString = jdlString.strip() gLogger.debug("Submitting JDL", jdlString) # Strip of comments in the jdl string newJdlList = [] for line in jdlString.split("\n"): if not line.strip().startswith("#"): newJdlList.append(line) jdlString = "\n".join(newJdlList) # Check the validity of the input JDL if jdlString.find("[") != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd(jdlString) if not classAdJob.isOK(): return S_ERROR(EWMSJDL, "Invalid job JDL") # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject) if not result["OK"]: return result # Submit the job now and get the new job ID result = getParameterVectorLength(classAdJob) if not result["OK"]: return result nJobs = result["Value"] result = self.jobManager.submitJob(classAdJob.asJDL()) if nJobs: gLogger.debug("Applying transactional job submission") # The server applies transactional bulk submission, we should confirm the jobs if result["OK"]: jobIDList = result["Value"] if len(jobIDList) == nJobs: # Confirm the submitted jobs confirmed = False for _attempt in range(3): result = self.jobManager.confirmBulkSubmission(jobIDList) if result["OK"]: confirmed = True break time.sleep(1) if not confirmed: # The bulk submission failed, try to remove the created jobs resultDelete = self.jobManager.removeJob(jobIDList) error = "Job submission failed to confirm bulk transaction" if not resultDelete["OK"]: error += "; removal of created jobs failed" return S_ERROR(EWMSSUBM, error) else: return S_ERROR(EWMSSUBM, "The number of submitted jobs does not match job description") if result.get("requireProxyUpload"): gLogger.warn("Need to upload the proxy") return result
def export_submitJob(self, jobDesc): """ Submit a job to DIRAC WMS. The job can be a single job, or a parametric job. If it is a parametric job, then the parameters will need to be unpacked. :param str jobDesc: job description JDL (of a single or parametric job) :return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK. """ if self.peerUsesLimitedProxy: return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR(EWMSSUBM, 'Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR(EWMSSUBM, 'Job submission not authorized') # jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parametric one jobClassAd = ClassAd(jobDesc) result = getParameterVectorLength(jobClassAd) if not result['OK']: return result nJobs = result['Value'] parametricJob = False if nJobs > 0: # if we are here, then jobDesc was the description of a parametric job. So we start unpacking parametricJob = True if nJobs > self.maxParametricJobs: return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs) result = generateParametricJobs(jobClassAd) if not result['OK']: return result jobDescList = result['Value'] else: # if we are here, then jobDesc was the description of a single job. jobDescList = [jobDesc] jobIDList = [] if parametricJob: initialStatus = 'Submitting' initialMinorStatus = 'Bulk transaction confirmation' else: initialStatus = 'Received' initialMinorStatus = 'Job accepted' for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup, initialStatus=initialStatus, initialMinorStatus=initialMinorStatus) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) # Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() return result
def generateParametricJobs(jobClassAd): """ Generate a series of ClassAd job descriptions expanding job parameters :param jobClassAd: ClassAd job description object :return: list of ClassAd job description objects """ if not jobClassAd.lookupAttribute('Parameters'): return S_OK([jobClassAd.asJDL()]) result = getParameterVectorLength(jobClassAd) if not result['OK']: return result nParValues = result['Value'] if nParValues is None: return S_ERROR(EWMSJDL, 'Can not determine the number of job parameters') parameterDict = {} attributes = jobClassAd.getAttributes() for attribute in attributes: for key in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]: if attribute.startswith(key): seqID = '0' if '.' not in attribute else attribute.split( '.')[1] parameterDict.setdefault(seqID, {}) if key == 'Parameters': if jobClassAd.isAttributeList(attribute): parList = jobClassAd.getListFromExpression(attribute) if len(parList) != nParValues: return S_ERROR( EWMSJDL, 'Inconsistent parametric job description') parameterDict[seqID]['ParameterList'] = parList else: if attribute != "Parameters": return S_ERROR( EWMSJDL, 'Inconsistent parametric job description') nPar = jobClassAd.getAttributeInt(attribute) if nPar is None: value = jobClassAd.get_expression(attribute) return S_ERROR( EWMSJDL, 'Inconsistent parametric job description: %s=%s' % (attribute, value)) parameterDict[seqID]['Parameters'] = nPar else: value = jobClassAd.getAttributeInt(attribute) if value is None: value = jobClassAd.getAttributeFloat(attribute) if value is None: value = jobClassAd.get_expression(attribute) return S_ERROR( 'Illegal value for %s JDL field: %s' % (attribute, value)) parameterDict[seqID][key] = value if '0' in parameterDict and not parameterDict.get('0'): parameterDict.pop('0') parameterLists = {} for seqID in parameterDict: parList = __getParameterSequence( nParValues, parList=parameterDict[seqID].get('ParameterList', []), parStart=parameterDict[seqID].get('ParameterStart', 1), parStep=parameterDict[seqID].get('ParameterStep', 0), parFactor=parameterDict[seqID].get('ParameterFactor', 1)) if not parList: return S_ERROR(EWMSJDL, 'Inconsistent parametric job description') parameterLists[seqID] = parList jobDescList = [] jobDesc = jobClassAd.asJDL() # Width of the sequential parameter number zLength = len(str(nParValues - 1)) for n in range(nParValues): newJobDesc = jobDesc newJobDesc = newJobDesc.replace('%n', str(n).zfill(zLength)) newClassAd = ClassAd(newJobDesc) for seqID in parameterLists: parameter = parameterLists[seqID][n] for attribute in newClassAd.getAttributes(): __updateAttribute(newClassAd, attribute, seqID, str(parameter)) for seqID in parameterLists: for attribute in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]: if seqID == '0': newClassAd.deleteAttribute(attribute) else: newClassAd.deleteAttribute('%s.%s' % (attribute, seqID)) parameter = parameterLists[seqID][n] if seqID == '0': attribute = 'Parameter' else: attribute = 'Parameter.%s' % seqID if isinstance(parameter, six.string_types) and parameter.startswith('{'): newClassAd.insertAttributeInt(attribute, str(parameter)) else: newClassAd.insertAttributeString(attribute, str(parameter)) newClassAd.insertAttributeInt('ParameterNumber', n) newJDL = newClassAd.asJDL() jobDescList.append(newJDL) return S_OK(jobDescList)
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ #Check if we have to do old bootstrap... classadJob = ClassAd( '[]' ) paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} scriptname = 'jobDescription.xml' arguments = [] if self.script: if os.path.exists( self.script ): scriptname = os.path.abspath( self.script ) self.log.verbose( 'Found script name %s' % scriptname ) else: if xmlFile: self.log.verbose( 'Found XML File %s' % xmlFile ) scriptname = xmlFile arguments.append( os.path.basename( scriptname ) ) self.addToInputSandbox.append( scriptname ) if paramsDict.has_key( 'LogLevel' ): if paramsDict['LogLevel']['value']: arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) ) else: self.log.warn( 'Job LogLevel defined with null value' ) if paramsDict.has_key( 'DIRACSetup' ): if paramsDict['DIRACSetup']['value']: arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) ) else: self.log.warn( 'Job DIRACSetup defined with null value' ) if paramsDict.has_key( 'JobMode' ): if paramsDict['JobMode']['value']: arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) ) else: self.log.warn( 'Job Mode defined with null value' ) if paramsDict.has_key( 'JobConfigArgs' ): if paramsDict['JobConfigArgs']['value']: arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) ) else: self.log.warn( 'JobConfigArgs defined with null value' ) classadJob.insertAttributeString( 'Executable', self.executable ) self.addToOutputSandbox.append( self.stderr ) self.addToOutputSandbox.append( self.stdout ) #Extract i/o sandbox parameters from steps and any input data parameters #to do when introducing step-level api... #To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ';'.join( self.addToInputSandbox ) if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] finalInputSandbox = currentFiles + ';' + extraFiles uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) ) paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox ) self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) ) else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = extraFiles paramsDict['InputSandbox']['type'] = 'JDL' if self.addToOutputSandbox: extraFiles = ';'.join( self.addToOutputSandbox ) if paramsDict.has_key( 'OutputSandbox' ): currentFiles = paramsDict['OutputSandbox']['value'] finalOutputSandbox = currentFiles + ';' + extraFiles uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) ) paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox ) self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) ) else: paramsDict['OutputSandbox'] = {} paramsDict['OutputSandbox']['value'] = extraFiles paramsDict['OutputSandbox']['type'] = 'JDL' if self.addToInputData: extraFiles = ';'.join( self.addToInputData ) if paramsDict.has_key( 'InputData' ): currentFiles = paramsDict['InputData']['value'] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ';' + extraFiles uniqueInputData = uniqueElements( finalInputData.split( ';' ) ) paramsDict['InputData']['value'] = ';'.join( uniqueInputData ) self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) ) else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = extraFiles paramsDict['InputData']['type'] = 'JDL' # Handle here the Parametric values if self.parametric: for pType in ['InputData', 'InputSandbox']: if self.parametric.has_key( pType ): if paramsDict.has_key( pType ) and paramsDict[pType]['value']: pData = self.parametric[pType] # List of lists case currentFiles = paramsDict[pType]['value'].split( ';' ) tmpList = [] if type( pData[0] ) == list: for pElement in pData: tmpList.append( currentFiles + pElement ) else: for pElement in pData: tmpList.append( currentFiles + [pElement] ) self.parametric[pType] = tmpList paramsDict[pType] = {} paramsDict[pType]['value'] = "%s" paramsDict[pType]['type'] = 'JDL' self.parametric['files'] = self.parametric[pType] arguments.append( ' -p Parametric' + pType + '=%s' ) break if self.parametric.has_key( 'files' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['files'] paramsDict['Parameters']['type'] = 'JDL' if self.parametric.has_key( 'GenericParameters' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['GenericParameters'] paramsDict['Parameters']['type'] = 'JDL' arguments.append( ' -p ParametricParameters=%s' ) ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) #Add any JDL parameters to classad obeying lists with ';' rule requirements = False for name, props in paramsDict.items(): ptype = props['type'] value = props['value'] if name.lower() == 'requirements' and ptype == 'JDL': self.log.verbose( 'Found existing requirements: %s' % ( value ) ) requirements = True if re.search( '^JDL', ptype ): if type( value ) == list: if type( value[0] ) == list: classadJob.insertAttributeVectorStringList( name, value ) else: classadJob.insertAttributeVectorString( name, value ) elif value == "%s": classadJob.insertAttributeInt( name, value ) elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) else: classadJob.insertAttributeVectorString( name, value.split( ';' ) ) if not requirements: reqtsDict = self.reqParams exprn = '' plus = '' for name, props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if not ptype == 'dict': if ptype == 'JDLReqt': if value and not value.lower() == 'any': plus = ' && ' if re.search( ';', value ): for val in value.split( ';' ): exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus else: exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus if len( plus ): exprn = exprn[:-len( plus )] if not exprn: exprn = 'true' self.log.verbose( 'Requirements: %s' % ( exprn ) ) #classadJob.set_expression('Requirements', exprn) self.addToInputSandbox.remove( scriptname ) self.addToOutputSandbox.remove( self.stdout ) self.addToOutputSandbox.remove( self.stderr ) jdl = classadJob.asJDL() start = jdl.find( '[' ) end = jdl.rfind( ']' ) return jdl[( start + 1 ):( end - 1 )]
def matchQueue(jobJDL, queueDict, fullMatch=False): """ Match the job description to the queue definition :param str job: JDL job description :param bool fullMatch: test matching on all the criteria :param dict queueDict: queue parameters dictionary :return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or S_ERROR with the reason for no match """ # Check the job description validity job = ClassAd(jobJDL) if not job.isOK(): return S_ERROR("Invalid job description") noMatchReasons = [] # Check job requirements to resource # 1. CPUTime cpuTime = job.getAttributeInt("CPUTime") if not cpuTime: cpuTime = 84600 if cpuTime > int(queueDict.get("CPUTime", 0)): noMatchReasons.append("Job CPUTime requirement not satisfied") if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Multi-value match requirements for parameter in ["Site", "GridCE", "Platform", "JobType"]: if parameter in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[parameter])) if valueSet and queueSet and not valueSet.intersection(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. Banned multi-value match requirements for par in ["Site", "GridCE", "Platform", "JobType"]: parameter = "Banned%s" % par if par in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[par])) if valueSet and queueSet and valueSet.issubset(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. Tags tags = set(job.getListFromExpression("Tag")) nProc = job.getAttributeInt("NumberOfProcessors") if nProc and nProc > 1: tags.add("MultiProcessor") wholeNode = job.getAttributeString("WholeNode") if wholeNode: tags.add("WholeNode") queueTags = set(queueDict.get("Tag", [])) if not tags.issubset(queueTags): noMatchReasons.append("Job Tag %s not satisfied" % ",".join(tags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. MultiProcessor requirements if nProc and nProc > int(queueDict.get("NumberOfProcessors", 1)): noMatchReasons.append("Job NumberOfProcessors %d requirement not satisfied" % nProc) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 5. RAM ram = job.getAttributeInt("RAM") # If MaxRAM is not specified in the queue description, assume 2GB if ram and ram > int(queueDict.get("MaxRAM", 2048) / 1024): noMatchReasons.append("Job RAM %d requirement not satisfied" % ram) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # Check resource requirements to job # 1. OwnerGroup - rare case but still if "OwnerGroup" in queueDict: result = getProxyInfo(disableVOMS=True) if not result["OK"]: return S_ERROR("No valid proxy available") ownerGroup = result["Value"]["group"] if ownerGroup != queueDict["OwnerGroup"]: noMatchReasons.append("Resource OwnerGroup %s requirement not satisfied" % queueDict["OwnerGroup"]) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Required tags requiredTags = set(queueDict.get("RequiredTags", [])) if not requiredTags.issubset(tags): noMatchReasons.append("Resource RequiredTags %s not satisfied" % ",".join(requiredTags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. RunningLimit site = queueDict["Site"] ce = queueDict.get("GridCE") opsHelper = Operations() result = opsHelper.getSections("JobScheduling/RunningLimit") if result["OK"] and site in result["Value"]: result = opsHelper.getSections("JobScheduling/RunningLimit/%s" % site) if result["OK"]: for parameter in result["Value"]: value = job.getAttributeString(parameter) if ( value and ( opsHelper.getValue("JobScheduling/RunningLimit/%s/%s/%s" % (site, parameter, value), 1) or opsHelper.getValue( "JobScheduling/RunningLimit/%s/CEs/%s/%s/%s" % (site, ce, parameter, value), 1 ) ) == 0 ): noMatchReasons.append("Resource operational %s requirement not satisfied" % parameter) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) return S_OK({"Match": not bool(noMatchReasons), "Reason": noMatchReasons})