def submitJob( self, jdl ): """ Submit one job specified by its JDL to WMS """ if os.path.exists( jdl ): fic = open ( jdl, "r" ) jdlString = fic.read() fic.close() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl # Check the validity of the input JDL jdlString = jdlString.strip() if jdlString.find( "[" ) != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd( jdlString ) if not classAdJob.isOK(): return S_ERROR( 'Invalid job JDL' ) # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox( classAdJob ) if not result['OK']: return result # Submit the job now and get the new job ID if not self.jobManager: self.jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = self.useCertificates, timeout = self.timeout ) result = self.jobManager.submitJob( classAdJob.asJDL() ) if 'requireProxyUpload' in result and result['requireProxyUpload']: gLogger.warn( "Need to upload the proxy" ) return result
def submitJob(self, jdl): """ Submit one job specified by its JDL to WMS """ if os.path.exists(jdl): fic = open(jdl, "r") jdlString = fic.read() fic.close() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl # Check the validity of the input JDL jdlString = jdlString.strip() if jdlString.find("[") != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd(jdlString) if not classAdJob.isOK(): return S_ERROR('Invalid job JDL') # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox(classAdJob) if not result['OK']: return result # Submit the job now and get the new job ID if not self.jobManager: self.jobManager = RPCClient('WorkloadManagement/JobManager', useCertificates=self.useCertificates, timeout=self.timeout) result = self.jobManager.submitJob(classAdJob.asJDL()) if 'requireProxyUpload' in result and result['requireProxyUpload']: gLogger.warn("Need to upload the proxy") return result
def _saveJobJDLRequest(self, jobID, jobJDL): """Save job JDL local to JobAgent.""" classAdJob = ClassAd(jobJDL) classAdJob.insertAttributeString("LocalCE", self.ceName) jdlFileName = jobID + ".jdl" jdlFile = open(jdlFileName, "w") jdl = classAdJob.asJDL() jdlFile.write(jdl) jdlFile.close()
def __saveJobJDLRequest( self, jobID, jobJDL ): """Save job JDL local to JobAgent. """ classAdJob = ClassAd( jobJDL ) classAdJob.insertAttributeString( 'LocalCE', self.ceName ) jdlFileName = jobID + '.jdl' jdlFile = open( jdlFileName, 'w' ) jdl = classAdJob.asJDL() jdlFile.write( jdl ) jdlFile.close()
def __processResourceDescription(self, resourceDescription): # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR('Illegal Resource JDL') gLogger.verbose(classAdAgent.asJDL()) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): if name == 'SubmitPool': resourceDict[ name] = classAdAgent.getListFromExpression(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization'): if classAdAgent.lookupAttribute(k): resourceDict[k] = classAdAgent.getAttributeString(k) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key('JobID'): resourceDict['JobID'] = resourceDescription['JobID'] for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotInfoReportedFlag', 'PilotBenchmark', 'LHCbPlatform'): if k in resourceDescription: resourceDict[k] = resourceDescription[k] return resourceDict
def __processResourceDescription(self, resourceDescription): # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR('Illegal Resource JDL') gLogger.verbose(classAdAgent.asJDL()) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): resourceDict[name] = classAdAgent.getAttributeString(name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') if classAdAgent.lookupAttribute('DIRACVersion'): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion') if classAdAgent.lookupAttribute('VirtualOrganization'): resourceDict[ 'VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization') else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key('JobID'): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key('DIRACVersion'): resourceDict['DIRACVersion'] = resourceDescription[ 'DIRACVersion'] if resourceDescription.has_key('VirtualOrganization'): resourceDict['VirtualOrganization'] = resourceDescription[ 'VirtualOrganization'] return resourceDict
def __saveJobJDLRequest(self, jobID, jobJDL): """Save job JDL local to JobAgent. """ classAdJob = ClassAd(jobJDL) classAdJob.insertAttributeString("LocalCE", self.ceName) jdlFileName = jobID + ".jdl" jdlFile = open(jdlFileName, "w") jdl = classAdJob.asJDL() jdlFile.write(jdl) jdlFile.close() return S_OK(jdlFileName)
def __saveJobJDLRequest(self, jobID, jobJDL): """Save job JDL local to JobAgent. """ classAdJob = ClassAd(jobJDL) classAdJob.insertAttributeString('LocalCE', self.ceName) jdlFileName = jobID + '.jdl' jdlFile = open(jdlFileName, 'w') jdl = classAdJob.asJDL() jdlFile.write(jdl) jdlFile.close() return S_OK(jdlFileName)
def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if isinstance( resourceDescription, basestring ): classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def __processResourceDescription( self, resourceDescription ): # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) if classAdAgent.lookupAttribute( 'DIRACVersion' ): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' ) if classAdAgent.lookupAttribute( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization' ) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key( 'DIRACVersion' ): resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion'] if resourceDescription.has_key( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = resourceDescription['VirtualOrganization'] return resourceDict
def __processResourceDescription( self, resourceDescription ): # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotInfoReportedFlag', 'PilotBenchmark' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def matchJob(self, resourceJDL): """ Use Matcher service to retrieve a MPI job from Task Queue. Returns: JobID, NumProc required, JDL and MPI flavor Input: resourceJDL Output: result = {JobID, JobJDL, NumProc, MPIFlavor} """ print "S37" matcher = RPCClient('WorkloadManagement/Matcher', timeout = 600) dictMatchMPI = {'Setup':'EELA-Production', 'CPUTime':6000, 'JobType':'MPI'} result = matcher.getMatchingTaskQueues(dictMatchMPI) if not result['OK']: print "S38" gLogger.info("-------------------------------------------------------------------") gLogger.error ("Here I have to call to get normal job") gLogger.info("-------------------------------------------------------------------") gLogger.error (("Match not found: %s") % (result['Message'])) gLogger.info("-------------------------------------------------------------------") return S_ERROR() else: if result['Value'] == {}: gLogger.info("-------------------------------------------------------------------") gLogger.info("Value == Empty") gLogger.info("-------------------------------------------------------------------") return S_ERROR() mpiTaskQueue = result['Value'] classAdAgent = ClassAd(resourceJDL) classAdAgent.insertAttributeString('JobType', 'MPI') resourceJDL = str(classAdAgent.asJDL()) result = matcher.requestJob(resourceJDL) if not result['OK']: gLogger.error (("Request Job Error: %s") % (result['Message'])) return S_ERROR() elif result['OK']==False: gLogger.error (("Request Job False: %s") % (result['Message'])) return S_ERROR() else: gLogger.error (("Request Job OK")) jobJDL = result['Value']['JDL'] ### Review how to optimize this part (Importante) jobID1 = ClassAd(jobJDL) jobID = jobID1.getAttributeString('JobID') numProc = jobID1.getAttributeString('CPUNumber') mpiFlavor = jobID1.getAttributeString('Flavor') result = {'JobID':jobID, 'JobJDL':jobJDL, 'NumProc': numProc, 'MPIFlavor': mpiFlavor} print "S39" print result return S_OK(result)
def submitJob(self, jdl): """ Submit one job specified by its JDL to WMS """ if not self.jobManagerClient: jobManager = RPCClient( "WorkloadManagement/JobManager", useCertificates=self.useCertificates, timeout=self.timeout ) else: jobManager = self.jobManagerClient if os.path.exists(jdl): fic = open(jdl, "r") jdlString = fic.read() fic.close() else: # If file JDL does not exist, assume that the JDL is # passed as a string jdlString = jdl # Check the validity of the input JDL jdlString = jdlString.strip() if jdlString.find("[") != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd(jdlString) if not classAdJob.isOK(): return S_ERROR("Invalid job JDL") # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox(classAdJob) if not result["OK"]: return result # Submit the job now and get the new job ID result = jobManager.submitJob(classAdJob.asJDL()) if not result["OK"]: return result jobID = result["Value"] if "requireProxyUpload" in result and result["requireProxyUpload"]: gProxyManager.uploadProxy() # print "Sandbox uploading" return S_OK(jobID)
def submitJob( self, jdl, jobDescriptionObject = None ): """ Submit one job specified by its JDL to WMS """ if os.path.exists( jdl ): fic = open ( jdl, "r" ) jdlString = fic.read() fic.close() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl jdlString = jdlString.strip() # Strip of comments in the jdl string newJdlList = [] for line in jdlString.split('\n'): if not line.strip().startswith( '#' ): newJdlList.append( line ) jdlString = '\n'.join( newJdlList ) # Check the validity of the input JDL if jdlString.find( "[" ) != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd( jdlString ) if not classAdJob.isOK(): return S_ERROR( 'Invalid job JDL' ) # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox( classAdJob, jobDescriptionObject ) if not result['OK']: return result # Submit the job now and get the new job ID if not self.jobManager: self.jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = self.useCertificates, timeout = self.timeout ) result = self.jobManager.submitJob( classAdJob.asJDL() ) if 'requireProxyUpload' in result and result['requireProxyUpload']: gLogger.warn( "Need to upload the proxy" ) return result
def submitJob(self, jdl, jobDescriptionObject=None): """ Submit one job specified by its JDL to WMS """ if os.path.exists(jdl): fic = open(jdl, "r") jdlString = fic.read() fic.close() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl jdlString = jdlString.strip() # Strip of comments in the jdl string newJdlList = [] for line in jdlString.split('\n'): if not line.strip().startswith('#'): newJdlList.append(line) jdlString = '\n'.join(newJdlList) # Check the validity of the input JDL if jdlString.find("[") != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd(jdlString) if not classAdJob.isOK(): return S_ERROR('Invalid job JDL') # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject) if not result['OK']: return result # Submit the job now and get the new job ID if not self.jobManager: self.jobManager = RPCClient('WorkloadManagement/JobManager', useCertificates=self.useCertificates, timeout=self.timeout) result = self.jobManager.submitJob(classAdJob.asJDL()) if 'requireProxyUpload' in result and result['requireProxyUpload']: gLogger.warn("Need to upload the proxy") return result
def __sendJobToTaskQueue( self, job, classAdJob, siteCandidates, bannedSites ): """This method sends jobs to the task queue agent and if candidate sites are defined, updates job JDL accordingly. """ reqJDL = classAdJob.get_expression( 'JobRequirements' ) classAddReq = ClassAd( reqJDL ) if siteCandidates: classAddReq.insertAttributeVectorString( 'Sites', siteCandidates ) if bannedSites: classAddReq.insertAttributeVectorString( 'BannedSites', bannedSites ) if classAdJob.lookupAttribute( "SubmitPools" ): classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression( 'SubmitPools' ) ) if classAdJob.lookupAttribute( "GridMiddleware" ): classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression( 'GridMiddleware' ) ) if classAdJob.lookupAttribute( "PilotTypes" ): classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotTypes' ) ) #HAck to migrate old jobs to new ones. #DELETE ON 08/09 else: if classAdJob.lookupAttribute( "PilotType" ): classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotType' ) ) if classAdJob.lookupAttribute( "JobType" ): jobTypes = [ jt for jt in classAdJob.getListFromExpression( 'JobType' ) if jt ] classAddReq.insertAttributeVectorString( 'JobTypes', jobTypes ) #Required CE's requirements gridCEs = [ ce for ce in classAdJob.getListFromExpression( 'GridRequiredCEs' ) if ce ] if gridCEs: classAddReq.insertAttributeVectorString( 'GridCEs', gridCEs ) if siteCandidates: sites = ','.join( siteCandidates ) classAdJob.insertAttributeString( "Site", sites ) reqJDL = classAddReq.asJDL() classAdJob.insertAttributeInt( 'JobRequirements', reqJDL ) jdl = classAdJob.asJDL() result = self.jobDB.setJobJDL( job, jdl ) if not result['OK']: return result if siteCandidates: if len( siteCandidates ) == 1: self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, siteCandidates[0] ) ) self.jobDB.setJobAttribute( job, 'Site', siteCandidates[0] ) elif bannedSites: remainingSites = [] for site in siteCandidates: if not site in bannedSites: remainingSites.append( site ) if remainingSites: if len( remainingSites ) == 1: self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, remainingSites[0] ) ) self.jobDB.setJobAttribute( job, 'Site', remainingSites[0] ) else: self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( remainingSites ) ) ) result = self.jobDB.getJobAttribute(job,'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute( job, 'Site', siteGroup ) else: self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( siteCandidates ) ) ) result = self.jobDB.getJobAttribute(job,'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute( job, 'Site', siteGroup ) else: self.log.verbose( 'All sites are eligible for job %s' % job ) self.jobDB.setJobAttribute( job, 'Site', 'ANY' ) return self.setNextOptimizer( job )
def export_submitJob(self, jobDesc): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR("Can't submit using a limited proxy! (bad boy!)") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR('Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR('Job submission not authorized') #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parameteric one jobClassAd = ClassAd(jobDesc) parametricJob = False if jobClassAd.lookupAttribute('Parameters'): parametricJob = True if jobClassAd.isAttributeList('Parameters'): parameterList = jobClassAd.getListFromExpression('Parameters') else: pStep = 0 pFactor = 1 pStart = 1 nParameters = jobClassAd.getAttributeInt('Parameters') if not nParameters: value = jobClassAd.get_expression('Parameters') return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterStart'): value = jobClassAd.get_expression( 'ParameterStart').replace('"', '') try: pStart = int(value) except: try: pStart = float(value) except: return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterStep'): pStep = jobClassAd.getAttributeInt('ParameterStep') if not pStep: pStep = jobClassAd.getAttributeFloat('ParameterStep') if not pStep: value = jobClassAd.get_expression('ParameterStep') return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterFactor'): pFactor = jobClassAd.getAttributeInt('ParameterFactor') if not pFactor: pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor') if not pFactor: value = jobClassAd.get_expression( 'ParameterFactor') return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value) parameterList = list() parameterList.append(pStart) for i in range(nParameters - 1): parameterList.append(parameterList[i] * pFactor + pStep) if len(parameterList) > self.maxParametricJobs: return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs) jobDescList = [] nParam = len(parameterList) - 1 for n, p in enumerate(parameterList): newJobDesc = jobDesc.replace('%s', str(p)).replace( '%n', str(n).zfill(len(str(nParam)))) newClassAd = ClassAd(newJobDesc) for attr in ['Parameters', 'ParameterStep', 'ParameterFactor']: newClassAd.deleteAttribute(attr) if type(p) == type(' ') and p.startswith('{'): newClassAd.insertAttributeInt('Parameter', str(p)) else: newClassAd.insertAttributeString('Parameter', str(p)) newClassAd.insertAttributeInt('ParameterNumber', n) newJDL = newClassAd.asJDL() jobDescList.append(newJDL) else: jobDescList = [jobDesc] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) #Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() self.__sendNewJobsToMind(jobIDList) return result
def _processResourceDescription(self, resourceDescription): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if isinstance(resourceDescription, basestring): classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): raise ValueError('Illegal Resource JDL') self.log.verbose(classAdAgent.asJDL()) for name in singleValueDefFields: if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in multiValueMatchFields: if classAdAgent.lookupAttribute(name): if name == 'SubmitPool': resourceDict[ name] = classAdAgent.getListFromExpression(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization'): if classAdAgent.lookupAttribute(k): resourceDict[k] = classAdAgent.getAttributeString(k) else: for name in singleValueDefFields: if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if 'JobID' in resourceDescription: resourceDict['JobID'] = resourceDescription['JobID'] # Convert MaxRAM and NumberOfCores parameters into a list of tags maxRAM = resourceDescription.get('MaxRAM') nCores = resourceDescription.get('NumberOfProcessors') for param, key in [(maxRAM, 'GB'), (nCores, 'Cores')]: if param: try: intValue = int(param) / 1000 if intValue <= 128: paramList = range(1, intValue + 1) paramTags = [ '%d%s' % (par, key) for par in paramList ] resourceDict.setdefault("Tag", []).extend(paramTags) except ValueError: pass if 'Tag' in resourceDict: resourceDict['Tag'] = list(set(resourceDict['Tag'])) for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag'): if k in resourceDescription: resourceDict[k] = resourceDescription[k] return resourceDict
def submitJob(self, jdl, jobDescriptionObject=None): """Submit one job specified by its JDL to WMS. The JDL may actually be the desciption of a parametric job, resulting in multiple DIRAC jobs submitted to the DIRAC WMS """ if os.path.exists(jdl): with open(jdl, "r") as fic: jdlString = fic.read() else: # If file JDL does not exist, assume that the JDL is passed as a string jdlString = jdl jdlString = jdlString.strip() gLogger.debug("Submitting JDL", jdlString) # Strip of comments in the jdl string newJdlList = [] for line in jdlString.split("\n"): if not line.strip().startswith("#"): newJdlList.append(line) jdlString = "\n".join(newJdlList) # Check the validity of the input JDL if jdlString.find("[") != 0: jdlString = "[%s]" % jdlString classAdJob = ClassAd(jdlString) if not classAdJob.isOK(): return S_ERROR(EWMSJDL, "Invalid job JDL") # Check the size and the contents of the input sandbox result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject) if not result["OK"]: return result # Submit the job now and get the new job ID result = getParameterVectorLength(classAdJob) if not result["OK"]: return result nJobs = result["Value"] result = self.jobManager.submitJob(classAdJob.asJDL()) if nJobs: gLogger.debug("Applying transactional job submission") # The server applies transactional bulk submission, we should confirm the jobs if result["OK"]: jobIDList = result["Value"] if len(jobIDList) == nJobs: # Confirm the submitted jobs confirmed = False for _attempt in range(3): result = self.jobManager.confirmBulkSubmission(jobIDList) if result["OK"]: confirmed = True break time.sleep(1) if not confirmed: # The bulk submission failed, try to remove the created jobs resultDelete = self.jobManager.removeJob(jobIDList) error = "Job submission failed to confirm bulk transaction" if not resultDelete["OK"]: error += "; removal of created jobs failed" return S_ERROR(EWMSSUBM, error) else: return S_ERROR(EWMSSUBM, "The number of submitted jobs does not match job description") if result.get("requireProxyUpload"): gLogger.warn("Need to upload the proxy") return result
def _toJDL( self, xmlFile = '', jobDescriptionObject = None ): # messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ #Check if we have to do old bootstrap... classadJob = ClassAd( '[]' ) paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} arguments = [] scriptName = 'jobDescription.xml' if jobDescriptionObject is None: # if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml' if self.script: if os.path.exists( self.script ): scriptName = os.path.abspath( self.script ) self.log.verbose( 'Found script name %s' % scriptName ) else: self.log.error( "File not found", self.script ) else: if xmlFile: self.log.verbose( 'Found XML File %s' % xmlFile ) scriptName = xmlFile self.addToInputSandbox.append( scriptName ) elif isinstance( jobDescriptionObject, StringIO.StringIO ): self.log.verbose( "jobDescription is passed in as a StringIO object" ) else: self.log.error( "Where's the job description?" ) arguments.append( os.path.basename( scriptName ) ) if paramsDict.has_key( 'LogLevel' ): if paramsDict['LogLevel']['value']: arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) ) else: self.log.warn( 'Job LogLevel defined with null value' ) if paramsDict.has_key( 'DIRACSetup' ): if paramsDict['DIRACSetup']['value']: arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) ) else: self.log.warn( 'Job DIRACSetup defined with null value' ) if paramsDict.has_key( 'JobMode' ): if paramsDict['JobMode']['value']: arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) ) else: self.log.warn( 'Job Mode defined with null value' ) if paramsDict.has_key( 'JobConfigArgs' ): if paramsDict['JobConfigArgs']['value']: arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) ) else: self.log.warn( 'JobConfigArgs defined with null value' ) if self.parametricWFArguments: for name, value in self.parametricWFArguments.items(): arguments.append( "-p %s='%s'" % ( name, value ) ) classadJob.insertAttributeString( 'Executable', self.executable ) self.addToOutputSandbox.append( self.stderr ) self.addToOutputSandbox.append( self.stdout ) #Extract i/o sandbox parameters from steps and any input data parameters #to do when introducing step-level api... #To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ';'.join( self.addToInputSandbox ) if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] finalInputSandbox = currentFiles + ';' + extraFiles uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) ) paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox ) self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) ) else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = extraFiles paramsDict['InputSandbox']['type'] = 'JDL' if self.addToOutputSandbox: extraFiles = ';'.join( self.addToOutputSandbox ) if paramsDict.has_key( 'OutputSandbox' ): currentFiles = paramsDict['OutputSandbox']['value'] finalOutputSandbox = currentFiles + ';' + extraFiles uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) ) paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox ) self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) ) else: paramsDict['OutputSandbox'] = {} paramsDict['OutputSandbox']['value'] = extraFiles paramsDict['OutputSandbox']['type'] = 'JDL' if self.addToInputData: extraFiles = ';'.join( self.addToInputData ) if paramsDict.has_key( 'InputData' ): currentFiles = paramsDict['InputData']['value'] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ';' + extraFiles uniqueInputData = uniqueElements( finalInputData.split( ';' ) ) paramsDict['InputData']['value'] = ';'.join( uniqueInputData ) self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) ) else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = extraFiles paramsDict['InputData']['type'] = 'JDL' # Handle parameter sequences if self.numberOfParameters > 0: paramsDict, arguments = self._handleParameterSequences( paramsDict, arguments ) classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) #Add any JDL parameters to classad obeying lists with ';' rule for name, props in paramsDict.iteritems(): ptype = props['type'] value = props['value'] if isinstance( value, basestring) and re.search( ';', value ): value = value.split( ';' ) if name.lower() == 'requirements' and ptype == 'JDL': self.log.verbose( 'Found existing requirements: %s' % ( value ) ) if re.search( '^JDL', ptype ): if isinstance( value, list ): if isinstance( value[0], list ): classadJob.insertAttributeVectorStringList( name, value ) else: classadJob.insertAttributeVectorInt( name, value ) elif isinstance( value, basestring ) and value: classadJob.insertAttributeInt( name, value ) elif isinstance( value, ( int, long, float ) ): classadJob.insertAttributeInt( name, value ) if self.numberOfParameters > 0: classadJob.insertAttributeInt( 'Parameters', self.numberOfParameters ) for fToBeRemoved in [scriptName, self.stdout, self.stderr]: try: self.addToInputSandbox.remove( fToBeRemoved ) except ValueError: pass jdl = classadJob.asJDL() start = jdl.find( '[' ) end = jdl.rfind( ']' ) return jdl[( start + 1 ):( end - 1 )]
def _toJDL(self, xmlFile='', jobDescriptionObject=None): """ Creates a JDL representation of itself as a Job. Example usage: >>> job = Job() >>> job._toJDL() :param xmlFile: location of the XML file :type xmlFile: str :param jobDescriptionObject: if not None, it must be a StringIO object :type jobDescriptionObject: StringIO :returns: JDL (str) """ # Check if we have to do old bootstrap... classadJob = ClassAd('[]') paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {'type': param.getType(), 'value': param.getValue()} arguments = [] scriptName = 'jobDescription.xml' if jobDescriptionObject is None: # if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml' # Messy but need to account for xml file being in /tmp/guid dir if self.script: if os.path.exists(self.script): scriptName = os.path.abspath(self.script) self.log.verbose('Found script name %s' % scriptName) else: self.log.warn("File not found", self.script) else: if xmlFile: if os.path.exists(xmlFile): self.log.verbose('Found XML File %s' % xmlFile) scriptName = xmlFile else: self.log.warn("File not found", xmlFile) else: if os.path.exists('jobDescription.xml'): scriptName = os.path.abspath('jobDescription.xml') self.log.verbose('Found script name %s' % scriptName) else: self.log.warn("Job description XML file not found") self.addToInputSandbox.append(scriptName) elif isinstance(jobDescriptionObject, StringIO.StringIO): self.log.verbose("jobDescription is passed in as a StringIO object") else: self.log.error("Where's the job description?") arguments.append(os.path.basename(scriptName)) if 'LogLevel' in paramsDict: if paramsDict['LogLevel']['value']: arguments.append('-o LogLevel=%s' % (paramsDict['LogLevel']['value'])) else: self.log.warn('Job LogLevel defined with null value') if 'DIRACSetup' in paramsDict: if paramsDict['DIRACSetup']['value']: arguments.append('-o DIRAC/Setup=%s' % (paramsDict['DIRACSetup']['value'])) else: self.log.warn('Job DIRACSetup defined with null value') if 'JobMode' in paramsDict: if paramsDict['JobMode']['value']: arguments.append('-o JobMode=%s' % (paramsDict['JobMode']['value'])) else: self.log.warn('Job Mode defined with null value') if 'JobConfigArgs' in paramsDict: if paramsDict['JobConfigArgs']['value']: arguments.append('%s' % (paramsDict['JobConfigArgs']['value'])) else: self.log.warn('JobConfigArgs defined with null value') if self.parametricWFArguments: for name, value in self.parametricWFArguments.iteritems(): arguments.append("-p %s='%s'" % (name, value)) classadJob.insertAttributeString('Executable', self.executable) self.addToOutputSandbox.append(self.stderr) self.addToOutputSandbox.append(self.stdout) # Extract i/o sandbox parameters from steps and any input data parameters # to do when introducing step-level api... # To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ';'.join(self.addToInputSandbox) if 'InputSandbox' in paramsDict: currentFiles = paramsDict['InputSandbox']['value'] finalInputSandbox = currentFiles + ';' + extraFiles uniqueInputSandbox = uniqueElements(finalInputSandbox.split(';')) paramsDict['InputSandbox']['value'] = ';'.join(uniqueInputSandbox) self.log.verbose('Final unique Input Sandbox %s' % (';'.join(uniqueInputSandbox))) else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = extraFiles paramsDict['InputSandbox']['type'] = 'JDL' if self.addToOutputSandbox: extraFiles = ';'.join(self.addToOutputSandbox) if 'OutputSandbox' in paramsDict: currentFiles = paramsDict['OutputSandbox']['value'] finalOutputSandbox = currentFiles + ';' + extraFiles uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(';')) paramsDict['OutputSandbox']['value'] = ';'.join(uniqueOutputSandbox) self.log.verbose('Final unique Output Sandbox %s' % (';'.join(uniqueOutputSandbox))) else: paramsDict['OutputSandbox'] = {} paramsDict['OutputSandbox']['value'] = extraFiles paramsDict['OutputSandbox']['type'] = 'JDL' if self.addToInputData: extraFiles = ';'.join(self.addToInputData) if 'InputData' in paramsDict: currentFiles = paramsDict['InputData']['value'] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ';' + extraFiles uniqueInputData = uniqueElements(finalInputData.split(';')) paramsDict['InputData']['value'] = ';'.join(uniqueInputData) self.log.verbose('Final unique Input Data %s' % (';'.join(uniqueInputData))) else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = extraFiles paramsDict['InputData']['type'] = 'JDL' # Handle parameter sequences if self.numberOfParameters > 0: paramsDict, arguments = self._handleParameterSequences(paramsDict, arguments) classadJob.insertAttributeString('Arguments', ' '.join(arguments)) # Add any JDL parameters to classad obeying lists with ';' rule for name, props in paramsDict.iteritems(): ptype = props['type'] value = props['value'] if isinstance(value, basestring) and re.search(';', value): value = value.split(';') if name.lower() == 'requirements' and ptype == 'JDL': self.log.verbose('Found existing requirements: %s' % (value)) if re.search('^JDL', ptype): if isinstance(value, list): if isinstance(value[0], list): classadJob.insertAttributeVectorStringList(name, value) else: classadJob.insertAttributeVectorInt(name, value) elif isinstance(value, basestring) and value: classadJob.insertAttributeInt(name, value) elif isinstance(value, (int, long, float)): classadJob.insertAttributeInt(name, value) if self.numberOfParameters > 0: classadJob.insertAttributeInt('Parameters', self.numberOfParameters) for fToBeRemoved in [scriptName, self.stdout, self.stderr]: try: self.addToInputSandbox.remove(fToBeRemoved) except ValueError: pass jdl = classadJob.asJDL() start = jdl.find('[') end = jdl.rfind(']') return jdl[(start + 1):(end - 1)]
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ #Check if we have to do old bootstrap... classadJob = ClassAd( '[]' ) paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} scriptname = 'jobDescription.xml' arguments = [] if self.script: if os.path.exists( self.script ): scriptname = os.path.abspath( self.script ) self.log.verbose( 'Found script name %s' % scriptname ) else: if xmlFile: self.log.verbose( 'Found XML File %s' % xmlFile ) scriptname = xmlFile arguments.append( os.path.basename( scriptname ) ) self.addToInputSandbox.append( scriptname ) if paramsDict.has_key( 'LogLevel' ): if paramsDict['LogLevel']['value']: arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) ) else: self.log.warn( 'Job LogLevel defined with null value' ) if paramsDict.has_key( 'DIRACSetup' ): if paramsDict['DIRACSetup']['value']: arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) ) else: self.log.warn( 'Job DIRACSetup defined with null value' ) if paramsDict.has_key( 'JobMode' ): if paramsDict['JobMode']['value']: arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) ) else: self.log.warn( 'Job Mode defined with null value' ) if paramsDict.has_key( 'JobConfigArgs' ): if paramsDict['JobConfigArgs']['value']: arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) ) else: self.log.warn( 'JobConfigArgs defined with null value' ) classadJob.insertAttributeString( 'Executable', self.executable ) self.addToOutputSandbox.append( self.stderr ) self.addToOutputSandbox.append( self.stdout ) #Extract i/o sandbox parameters from steps and any input data parameters #to do when introducing step-level api... #To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ';'.join( self.addToInputSandbox ) if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] finalInputSandbox = currentFiles + ';' + extraFiles uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) ) paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox ) self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) ) else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = extraFiles paramsDict['InputSandbox']['type'] = 'JDL' if self.addToOutputSandbox: extraFiles = ';'.join( self.addToOutputSandbox ) if paramsDict.has_key( 'OutputSandbox' ): currentFiles = paramsDict['OutputSandbox']['value'] finalOutputSandbox = currentFiles + ';' + extraFiles uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) ) paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox ) self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) ) else: paramsDict['OutputSandbox'] = {} paramsDict['OutputSandbox']['value'] = extraFiles paramsDict['OutputSandbox']['type'] = 'JDL' if self.addToInputData: extraFiles = ';'.join( self.addToInputData ) if paramsDict.has_key( 'InputData' ): currentFiles = paramsDict['InputData']['value'] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ';' + extraFiles uniqueInputData = uniqueElements( finalInputData.split( ';' ) ) paramsDict['InputData']['value'] = ';'.join( uniqueInputData ) self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) ) else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = extraFiles paramsDict['InputData']['type'] = 'JDL' # Handle here the Parametric values if self.parametric: for pType in ['InputData', 'InputSandbox']: if self.parametric.has_key( pType ): if paramsDict.has_key( pType ) and paramsDict[pType]['value']: pData = self.parametric[pType] # List of lists case currentFiles = paramsDict[pType]['value'].split( ';' ) tmpList = [] if type( pData[0] ) == list: for pElement in pData: tmpList.append( currentFiles + pElement ) else: for pElement in pData: tmpList.append( currentFiles + [pElement] ) self.parametric[pType] = tmpList paramsDict[pType] = {} paramsDict[pType]['value'] = "%s" paramsDict[pType]['type'] = 'JDL' self.parametric['files'] = self.parametric[pType] arguments.append( ' -p Parametric' + pType + '=%s' ) break if self.parametric.has_key( 'files' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['files'] paramsDict['Parameters']['type'] = 'JDL' if self.parametric.has_key( 'GenericParameters' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['GenericParameters'] paramsDict['Parameters']['type'] = 'JDL' arguments.append( ' -p ParametricParameters=%s' ) ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) #Add any JDL parameters to classad obeying lists with ';' rule requirements = False for name, props in paramsDict.items(): ptype = props['type'] value = props['value'] if name.lower() == 'requirements' and ptype == 'JDL': self.log.verbose( 'Found existing requirements: %s' % ( value ) ) requirements = True if re.search( '^JDL', ptype ): if type( value ) == list: if type( value[0] ) == list: classadJob.insertAttributeVectorStringList( name, value ) else: classadJob.insertAttributeVectorString( name, value ) elif value == "%s": classadJob.insertAttributeInt( name, value ) elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) else: classadJob.insertAttributeVectorString( name, value.split( ';' ) ) if not requirements: reqtsDict = self.reqParams exprn = '' plus = '' for name, props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if not ptype == 'dict': if ptype == 'JDLReqt': if value and not value.lower() == 'any': plus = ' && ' if re.search( ';', value ): for val in value.split( ';' ): exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus else: exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus if len( plus ): exprn = exprn[:-len( plus )] if not exprn: exprn = 'true' self.log.verbose( 'Requirements: %s' % ( exprn ) ) #classadJob.set_expression('Requirements', exprn) self.addToInputSandbox.remove( scriptname ) self.addToOutputSandbox.remove( self.stdout ) self.addToOutputSandbox.remove( self.stderr ) jdl = classadJob.asJDL() start = jdl.find( '[' ) end = jdl.rfind( ']' ) return jdl[( start + 1 ):( end - 1 )]
def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if isinstance( resourceDescription, basestring ): classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if 'JobID' in resourceDescription: resourceDict['JobID'] = resourceDescription['JobID'] # Convert MaxRAM and NumberOfCores parameters into a list of tags maxRAM = resourceDescription.get( 'MaxRAM' ) nCores = resourceDescription.get( 'NumberOfProcessors' ) for param, key in [ ( maxRAM, 'GB' ), ( nCores, 'Cores' ) ]: if param: try: intValue = int( param )/1000 if intValue <= 128: paramList = range( 1, intValue + 1 ) paramTags = [ '%d%s' % ( par, key ) for par in paramList ] resourceDict.setdefault( "Tag", [] ).extend( paramTags ) except ValueError: pass if 'Tag' in resourceDict: resourceDict['Tag'] = list( set( resourceDict['Tag'] ) ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def _toJDL( self, xmlFile="", jobDescriptionObject=None ): # messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ # Check if we have to do old bootstrap... classadJob = ClassAd("[]") paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {"type": param.getType(), "value": param.getValue()} arguments = [] scriptname = "jobDescription.xml" if jobDescriptionObject is None: # if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml' if self.script: if os.path.exists(self.script): scriptname = os.path.abspath(self.script) self.log.verbose("Found script name %s" % scriptname) else: self.log.error("File not found", self.script) else: if xmlFile: self.log.verbose("Found XML File %s" % xmlFile) scriptname = xmlFile self.addToInputSandbox.append(scriptname) elif isinstance(jobDescriptionObject, StringIO.StringIO): self.log.verbose("jobDescription is passed in as a StringIO object") else: self.log.error("Where's the job description?") arguments.append(os.path.basename(scriptname)) if paramsDict.has_key("LogLevel"): if paramsDict["LogLevel"]["value"]: arguments.append("-o LogLevel=%s" % (paramsDict["LogLevel"]["value"])) else: self.log.warn("Job LogLevel defined with null value") if paramsDict.has_key("DIRACSetup"): if paramsDict["DIRACSetup"]["value"]: arguments.append("-o DIRAC/Setup=%s" % (paramsDict["DIRACSetup"]["value"])) else: self.log.warn("Job DIRACSetup defined with null value") if paramsDict.has_key("JobMode"): if paramsDict["JobMode"]["value"]: arguments.append("-o JobMode=%s" % (paramsDict["JobMode"]["value"])) else: self.log.warn("Job Mode defined with null value") if paramsDict.has_key("JobConfigArgs"): if paramsDict["JobConfigArgs"]["value"]: arguments.append("%s" % (paramsDict["JobConfigArgs"]["value"])) else: self.log.warn("JobConfigArgs defined with null value") classadJob.insertAttributeString("Executable", self.executable) self.addToOutputSandbox.append(self.stderr) self.addToOutputSandbox.append(self.stdout) # Extract i/o sandbox parameters from steps and any input data parameters # to do when introducing step-level api... # To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ";".join(self.addToInputSandbox) if paramsDict.has_key("InputSandbox"): currentFiles = paramsDict["InputSandbox"]["value"] finalInputSandbox = currentFiles + ";" + extraFiles uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";")) paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox) self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox))) else: paramsDict["InputSandbox"] = {} paramsDict["InputSandbox"]["value"] = extraFiles paramsDict["InputSandbox"]["type"] = "JDL" if self.addToOutputSandbox: extraFiles = ";".join(self.addToOutputSandbox) if paramsDict.has_key("OutputSandbox"): currentFiles = paramsDict["OutputSandbox"]["value"] finalOutputSandbox = currentFiles + ";" + extraFiles uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";")) paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox) self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox))) else: paramsDict["OutputSandbox"] = {} paramsDict["OutputSandbox"]["value"] = extraFiles paramsDict["OutputSandbox"]["type"] = "JDL" if self.addToInputData: extraFiles = ";".join(self.addToInputData) if paramsDict.has_key("InputData"): currentFiles = paramsDict["InputData"]["value"] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ";" + extraFiles uniqueInputData = uniqueElements(finalInputData.split(";")) paramsDict["InputData"]["value"] = ";".join(uniqueInputData) self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData))) else: paramsDict["InputData"] = {} paramsDict["InputData"]["value"] = extraFiles paramsDict["InputData"]["type"] = "JDL" # Handle here the Parametric values if self.parametric: for pType in ["InputData", "InputSandbox"]: if self.parametric.has_key(pType): if paramsDict.has_key(pType) and paramsDict[pType]["value"]: pData = self.parametric[pType] # List of lists case currentFiles = paramsDict[pType]["value"].split(";") tmpList = [] if type(pData[0]) == list: for pElement in pData: tmpList.append(currentFiles + pElement) else: for pElement in pData: tmpList.append(currentFiles + [pElement]) self.parametric[pType] = tmpList paramsDict[pType] = {} paramsDict[pType]["value"] = "%s" paramsDict[pType]["type"] = "JDL" self.parametric["files"] = self.parametric[pType] arguments.append(" -p Parametric" + pType + "=%s") break if self.parametric.has_key("files"): paramsDict["Parameters"] = {} paramsDict["Parameters"]["value"] = self.parametric["files"] paramsDict["Parameters"]["type"] = "JDL" if self.parametric.has_key("GenericParameters"): paramsDict["Parameters"] = {} paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"] paramsDict["Parameters"]["type"] = "JDL" arguments.append(" -p ParametricParameters=%s") ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString("Arguments", " ".join(arguments)) # Add any JDL parameters to classad obeying lists with ';' rule for name, props in paramsDict.items(): ptype = props["type"] value = props["value"] if name.lower() == "requirements" and ptype == "JDL": self.log.verbose("Found existing requirements: %s" % (value)) if re.search("^JDL", ptype): if type(value) == list: if type(value[0]) == list: classadJob.insertAttributeVectorStringList(name, value) else: classadJob.insertAttributeVectorString(name, value) elif value == "%s": classadJob.insertAttributeInt(name, value) elif not re.search(";", value) or name == "GridRequirements": # not a nice fix... classadJob.insertAttributeString(name, value) else: classadJob.insertAttributeVectorString(name, value.split(";")) for fToBeRemoved in [scriptname, self.stdout, self.stderr]: try: self.addToInputSandbox.remove(fToBeRemoved) except ValueError: pass jdl = classadJob.asJDL() start = jdl.find("[") end = jdl.rfind("]") return jdl[(start + 1) : (end - 1)]
def selectJob(self, resourceDescription): """ Main job selection function to find the highest priority job matching the resource capacity """ startTime = time.time() # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR("Illegal Resource JDL") gLogger.verbose(classAdAgent.asJDL()) for name in taskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == "CPUTime": resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString(name) for name in taskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): resourceDict[name] = classAdAgent.getAttributeString(name) # Check if a JobID is requested if classAdAgent.lookupAttribute("JobID"): resourceDict["JobID"] = classAdAgent.getAttributeInt("JobID") if classAdAgent.lookupAttribute("DIRACVersion"): resourceDict["DIRACVersion"] = classAdAgent.getAttributeString("DIRACVersion") else: for name in taskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in taskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key("JobID"): resourceDict["JobID"] = resourceDescription["JobID"] if resourceDescription.has_key("DIRACVersion"): resourceDict["DIRACVersion"] = resourceDescription["DIRACVersion"] # Check the pilot DIRAC version if self.checkPilotVersion: if "DIRACVersion" in resourceDict: if self.pilotVersion and resourceDict["DIRACVersion"] != self.pilotVersion: return S_ERROR( "Pilot version does not match the production version %s:%s" % (resourceDict["DIRACVersion"], self.pilotVersion) ) # Get common site mask and check the agent site result = jobDB.getSiteMask(siteState="Active") if result["OK"]: maskList = result["Value"] else: return S_ERROR("Internal error: can not get site mask") if not "Site" in resourceDict: return S_ERROR("Missing Site Name in Resource JDL") siteName = resourceDict["Site"] if resourceDict["Site"] not in maskList: if "GridCE" in resourceDict: del resourceDict["Site"] else: return S_ERROR("Site not in mask and GridCE not specified") resourceDict["Setup"] = self.serviceInfoDict["clientSetup"] if DEBUG: print "Resource description:" for k, v in resourceDict.items(): print k.rjust(20), v # Check if Job Limits are imposed onto the site extraConditions = {} if self.siteJobLimits: result = self.getExtraConditions(siteName) if result["OK"]: extraConditions = result["Value"] if extraConditions: gLogger.info("Job Limits for site %s are: %s" % (siteName, str(extraConditions))) result = taskQueueDB.matchAndGetJob(resourceDict, extraConditions=extraConditions) if DEBUG: print result if not result["OK"]: return result result = result["Value"] if not result["matchFound"]: return S_ERROR("No match found") jobID = result["jobId"] resAtt = jobDB.getJobAttributes(jobID, ["OwnerDN", "OwnerGroup", "Status"]) if not resAtt["OK"]: return S_ERROR("Could not retrieve job attributes") if not resAtt["Value"]: return S_ERROR("No attributes returned for job") if not resAtt["Value"]["Status"] == "Waiting": gLogger.error("Job %s matched by the TQ is not in Waiting state" % str(jobID)) result = taskQueueDB.deleteJob(jobID) result = jobDB.setJobStatus(jobID, status="Matched", minor="Assigned") result = jobLoggingDB.addLoggingRecord(jobID, status="Matched", minor="Assigned", source="Matcher") result = jobDB.getJobJDL(jobID) if not result["OK"]: return S_ERROR("Failed to get the job JDL") resultDict = {} resultDict["JDL"] = result["Value"] resultDict["JobID"] = jobID matchTime = time.time() - startTime gLogger.info("Match time: [%s]" % str(matchTime)) gMonitor.addMark("matchTime", matchTime) # Get some extra stuff into the response returned resOpt = jobDB.getJobOptParameters(jobID) if resOpt["OK"]: for key, value in resOpt["Value"].items(): resultDict[key] = value resAtt = jobDB.getJobAttributes(jobID, ["OwnerDN", "OwnerGroup"]) if not resAtt["OK"]: return S_ERROR("Could not retrieve job attributes") if not resAtt["Value"]: return S_ERROR("No attributes returned for job") resultDict["DN"] = resAtt["Value"]["OwnerDN"] resultDict["Group"] = resAtt["Value"]["OwnerGroup"] return S_OK(resultDict)
def generateParametricJobs( jobClassAd ): """ Generate a series of ClassAd job descriptions expanding job parameters :param jobClassAd: ClassAd job description object :return: list of ClassAd job description objects """ if not jobClassAd.lookupAttribute( 'Parameters' ): return S_OK( [ jobClassAd.asJDL() ] ) result = getParameterVectorLength( jobClassAd ) if not result['OK']: return result nParValues = result['Value'] if nParValues is None: return S_ERROR(EWMSJDL, 'Can not determine the number of job parameters') parameterDict = {} attributes = jobClassAd.getAttributes() for attribute in attributes: for key in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]: if attribute.startswith( key ): seqID = '0' if not '.' in attribute else attribute.split( '.' )[1] parameterDict.setdefault( seqID, {} ) if key == 'Parameters': if jobClassAd.isAttributeList( attribute ): parList = jobClassAd.getListFromExpression( attribute ) if len( parList ) != nParValues: return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' ) parameterDict[seqID]['ParameterList'] = parList else: if attribute != "Parameters": return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' ) nPar = jobClassAd.getAttributeInt( attribute ) if nPar is None: value = jobClassAd.get_expression( attribute ) return S_ERROR( EWMSJDL, 'Inconsistent parametric job description: %s=%s' % ( attribute, value ) ) parameterDict[seqID]['Parameters'] = nPar else: value = jobClassAd.getAttributeInt( attribute ) if value is None: value = jobClassAd.getAttributeFloat( attribute ) if value is None: value = jobClassAd.get_expression( attribute ) return S_ERROR( 'Illegal value for %s JDL field: %s' % ( attribute, value ) ) parameterDict[seqID][key] = value if '0' in parameterDict and not parameterDict.get( '0' ): parameterDict.pop( '0' ) parameterLists = {} for seqID in parameterDict: parList = __getParameterSequence( nParValues, parList = parameterDict[seqID].get( 'ParameterList', [] ), parStart = parameterDict[seqID].get( 'ParameterStart', 1 ), parStep = parameterDict[seqID].get( 'ParameterStep', 0 ), parFactor = parameterDict[seqID].get( 'ParameterFactor', 1 ) ) if not parList: return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' ) parameterLists[seqID] = parList jobDescList = [] jobDesc = jobClassAd.asJDL() # Width of the sequential parameter number zLength = len( str( nParValues - 1 ) ) for n in range( nParValues ): newJobDesc = jobDesc newJobDesc = newJobDesc.replace( '%n', str( n ).zfill( zLength ) ) newClassAd = ClassAd( newJobDesc ) for seqID in parameterLists: parameter = parameterLists[seqID][n] for attribute in newClassAd.getAttributes(): __updateAttribute( newClassAd, attribute, seqID, str( parameter ) ) for seqID in parameterLists: for attribute in ['Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor']: if seqID == '0': newClassAd.deleteAttribute( attribute ) else: newClassAd.deleteAttribute( '%s.%s' % ( attribute, seqID ) ) parameter = parameterLists[seqID][n] if seqID == '0': attribute = 'Parameter' else: attribute = 'Parameter.%s' % seqID if isinstance( parameter, basestring) and parameter.startswith( '{' ): newClassAd.insertAttributeInt( attribute, str( parameter ) ) else: newClassAd.insertAttributeString( attribute, str( parameter ) ) newClassAd.insertAttributeInt( 'ParameterNumber', n ) newJDL = newClassAd.asJDL() jobDescList.append( newJDL ) return S_OK( jobDescList )
def generateParametricJobs(jobClassAd): """Generate a series of ClassAd job descriptions expanding job parameters :param jobClassAd: ClassAd job description object :return: list of ClassAd job description objects """ if not jobClassAd.lookupAttribute("Parameters"): return S_OK([jobClassAd.asJDL()]) result = getParameterVectorLength(jobClassAd) if not result["OK"]: return result nParValues = result["Value"] if nParValues is None: return S_ERROR(EWMSJDL, "Can not determine the number of job parameters") parameterDict = {} attributes = jobClassAd.getAttributes() for attribute in attributes: for key in [ "Parameters", "ParameterStart", "ParameterStep", "ParameterFactor" ]: if attribute.startswith(key): seqID = "0" if "." not in attribute else attribute.split( ".")[1] parameterDict.setdefault(seqID, {}) if key == "Parameters": if jobClassAd.isAttributeList(attribute): parList = jobClassAd.getListFromExpression(attribute) if len(parList) != nParValues: return S_ERROR( EWMSJDL, "Inconsistent parametric job description") parameterDict[seqID]["ParameterList"] = parList else: if attribute != "Parameters": return S_ERROR( EWMSJDL, "Inconsistent parametric job description") nPar = jobClassAd.getAttributeInt(attribute) if nPar is None: value = jobClassAd.get_expression(attribute) return S_ERROR( EWMSJDL, "Inconsistent parametric job description: %s=%s" % (attribute, value)) parameterDict[seqID]["Parameters"] = nPar else: value = jobClassAd.getAttributeInt(attribute) if value is None: value = jobClassAd.getAttributeFloat(attribute) if value is None: value = jobClassAd.get_expression(attribute) return S_ERROR( "Illegal value for %s JDL field: %s" % (attribute, value)) parameterDict[seqID][key] = value if "0" in parameterDict and not parameterDict.get("0"): parameterDict.pop("0") parameterLists = {} for seqID in parameterDict: parList = __getParameterSequence( nParValues, parList=parameterDict[seqID].get("ParameterList", []), parStart=parameterDict[seqID].get("ParameterStart", 1), parStep=parameterDict[seqID].get("ParameterStep", 0), parFactor=parameterDict[seqID].get("ParameterFactor", 1), ) if not parList: return S_ERROR(EWMSJDL, "Inconsistent parametric job description") parameterLists[seqID] = parList jobDescList = [] jobDesc = jobClassAd.asJDL() # Width of the sequential parameter number zLength = len(str(nParValues - 1)) for n in range(nParValues): newJobDesc = jobDesc newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength)) newClassAd = ClassAd(newJobDesc) for seqID in parameterLists: parameter = parameterLists[seqID][n] for attribute in newClassAd.getAttributes(): __updateAttribute(newClassAd, attribute, seqID, str(parameter)) for seqID in parameterLists: for attribute in [ "Parameters", "ParameterStart", "ParameterStep", "ParameterFactor" ]: if seqID == "0": newClassAd.deleteAttribute(attribute) else: newClassAd.deleteAttribute("%s.%s" % (attribute, seqID)) parameter = parameterLists[seqID][n] if seqID == "0": attribute = "Parameter" else: attribute = "Parameter.%s" % seqID if isinstance(parameter, str) and parameter.startswith("{"): newClassAd.insertAttributeInt(attribute, str(parameter)) else: newClassAd.insertAttributeString(attribute, str(parameter)) newClassAd.insertAttributeInt("ParameterNumber", n) newJDL = newClassAd.asJDL() jobDescList.append(newJDL) return S_OK(jobDescList)
def selectJob( self, resourceDescription ): """ Main job selection function to find the highest priority job matching the resource capacity """ startTime = time.time() # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) if classAdAgent.lookupAttribute( 'DIRACVersion' ): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' ) if classAdAgent.lookupAttribute( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization' ) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key( 'DIRACVersion' ): resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion'] if resourceDescription.has_key( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = resourceDescription['VirtualOrganization'] # Check the pilot DIRAC version if self.checkPilotVersion: if not 'DIRACVersion' in resourceDict: return S_ERROR( 'Version check requested and not provided by Pilot' ) # Check if the matching Request provides a VirtualOrganization if 'VirtualOrganization' in resourceDict: voName = resourceDict['VirtualOrganization'] # Check if the matching Request provides an OwnerGroup elif 'OwnerGroup' in resourceDict: voName = getVOForGroup( resourceDict['OwnerGroup'] ) # else take the default VirtualOrganization for the installation else: voName = getVOForGroup( '' ) self.pilotVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( voName, self.setup ), '' ) if self.pilotVersion and resourceDict['DIRACVersion'] != self.pilotVersion: return S_ERROR( 'Pilot version does not match the production version %s:%s' % \ ( resourceDict['DIRACVersion'], self.pilotVersion ) ) # Get common site mask and check the agent site result = gJobDB.getSiteMask( siteState = 'Active' ) if result['OK']: maskList = result['Value'] else: return S_ERROR( 'Internal error: can not get site mask' ) if not 'Site' in resourceDict: return S_ERROR( 'Missing Site Name in Resource JDL' ) siteName = resourceDict['Site'] if resourceDict['Site'] not in maskList: if 'GridCE' in resourceDict: del resourceDict['Site'] else: return S_ERROR( 'Site not in mask and GridCE not specified' ) resourceDict['Setup'] = self.serviceInfoDict['clientSetup'] if DEBUG: print "Resource description:" for key, value in resourceDict.items(): print key.rjust( 20 ), value # Check if Job Limits are imposed onto the site extraConditions = {} if self.siteJobLimits: result = self.getExtraConditions( siteName ) if result['OK']: extraConditions = result['Value'] if extraConditions: gLogger.info( 'Job Limits for site %s are: %s' % ( siteName, str( extraConditions ) ) ) result = gTaskQueueDB.matchAndGetJob( resourceDict, extraConditions = extraConditions ) if DEBUG: print result if not result['OK']: return result result = result['Value'] if not result['matchFound']: return S_ERROR( 'No match found' ) jobID = result['jobId'] resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) if not resAtt['Value']['Status'] == 'Waiting': gLogger.error( 'Job %s matched by the TQ is not in Waiting state' % str( jobID ) ) result = gTaskQueueDB.deleteJob( jobID ) result = gJobDB.setJobStatus( jobID, status = 'Matched', minor = 'Assigned' ) result = gJobLoggingDB.addLoggingRecord( jobID, status = 'Matched', minor = 'Assigned', source = 'Matcher' ) result = gJobDB.getJobJDL( jobID ) if not result['OK']: return S_ERROR( 'Failed to get the job JDL' ) resultDict = {} resultDict['JDL'] = result['Value'] resultDict['JobID'] = jobID matchTime = time.time() - startTime gLogger.info( "Match time: [%s]" % str( matchTime ) ) gMonitor.addMark( "matchTime", matchTime ) # Get some extra stuff into the response returned resOpt = gJobDB.getJobOptParameters( jobID ) if resOpt['OK']: for key, value in resOpt['Value'].items(): resultDict[key] = value resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) resultDict['DN'] = resAtt['Value']['OwnerDN'] resultDict['Group'] = resAtt['Value']['OwnerGroup'] return S_OK( resultDict )
def export_submitJob( self, jobDesc ): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" ) # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR( 'Failed to get job policies' ) policyDict = result['Value'] if not policyDict[ RIGHT_SUBMIT ]: return S_ERROR( 'Job submission not authorized' ) #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parameteric one jobClassAd = ClassAd( jobDesc ) parametricJob = False if jobClassAd.lookupAttribute( 'Parameters' ): parametricJob = True if jobClassAd.isAttributeList( 'Parameters' ): parameterList = jobClassAd.getListFromExpression( 'Parameters' ) else: pStep = 0 pFactor = 1 pStart = 1 nParameters = jobClassAd.getAttributeInt( 'Parameters' ) if not nParameters: value = jobClassAd.get_expression( 'Parameters' ) return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value ) if jobClassAd.lookupAttribute( 'ParameterStart' ): value = jobClassAd.get_expression( 'ParameterStart' ).replace( '"', '' ) try: pStart = int( value ) except: try: pStart = float( value ) except: return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value ) if jobClassAd.lookupAttribute( 'ParameterStep' ): pStep = jobClassAd.getAttributeInt( 'ParameterStep' ) if not pStep: pStep = jobClassAd.getAttributeFloat( 'ParameterStep' ) if not pStep: value = jobClassAd.get_expression( 'ParameterStep' ) return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value ) if jobClassAd.lookupAttribute( 'ParameterFactor' ): pFactor = jobClassAd.getAttributeInt( 'ParameterFactor' ) if not pFactor: pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor' ) if not pFactor: value = jobClassAd.get_expression( 'ParameterFactor' ) return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value ) parameterList = list() parameterList.append( pStart ) for i in range( nParameters - 1 ): parameterList.append( parameterList[i] * pFactor + pStep ) if len( parameterList ) > self.maxParametricJobs: return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs ) jobDescList = [] nParam = len(parameterList) - 1 for n,p in enumerate(parameterList): newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n).zfill(len(str(nParam)))) newClassAd = ClassAd(newJobDesc) for attr in ['Parameters','ParameterStep','ParameterFactor']: newClassAd.deleteAttribute(attr) if type( p ) == type ( ' ' ) and p.startswith('{'): newClassAd.insertAttributeInt( 'Parameter',str(p) ) else: newClassAd.insertAttributeString( 'Parameter', str( p ) ) newClassAd.insertAttributeInt( 'ParameterNumber', n ) newJDL = newClassAd.asJDL() jobDescList.append( newJDL ) else: jobDescList = [ jobDesc ] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup ) if not result['OK']: return result jobID = result['JobID'] gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) ) gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' ) jobIDList.append( jobID ) #Set persistency flag retVal = gProxyManager.getUserPersistence( self.ownerDN, self.ownerGroup ) if 'Value' not in retVal or not retVal[ 'Value' ]: gProxyManager.setPersistency( self.ownerDN, self.ownerGroup, True ) if parametricJob: result = S_OK( jobIDList ) else: result = S_OK( jobIDList[0] ) result['JobID'] = result['Value'] result[ 'requireProxyUpload' ] = self.__checkIfProxyUploadIsRequired() self.__sendNewJobsToMind( jobIDList ) return result
def __sendJobToTaskQueue(self, job, classAdJob, siteCandidates, bannedSites): """This method sends jobs to the task queue agent and if candidate sites are defined, updates job JDL accordingly. """ reqJDL = classAdJob.get_expression('JobRequirements') classAddReq = ClassAd(reqJDL) if siteCandidates: classAddReq.insertAttributeVectorString('Sites', siteCandidates) if bannedSites: classAddReq.insertAttributeVectorString('BannedSites', bannedSites) if classAdJob.lookupAttribute("SubmitPools"): classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression('SubmitPools')) if classAdJob.lookupAttribute("GridMiddleware"): classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression('GridMiddleware')) if classAdJob.lookupAttribute("PilotTypes"): classAddReq.set_expression('PilotTypes', classAdJob.get_expression('PilotTypes')) #HAck to migrate old jobs to new ones. #DELETE ON 08/09 else: if classAdJob.lookupAttribute("PilotType"): classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression('PilotType')) if classAdJob.lookupAttribute("JobType"): jobTypes = [ jt for jt in classAdJob.getListFromExpression('JobType') if jt ] classAddReq.insertAttributeVectorString('JobTypes', jobTypes) #Required CE's requirements gridCEs = [ ce for ce in classAdJob.getListFromExpression('GridRequiredCEs') if ce ] if gridCEs: classAddReq.insertAttributeVectorString('GridCEs', gridCEs) if siteCandidates: sites = ','.join(siteCandidates) classAdJob.insertAttributeString("Site", sites) reqJDL = classAddReq.asJDL() classAdJob.insertAttributeInt('JobRequirements', reqJDL) jdl = classAdJob.asJDL() result = self.jobDB.setJobJDL(job, jdl) if not result['OK']: return result if siteCandidates: if len(siteCandidates) == 1: self.log.verbose('Individual site candidate for job %s is %s' % (job, siteCandidates[0])) self.jobDB.setJobAttribute(job, 'Site', siteCandidates[0]) elif bannedSites: remainingSites = [] for site in siteCandidates: if not site in bannedSites: remainingSites.append(site) if remainingSites: if len(remainingSites) == 1: self.log.verbose( 'Individual site candidate for job %s is %s' % (job, remainingSites[0])) self.jobDB.setJobAttribute(job, 'Site', remainingSites[0]) else: self.log.verbose('Site candidates for job %s are %s' % (job, str(remainingSites))) result = self.jobDB.getJobAttribute(job, 'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute(job, 'Site', siteGroup) else: self.log.verbose('Site candidates for job %s are %s' % (job, str(siteCandidates))) result = self.jobDB.getJobAttribute(job, 'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute(job, 'Site', siteGroup) else: self.log.verbose('All sites are eligible for job %s' % job) self.jobDB.setJobAttribute(job, 'Site', 'ANY') return self.setNextOptimizer(job)
def generateParametricJobs(jobClassAd): """ Generate a series of ClassAd job descriptions expanding job parameters :param jobClassAd: ClassAd job description object :return: list of ClassAd job description objects """ if not jobClassAd.lookupAttribute('Parameters'): return S_OK([jobClassAd.asJDL()]) result = getParameterVectorLength(jobClassAd) if not result['OK']: return result nParValues = result['Value'] if nParValues is None: return S_ERROR(EWMSJDL, 'Can not determine the number of job parameters') parameterDict = {} attributes = jobClassAd.getAttributes() for attribute in attributes: for key in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]: if attribute.startswith(key): seqID = '0' if '.' not in attribute else attribute.split( '.')[1] parameterDict.setdefault(seqID, {}) if key == 'Parameters': if jobClassAd.isAttributeList(attribute): parList = jobClassAd.getListFromExpression(attribute) if len(parList) != nParValues: return S_ERROR( EWMSJDL, 'Inconsistent parametric job description') parameterDict[seqID]['ParameterList'] = parList else: if attribute != "Parameters": return S_ERROR( EWMSJDL, 'Inconsistent parametric job description') nPar = jobClassAd.getAttributeInt(attribute) if nPar is None: value = jobClassAd.get_expression(attribute) return S_ERROR( EWMSJDL, 'Inconsistent parametric job description: %s=%s' % (attribute, value)) parameterDict[seqID]['Parameters'] = nPar else: value = jobClassAd.getAttributeInt(attribute) if value is None: value = jobClassAd.getAttributeFloat(attribute) if value is None: value = jobClassAd.get_expression(attribute) return S_ERROR( 'Illegal value for %s JDL field: %s' % (attribute, value)) parameterDict[seqID][key] = value if '0' in parameterDict and not parameterDict.get('0'): parameterDict.pop('0') parameterLists = {} for seqID in parameterDict: parList = __getParameterSequence( nParValues, parList=parameterDict[seqID].get('ParameterList', []), parStart=parameterDict[seqID].get('ParameterStart', 1), parStep=parameterDict[seqID].get('ParameterStep', 0), parFactor=parameterDict[seqID].get('ParameterFactor', 1)) if not parList: return S_ERROR(EWMSJDL, 'Inconsistent parametric job description') parameterLists[seqID] = parList jobDescList = [] jobDesc = jobClassAd.asJDL() # Width of the sequential parameter number zLength = len(str(nParValues - 1)) for n in range(nParValues): newJobDesc = jobDesc newJobDesc = newJobDesc.replace('%n', str(n).zfill(zLength)) newClassAd = ClassAd(newJobDesc) for seqID in parameterLists: parameter = parameterLists[seqID][n] for attribute in newClassAd.getAttributes(): __updateAttribute(newClassAd, attribute, seqID, str(parameter)) for seqID in parameterLists: for attribute in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]: if seqID == '0': newClassAd.deleteAttribute(attribute) else: newClassAd.deleteAttribute('%s.%s' % (attribute, seqID)) parameter = parameterLists[seqID][n] if seqID == '0': attribute = 'Parameter' else: attribute = 'Parameter.%s' % seqID if isinstance(parameter, six.string_types) and parameter.startswith('{'): newClassAd.insertAttributeInt(attribute, str(parameter)) else: newClassAd.insertAttributeString(attribute, str(parameter)) newClassAd.insertAttributeInt('ParameterNumber', n) newJDL = newClassAd.asJDL() jobDescList.append(newJDL) return S_OK(jobDescList)
def selectJob( self, resourceDescription ): """ Main job selection function to find the highest priority job matching the resource capacity """ startTime = time.time() # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in taskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in taskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) if classAdAgent.lookupAttribute( 'DIRACVersion' ): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' ) else: for name in taskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in taskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key( 'DIRACVersion' ): resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion'] # Check the pilot DIRAC version if self.checkPilotVersion: if 'DIRACVersion' in resourceDict: if self.pilotVersion and resourceDict['DIRACVersion'] != self.pilotVersion: return S_ERROR( 'Pilot version does not match the production version %s:%s' % \ ( resourceDict['DIRACVersion'], self.pilotVersion ) ) # Get common site mask and check the agent site result = jobDB.getSiteMask( siteState = 'Active' ) if result['OK']: maskList = result['Value'] else: return S_ERROR( 'Internal error: can not get site mask' ) if not 'Site' in resourceDict: return S_ERROR( 'Missing Site Name in Resource JDL' ) siteName = resourceDict['Site'] if resourceDict['Site'] not in maskList: if 'GridCE' in resourceDict: del resourceDict['Site'] else: return S_ERROR( 'Site not in mask and GridCE not specified' ) resourceDict['Setup'] = self.serviceInfoDict['clientSetup'] if DEBUG: print "Resource description:" for k, v in resourceDict.items(): print k.rjust( 20 ), v # Check if Job Limits are imposed onto the site extraConditions = {} if self.siteJobLimits: result = self.getExtraConditions( siteName ) if result['OK']: extraConditions = result['Value'] if extraConditions: gLogger.info( 'Job Limits for site %s are: %s' % ( siteName, str( extraConditions ) ) ) result = taskQueueDB.matchAndGetJob( resourceDict, extraConditions = extraConditions ) if DEBUG: print result if not result['OK']: return result result = result['Value'] if not result['matchFound']: return S_ERROR( 'No match found' ) jobID = result['jobId'] resAtt = jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) if not resAtt['Value']['Status'] == 'Waiting': gLogger.error( 'Job %s matched by the TQ is not in Waiting state' % str( jobID ) ) result = taskQueueDB.deleteJob( jobID ) result = jobDB.setJobStatus( jobID, status = 'Matched', minor = 'Assigned' ) result = jobLoggingDB.addLoggingRecord( jobID, status = 'Matched', minor = 'Assigned', source = 'Matcher' ) result = jobDB.getJobJDL( jobID ) if not result['OK']: return S_ERROR( 'Failed to get the job JDL' ) resultDict = {} resultDict['JDL'] = result['Value'] resultDict['JobID'] = jobID matchTime = time.time() - startTime gLogger.info( "Match time: [%s]" % str( matchTime ) ) gMonitor.addMark( "matchTime", matchTime ) # Get some extra stuff into the response returned resOpt = jobDB.getJobOptParameters( jobID ) if resOpt['OK']: for key, value in resOpt['Value'].items(): resultDict[key] = value resAtt = jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) resultDict['DN'] = resAtt['Value']['OwnerDN'] resultDict['Group'] = resAtt['Value']['OwnerGroup'] return S_OK( resultDict )