def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ #Check if we have to do old bootstrap... classadJob = ClassAd( '[]' ) paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} scriptname = 'jobDescription.xml' arguments = [] if self.script: if os.path.exists( self.script ): scriptname = os.path.abspath( self.script ) self.log.verbose( 'Found script name %s' % scriptname ) else: if xmlFile: self.log.verbose( 'Found XML File %s' % xmlFile ) scriptname = xmlFile arguments.append( os.path.basename( scriptname ) ) self.addToInputSandbox.append( scriptname ) if paramsDict.has_key( 'LogLevel' ): if paramsDict['LogLevel']['value']: arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) ) else: self.log.warn( 'Job LogLevel defined with null value' ) if paramsDict.has_key( 'DIRACSetup' ): if paramsDict['DIRACSetup']['value']: arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) ) else: self.log.warn( 'Job DIRACSetup defined with null value' ) if paramsDict.has_key( 'JobMode' ): if paramsDict['JobMode']['value']: arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) ) else: self.log.warn( 'Job Mode defined with null value' ) if paramsDict.has_key( 'JobConfigArgs' ): if paramsDict['JobConfigArgs']['value']: arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) ) else: self.log.warn( 'JobConfigArgs defined with null value' ) classadJob.insertAttributeString( 'Executable', self.executable ) self.addToOutputSandbox.append( self.stderr ) self.addToOutputSandbox.append( self.stdout ) #Extract i/o sandbox parameters from steps and any input data parameters #to do when introducing step-level api... #To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ';'.join( self.addToInputSandbox ) if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] finalInputSandbox = currentFiles + ';' + extraFiles uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) ) paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox ) self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) ) else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = extraFiles paramsDict['InputSandbox']['type'] = 'JDL' if self.addToOutputSandbox: extraFiles = ';'.join( self.addToOutputSandbox ) if paramsDict.has_key( 'OutputSandbox' ): currentFiles = paramsDict['OutputSandbox']['value'] finalOutputSandbox = currentFiles + ';' + extraFiles uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) ) paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox ) self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) ) else: paramsDict['OutputSandbox'] = {} paramsDict['OutputSandbox']['value'] = extraFiles paramsDict['OutputSandbox']['type'] = 'JDL' if self.addToInputData: extraFiles = ';'.join( self.addToInputData ) if paramsDict.has_key( 'InputData' ): currentFiles = paramsDict['InputData']['value'] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ';' + extraFiles uniqueInputData = uniqueElements( finalInputData.split( ';' ) ) paramsDict['InputData']['value'] = ';'.join( uniqueInputData ) self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) ) else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = extraFiles paramsDict['InputData']['type'] = 'JDL' # Handle here the Parametric values if self.parametric: for pType in ['InputData', 'InputSandbox']: if self.parametric.has_key( pType ): if paramsDict.has_key( pType ) and paramsDict[pType]['value']: pData = self.parametric[pType] # List of lists case currentFiles = paramsDict[pType]['value'].split( ';' ) tmpList = [] if type( pData[0] ) == list: for pElement in pData: tmpList.append( currentFiles + pElement ) else: for pElement in pData: tmpList.append( currentFiles + [pElement] ) self.parametric[pType] = tmpList paramsDict[pType] = {} paramsDict[pType]['value'] = "%s" paramsDict[pType]['type'] = 'JDL' self.parametric['files'] = self.parametric[pType] arguments.append( ' -p Parametric' + pType + '=%s' ) break if self.parametric.has_key( 'files' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['files'] paramsDict['Parameters']['type'] = 'JDL' if self.parametric.has_key( 'GenericParameters' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['GenericParameters'] paramsDict['Parameters']['type'] = 'JDL' arguments.append( ' -p ParametricParameters=%s' ) ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) #Add any JDL parameters to classad obeying lists with ';' rule requirements = False for name, props in paramsDict.items(): ptype = props['type'] value = props['value'] if name.lower() == 'requirements' and ptype == 'JDL': self.log.verbose( 'Found existing requirements: %s' % ( value ) ) requirements = True if re.search( '^JDL', ptype ): if type( value ) == list: if type( value[0] ) == list: classadJob.insertAttributeVectorStringList( name, value ) else: classadJob.insertAttributeVectorString( name, value ) elif value == "%s": classadJob.insertAttributeInt( name, value ) elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) else: classadJob.insertAttributeVectorString( name, value.split( ';' ) ) if not requirements: reqtsDict = self.reqParams exprn = '' plus = '' for name, props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if not ptype == 'dict': if ptype == 'JDLReqt': if value and not value.lower() == 'any': plus = ' && ' if re.search( ';', value ): for val in value.split( ';' ): exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus else: exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus if len( plus ): exprn = exprn[:-len( plus )] if not exprn: exprn = 'true' self.log.verbose( 'Requirements: %s' % ( exprn ) ) #classadJob.set_expression('Requirements', exprn) self.addToInputSandbox.remove( scriptname ) self.addToOutputSandbox.remove( self.stdout ) self.addToOutputSandbox.remove( self.stderr ) jdl = classadJob.asJDL() start = jdl.find( '[' ) end = jdl.rfind( ']' ) return jdl[( start + 1 ):( end - 1 )]
def __sendJobToTaskQueue( self, job, classAdJob, siteCandidates, bannedSites ): """This method sends jobs to the task queue agent and if candidate sites are defined, updates job JDL accordingly. """ reqJDL = classAdJob.get_expression( 'JobRequirements' ) classAddReq = ClassAd( reqJDL ) if siteCandidates: classAddReq.insertAttributeVectorString( 'Sites', siteCandidates ) if bannedSites: classAddReq.insertAttributeVectorString( 'BannedSites', bannedSites ) if classAdJob.lookupAttribute( "SubmitPools" ): classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression( 'SubmitPools' ) ) if classAdJob.lookupAttribute( "GridMiddleware" ): classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression( 'GridMiddleware' ) ) if classAdJob.lookupAttribute( "PilotTypes" ): classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotTypes' ) ) #HAck to migrate old jobs to new ones. #DELETE ON 08/09 else: if classAdJob.lookupAttribute( "PilotType" ): classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotType' ) ) if classAdJob.lookupAttribute( "JobType" ): jobTypes = [ jt for jt in classAdJob.getListFromExpression( 'JobType' ) if jt ] classAddReq.insertAttributeVectorString( 'JobTypes', jobTypes ) #Required CE's requirements gridCEs = [ ce for ce in classAdJob.getListFromExpression( 'GridRequiredCEs' ) if ce ] if gridCEs: classAddReq.insertAttributeVectorString( 'GridCEs', gridCEs ) if siteCandidates: sites = ','.join( siteCandidates ) classAdJob.insertAttributeString( "Site", sites ) reqJDL = classAddReq.asJDL() classAdJob.insertAttributeInt( 'JobRequirements', reqJDL ) jdl = classAdJob.asJDL() result = self.jobDB.setJobJDL( job, jdl ) if not result['OK']: return result if siteCandidates: if len( siteCandidates ) == 1: self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, siteCandidates[0] ) ) self.jobDB.setJobAttribute( job, 'Site', siteCandidates[0] ) elif bannedSites: remainingSites = [] for site in siteCandidates: if not site in bannedSites: remainingSites.append( site ) if remainingSites: if len( remainingSites ) == 1: self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, remainingSites[0] ) ) self.jobDB.setJobAttribute( job, 'Site', remainingSites[0] ) else: self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( remainingSites ) ) ) result = self.jobDB.getJobAttribute(job,'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute( job, 'Site', siteGroup ) else: self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( siteCandidates ) ) ) result = self.jobDB.getJobAttribute(job,'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute( job, 'Site', siteGroup ) else: self.log.verbose( 'All sites are eligible for job %s' % job ) self.jobDB.setJobAttribute( job, 'Site', 'ANY' ) return self.setNextOptimizer( job )
def _toJDL( self, xmlFile="", jobDescriptionObject=None ): # messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ # Check if we have to do old bootstrap... classadJob = ClassAd("[]") paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {"type": param.getType(), "value": param.getValue()} arguments = [] scriptname = "jobDescription.xml" if jobDescriptionObject is None: # if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml' if self.script: if os.path.exists(self.script): scriptname = os.path.abspath(self.script) self.log.verbose("Found script name %s" % scriptname) else: self.log.error("File not found", self.script) else: if xmlFile: self.log.verbose("Found XML File %s" % xmlFile) scriptname = xmlFile self.addToInputSandbox.append(scriptname) elif isinstance(jobDescriptionObject, StringIO.StringIO): self.log.verbose("jobDescription is passed in as a StringIO object") else: self.log.error("Where's the job description?") arguments.append(os.path.basename(scriptname)) if paramsDict.has_key("LogLevel"): if paramsDict["LogLevel"]["value"]: arguments.append("-o LogLevel=%s" % (paramsDict["LogLevel"]["value"])) else: self.log.warn("Job LogLevel defined with null value") if paramsDict.has_key("DIRACSetup"): if paramsDict["DIRACSetup"]["value"]: arguments.append("-o DIRAC/Setup=%s" % (paramsDict["DIRACSetup"]["value"])) else: self.log.warn("Job DIRACSetup defined with null value") if paramsDict.has_key("JobMode"): if paramsDict["JobMode"]["value"]: arguments.append("-o JobMode=%s" % (paramsDict["JobMode"]["value"])) else: self.log.warn("Job Mode defined with null value") if paramsDict.has_key("JobConfigArgs"): if paramsDict["JobConfigArgs"]["value"]: arguments.append("%s" % (paramsDict["JobConfigArgs"]["value"])) else: self.log.warn("JobConfigArgs defined with null value") classadJob.insertAttributeString("Executable", self.executable) self.addToOutputSandbox.append(self.stderr) self.addToOutputSandbox.append(self.stdout) # Extract i/o sandbox parameters from steps and any input data parameters # to do when introducing step-level api... # To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ";".join(self.addToInputSandbox) if paramsDict.has_key("InputSandbox"): currentFiles = paramsDict["InputSandbox"]["value"] finalInputSandbox = currentFiles + ";" + extraFiles uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";")) paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox) self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox))) else: paramsDict["InputSandbox"] = {} paramsDict["InputSandbox"]["value"] = extraFiles paramsDict["InputSandbox"]["type"] = "JDL" if self.addToOutputSandbox: extraFiles = ";".join(self.addToOutputSandbox) if paramsDict.has_key("OutputSandbox"): currentFiles = paramsDict["OutputSandbox"]["value"] finalOutputSandbox = currentFiles + ";" + extraFiles uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";")) paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox) self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox))) else: paramsDict["OutputSandbox"] = {} paramsDict["OutputSandbox"]["value"] = extraFiles paramsDict["OutputSandbox"]["type"] = "JDL" if self.addToInputData: extraFiles = ";".join(self.addToInputData) if paramsDict.has_key("InputData"): currentFiles = paramsDict["InputData"]["value"] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ";" + extraFiles uniqueInputData = uniqueElements(finalInputData.split(";")) paramsDict["InputData"]["value"] = ";".join(uniqueInputData) self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData))) else: paramsDict["InputData"] = {} paramsDict["InputData"]["value"] = extraFiles paramsDict["InputData"]["type"] = "JDL" # Handle here the Parametric values if self.parametric: for pType in ["InputData", "InputSandbox"]: if self.parametric.has_key(pType): if paramsDict.has_key(pType) and paramsDict[pType]["value"]: pData = self.parametric[pType] # List of lists case currentFiles = paramsDict[pType]["value"].split(";") tmpList = [] if type(pData[0]) == list: for pElement in pData: tmpList.append(currentFiles + pElement) else: for pElement in pData: tmpList.append(currentFiles + [pElement]) self.parametric[pType] = tmpList paramsDict[pType] = {} paramsDict[pType]["value"] = "%s" paramsDict[pType]["type"] = "JDL" self.parametric["files"] = self.parametric[pType] arguments.append(" -p Parametric" + pType + "=%s") break if self.parametric.has_key("files"): paramsDict["Parameters"] = {} paramsDict["Parameters"]["value"] = self.parametric["files"] paramsDict["Parameters"]["type"] = "JDL" if self.parametric.has_key("GenericParameters"): paramsDict["Parameters"] = {} paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"] paramsDict["Parameters"]["type"] = "JDL" arguments.append(" -p ParametricParameters=%s") ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString("Arguments", " ".join(arguments)) # Add any JDL parameters to classad obeying lists with ';' rule for name, props in paramsDict.items(): ptype = props["type"] value = props["value"] if name.lower() == "requirements" and ptype == "JDL": self.log.verbose("Found existing requirements: %s" % (value)) if re.search("^JDL", ptype): if type(value) == list: if type(value[0]) == list: classadJob.insertAttributeVectorStringList(name, value) else: classadJob.insertAttributeVectorString(name, value) elif value == "%s": classadJob.insertAttributeInt(name, value) elif not re.search(";", value) or name == "GridRequirements": # not a nice fix... classadJob.insertAttributeString(name, value) else: classadJob.insertAttributeVectorString(name, value.split(";")) for fToBeRemoved in [scriptname, self.stdout, self.stderr]: try: self.addToInputSandbox.remove(fToBeRemoved) except ValueError: pass jdl = classadJob.asJDL() start = jdl.find("[") end = jdl.rfind("]") return jdl[(start + 1) : (end - 1)]
def __sendJobToTaskQueue(self, job, classAdJob, siteCandidates, bannedSites): """This method sends jobs to the task queue agent and if candidate sites are defined, updates job JDL accordingly. """ reqJDL = classAdJob.get_expression('JobRequirements') classAddReq = ClassAd(reqJDL) if siteCandidates: classAddReq.insertAttributeVectorString('Sites', siteCandidates) if bannedSites: classAddReq.insertAttributeVectorString('BannedSites', bannedSites) if classAdJob.lookupAttribute("SubmitPools"): classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression('SubmitPools')) if classAdJob.lookupAttribute("GridMiddleware"): classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression('GridMiddleware')) if classAdJob.lookupAttribute("PilotTypes"): classAddReq.set_expression('PilotTypes', classAdJob.get_expression('PilotTypes')) #HAck to migrate old jobs to new ones. #DELETE ON 08/09 else: if classAdJob.lookupAttribute("PilotType"): classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression('PilotType')) if classAdJob.lookupAttribute("JobType"): jobTypes = [ jt for jt in classAdJob.getListFromExpression('JobType') if jt ] classAddReq.insertAttributeVectorString('JobTypes', jobTypes) #Required CE's requirements gridCEs = [ ce for ce in classAdJob.getListFromExpression('GridRequiredCEs') if ce ] if gridCEs: classAddReq.insertAttributeVectorString('GridCEs', gridCEs) if siteCandidates: sites = ','.join(siteCandidates) classAdJob.insertAttributeString("Site", sites) reqJDL = classAddReq.asJDL() classAdJob.insertAttributeInt('JobRequirements', reqJDL) jdl = classAdJob.asJDL() result = self.jobDB.setJobJDL(job, jdl) if not result['OK']: return result if siteCandidates: if len(siteCandidates) == 1: self.log.verbose('Individual site candidate for job %s is %s' % (job, siteCandidates[0])) self.jobDB.setJobAttribute(job, 'Site', siteCandidates[0]) elif bannedSites: remainingSites = [] for site in siteCandidates: if not site in bannedSites: remainingSites.append(site) if remainingSites: if len(remainingSites) == 1: self.log.verbose( 'Individual site candidate for job %s is %s' % (job, remainingSites[0])) self.jobDB.setJobAttribute(job, 'Site', remainingSites[0]) else: self.log.verbose('Site candidates for job %s are %s' % (job, str(remainingSites))) result = self.jobDB.getJobAttribute(job, 'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute(job, 'Site', siteGroup) else: self.log.verbose('Site candidates for job %s are %s' % (job, str(siteCandidates))) result = self.jobDB.getJobAttribute(job, 'Site') siteGroup = "Multiple" if result['OK']: if result['Value'].startswith('Group'): siteGroup = result['Value'] self.jobDB.setJobAttribute(job, 'Site', siteGroup) else: self.log.verbose('All sites are eligible for job %s' % job) self.jobDB.setJobAttribute(job, 'Site', 'ANY') return self.setNextOptimizer(job)