def _getCurrentConfig(self): """Return the current system configuration.""" from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData gConfig.forceRefresh() fullCfg = CFG() setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if setup not in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict for system, setup in serviceSetups.items(): if self.systems and system not in self.systems: continue systemCfg = gConfigurationData.remoteCFG.getAsCFG("/Systems/%s/%s" % (system, setup)) for section in systemCfg.listSections(): if section not in ('Agents', 'Services', 'Executors'): systemCfg.deleteKey(section) fullCfg.createNewSection("/%s" % system, contents=systemCfg) return S_OK(fullCfg)
def loadFile( self, fileName ): try: fileCFG = CFG() fileCFG.loadFromFile( fileName ) except IOError: self.localCFG = self.localCFG.mergeWith( fileCFG ) return S_ERROR( "Can't load a cfg file '%s'" % fileName ) return self.mergeWithLocal( fileCFG )
def mergeWithServer( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteCFG = CFG() remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) serverVersion = gConfigurationData.getVersion( remoteCFG ) self.cfgData = remoteCFG.mergeWith( self.cfgData ) gConfigurationData.setVersion( serverVersion, self.cfgData ) return retVal
def _parseConfigTemplate(self, templatePath, cfg=None): """Parse the ConfigTemplate.cfg files. :param str templatePath: path to the folder containing a ConfigTemplate.cfg file :param CFG cfg: cfg to merge with the systems config :returns: CFG object """ cfg = CFG() if cfg is None else cfg system = os.path.split(templatePath.rstrip("/"))[1] if system.lower().endswith('system'): system = system[:-len('System')] if self.systems and system not in self.systems: return S_OK(cfg) templatePath = os.path.join(templatePath, 'ConfigTemplate.cfg') if not os.path.exists(templatePath): return S_ERROR("File not found: %s" % templatePath) loadCfg = CFG() loadCfg.loadFromFile(templatePath) newCfg = CFG() newCfg.createNewSection("/%s" % system, contents=loadCfg) cfg = cfg.mergeWith(newCfg) return S_OK(cfg)
class JobManifest( object ): def __init__( self, manifest = "" ): self.__manifest = CFG() self.__dirty = False if manifest: result = self.loadManifest( manifest ) if not result[ 'OK' ]: raise Exception( result[ 'Message' ] ) def isDirty( self ): return self.__dirty def setDirty( self ): self.__dirty = True def clearDirty( self ): self.__dirty = False def load( self, dataString ): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadJDL( dataString ) else: return self.loadCFG( dataString ) def loadJDL( self, jdlString ): """ Load job manifest from JDL format """ result = loadJDLAsCFG( jdlString.strip() ) if not result[ 'OK' ]: self.__manifest = CFG() return result self.__manifest = result[ 'Value' ][0] return S_OK() def loadCFG( self, cfgString ): """ Load job manifest from CFG format """ try: self.__manifest.loadFromBuffer( cfgString ) except Exception, e: return S_ERROR( "Can't load manifest from cfg: %s" % str( e ) ) return S_OK()
def showTextConfiguration(self): response.headers["Content-type"] = "text/plain" if "download" in request.params and request.params["download"] in ("yes", "true", "1"): version = "" try: cfg = CFG() cfg.loadFromBuffer(session["cfgData"]) cfg = cfg["DIRAC"]["Configuration"] version = ".%s.%s" % (cfg["Name"], cfg["Version"].replace(":", "").replace("-", "")) except Exception, e: print e print 'attachment; filename="cs%s.cfg"' % version.replace(" ", "_") response.headers["Content-Disposition"] = 'attachment; filename="cs%s.cfg"' % version.replace(" ", "") response.headers["Content-Length"] = len(session["cfgData"]) response.headers["Content-Transfer-Encoding"] = "Binary"
def appendToRepository( self, repoLocation ): if not os.path.exists( repoLocation ): gLogger.error( "Secondary repository does not exist", repoLocation ) return S_ERROR( "Secondary repository does not exist" ) self.repo = CFG().loadFromFile( repoLocation ).mergeWith( self.repo ) self._writeRepository( self.location ) return S_OK()
def __init__( self, rpcClient = False, commiterId = "unknown" ): self.commiterTag = "@@-" self.commiterId = commiterId self.cfgData = CFG() self.rpcClient = None if rpcClient: self.setRPCClient( rpcClient )
def __init__( self, manifest = "" ): self.__manifest = CFG() self.__dirty = False if manifest: result = self.loadManifest( manifest ) if not result[ 'OK' ]: raise Exception( result[ 'Message' ] )
def __init__(self, manifest=""): self.__manifest = CFG() self.__dirty = False self.__ops = False if manifest: result = self.load(manifest) if not result["OK"]: raise Exception(result["Message"])
def __gConfigDefaults(defaultPath): """ Build a cfg from a Default Section """ from DIRAC import gConfig cfgDefaults = CFG() result = gConfig.getSections(defaultPath) if not result['OK']: return cfgDefaults for name in result['Value']: typePath = cfgPath(defaultPath, name) cfgDefaults.createNewSection(name) result = gConfig.getOptionsDict(typePath) if result['OK']: optionsDict = result['Value'] for option, value in optionsDict.items(): cfgDefaults[name].setOption(option, value) return cfgDefaults
def loadWebAppCFGFiles(): """ Load WebApp/web.cfg definitions """ exts = [] for ext in CSGlobals.getCSExtensions(): if ext == "DIRAC": continue if ext[-5:] != "DIRAC": ext = "%sDIRAC" % ext if ext != "WebAppDIRAC": exts.append( ext ) exts.append( "DIRAC" ) exts.append( "WebAppDIRAC" ) webCFG = CFG() for modName in reversed( exts ): try: modPath = imp.find_module( modName )[1] except ImportError: continue gLogger.verbose( "Found module %s at %s" % ( modName, modPath ) ) cfgPath = os.path.join( modPath, "WebApp", "web.cfg" ) if not os.path.isfile( cfgPath ): gLogger.verbose( "Inexistant %s" % cfgPath ) continue try: modCFG = CFG().loadFromFile( cfgPath ) except Exception, excp: gLogger.error( "Could not load %s: %s" % ( cfgPath, excp ) ) continue gLogger.verbose( "Loaded %s" % cfgPath ) expl = [ BASECS ] while len( expl ): current = expl.pop( 0 ) if not modCFG.isSection( current ): continue if modCFG.getOption( "%s/AbsoluteDefinition" % current, False ): gLogger.verbose( "%s:%s is an absolute definition" % ( modName, current ) ) try: webCFG.deleteKey( current ) except: pass modCFG.deleteKey( "%s/AbsoluteDefinition" % current ) else: for sec in modCFG[ current ].listSections(): expl.append( "%s/%s" % ( current, sec ) ) #Add the modCFG webCFG = webCFG.mergeWith( modCFG )
def setUp( self ): self.authMgr = AuthManager( '/Systems/Service/Authorization' ) cfg = CFG() cfg.loadFromBuffer( testSystemsCFG ) gConfig.loadCFG( cfg ) cfg.loadFromBuffer( testRegistryCFG ) gConfig.loadCFG( cfg ) self.noAuthCredDict = { 'group': 'group_test' } self.userCredDict = { 'DN': '/User/test/DN/CN=userA', 'group': 'group_test' } self.badUserCredDict = { 'DN': '/User/test/DN/CN=userB', 'group': 'group_bad' } self.hostCredDict = { 'DN': '/User/test/DN/CN=test.hostA.ch', 'group': 'hosts' } self.badHostCredDict = { 'DN': '/User/test/DN/CN=test.hostB.ch', 'group': 'hosts' }
def loadJDL(self, jdlString): """ Load job manifest from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result["OK"]: self.__manifest = CFG() return result self.__manifest = result["Value"][0] return S_OK()
def __init__(self, location): self.cfg = CFG() self.location = location self.OK = True if os.path.exists(self.location): self.cfg.loadFromFile(self.location) if not self.cfg.existsKey('Processes'): self.cfg.createNewSection('Processes') else: self.OK = False
def loadDescriptionFromJDL( self, jdlString ): """ Load job description from JDL format """ result = loadJDLAsCFG( jdlString.strip() ) if not result[ 'OK' ]: self.__description = CFG() return result self.__description = result[ 'Value' ][0] return S_OK()
def __init__( self, loadDefaultCFG = True ): lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.compressedConfigurationData = "" self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" ) self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" ) gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile ) retVal = self.loadFile( defaultCFGFile ) if not retVal[ 'OK' ]: gLogger.warn( "Can't load %s file" % defaultCFGFile ) self.sync()
class ProcessList(object): """ The ProcessList uses internally the CFG utility to store the processes and their properties. """ def __init__(self, location): self.cfg = CFG() self.location = location self.goodProcessList = True if os.path.exists(self.location): self.cfg.loadFromFile(self.location) if not self.cfg.existsKey('Processes'): self.cfg.createNewSection('Processes') else: self.goodProcessList = False def _writeProcessList(self, path): """ Write to text """ handle, tmpName = tempfile.mkstemp() written = self.cfg.writeToFile(tmpName) os.close(handle) if not written: if os.path.exists(tmpName): os.remove(tmpName) return written if os.path.exists(path): LOG.debug("Replacing %s" % path) try: shutil.move(tmpName, path) return True except OSError, err: LOG.error("Failed to overwrite process list.", err) LOG.info("If your process list is corrupted a backup can be found %s" % tmpName) return False
class JobDescription: def __init__( self ): self.__description = CFG() self.__dirty = False def isDirty( self ): return self.__dirty def loadDescription( self, dataString ): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadDescriptionFromJDL( dataString ) else: return self.loadDescriptionFromCFG( dataString ) def loadDescriptionFromJDL( self, jdlString ): """ Load job description from JDL format """ result = loadJDLAsCFG( jdlString.strip() ) if not result[ 'OK' ]: self.__description = CFG() return result self.__description = result[ 'Value' ][0] return S_OK() def loadDescriptionFromCFG( self, cfgString ): """ Load job description from CFG format """ try: self.__description.loadFromBuffer( cfgString ) except Exception, e: return S_ERROR( "Can't load description from cfg: %s" % str( e ) ) return S_OK()
def __saveVomsMapping(self, params): sectionPath = "/Registry/VOMS/Mapping" sectionCfg = self.getSectionCfg(sectionPath) for opt in sectionCfg.listAll(): if not sectionCfg.isSection(opt): self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt) configText = "" for newOpt in params: if newOpt != "op": if configText == "": configText = newOpt + "=" + params[newOpt] + "\n" else: configText = configText + newOpt + "=" + params[newOpt] + "\n" newCFG = CFG() newCFG.loadFromBuffer(configText) self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG) return {"op":"saveVomsMapping", "success": 1}
def execute( self ): """ execute """ stopAgents = self.findStopAgents()[ 'Value' ] if stopAgents: self.log.info( 'Aborting, there are stop_agents to be picked' ) return S_OK() pilotVersion = self.opHelper.getValue( 'Pilot/Version', '' ) if not pilotVersion: self.log.error( 'There is no pilot version on the CS' ) return S_OK() pilotVersion = self.getNewestPilotVersion() if not pilotVersion[ 'OK' ]: self.log.error( pilotVersion[ 'Message' ] ) return S_ERROR( pilotVersion[ 'Message' ] ) pilotVersion = pilotVersion[ 'Value' ] localCFG = CFG() #load local CFG localCFG.loadFromFile( self.cfgToUpdate ) releaseVersion = localCFG.getRecursive( 'LocalSite/ReleaseVersion' )[ 'value' ] self.log.info( 'PilotVersion : %s' % pilotVersion ) self.log.info( 'ReleaseVersion : %s' % releaseVersion ) if LooseVersion( pilotVersion ) > LooseVersion( releaseVersion ): self.log.info( 'UPDATING %s > %s' % ( pilotVersion, releaseVersion ) ) localCFG.setOption( 'LocalSite/ReleaseVersion', pilotVersion ) localCFG.writeToFile( self.cfgToUpdate ) self.touchStopAgents() else: self.log.info( 'Nothing to do' ) return S_OK()
def __init__( self, repository = None ): self.location = repository if not self.location: if "HOME" in os.environ: self.location = '%s/.dirac.repo.rep' % os.environ['HOME'] else: self.location = '%s/.dirac.repo.rep' % os.getcwd() self.repo = CFG() if os.path.exists( self.location ): self.repo.loadFromFile( self.location ) if not self.repo.existsKey( 'Jobs' ): self.repo.createNewSection( 'Jobs' ) else: self.repo.createNewSection( 'Jobs' ) self.OK = True written = self._writeRepository( self.location ) if not written: self.OK = False
def _loadWebAppCFGFiles(self, extension): """ Load WebApp/web.cfg definitions :param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC """ exts = [extension, "WebAppDIRAC"] webCFG = CFG() for modName in reversed(exts): cfgPath = os.path.join(self.__params.destination, "%s/WebApp" % modName, "web.cfg") if not os.path.isfile(cfgPath): gLogger.verbose("Web configuration file %s does not exists!" % cfgPath) continue try: modCFG = CFG().loadFromFile(cfgPath) except Exception, excp: gLogger.error("Could not load %s: %s" % (cfgPath, excp)) continue gLogger.verbose("Loaded %s" % cfgPath) expl = ["/WebApp"] while len(expl): current = expl.pop(0) if not modCFG.isSection(current): continue if modCFG.getOption("%s/AbsoluteDefinition" % current, False): gLogger.verbose("%s:%s is an absolute definition" % (modName, current)) try: webCFG.deleteKey(current) except: pass modCFG.deleteKey("%s/AbsoluteDefinition" % current) else: for sec in modCFG[current].listSections(): expl.append("%s/%s" % (current, sec)) # Add the modCFG webCFG = webCFG.mergeWith(modCFG)
def getComputingElementDefaults(ceName="", ceType="", cfg=None, currentSectionPath=""): """ Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg """ cesCfg = CFG() if cfg: try: cesCfg.loadFromFile(cfg) cesPath = cfgInstallPath("ComputingElements") if cesCfg.isSection(cesPath): for section in cfgPathToList(cesPath): cesCfg = cesCfg[section] except: return CFG() # Overwrite the cfg with Command line arguments if ceName: if not cesCfg.isSection(ceName): cesCfg.createNewSection(ceName) if currentSectionPath: # Add Options from Command Line optionsDict = __getExtraOptions(currentSectionPath) for name, value in optionsDict.items(): cesCfg[ceName].setOption(name, value) if ceType: cesCfg[ceName].setOption("CEType", ceType) ceDefaultSection = cfgPath(defaultSection("ComputingElements")) # Load Default for the given type from Central configuration is defined ceDefaults = __gConfigDefaults(ceDefaultSection) for ceName in cesCfg.listSections(): if "CEType" in cesCfg[ceName]: ceType = cesCfg[ceName]["CEType"] if ceType in ceDefaults: for option in ceDefaults[ceType].listOptions(): if option not in cesCfg[ceName]: cesCfg[ceName].setOption(option, ceDefaults[ceType][option]) return cesCfg
def getComputingElementDefaults(ceName='', ceType='', cfg=None, currentSectionPath=''): """ Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg """ cesCfg = CFG() if cfg: try: cesCfg.loadFromFile(cfg) cesPath = cfgInstallPath('ComputingElements') if cesCfg.isSection(cesPath): for section in cfgPathToList(cesPath): cesCfg = cesCfg[section] except BaseException: return CFG() # Overwrite the cfg with Command line arguments if ceName: if not cesCfg.isSection(ceName): cesCfg.createNewSection(ceName) if currentSectionPath: # Add Options from Command Line optionsDict = __getExtraOptions(currentSectionPath) for name, value in optionsDict.items(): cesCfg[ceName].setOption(name, value) # pylint: disable=no-member if ceType: cesCfg[ceName].setOption('CEType', ceType) # pylint: disable=no-member ceDefaultSection = cfgPath(defaultSection('ComputingElements')) # Load Default for the given type from Central configuration is defined ceDefaults = __gConfigDefaults(ceDefaultSection) for ceName in cesCfg.listSections(): if 'CEType' in cesCfg[ceName]: ceType = cesCfg[ceName]['CEType'] if ceType in ceDefaults: for option in ceDefaults[ceType].listOptions(): # pylint: disable=no-member if option not in cesCfg[ceName]: cesCfg[ceName].setOption(option, ceDefaults[ceType][option]) # pylint: disable=unsubscriptable-object return cesCfg
class JobRepository(object): def __init__(self, repository=None): self.location = repository if not self.location: if "HOME" in os.environ: self.location = '%s/.dirac.repo.rep' % os.environ['HOME'] else: self.location = '%s/.dirac.repo.rep' % os.getcwd() self.repo = CFG() if os.path.exists(self.location): self.repo.loadFromFile(self.location) if not self.repo.existsKey('Jobs'): self.repo.createNewSection('Jobs') else: self.repo.createNewSection('Jobs') self.OK = True written = self._writeRepository(self.location) if not written: self.OK = False def isOK(self): return self.OK def readRepository(self): return S_OK(self.repo.getAsDict('Jobs')) def writeRepository(self, alternativePath=None): destination = self.location if alternativePath: destination = alternativePath written = self._writeRepository(destination) if not written: return S_ERROR("Failed to write repository") return S_OK(destination) def resetRepository(self, jobIDs=[]): if not jobIDs: jobs = self.readRepository()['Value'] jobIDs = jobs.keys() paramDict = {'State': 'Submitted', 'Retrieved': 0, 'OutputData': 0} for jobID in jobIDs: self._writeJob(jobID, paramDict, True) self._writeRepository(self.location) return S_OK() def _writeRepository(self, path): handle, tmpName = tempfile.mkstemp() written = self.repo.writeToFile(tmpName) os.close(handle) if not written: if os.path.exists(tmpName): os.remove(tmpName) return written if os.path.exists(path): gLogger.debug("Replacing %s" % path) try: shutil.move(tmpName, path) return True except Exception as x: gLogger.error("Failed to overwrite repository.", x) gLogger.info( "If your repository is corrupted a backup can be found %s" % tmpName) return False def appendToRepository(self, repoLocation): if not os.path.exists(repoLocation): gLogger.error("Secondary repository does not exist", repoLocation) return S_ERROR("Secondary repository does not exist") self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo) self._writeRepository(self.location) return S_OK() def addJob(self, jobID, state='Submitted', retrieved=0, outputData=0, update=False): paramDict = { 'State': state, 'Time': self._getTime(), 'Retrieved': int(retrieved), 'OutputData': outputData } self._writeJob(jobID, paramDict, update) self._writeRepository(self.location) return S_OK(jobID) def updateJob(self, jobID, paramDict): if self._existsJob(jobID): paramDict['Time'] = self._getTime() self._writeJob(jobID, paramDict, True) self._writeRepository(self.location) return S_OK() def updateJobs(self, jobDict): for jobID, paramDict in jobDict.items(): if self._existsJob(jobID): paramDict['Time'] = self._getTime() self._writeJob(jobID, paramDict, True) self._writeRepository(self.location) return S_OK() def _getTime(self): runtime = time.ctime() return runtime.replace(" ", "_") def _writeJob(self, jobID, paramDict, update): jobID = str(jobID) jobExists = self._existsJob(jobID) if jobExists and (not update): gLogger.warn("Job exists and not overwriting") return S_ERROR("Job exists and not overwriting") if not jobExists: self.repo.createNewSection('Jobs/%s' % jobID) for key, value in paramDict.items(): self.repo.setOption('Jobs/%s/%s' % (jobID, key), value) return S_OK() def removeJob(self, jobID): res = self.repo['Jobs'].deleteKey(str(jobID)) #pylint: disable=no-member if res: self._writeRepository(self.location) return S_OK() def existsJob(self, jobID): return S_OK(self._existsJob(jobID)) def _existsJob(self, jobID): return self.repo.isSection('Jobs/%s' % jobID) def getLocation(self): return S_OK(self.location) def getSize(self): return S_OK(len(self.repo.getAsDict('Jobs')))
def __editItem(self, params): ret = self.__deleteItem(params) if ret["success"] == 1: ret = self.__addItem(params) ret["op"] = "editItem" return ret ret["op"] = "editItem" return ret sectionPath = "/Registry/" configText = "" if params["type"] == "users": sectionPath = sectionPath + "Users" if params["dn"].strip() != "": configText = "DN = " + params["dn"].strip() + "\n" if params["ca"].strip() != "": configText = configText + "CA = " + params["ca"].strip() + "\n" if params["email"].strip() != "": configText = configText + "Email = " + params["email"].strip() elif params["type"] == "groups": sectionPath = sectionPath + "Groups" if params["users"].strip() != "": configText = "Users = " + params["users"].strip() + "\n" if params["properties"].strip() != "": configText = configText + "Properties = " + params["properties"].strip() + "\n" if str(params["jobshare"]).strip() != "": configText = configText + "JobShare = " + str(params["jobshare"]) + "\n" if params["autouploadproxy"].strip() != "": configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n" if params["autouploadpilotproxy"].strip() != "": configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n" if params["autoaddvoms"].strip() != "": configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip() elif params["type"] == "hosts": sectionPath = sectionPath + "Hosts" if params["dn"].strip() != "": configText = "DN = " + params["dn"].strip() + "\n" if params["properties"].strip() != "": configText = configText + "Properties = " + params["properties"].strip() sectionPath = sectionPath + "/" + params["name"] # deleting the options underneath sectionCfg = self.getSectionCfg(sectionPath) for opt in sectionCfg.listAll(): print "deleting " + opt + "\n" self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt) newCFG = CFG() newCFG.loadFromBuffer(configText) self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG) return {"success":1, "op": "editItem"}
class JobManifest(object): def __init__(self, manifest=""): self.__manifest = CFG() self.__dirty = False self.__ops = False if manifest: result = self.load(manifest) if not result["OK"]: raise Exception(result["Message"]) def isDirty(self): return self.__dirty def setDirty(self): self.__dirty = True def clearDirty(self): self.__dirty = False def load(self, dataString): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadJDL(dataString) else: return self.loadCFG(dataString) def loadJDL(self, jdlString): """ Load job manifest from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result["OK"]: self.__manifest = CFG() return result self.__manifest = result["Value"][0] return S_OK() def loadCFG(self, cfgString): """ Load job manifest from CFG format """ try: self.__manifest.loadFromBuffer(cfgString) except Exception as e: return S_ERROR("Can't load manifest from cfg: %s" % str(e)) return S_OK() def dumpAsCFG(self): return str(self.__manifest) def getAsCFG(self): return self.__manifest.clone() def dumpAsJDL(self): return dumpCFGAsJDL(self.__manifest) def __getCSValue(self, varName, defaultVal=None): if not self.__ops: self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"]) if varName[0] != "/": varName = "JobDescription/%s" % varName return self.__ops.getValue(varName, defaultVal) def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal): """ Check a numerical var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue try: varValue = long(varValue) except: return S_ERROR("%s must be a number" % varName) minVal = self.__getCSValue("Min%s" % varName, minVal) maxVal = self.__getCSValue("Max%s" % varName, maxVal) varValue = max(minVal, min(varValue, maxVal)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkChoiceVar(self, varName, defaultVal, choices): """ Check a choice var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue if varValue not in self.__getCSValue("Choices%s" % varName, choices): return S_ERROR("%s is not a valid value for %s" % (varValue, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMultiChoice(self, varName, choices): """ Check a multi choice var """ initialVal = False if varName not in self.__manifest: return S_OK() else: varValue = self.__manifest[varName] initialVal = varValue choices = self.__getCSValue("Choices%s" % varName, choices) for v in List.fromChar(varValue): if v not in choices: return S_ERROR("%s is not a valid value for %s" % (v, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMaxInputData(self, maxNumber): """ Check Maximum Number of Input Data files allowed """ varName = "InputData" if varName not in self.__manifest: return S_OK() varValue = self.__manifest[varName] if len(List.fromChar(varValue)) > maxNumber: return S_ERROR( "Number of Input Data Files (%s) greater than current limit: %s" % (len(List.fromChar(varValue)), maxNumber) ) return S_OK() def __contains__(self, key): """ Check if the manifest has the required key """ return key in self.__manifest def setOptionsFromDict(self, varDict): for k in sorted(varDict): self.setOption(k, varDict[k]) def check(self): """ Check that the manifest is OK """ for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]: if k not in self.__manifest: return S_ERROR("Missing var %s in manifest" % k) # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) if not result["OK"]: return result result = self.__checkNumericalVar("Priority", 1, 0, 10) if not result["OK"]: return result allowedSubmitPools = getSubmitPools(self.__manifest["OwnerGroup"]) result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools))) if not result["OK"]: return result result = self.__checkMultiChoice("PilotTypes", ["private"]) if not result["OK"]: return result maxInputData = Operations().getValue("JobDescription/MaxInputData", 500) result = self.__checkMaxInputData(maxInputData) if not result["OK"]: return result transformationTypes = Operations().getValue("Transformations/DataProcessing", []) result = self.__checkMultiChoice("JobType", ["User", "Test", "Hospital"] + transformationTypes) if not result["OK"]: return result return S_OK() def createSection(self, secName, contents=False): if secName not in self.__manifest: if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True return S_OK(self.__manifest.createNewSection(secName, contents=contents)) return S_ERROR("Section %s already exists" % secName) def getSection(self, secName): self.__dirty = True sec = self.__manifest[secName] if not sec: return S_ERROR("%s does not exist") return S_OK(sec) def setSectionContents(self, secName, contents): if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True if secName in self.__manifest: self.__manifest[secName].reset() self.__manifest[secName].mergeWith(contents) else: self.__manifest.createNewSection(secName, contents=contents) def setOption(self, varName, varValue): """ Set a var in job manifest """ self.__dirty = True levels = List.fromChar(varName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: cfg.createNewSection(l) cfg = cfg[l] cfg.setOption(levels[-1], varValue) def remove(self, opName): levels = List.fromChar(opName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: return S_ERROR("%s does not exist" % opName) cfg = cfg[l] if cfg.deleteKey(levels[-1]): self.__dirty = True return S_OK() return S_ERROR("%s does not exist" % opName) def getOption(self, varName, defaultValue=None): """ Get a variable from the job manifest """ cfg = self.__manifest return cfg.getOption(varName, defaultValue) def getOptionList(self, section=""): """ Get a list of variables in a section of the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listOptions() def isOption(self, opName): """ Check if it is a valid option """ return self.__manifest.isOption(opName) def getSectionList(self, section=""): """ Get a list of sections in the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listSections()
def toCFG(self): """ Get the full description of the file in CFG format """ oCFG = CFG() strippedLFN = self.lfn.replace('/', '&&') oCFG.createNewSection(strippedLFN) oCFG.setOption('%s/Status' % (strippedLFN), self.status) oCFG.setOption('%s/Size' % (strippedLFN), self.size) oCFG.setOption('%s/GUID' % (strippedLFN), self.guid) oCFG.setOption('%s/Checksum' % (strippedLFN), self.checksum) #TODO: still have to include the CFG from the replica objects if self.catalogReplicas: oCFG.createNewSection('%s/CatalogReplicas' % strippedLFN) for replica in self.catalogReplicas: pass # rCFG.mergeWith(CFG().loadFromBuffer(replica.toCFG()['Value'])) return S_OK(str(oCFG))
class ConfigurationData: def __init__( self, loadDefaultCFG = True ): lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.compressedConfigurationData = "" self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" ) self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" ) gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile ) retVal = self.loadFile( defaultCFGFile ) if not retVal[ 'OK' ]: gLogger.warn( "Can't load %s file" % defaultCFGFile ) self.sync() def getBackupDir( self ): return self.backupsDir def sync( self ): gLogger.debug( "Updating configuration internals" ) self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG ) self.remoteServerList = [] localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones = True ) if localServers: self.remoteServerList.extend( List.fromChar( localServers, "," ) ) remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones = True ) if remoteServers: self.remoteServerList.extend( List.fromChar( remoteServers, "," ) ) self.remoteServerList = List.uniqueElements( self.remoteServerList ) self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 ) def loadFile( self, fileName ): try: fileCFG = CFG() fileCFG.loadFromFile( fileName ) except IOError: self.localCFG = self.localCFG.mergeWith( fileCFG ) return S_ERROR( "Can't load a cfg file '%s'" % fileName ) return self.mergeWithLocal( fileCFG ) def mergeWithLocal( self, extraCFG ): self.lock() try: self.localCFG = self.localCFG.mergeWith( extraCFG ) self.unlock() gLogger.debug( "CFG merged" ) except Exception, e: self.unlock() return S_ERROR( "Cannot merge with new cfg: %s" % str( e ) ) self.sync() return S_OK()
def setUpClass(cls): cls.failed = False # Add configuration cfg = CFG() cfg.loadFromBuffer(diracTestCACFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(userCFG) gConfig.loadCFG(cfg) # Prepare CA lines = [] cfgDict = {} cls.caPath = os.path.join(certsPath, 'ca') cls.caConfigFile = os.path.join(cls.caPath, 'openssl_config_ca.cnf') # Save original configuration file shutil.copyfile(cls.caConfigFile, cls.caConfigFile + 'bak') # Parse fields = [ 'dir', 'database', 'serial', 'new_certs_dir', 'private_key', 'certificate' ] with open(cls.caConfigFile, "r") as caCFG: for line in caCFG: if re.findall('=', re.sub(r'#.*', '', line)): field = re.sub(r'#.*', '', line).replace(' ', '').rstrip().split('=')[0] line = 'dir = %s #PUT THE RIGHT DIR HERE!\n' % ( cls.caPath) if field == 'dir' else line val = re.sub(r'#.*', '', line).replace(' ', '').rstrip().split('=')[1] if field in fields: for i in fields: if cfgDict.get(i): val = val.replace('$%s' % i, cfgDict[i]) cfgDict[field] = val if not cfgDict[field]: cls.failed = '%s have empty value in %s' % ( field, cls.caConfigFile) lines.append(line) with open(cls.caConfigFile, "w") as caCFG: caCFG.writelines(lines) for field in fields: if field not in cfgDict.keys(): cls.failed = '%s value is absent in %s' % (field, cls.caConfigFile) cls.hostCert = os.path.join(certsPath, 'host/hostcert.pem') cls.hostKey = os.path.join(certsPath, 'host/hostkey.pem') cls.caCert = cfgDict['certificate'] cls.caKey = cfgDict['private_key'] os.chmod(cls.caKey, stat.S_IREAD) # Check directory for new certificates cls.newCertDir = cfgDict['new_certs_dir'] if not os.path.exists(cls.newCertDir): os.makedirs(cls.newCertDir) for f in os.listdir(cls.newCertDir): os.remove(os.path.join(cls.newCertDir, f)) # Empty the certificate database cls.index = cfgDict['database'] with open(cls.index, 'w') as indx: indx.write('') # Write down serial cls.serial = cfgDict['serial'] with open(cls.serial, 'w') as serialFile: serialFile.write('1000') # Create temporaly directory for users certificates cls.userDir = tempfile.mkdtemp(dir=certsPath) # Create user certificates for userName in ['no_user', 'user', 'user_1', 'user_2', 'user_3']: userConf = """[ req ] default_bits = 4096 encrypt_key = yes distinguished_name = req_dn prompt = no req_extensions = v3_req [ req_dn ] C = CC O = DN 0.O = DIRAC CN = %s [ v3_req ] # Extensions for client certificates (`man x509v3_config`). nsComment = "OpenSSL Generated Client Certificate" keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth """ % (userName) userConfFile = os.path.join(cls.userDir, userName + '.cnf') userReqFile = os.path.join(cls.userDir, userName + '.req') userKeyFile = os.path.join(cls.userDir, userName + '.key.pem') userCertFile = os.path.join(cls.userDir, userName + '.cert.pem') with open(userConfFile, "w") as f: f.write(userConf) status, output = commands.getstatusoutput( 'openssl genrsa -out %s' % userKeyFile) if status: gLogger.error(output) exit() gLogger.debug(output) os.chmod(userKeyFile, stat.S_IREAD) status, output = commands.getstatusoutput( 'openssl req -config %s -key %s -new -out %s' % (userConfFile, userKeyFile, userReqFile)) if status: gLogger.error(output) exit() gLogger.debug(output) cmd = 'openssl ca -config %s -extensions usr_cert -batch -days 375 -in %s -out %s' cmd = cmd % (cls.caConfigFile, userReqFile, userCertFile) status, output = commands.getstatusoutput(cmd) if status: gLogger.error(output) exit() gLogger.debug(output) # Result status, output = commands.getstatusoutput('ls -al %s' % cls.userDir) if status: gLogger.error(output) exit() gLogger.debug('User certificates:\n', output)
def __addItem(self, params): sectionPath = "/Registry/" configText = "" if params["type"] == "users": sectionPath = sectionPath + "Users" if params["dn"].strip() != "": configText = "DN = " + params["dn"].strip() + "\n" if params["ca"].strip() != "": configText = configText + "CA = " + params["ca"].strip() + "\n" if params["email"].strip() != "": configText = configText + "Email = " + params["email"].strip() elif params["type"] == "groups": sectionPath = sectionPath + "Groups" if params["users"].strip() != "": configText = "Users = " + params["users"].strip() + "\n" if params["properties"].strip() != "": configText = configText + "Properties = " + params["properties"].strip() + "\n" if str(params["jobshare"]).strip() != "": configText = configText + "JobShare = " + str(params["jobshare"]) + "\n" if params["autouploadproxy"].strip() != "": configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n" if params["autouploadpilotproxy"].strip() != "": configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n" if params["autoaddvoms"].strip() != "": configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip() elif params["type"] == "hosts": sectionPath = sectionPath + "Hosts" if params["dn"].strip() != "": configText = "DN = " + params["dn"].strip() + "\n" if params["properties"].strip() != "": configText = configText + "Properties = " + params["properties"].strip() elif params["type"] == "voms": sectionPath = sectionPath + "VOMS/Servers" elif params["type"] == "servers": sectionPath = sectionPath + "VOMS/Servers/" + params["vom"] if params["dn"].strip() != "": configText = "DN = " + params["dn"].strip() + "\n" if params["port"].strip() != "": configText = configText + "Port = " + params["port"].strip() + "\n" if params["ca"].strip() != "": configText = configText + "CA = " + params["ca"].strip() sectionPath = sectionPath + "/" + params["name"] if self.__configData[ 'cfgData' ].createSection(sectionPath): cfgData = self.__configData[ 'cfgData' ].getCFG() newCFG = CFG() newCFG.loadFromBuffer(configText) self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG) return {"success":1, "op": "addItem"} else: return {"success":0, "op":"addItem", "message":"Section can't be created. It already exists?"}
# $HeadURL$ __RCSID__ = "7b8878b (2009-11-05 19:40:01 +0000) Adria Casajus <*****@*****.**>" from dirac import DIRAC from DIRAC.Core.Utilities.CFG import CFG DIRAC.gLogger.initialize('test_gConfig','/testSectionDebug') testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath cfg1 = CFG() cfg1.loadFromFile( testconfig ) fd = file( testconfig ) cfg1String = fd.read() fd.close() cfg2 = CFG() cfg2.loadFromBuffer( cfg1.serialize() ) cfg3 = cfg1.mergeWith( cfg2 ) testList = [{ 'method' : DIRAC.gConfig.loadFile, 'arguments' : ( testconfig, ), 'output' : {'OK': True, 'Value': ''} }, { 'method' : DIRAC.gConfig.dumpLocalCFGToFile, 'arguments' : ( dumpconfig, ), 'output' : {'OK': True, 'Value': ''} },
class JobManifest(object): def __init__(self, manifest=""): self.__manifest = CFG() self.__dirty = False self.__ops = False if manifest: result = self.load(manifest) if not result['OK']: raise Exception(result['Message']) def isDirty(self): return self.__dirty def setDirty(self): self.__dirty = True def clearDirty(self): self.__dirty = False def load(self, dataString): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadJDL(dataString) else: return self.loadCFG(dataString) def loadJDL(self, jdlString): """ Load job manifest from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result['OK']: self.__manifest = CFG() return result self.__manifest = result['Value'][0] return S_OK() def loadCFG(self, cfgString): """ Load job manifest from CFG format """ try: self.__manifest.loadFromBuffer(cfgString) except Exception as e: return S_ERROR("Can't load manifest from cfg: %s" % str(e)) return S_OK() def dumpAsCFG(self): return str(self.__manifest) def getAsCFG(self): return self.__manifest.clone() def dumpAsJDL(self): return dumpCFGAsJDL(self.__manifest) def __getCSValue(self, varName, defaultVal=None): if not self.__ops: self.__ops = Operations(group=self.__manifest['OwnerGroup'], setup=self.__manifest['DIRACSetup']) if varName[0] != "/": varName = "JobDescription/%s" % varName return self.__ops.getValue(varName, defaultVal) def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal): """ Check a numerical var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue try: varValue = long(varValue) except BaseException: return S_ERROR("%s must be a number" % varName) minVal = self.__getCSValue("Min%s" % varName, minVal) maxVal = self.__getCSValue("Max%s" % varName, maxVal) varValue = max(minVal, min(varValue, maxVal)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkChoiceVar(self, varName, defaultVal, choices): """ Check a choice var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue if varValue not in self.__getCSValue("Choices%s" % varName, choices): return S_ERROR("%s is not a valid value for %s" % (varValue, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMultiChoice(self, varName, choices): """ Check a multi choice var """ initialVal = False if varName not in self.__manifest: return S_OK() else: varValue = self.__manifest[varName] initialVal = varValue choices = self.__getCSValue("Choices%s" % varName, choices) for v in List.fromChar(varValue): if v not in choices: return S_ERROR("%s is not a valid value for %s" % (v, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMaxInputData(self, maxNumber): """ Check Maximum Number of Input Data files allowed """ varName = "InputData" if varName not in self.__manifest: return S_OK() varValue = self.__manifest[varName] if len(List.fromChar(varValue)) > maxNumber: return S_ERROR('Number of Input Data Files (%s) greater than current limit: %s' % (len(List.fromChar(varValue)), maxNumber)) return S_OK() def __contains__(self, key): """ Check if the manifest has the required key """ return key in self.__manifest def setOptionsFromDict(self, varDict): for k in sorted(varDict): self.setOption(k, varDict[k]) def check(self): """ Check that the manifest is OK """ for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']: if k not in self.__manifest: return S_ERROR("Missing var %s in manifest" % k) # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) if not result['OK']: return result result = self.__checkNumericalVar("Priority", 1, 0, 10) if not result['OK']: return result allowedSubmitPools = getSubmitPools(self.__manifest['OwnerGroup']) result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools))) if not result['OK']: return result result = self.__checkMultiChoice("PilotTypes", ['private']) if not result['OK']: return result maxInputData = Operations().getValue("JobDescription/MaxInputData", 500) result = self.__checkMaxInputData(maxInputData) if not result['OK']: return result transformationTypes = Operations().getValue("Transformations/DataProcessing", []) result = self.__checkMultiChoice("JobType", ['User', 'Test', 'Hospital'] + transformationTypes) if not result['OK']: return result return S_OK() def createSection(self, secName, contents=False): if secName not in self.__manifest: if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True return S_OK(self.__manifest.createNewSection(secName, contents=contents)) return S_ERROR("Section %s already exists" % secName) def getSection(self, secName): self.__dirty = True if secName not in self.__manifest: return S_ERROR("%s does not exist" % secName) sec = self.__manifest[secName] if not sec: return S_ERROR("%s section empty" % secName) return S_OK(sec) def setSectionContents(self, secName, contents): if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True if secName in self.__manifest: self.__manifest[secName].reset() self.__manifest[secName].mergeWith(contents) else: self.__manifest.createNewSection(secName, contents=contents) def setOption(self, varName, varValue): """ Set a var in job manifest """ self.__dirty = True levels = List.fromChar(varName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: cfg.createNewSection(l) cfg = cfg[l] cfg.setOption(levels[-1], varValue) def remove(self, opName): levels = List.fromChar(opName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: return S_ERROR("%s does not exist" % opName) cfg = cfg[l] if cfg.deleteKey(levels[-1]): self.__dirty = True return S_OK() return S_ERROR("%s does not exist" % opName) def getOption(self, varName, defaultValue=None): """ Get a variable from the job manifest """ cfg = self.__manifest return cfg.getOption(varName, defaultValue) def getOptionList(self, section=""): """ Get a list of variables in a section of the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or 'value' not in cfg: return [] cfg = cfg['value'] return cfg.listOptions() def isOption(self, opName): """ Check if it is a valid option """ return self.__manifest.isOption(opName) def getSectionList(self, section=""): """ Get a list of sections in the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or 'value' not in cfg: return [] cfg = cfg['value'] return cfg.listSections()
def execute(self): """The JobAgent execution method. """ if self.jobCount: # Temporary mechanism to pass a shutdown message to the agent if os.path.exists('/var/lib/dirac_drain'): return self.__finish('Node is being drained by an operator') # Only call timeLeft utility after a job has been picked up self.log.info('Attempting to check CPU time left for filling mode') if self.fillingMode: if self.timeLeftError: self.log.warn(self.timeLeftError) return self.__finish(self.timeLeftError) self.log.info('%s normalized CPU units remaining in slot' % (self.timeLeft)) if self.timeLeft <= self.minimumTimeLeft: return self.__finish('No more time left') # Need to update the Configuration so that the new value is published in the next matching request result = self.computingElement.setCPUTimeLeft( cpuTimeLeft=self.timeLeft) if not result['OK']: return self.__finish(result['Message']) # Update local configuration to be used by submitted job wrappers localCfg = CFG() if self.extraOptions: localConfigFile = os.path.join('.', self.extraOptions) else: localConfigFile = os.path.join(rootPath, "etc", "dirac.cfg") localCfg.loadFromFile(localConfigFile) if not localCfg.isSection('/LocalSite'): localCfg.createNewSection('/LocalSite') localCfg.setOption('/LocalSite/CPUTimeLeft', self.timeLeft) localCfg.writeToFile(localConfigFile) else: return self.__finish('Filling Mode is Disabled') self.log.verbose('Job Agent execution loop') result = self.computingElement.available() if not result['OK']: self.log.info('Resource is not available') self.log.info(result['Message']) return self.__finish('CE Not Available') self.log.info(result['Message']) ceInfoDict = result['CEInfoDict'] runningJobs = ceInfoDict.get("RunningJobs") availableSlots = result['Value'] if not availableSlots: if runningJobs: self.log.info('No available slots with %d running jobs' % runningJobs) return S_OK('Job Agent cycle complete with %d running jobs' % runningJobs) else: self.log.info('CE is not available') return self.__finish('CE Not Available') result = self.computingElement.getDescription() if not result['OK']: return result # We can have several prioritized job retrieval strategies if isinstance(result['Value'], dict): ceDictList = [result['Value']] elif isinstance(result['Value'], list): ceDictList = result['Value'] for ceDict in ceDictList: # Add pilot information gridCE = gConfig.getValue('LocalSite/GridCE', 'Unknown') if gridCE != 'Unknown': ceDict['GridCE'] = gridCE if 'PilotReference' not in ceDict: ceDict['PilotReference'] = str(self.pilotReference) ceDict['PilotBenchmark'] = self.cpuFactor ceDict['PilotInfoReportedFlag'] = self.pilotInfoReportedFlag # Add possible job requirements result = gConfig.getOptionsDict('/AgentJobRequirements') if result['OK']: requirementsDict = result['Value'] ceDict.update(requirementsDict) self.log.info('Requirements:', requirementsDict) self.log.verbose(ceDict) start = time.time() jobRequest = MatcherClient().requestJob(ceDict) matchTime = time.time() - start self.log.info('MatcherTime = %.2f (s)' % (matchTime)) if jobRequest['OK']: break self.stopAfterFailedMatches = self.am_getOption( 'StopAfterFailedMatches', self.stopAfterFailedMatches) if not jobRequest['OK']: if re.search('No match found', jobRequest['Message']): self.log.notice('Job request OK: %s' % (jobRequest['Message'])) self.matchFailedCount += 1 if self.matchFailedCount > self.stopAfterFailedMatches: return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches) return S_OK(jobRequest['Message']) elif jobRequest['Message'].find("seconds timeout") != -1: self.log.error('Timeout while requesting job', jobRequest['Message']) self.matchFailedCount += 1 if self.matchFailedCount > self.stopAfterFailedMatches: return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches) return S_OK(jobRequest['Message']) elif jobRequest['Message'].find( "Pilot version does not match") != -1: errorMsg = 'Pilot version does not match the production version' self.log.error(errorMsg, jobRequest['Message'].replace(errorMsg, '')) return S_ERROR(jobRequest['Message']) else: self.log.notice('Failed to get jobs: %s' % (jobRequest['Message'])) self.matchFailedCount += 1 if self.matchFailedCount > self.stopAfterFailedMatches: return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches) return S_OK(jobRequest['Message']) # Reset the Counter self.matchFailedCount = 0 matcherInfo = jobRequest['Value'] if not self.pilotInfoReportedFlag: # Check the flag after the first access to the Matcher self.pilotInfoReportedFlag = matcherInfo.get( 'PilotInfoReportedFlag', False) jobID = matcherInfo['JobID'] matcherParams = ['JDL', 'DN', 'Group'] for param in matcherParams: if param not in matcherInfo: self.__report(jobID, 'Failed', 'Matcher did not return %s' % (param)) return self.__finish('Matcher Failed') elif not matcherInfo[param]: self.__report(jobID, 'Failed', 'Matcher returned null %s' % (param)) return self.__finish('Matcher Failed') else: self.log.verbose('Matcher returned %s = %s ' % (param, matcherInfo[param])) jobJDL = matcherInfo['JDL'] jobGroup = matcherInfo['Group'] ownerDN = matcherInfo['DN'] optimizerParams = {} for key in matcherInfo: if key not in matcherParams: optimizerParams[key] = matcherInfo[key] parameters = self.__getJDLParameters(jobJDL) if not parameters['OK']: self.__report(jobID, 'Failed', 'Could Not Extract JDL Parameters') self.log.warn(parameters['Message']) return self.__finish('JDL Problem') params = parameters['Value'] if 'JobID' not in params: msg = 'Job has not JobID defined in JDL parameters' self.__report(jobID, 'Failed', msg) self.log.warn(msg) return self.__finish('JDL Problem') else: jobID = params['JobID'] if 'JobType' not in params: self.log.warn('Job has no JobType defined in JDL parameters') jobType = 'Unknown' else: jobType = params['JobType'] if 'CPUTime' not in params: self.log.warn( 'Job has no CPU requirement defined in JDL parameters') # Job requirement for a number of processors processors = int(params.get('NumberOfProcessors', 1)) wholeNode = 'WholeNode' in params if self.extraOptions: params['Arguments'] += ' ' + self.extraOptions params['ExtraOptions'] = self.extraOptions self.log.verbose('Job request successful: \n', jobRequest['Value']) self.log.info('Received JobID=%s, JobType=%s' % (jobID, jobType)) self.log.info('OwnerDN: %s JobGroup: %s' % (ownerDN, jobGroup)) self.jobCount += 1 try: jobReport = JobReport(jobID, 'JobAgent@%s' % self.siteName) jobReport.setJobParameter('MatcherServiceTime', str(matchTime), sendFlag=False) if 'BOINC_JOB_ID' in os.environ: # Report BOINC environment for thisp in ('BoincUserID', 'BoincHostID', 'BoincHostPlatform', 'BoincHostName'): jobReport.setJobParameter(thisp, gConfig.getValue( '/LocalSite/%s' % thisp, 'Unknown'), sendFlag=False) jobReport.setJobStatus('Matched', 'Job Received by Agent') result = self.__setupProxy(ownerDN, jobGroup) if not result['OK']: return self.__rescheduleFailedJob( jobID, result['Message'], self.stopOnApplicationFailure) proxyChain = result.get('Value') # Save the job jdl for external monitoring self.__saveJobJDLRequest(jobID, jobJDL) software = self.__checkInstallSoftware(jobID, params, ceDict) if not software['OK']: self.log.error('Failed to install software for job', '%s' % (jobID)) errorMsg = software['Message'] if not errorMsg: errorMsg = 'Failed software installation' return self.__rescheduleFailedJob( jobID, errorMsg, self.stopOnApplicationFailure) self.log.debug('Before %sCE submitJob()' % (self.ceName)) result = self.__submitJob(jobID, params, ceDict, optimizerParams, proxyChain, processors, wholeNode) if not result['OK']: self.__report(jobID, 'Failed', result['Message']) return self.__finish(result['Message']) elif 'PayloadFailed' in result: # Do not keep running and do not overwrite the Payload error message = 'Payload execution failed with error code %s' % result[ 'PayloadFailed'] if self.stopOnApplicationFailure: return self.__finish(message, self.stopOnApplicationFailure) else: self.log.info(message) self.log.debug('After %sCE submitJob()' % (self.ceName)) except Exception as subExcept: # pylint: disable=broad-except self.log.exception("Exception in submission", "", lException=subExcept, lExcInfo=True) return self.__rescheduleFailedJob( jobID, 'Job processing failed with exception', self.stopOnApplicationFailure) # Sum all times but the last one (elapsed_time) and remove times at init (is this correct?) cpuTime = sum(os.times()[:-1]) - sum(self.initTimes[:-1]) result = self.timeLeftUtil.getTimeLeft(cpuTime, processors) if result['OK']: self.timeLeft = result['Value'] else: if result['Message'] != 'Current batch system is not supported': self.timeLeftError = result['Message'] else: # if the batch system is not defined, use the process time and the CPU normalization defined locally self.timeLeft = self.__getCPUTimeLeft() return S_OK('Job Agent cycle complete')
class JobDescription(object): def __init__(self): self.__description = CFG() self.__dirty = False def isDirty(self): return self.__dirty def loadDescription(self, dataString): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadDescriptionFromJDL(dataString) else: return self.loadDescriptionFromCFG(dataString) def loadDescriptionFromJDL(self, jdlString): """ Load job description from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result['OK']: self.__description = CFG() return result self.__description = result['Value'][0] return S_OK() def loadDescriptionFromCFG(self, cfgString): """ Load job description from CFG format """ try: self.__description.loadFromBuffer(cfgString) except Exception as e: return S_ERROR("Can't load description from cfg: %s" % str(e)) return S_OK() def dumpDescriptionAsCFG(self): return str(self.__description) def dumpDescriptionAsJDL(self): return dumpCFGAsJDL(self.__description) def __checkNumericalVarInDescription(self, varName, defaultVal, minVal, maxVal): """ Check a numerical var """ initialVal = 0 if varName not in self.__description: varValue = Operations().getValue( "JobDescription/Default%s" % varName, defaultVal) else: varValue = self.__description[varName] initialVal = varValue try: varValue = long(varValue) except BaseException: return S_ERROR("%s must be a number" % varName) minVal = Operations().getValue("JobDescription/Min%s" % varName, minVal) maxVal = Operations().getValue("JobDescription/Max%s" % varName, maxVal) varValue = max(minVal, min(varValue, maxVal)) if initialVal != varValue: self.__description.setOption(varName, varValue) return S_OK(varValue) def __checkChoiceVarInDescription(self, varName, defaultVal, choices): """ Check a choice var """ initialVal = False if varName not in self.__description: varValue = Operations().getValue( "JobDescription/Default%s" % varName, defaultVal) else: varValue = self.__description[varName] initialVal = varValue if varValue not in Operations().getValue( "JobDescription/Choices%s" % varName, choices): return S_ERROR("%s is not a valid value for %s" % (varValue, varName)) if initialVal != varValue: self.__description.setOption(varName, varValue) return S_OK(varValue) def __checkMultiChoiceInDescription(self, varName, choices): """ Check a multi choice var """ initialVal = False if varName not in self.__description: return S_OK() else: varValue = self.__description[varName] initialVal = varValue choices = Operations().getValue("JobDescription/Choices%s" % varName, choices) for v in List.fromChar(varValue): if v not in choices: return S_ERROR("%s is not a valid value for %s" % (v, varName)) if initialVal != varValue: self.__description.setOption(varName, varValue) return S_OK(varValue) def __checkMaxInputData(self, maxNumber): """ Check Maximum Number of Input Data files allowed """ varName = "InputData" if varName not in self.__description: return S_OK() varValue = self.__description[varName] if len(List.fromChar(varValue)) > maxNumber: return S_ERROR( 'Number of Input Data Files (%s) greater than current limit: %s' % (len(List.fromChar(varValue)), maxNumber)) return S_OK() def setVarsFromDict(self, varDict): for k in sorted(varDict): self.setVar(k, varDict[k]) def checkDescription(self): """ Check that the description is OK """ for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']: if k not in self.__description: return S_ERROR("Missing var %s in description" % k) # Check CPUTime result = self.__checkNumericalVarInDescription("CPUTime", 86400, 0, 500000) if not result['OK']: return result result = self.__checkNumericalVarInDescription("Priority", 1, 0, 10) if not result['OK']: return result allowedSubmitPools = getSubmitPools(self.__description['OwnerGroup']) result = self.__checkMultiChoiceInDescription( "SubmitPools", list(set(allowedSubmitPools))) if not result['OK']: return result result = self.__checkMultiChoiceInDescription( "SubmitPools", list(set(allowedSubmitPools))) if not result['OK']: return result result = self.__checkMultiChoiceInDescription("PilotTypes", ['private']) if not result['OK']: return result maxInputData = Operations().getValue("JobDescription/MaxInputData", 500) result = self.__checkMaxInputData(maxInputData) if not result['OK']: return result transformationTypes = Operations().getValue( "Transformations/DataProcessing", []) result = self.__checkMultiChoiceInDescription( "JobType", ['User', 'Test', 'Hospital'] + transformationTypes) return S_OK() def setVar(self, varName, varValue): """ Set a var in job description """ self.__dirty = True levels = List.fromChar(varName, "/") cfg = self.__description for l in levels[:-1]: if l not in cfg: cfg.createNewSection(l) cfg = cfg[l] cfg.setOption(levels[-1], varValue) def getVar(self, varName, defaultValue=None): cfg = self.__description return cfg.getOption(varName, defaultValue) def getOptionList(self, section=""): cfg = self.__description.getRecursive(section) if not cfg or 'value' not in cfg: return [] cfg = cfg['value'] return cfg.listOptions() def getSectionList(self, section=""): cfg = self.__description.getRecursive(section) if not cfg or 'value' not in cfg: return [] cfg = cfg['value'] return cfg.listSections()
def __init__(self): self.__description = CFG() self.__dirty = False