def _getCurrentConfig(self):
"""Return the current system configuration."""
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfig.forceRefresh()
fullCfg = CFG()
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
for system, setup in serviceSetups.items():
if self.systems and system not in self.systems:
continue
systemCfg = gConfigurationData.remoteCFG.getAsCFG("/Systems/%s/%s" % (system, setup))
for section in systemCfg.listSections():
if section not in ('Agents', 'Services', 'Executors'):
systemCfg.deleteKey(section)
fullCfg.createNewSection("/%s" % system, contents=systemCfg)
return S_OK(fullCfg)
def loadFile( self, fileName ):
try:
fileCFG = CFG()
fileCFG.loadFromFile( fileName )
except IOError:
self.localCFG = self.localCFG.mergeWith( fileCFG )
return S_ERROR( "Can't load a cfg file '%s'" % fileName )
return self.mergeWithLocal( fileCFG )
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def _parseConfigTemplate(self, templatePath, cfg=None):
"""Parse the ConfigTemplate.cfg files.
:param str templatePath: path to the folder containing a ConfigTemplate.cfg file
:param CFG cfg: cfg to merge with the systems config
:returns: CFG object
"""
cfg = CFG() if cfg is None else cfg
system = os.path.split(templatePath.rstrip("/"))[1]
if system.lower().endswith('system'):
system = system[:-len('System')]
if self.systems and system not in self.systems:
return S_OK(cfg)
templatePath = os.path.join(templatePath, 'ConfigTemplate.cfg')
if not os.path.exists(templatePath):
return S_ERROR("File not found: %s" % templatePath)
loadCfg = CFG()
loadCfg.loadFromFile(templatePath)
newCfg = CFG()
newCfg.createNewSection("/%s" % system, contents=loadCfg)
cfg = cfg.mergeWith(newCfg)
return S_OK(cfg)
class JobManifest( object ):
def __init__( self, manifest = "" ):
self.__manifest = CFG()
self.__dirty = False
if manifest:
result = self.loadManifest( manifest )
if not result[ 'OK' ]:
raise Exception( result[ 'Message' ] )
def isDirty( self ):
return self.__dirty
def setDirty( self ):
self.__dirty = True
def clearDirty( self ):
self.__dirty = False
def load( self, dataString ):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL( dataString )
else:
return self.loadCFG( dataString )
def loadJDL( self, jdlString ):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__manifest = CFG()
return result
self.__manifest = result[ 'Value' ][0]
return S_OK()
def loadCFG( self, cfgString ):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer( cfgString )
except Exception, e:
return S_ERROR( "Can't load manifest from cfg: %s" % str( e ) )
return S_OK()
def showTextConfiguration(self):
response.headers["Content-type"] = "text/plain"
if "download" in request.params and request.params["download"] in ("yes", "true", "1"):
version = ""
try:
cfg = CFG()
cfg.loadFromBuffer(session["cfgData"])
cfg = cfg["DIRAC"]["Configuration"]
version = ".%s.%s" % (cfg["Name"], cfg["Version"].replace(":", "").replace("-", ""))
except Exception, e:
print e
print 'attachment; filename="cs%s.cfg"' % version.replace(" ", "_")
response.headers["Content-Disposition"] = 'attachment; filename="cs%s.cfg"' % version.replace(" ", "")
response.headers["Content-Length"] = len(session["cfgData"])
response.headers["Content-Transfer-Encoding"] = "Binary"
def appendToRepository( self, repoLocation ):
if not os.path.exists( repoLocation ):
gLogger.error( "Secondary repository does not exist", repoLocation )
return S_ERROR( "Secondary repository does not exist" )
self.repo = CFG().loadFromFile( repoLocation ).mergeWith( self.repo )
self._writeRepository( self.location )
return S_OK()
def __init__( self, rpcClient = False, commiterId = "unknown" ):
self.commiterTag = "@@-"
self.commiterId = commiterId
self.cfgData = CFG()
self.rpcClient = None
if rpcClient:
self.setRPCClient( rpcClient )
def __init__( self, manifest = "" ):
self.__manifest = CFG()
self.__dirty = False
if manifest:
result = self.loadManifest( manifest )
if not result[ 'OK' ]:
raise Exception( result[ 'Message' ] )
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result["OK"]:
raise Exception(result["Message"])
def __gConfigDefaults(defaultPath):
"""
Build a cfg from a Default Section
"""
from DIRAC import gConfig
cfgDefaults = CFG()
result = gConfig.getSections(defaultPath)
if not result['OK']:
return cfgDefaults
for name in result['Value']:
typePath = cfgPath(defaultPath, name)
cfgDefaults.createNewSection(name)
result = gConfig.getOptionsDict(typePath)
if result['OK']:
optionsDict = result['Value']
for option, value in optionsDict.items():
cfgDefaults[name].setOption(option, value)
return cfgDefaults
def loadWebAppCFGFiles():
"""
Load WebApp/web.cfg definitions
"""
exts = []
for ext in CSGlobals.getCSExtensions():
if ext == "DIRAC":
continue
if ext[-5:] != "DIRAC":
ext = "%sDIRAC" % ext
if ext != "WebAppDIRAC":
exts.append( ext )
exts.append( "DIRAC" )
exts.append( "WebAppDIRAC" )
webCFG = CFG()
for modName in reversed( exts ):
try:
modPath = imp.find_module( modName )[1]
except ImportError:
continue
gLogger.verbose( "Found module %s at %s" % ( modName, modPath ) )
cfgPath = os.path.join( modPath, "WebApp", "web.cfg" )
if not os.path.isfile( cfgPath ):
gLogger.verbose( "Inexistant %s" % cfgPath )
continue
try:
modCFG = CFG().loadFromFile( cfgPath )
except Exception, excp:
gLogger.error( "Could not load %s: %s" % ( cfgPath, excp ) )
continue
gLogger.verbose( "Loaded %s" % cfgPath )
expl = [ BASECS ]
while len( expl ):
current = expl.pop( 0 )
if not modCFG.isSection( current ):
continue
if modCFG.getOption( "%s/AbsoluteDefinition" % current, False ):
gLogger.verbose( "%s:%s is an absolute definition" % ( modName, current ) )
try:
webCFG.deleteKey( current )
except:
pass
modCFG.deleteKey( "%s/AbsoluteDefinition" % current )
else:
for sec in modCFG[ current ].listSections():
expl.append( "%s/%s" % ( current, sec ) )
#Add the modCFG
webCFG = webCFG.mergeWith( modCFG )
def setUp( self ):
self.authMgr = AuthManager( '/Systems/Service/Authorization' )
cfg = CFG()
cfg.loadFromBuffer( testSystemsCFG )
gConfig.loadCFG( cfg )
cfg.loadFromBuffer( testRegistryCFG )
gConfig.loadCFG( cfg )
self.noAuthCredDict = { 'group': 'group_test' }
self.userCredDict = { 'DN': '/User/test/DN/CN=userA',
'group': 'group_test' }
self.badUserCredDict = { 'DN': '/User/test/DN/CN=userB',
'group': 'group_bad' }
self.hostCredDict = { 'DN': '/User/test/DN/CN=test.hostA.ch',
'group': 'hosts' }
self.badHostCredDict = { 'DN': '/User/test/DN/CN=test.hostB.ch',
'group': 'hosts' }
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result["OK"]:
self.__manifest = CFG()
return result
self.__manifest = result["Value"][0]
return S_OK()
def __init__(self, location):
self.cfg = CFG()
self.location = location
self.OK = True
if os.path.exists(self.location):
self.cfg.loadFromFile(self.location)
if not self.cfg.existsKey('Processes'):
self.cfg.createNewSection('Processes')
else:
self.OK = False
def loadDescriptionFromJDL( self, jdlString ):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__description = CFG()
return result
self.__description = result[ 'Value' ][0]
return S_OK()
def __init__( self, loadDefaultCFG = True ):
lr = LockRing()
self.threadingEvent = lr.getEvent()
self.threadingEvent.set()
self.threadingLock = lr.getLock()
self.runningThreadsNumber = 0
self.compressedConfigurationData = ""
self.configurationPath = "/DIRAC/Configuration"
self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" )
self._isService = False
self.localCFG = CFG()
self.remoteCFG = CFG()
self.mergedCFG = CFG()
self.remoteServerList = []
if loadDefaultCFG:
defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" )
gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile )
retVal = self.loadFile( defaultCFGFile )
if not retVal[ 'OK' ]:
gLogger.warn( "Can't load %s file" % defaultCFGFile )
self.sync()
class ProcessList(object):
""" The ProcessList uses internally the CFG utility to store the processes and their properties.
"""
def __init__(self, location):
self.cfg = CFG()
self.location = location
self.goodProcessList = True
if os.path.exists(self.location):
self.cfg.loadFromFile(self.location)
if not self.cfg.existsKey('Processes'):
self.cfg.createNewSection('Processes')
else:
self.goodProcessList = False
def _writeProcessList(self, path):
""" Write to text
"""
handle, tmpName = tempfile.mkstemp()
written = self.cfg.writeToFile(tmpName)
os.close(handle)
if not written:
if os.path.exists(tmpName):
os.remove(tmpName)
return written
if os.path.exists(path):
LOG.debug("Replacing %s" % path)
try:
shutil.move(tmpName, path)
return True
except OSError, err:
LOG.error("Failed to overwrite process list.", err)
LOG.info("If your process list is corrupted a backup can be found %s" % tmpName)
return False
class JobDescription:
def __init__( self ):
self.__description = CFG()
self.__dirty = False
def isDirty( self ):
return self.__dirty
def loadDescription( self, dataString ):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL( dataString )
else:
return self.loadDescriptionFromCFG( dataString )
def loadDescriptionFromJDL( self, jdlString ):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__description = CFG()
return result
self.__description = result[ 'Value' ][0]
return S_OK()
def loadDescriptionFromCFG( self, cfgString ):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer( cfgString )
except Exception, e:
return S_ERROR( "Can't load description from cfg: %s" % str( e ) )
return S_OK()
def __saveVomsMapping(self, params):
sectionPath = "/Registry/VOMS/Mapping"
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
if not sectionCfg.isSection(opt):
self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)
configText = ""
for newOpt in params:
if newOpt != "op":
if configText == "":
configText = newOpt + "=" + params[newOpt] + "\n"
else:
configText = configText + newOpt + "=" + params[newOpt] + "\n"
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"op":"saveVomsMapping", "success": 1}
def execute( self ):
""" execute
"""
stopAgents = self.findStopAgents()[ 'Value' ]
if stopAgents:
self.log.info( 'Aborting, there are stop_agents to be picked' )
return S_OK()
pilotVersion = self.opHelper.getValue( 'Pilot/Version', '' )
if not pilotVersion:
self.log.error( 'There is no pilot version on the CS' )
return S_OK()
pilotVersion = self.getNewestPilotVersion()
if not pilotVersion[ 'OK' ]:
self.log.error( pilotVersion[ 'Message' ] )
return S_ERROR( pilotVersion[ 'Message' ] )
pilotVersion = pilotVersion[ 'Value' ]
localCFG = CFG()
#load local CFG
localCFG.loadFromFile( self.cfgToUpdate )
releaseVersion = localCFG.getRecursive( 'LocalSite/ReleaseVersion' )[ 'value' ]
self.log.info( 'PilotVersion : %s' % pilotVersion )
self.log.info( 'ReleaseVersion : %s' % releaseVersion )
if LooseVersion( pilotVersion ) > LooseVersion( releaseVersion ):
self.log.info( 'UPDATING %s > %s' % ( pilotVersion, releaseVersion ) )
localCFG.setOption( 'LocalSite/ReleaseVersion', pilotVersion )
localCFG.writeToFile( self.cfgToUpdate )
self.touchStopAgents()
else:
self.log.info( 'Nothing to do' )
return S_OK()
def __init__( self, repository = None ):
self.location = repository
if not self.location:
if "HOME" in os.environ:
self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
else:
self.location = '%s/.dirac.repo.rep' % os.getcwd()
self.repo = CFG()
if os.path.exists( self.location ):
self.repo.loadFromFile( self.location )
if not self.repo.existsKey( 'Jobs' ):
self.repo.createNewSection( 'Jobs' )
else:
self.repo.createNewSection( 'Jobs' )
self.OK = True
written = self._writeRepository( self.location )
if not written:
self.OK = False
def _loadWebAppCFGFiles(self, extension):
"""
Load WebApp/web.cfg definitions
:param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC
"""
exts = [extension, "WebAppDIRAC"]
webCFG = CFG()
for modName in reversed(exts):
cfgPath = os.path.join(self.__params.destination, "%s/WebApp" % modName, "web.cfg")
if not os.path.isfile(cfgPath):
gLogger.verbose("Web configuration file %s does not exists!" % cfgPath)
continue
try:
modCFG = CFG().loadFromFile(cfgPath)
except Exception, excp:
gLogger.error("Could not load %s: %s" % (cfgPath, excp))
continue
gLogger.verbose("Loaded %s" % cfgPath)
expl = ["/WebApp"]
while len(expl):
current = expl.pop(0)
if not modCFG.isSection(current):
continue
if modCFG.getOption("%s/AbsoluteDefinition" % current, False):
gLogger.verbose("%s:%s is an absolute definition" % (modName, current))
try:
webCFG.deleteKey(current)
except:
pass
modCFG.deleteKey("%s/AbsoluteDefinition" % current)
else:
for sec in modCFG[current].listSections():
expl.append("%s/%s" % (current, sec))
# Add the modCFG
webCFG = webCFG.mergeWith(modCFG)
def getComputingElementDefaults(ceName="", ceType="", cfg=None, currentSectionPath=""):
"""
Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg
"""
cesCfg = CFG()
if cfg:
try:
cesCfg.loadFromFile(cfg)
cesPath = cfgInstallPath("ComputingElements")
if cesCfg.isSection(cesPath):
for section in cfgPathToList(cesPath):
cesCfg = cesCfg[section]
except:
return CFG()
# Overwrite the cfg with Command line arguments
if ceName:
if not cesCfg.isSection(ceName):
cesCfg.createNewSection(ceName)
if currentSectionPath:
# Add Options from Command Line
optionsDict = __getExtraOptions(currentSectionPath)
for name, value in optionsDict.items():
cesCfg[ceName].setOption(name, value)
if ceType:
cesCfg[ceName].setOption("CEType", ceType)
ceDefaultSection = cfgPath(defaultSection("ComputingElements"))
# Load Default for the given type from Central configuration is defined
ceDefaults = __gConfigDefaults(ceDefaultSection)
for ceName in cesCfg.listSections():
if "CEType" in cesCfg[ceName]:
ceType = cesCfg[ceName]["CEType"]
if ceType in ceDefaults:
for option in ceDefaults[ceType].listOptions():
if option not in cesCfg[ceName]:
cesCfg[ceName].setOption(option, ceDefaults[ceType][option])
return cesCfg
def getComputingElementDefaults(ceName='', ceType='', cfg=None, currentSectionPath=''):
"""
Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg
"""
cesCfg = CFG()
if cfg:
try:
cesCfg.loadFromFile(cfg)
cesPath = cfgInstallPath('ComputingElements')
if cesCfg.isSection(cesPath):
for section in cfgPathToList(cesPath):
cesCfg = cesCfg[section]
except BaseException:
return CFG()
# Overwrite the cfg with Command line arguments
if ceName:
if not cesCfg.isSection(ceName):
cesCfg.createNewSection(ceName)
if currentSectionPath:
# Add Options from Command Line
optionsDict = __getExtraOptions(currentSectionPath)
for name, value in optionsDict.items():
cesCfg[ceName].setOption(name, value) # pylint: disable=no-member
if ceType:
cesCfg[ceName].setOption('CEType', ceType) # pylint: disable=no-member
ceDefaultSection = cfgPath(defaultSection('ComputingElements'))
# Load Default for the given type from Central configuration is defined
ceDefaults = __gConfigDefaults(ceDefaultSection)
for ceName in cesCfg.listSections():
if 'CEType' in cesCfg[ceName]:
ceType = cesCfg[ceName]['CEType']
if ceType in ceDefaults:
for option in ceDefaults[ceType].listOptions(): # pylint: disable=no-member
if option not in cesCfg[ceName]:
cesCfg[ceName].setOption(option, ceDefaults[ceType][option]) # pylint: disable=unsubscriptable-object
return cesCfg
class JobRepository(object):
def __init__(self, repository=None):
self.location = repository
if not self.location:
if "HOME" in os.environ:
self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
else:
self.location = '%s/.dirac.repo.rep' % os.getcwd()
self.repo = CFG()
if os.path.exists(self.location):
self.repo.loadFromFile(self.location)
if not self.repo.existsKey('Jobs'):
self.repo.createNewSection('Jobs')
else:
self.repo.createNewSection('Jobs')
self.OK = True
written = self._writeRepository(self.location)
if not written:
self.OK = False
def isOK(self):
return self.OK
def readRepository(self):
return S_OK(self.repo.getAsDict('Jobs'))
def writeRepository(self, alternativePath=None):
destination = self.location
if alternativePath:
destination = alternativePath
written = self._writeRepository(destination)
if not written:
return S_ERROR("Failed to write repository")
return S_OK(destination)
def resetRepository(self, jobIDs=[]):
if not jobIDs:
jobs = self.readRepository()['Value']
jobIDs = jobs.keys()
paramDict = {'State': 'Submitted', 'Retrieved': 0, 'OutputData': 0}
for jobID in jobIDs:
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def _writeRepository(self, path):
handle, tmpName = tempfile.mkstemp()
written = self.repo.writeToFile(tmpName)
os.close(handle)
if not written:
if os.path.exists(tmpName):
os.remove(tmpName)
return written
if os.path.exists(path):
gLogger.debug("Replacing %s" % path)
try:
shutil.move(tmpName, path)
return True
except Exception as x:
gLogger.error("Failed to overwrite repository.", x)
gLogger.info(
"If your repository is corrupted a backup can be found %s" %
tmpName)
return False
def appendToRepository(self, repoLocation):
if not os.path.exists(repoLocation):
gLogger.error("Secondary repository does not exist", repoLocation)
return S_ERROR("Secondary repository does not exist")
self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo)
self._writeRepository(self.location)
return S_OK()
def addJob(self,
jobID,
state='Submitted',
retrieved=0,
outputData=0,
update=False):
paramDict = {
'State': state,
'Time': self._getTime(),
'Retrieved': int(retrieved),
'OutputData': outputData
}
self._writeJob(jobID, paramDict, update)
self._writeRepository(self.location)
return S_OK(jobID)
def updateJob(self, jobID, paramDict):
if self._existsJob(jobID):
paramDict['Time'] = self._getTime()
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def updateJobs(self, jobDict):
for jobID, paramDict in jobDict.items():
if self._existsJob(jobID):
paramDict['Time'] = self._getTime()
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def _getTime(self):
runtime = time.ctime()
return runtime.replace(" ", "_")
def _writeJob(self, jobID, paramDict, update):
jobID = str(jobID)
jobExists = self._existsJob(jobID)
if jobExists and (not update):
gLogger.warn("Job exists and not overwriting")
return S_ERROR("Job exists and not overwriting")
if not jobExists:
self.repo.createNewSection('Jobs/%s' % jobID)
for key, value in paramDict.items():
self.repo.setOption('Jobs/%s/%s' % (jobID, key), value)
return S_OK()
def removeJob(self, jobID):
res = self.repo['Jobs'].deleteKey(str(jobID)) #pylint: disable=no-member
if res:
self._writeRepository(self.location)
return S_OK()
def existsJob(self, jobID):
return S_OK(self._existsJob(jobID))
def _existsJob(self, jobID):
return self.repo.isSection('Jobs/%s' % jobID)
def getLocation(self):
return S_OK(self.location)
def getSize(self):
return S_OK(len(self.repo.getAsDict('Jobs')))
def __editItem(self, params):
ret = self.__deleteItem(params)
if ret["success"] == 1:
ret = self.__addItem(params)
ret["op"] = "editItem"
return ret
ret["op"] = "editItem"
return ret
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip()
sectionPath = sectionPath + "/" + params["name"]
# deleting the options underneath
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
print "deleting " + opt + "\n"
self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"success":1, "op": "editItem"}
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result["OK"]:
raise Exception(result["Message"])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result["OK"]:
self.__manifest = CFG()
return result
self.__manifest = result["Value"][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def dumpAsCFG(self):
return str(self.__manifest)
def getAsCFG(self):
return self.__manifest.clone()
def dumpAsJDL(self):
return dumpCFGAsJDL(self.__manifest)
def __getCSValue(self, varName, defaultVal=None):
if not self.__ops:
self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"])
if varName[0] != "/":
varName = "JobDescription/%s" % varName
return self.__ops.getValue(varName, defaultVal)
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
"""
Check a numerical var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
try:
varValue = long(varValue)
except:
return S_ERROR("%s must be a number" % varName)
minVal = self.__getCSValue("Min%s" % varName, minVal)
maxVal = self.__getCSValue("Max%s" % varName, maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVar(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
if varValue not in self.__getCSValue("Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoice(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__manifest:
return S_OK()
else:
varValue = self.__manifest[varName]
initialVal = varValue
choices = self.__getCSValue("Choices%s" % varName, choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
"Number of Input Data Files (%s) greater than current limit: %s"
% (len(List.fromChar(varValue)), maxNumber)
)
return S_OK()
def __contains__(self, key):
""" Check if the manifest has the required key
"""
return key in self.__manifest
def setOptionsFromDict(self, varDict):
for k in sorted(varDict):
self.setOption(k, varDict[k])
def check(self):
"""
Check that the manifest is OK
"""
for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result["OK"]:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result["OK"]:
return result
allowedSubmitPools = getSubmitPools(self.__manifest["OwnerGroup"])
result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools)))
if not result["OK"]:
return result
result = self.__checkMultiChoice("PilotTypes", ["private"])
if not result["OK"]:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
result = self.__checkMaxInputData(maxInputData)
if not result["OK"]:
return result
transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
result = self.__checkMultiChoice("JobType", ["User", "Test", "Hospital"] + transformationTypes)
if not result["OK"]:
return result
return S_OK()
def createSection(self, secName, contents=False):
if secName not in self.__manifest:
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
return S_OK(self.__manifest.createNewSection(secName, contents=contents))
return S_ERROR("Section %s already exists" % secName)
def getSection(self, secName):
self.__dirty = True
sec = self.__manifest[secName]
if not sec:
return S_ERROR("%s does not exist")
return S_OK(sec)
def setSectionContents(self, secName, contents):
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
if secName in self.__manifest:
self.__manifest[secName].reset()
self.__manifest[secName].mergeWith(contents)
else:
self.__manifest.createNewSection(secName, contents=contents)
def setOption(self, varName, varValue):
"""
Set a var in job manifest
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def remove(self, opName):
levels = List.fromChar(opName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
return S_ERROR("%s does not exist" % opName)
cfg = cfg[l]
if cfg.deleteKey(levels[-1]):
self.__dirty = True
return S_OK()
return S_ERROR("%s does not exist" % opName)
def getOption(self, varName, defaultValue=None):
"""
Get a variable from the job manifest
"""
cfg = self.__manifest
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
"""
Get a list of variables in a section of the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listOptions()
def isOption(self, opName):
"""
Check if it is a valid option
"""
return self.__manifest.isOption(opName)
def getSectionList(self, section=""):
"""
Get a list of sections in the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listSections()
def toCFG(self):
""" Get the full description of the file in CFG format
"""
oCFG = CFG()
strippedLFN = self.lfn.replace('/', '&&')
oCFG.createNewSection(strippedLFN)
oCFG.setOption('%s/Status' % (strippedLFN), self.status)
oCFG.setOption('%s/Size' % (strippedLFN), self.size)
oCFG.setOption('%s/GUID' % (strippedLFN), self.guid)
oCFG.setOption('%s/Checksum' % (strippedLFN), self.checksum)
#TODO: still have to include the CFG from the replica objects
if self.catalogReplicas:
oCFG.createNewSection('%s/CatalogReplicas' % strippedLFN)
for replica in self.catalogReplicas:
pass
# rCFG.mergeWith(CFG().loadFromBuffer(replica.toCFG()['Value']))
return S_OK(str(oCFG))
def _loadWebAppCFGFiles(self, extension):
"""
Load WebApp/web.cfg definitions
:param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC
"""
exts = [extension, "WebAppDIRAC"]
webCFG = CFG()
for modName in reversed(exts):
cfgPath = os.path.join(self.__params.destination,
"%s/WebApp" % modName, "web.cfg")
if not os.path.isfile(cfgPath):
gLogger.verbose("Web configuration file %s does not exists!" %
cfgPath)
continue
try:
modCFG = CFG().loadFromFile(cfgPath)
except Exception, excp:
gLogger.error("Could not load %s: %s" % (cfgPath, excp))
continue
gLogger.verbose("Loaded %s" % cfgPath)
expl = ["/WebApp"]
while len(expl):
current = expl.pop(0)
if not modCFG.isSection(current):
continue
if modCFG.getOption("%s/AbsoluteDefinition" % current, False):
gLogger.verbose("%s:%s is an absolute definition" %
(modName, current))
try:
webCFG.deleteKey(current)
except:
pass
modCFG.deleteKey("%s/AbsoluteDefinition" % current)
else:
for sec in modCFG[current].listSections():
expl.append("%s/%s" % (current, sec))
# Add the modCFG
webCFG = webCFG.mergeWith(modCFG)
class ConfigurationData:
def __init__( self, loadDefaultCFG = True ):
lr = LockRing()
self.threadingEvent = lr.getEvent()
self.threadingEvent.set()
self.threadingLock = lr.getLock()
self.runningThreadsNumber = 0
self.compressedConfigurationData = ""
self.configurationPath = "/DIRAC/Configuration"
self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" )
self._isService = False
self.localCFG = CFG()
self.remoteCFG = CFG()
self.mergedCFG = CFG()
self.remoteServerList = []
if loadDefaultCFG:
defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" )
gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile )
retVal = self.loadFile( defaultCFGFile )
if not retVal[ 'OK' ]:
gLogger.warn( "Can't load %s file" % defaultCFGFile )
self.sync()
def getBackupDir( self ):
return self.backupsDir
def sync( self ):
gLogger.debug( "Updating configuration internals" )
self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG )
self.remoteServerList = []
localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.localCFG,
disableDangerZones = True )
if localServers:
self.remoteServerList.extend( List.fromChar( localServers, "," ) )
remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.remoteCFG,
disableDangerZones = True )
if remoteServers:
self.remoteServerList.extend( List.fromChar( remoteServers, "," ) )
self.remoteServerList = List.uniqueElements( self.remoteServerList )
self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 )
def loadFile( self, fileName ):
try:
fileCFG = CFG()
fileCFG.loadFromFile( fileName )
except IOError:
self.localCFG = self.localCFG.mergeWith( fileCFG )
return S_ERROR( "Can't load a cfg file '%s'" % fileName )
return self.mergeWithLocal( fileCFG )
def mergeWithLocal( self, extraCFG ):
self.lock()
try:
self.localCFG = self.localCFG.mergeWith( extraCFG )
self.unlock()
gLogger.debug( "CFG merged" )
except Exception, e:
self.unlock()
return S_ERROR( "Cannot merge with new cfg: %s" % str( e ) )
self.sync()
return S_OK()
def setUpClass(cls):
cls.failed = False
# Add configuration
cfg = CFG()
cfg.loadFromBuffer(diracTestCACFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(userCFG)
gConfig.loadCFG(cfg)
# Prepare CA
lines = []
cfgDict = {}
cls.caPath = os.path.join(certsPath, 'ca')
cls.caConfigFile = os.path.join(cls.caPath, 'openssl_config_ca.cnf')
# Save original configuration file
shutil.copyfile(cls.caConfigFile, cls.caConfigFile + 'bak')
# Parse
fields = [
'dir', 'database', 'serial', 'new_certs_dir', 'private_key',
'certificate'
]
with open(cls.caConfigFile, "r") as caCFG:
for line in caCFG:
if re.findall('=', re.sub(r'#.*', '', line)):
field = re.sub(r'#.*', '',
line).replace(' ',
'').rstrip().split('=')[0]
line = 'dir = %s #PUT THE RIGHT DIR HERE!\n' % (
cls.caPath) if field == 'dir' else line
val = re.sub(r'#.*', '',
line).replace(' ', '').rstrip().split('=')[1]
if field in fields:
for i in fields:
if cfgDict.get(i):
val = val.replace('$%s' % i, cfgDict[i])
cfgDict[field] = val
if not cfgDict[field]:
cls.failed = '%s have empty value in %s' % (
field, cls.caConfigFile)
lines.append(line)
with open(cls.caConfigFile, "w") as caCFG:
caCFG.writelines(lines)
for field in fields:
if field not in cfgDict.keys():
cls.failed = '%s value is absent in %s' % (field,
cls.caConfigFile)
cls.hostCert = os.path.join(certsPath, 'host/hostcert.pem')
cls.hostKey = os.path.join(certsPath, 'host/hostkey.pem')
cls.caCert = cfgDict['certificate']
cls.caKey = cfgDict['private_key']
os.chmod(cls.caKey, stat.S_IREAD)
# Check directory for new certificates
cls.newCertDir = cfgDict['new_certs_dir']
if not os.path.exists(cls.newCertDir):
os.makedirs(cls.newCertDir)
for f in os.listdir(cls.newCertDir):
os.remove(os.path.join(cls.newCertDir, f))
# Empty the certificate database
cls.index = cfgDict['database']
with open(cls.index, 'w') as indx:
indx.write('')
# Write down serial
cls.serial = cfgDict['serial']
with open(cls.serial, 'w') as serialFile:
serialFile.write('1000')
# Create temporaly directory for users certificates
cls.userDir = tempfile.mkdtemp(dir=certsPath)
# Create user certificates
for userName in ['no_user', 'user', 'user_1', 'user_2', 'user_3']:
userConf = """[ req ]
default_bits = 4096
encrypt_key = yes
distinguished_name = req_dn
prompt = no
req_extensions = v3_req
[ req_dn ]
C = CC
O = DN
0.O = DIRAC
CN = %s
[ v3_req ]
# Extensions for client certificates (`man x509v3_config`).
nsComment = "OpenSSL Generated Client Certificate"
keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth
""" % (userName)
userConfFile = os.path.join(cls.userDir, userName + '.cnf')
userReqFile = os.path.join(cls.userDir, userName + '.req')
userKeyFile = os.path.join(cls.userDir, userName + '.key.pem')
userCertFile = os.path.join(cls.userDir, userName + '.cert.pem')
with open(userConfFile, "w") as f:
f.write(userConf)
status, output = commands.getstatusoutput(
'openssl genrsa -out %s' % userKeyFile)
if status:
gLogger.error(output)
exit()
gLogger.debug(output)
os.chmod(userKeyFile, stat.S_IREAD)
status, output = commands.getstatusoutput(
'openssl req -config %s -key %s -new -out %s' %
(userConfFile, userKeyFile, userReqFile))
if status:
gLogger.error(output)
exit()
gLogger.debug(output)
cmd = 'openssl ca -config %s -extensions usr_cert -batch -days 375 -in %s -out %s'
cmd = cmd % (cls.caConfigFile, userReqFile, userCertFile)
status, output = commands.getstatusoutput(cmd)
if status:
gLogger.error(output)
exit()
gLogger.debug(output)
# Result
status, output = commands.getstatusoutput('ls -al %s' % cls.userDir)
if status:
gLogger.error(output)
exit()
gLogger.debug('User certificates:\n', output)
def __editItem(self, params):
ret = self.__deleteItem(params)
if ret["success"] == 1:
ret = self.__addItem(params)
ret["op"] = "editItem"
return ret
ret["op"] = "editItem"
return ret
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip()
sectionPath = sectionPath + "/" + params["name"]
# deleting the options underneath
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
print "deleting " + opt + "\n"
self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"success":1, "op": "editItem"}
def __addItem(self, params):
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip()
elif params["type"] == "voms":
sectionPath = sectionPath + "VOMS/Servers"
elif params["type"] == "servers":
sectionPath = sectionPath + "VOMS/Servers/" + params["vom"]
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["port"].strip() != "":
configText = configText + "Port = " + params["port"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip()
sectionPath = sectionPath + "/" + params["name"]
if self.__configData[ 'cfgData' ].createSection(sectionPath):
cfgData = self.__configData[ 'cfgData' ].getCFG()
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"success":1, "op": "addItem"}
else:
return {"success":0, "op":"addItem", "message":"Section can't be created. It already exists?"}
# $HeadURL$
__RCSID__ = "7b8878b (2009-11-05 19:40:01 +0000) Adria Casajus <*****@*****.**>"
from dirac import DIRAC
from DIRAC.Core.Utilities.CFG import CFG
DIRAC.gLogger.initialize('test_gConfig','/testSectionDebug')
testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath
dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath
cfg1 = CFG()
cfg1.loadFromFile( testconfig )
fd = file( testconfig )
cfg1String = fd.read()
fd.close()
cfg2 = CFG()
cfg2.loadFromBuffer( cfg1.serialize() )
cfg3 = cfg1.mergeWith( cfg2 )
testList = [{ 'method' : DIRAC.gConfig.loadFile,
'arguments' : ( testconfig, ),
'output' : {'OK': True, 'Value': ''}
},
{ 'method' : DIRAC.gConfig.dumpLocalCFGToFile,
'arguments' : ( dumpconfig, ),
'output' : {'OK': True, 'Value': ''}
},
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result['OK']:
raise Exception(result['Message'])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__manifest = CFG()
return result
self.__manifest = result['Value'][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def dumpAsCFG(self):
return str(self.__manifest)
def getAsCFG(self):
return self.__manifest.clone()
def dumpAsJDL(self):
return dumpCFGAsJDL(self.__manifest)
def __getCSValue(self, varName, defaultVal=None):
if not self.__ops:
self.__ops = Operations(group=self.__manifest['OwnerGroup'], setup=self.__manifest['DIRACSetup'])
if varName[0] != "/":
varName = "JobDescription/%s" % varName
return self.__ops.getValue(varName, defaultVal)
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
"""
Check a numerical var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
try:
varValue = long(varValue)
except BaseException:
return S_ERROR("%s must be a number" % varName)
minVal = self.__getCSValue("Min%s" % varName, minVal)
maxVal = self.__getCSValue("Max%s" % varName, maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVar(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
if varValue not in self.__getCSValue("Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoice(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__manifest:
return S_OK()
else:
varValue = self.__manifest[varName]
initialVal = varValue
choices = self.__getCSValue("Choices%s" % varName, choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR('Number of Input Data Files (%s) greater than current limit: %s' %
(len(List.fromChar(varValue)), maxNumber))
return S_OK()
def __contains__(self, key):
""" Check if the manifest has the required key
"""
return key in self.__manifest
def setOptionsFromDict(self, varDict):
for k in sorted(varDict):
self.setOption(k, varDict[k])
def check(self):
"""
Check that the manifest is OK
"""
for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result['OK']:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result['OK']:
return result
allowedSubmitPools = getSubmitPools(self.__manifest['OwnerGroup'])
result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoice("PilotTypes", ['private'])
if not result['OK']:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
result = self.__checkMaxInputData(maxInputData)
if not result['OK']:
return result
transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
result = self.__checkMultiChoice("JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
if not result['OK']:
return result
return S_OK()
def createSection(self, secName, contents=False):
if secName not in self.__manifest:
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
return S_OK(self.__manifest.createNewSection(secName, contents=contents))
return S_ERROR("Section %s already exists" % secName)
def getSection(self, secName):
self.__dirty = True
if secName not in self.__manifest:
return S_ERROR("%s does not exist" % secName)
sec = self.__manifest[secName]
if not sec:
return S_ERROR("%s section empty" % secName)
return S_OK(sec)
def setSectionContents(self, secName, contents):
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
if secName in self.__manifest:
self.__manifest[secName].reset()
self.__manifest[secName].mergeWith(contents)
else:
self.__manifest.createNewSection(secName, contents=contents)
def setOption(self, varName, varValue):
"""
Set a var in job manifest
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def remove(self, opName):
levels = List.fromChar(opName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
return S_ERROR("%s does not exist" % opName)
cfg = cfg[l]
if cfg.deleteKey(levels[-1]):
self.__dirty = True
return S_OK()
return S_ERROR("%s does not exist" % opName)
def getOption(self, varName, defaultValue=None):
"""
Get a variable from the job manifest
"""
cfg = self.__manifest
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
"""
Get a list of variables in a section of the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listOptions()
def isOption(self, opName):
"""
Check if it is a valid option
"""
return self.__manifest.isOption(opName)
def getSectionList(self, section=""):
"""
Get a list of sections in the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listSections()
def execute(self):
"""The JobAgent execution method.
"""
if self.jobCount:
# Temporary mechanism to pass a shutdown message to the agent
if os.path.exists('/var/lib/dirac_drain'):
return self.__finish('Node is being drained by an operator')
# Only call timeLeft utility after a job has been picked up
self.log.info('Attempting to check CPU time left for filling mode')
if self.fillingMode:
if self.timeLeftError:
self.log.warn(self.timeLeftError)
return self.__finish(self.timeLeftError)
self.log.info('%s normalized CPU units remaining in slot' %
(self.timeLeft))
if self.timeLeft <= self.minimumTimeLeft:
return self.__finish('No more time left')
# Need to update the Configuration so that the new value is published in the next matching request
result = self.computingElement.setCPUTimeLeft(
cpuTimeLeft=self.timeLeft)
if not result['OK']:
return self.__finish(result['Message'])
# Update local configuration to be used by submitted job wrappers
localCfg = CFG()
if self.extraOptions:
localConfigFile = os.path.join('.', self.extraOptions)
else:
localConfigFile = os.path.join(rootPath, "etc",
"dirac.cfg")
localCfg.loadFromFile(localConfigFile)
if not localCfg.isSection('/LocalSite'):
localCfg.createNewSection('/LocalSite')
localCfg.setOption('/LocalSite/CPUTimeLeft', self.timeLeft)
localCfg.writeToFile(localConfigFile)
else:
return self.__finish('Filling Mode is Disabled')
self.log.verbose('Job Agent execution loop')
result = self.computingElement.available()
if not result['OK']:
self.log.info('Resource is not available')
self.log.info(result['Message'])
return self.__finish('CE Not Available')
self.log.info(result['Message'])
ceInfoDict = result['CEInfoDict']
runningJobs = ceInfoDict.get("RunningJobs")
availableSlots = result['Value']
if not availableSlots:
if runningJobs:
self.log.info('No available slots with %d running jobs' %
runningJobs)
return S_OK('Job Agent cycle complete with %d running jobs' %
runningJobs)
else:
self.log.info('CE is not available')
return self.__finish('CE Not Available')
result = self.computingElement.getDescription()
if not result['OK']:
return result
# We can have several prioritized job retrieval strategies
if isinstance(result['Value'], dict):
ceDictList = [result['Value']]
elif isinstance(result['Value'], list):
ceDictList = result['Value']
for ceDict in ceDictList:
# Add pilot information
gridCE = gConfig.getValue('LocalSite/GridCE', 'Unknown')
if gridCE != 'Unknown':
ceDict['GridCE'] = gridCE
if 'PilotReference' not in ceDict:
ceDict['PilotReference'] = str(self.pilotReference)
ceDict['PilotBenchmark'] = self.cpuFactor
ceDict['PilotInfoReportedFlag'] = self.pilotInfoReportedFlag
# Add possible job requirements
result = gConfig.getOptionsDict('/AgentJobRequirements')
if result['OK']:
requirementsDict = result['Value']
ceDict.update(requirementsDict)
self.log.info('Requirements:', requirementsDict)
self.log.verbose(ceDict)
start = time.time()
jobRequest = MatcherClient().requestJob(ceDict)
matchTime = time.time() - start
self.log.info('MatcherTime = %.2f (s)' % (matchTime))
if jobRequest['OK']:
break
self.stopAfterFailedMatches = self.am_getOption(
'StopAfterFailedMatches', self.stopAfterFailedMatches)
if not jobRequest['OK']:
if re.search('No match found', jobRequest['Message']):
self.log.notice('Job request OK: %s' % (jobRequest['Message']))
self.matchFailedCount += 1
if self.matchFailedCount > self.stopAfterFailedMatches:
return self.__finish(
'Nothing to do for more than %d cycles' %
self.stopAfterFailedMatches)
return S_OK(jobRequest['Message'])
elif jobRequest['Message'].find("seconds timeout") != -1:
self.log.error('Timeout while requesting job',
jobRequest['Message'])
self.matchFailedCount += 1
if self.matchFailedCount > self.stopAfterFailedMatches:
return self.__finish(
'Nothing to do for more than %d cycles' %
self.stopAfterFailedMatches)
return S_OK(jobRequest['Message'])
elif jobRequest['Message'].find(
"Pilot version does not match") != -1:
errorMsg = 'Pilot version does not match the production version'
self.log.error(errorMsg,
jobRequest['Message'].replace(errorMsg, ''))
return S_ERROR(jobRequest['Message'])
else:
self.log.notice('Failed to get jobs: %s' %
(jobRequest['Message']))
self.matchFailedCount += 1
if self.matchFailedCount > self.stopAfterFailedMatches:
return self.__finish(
'Nothing to do for more than %d cycles' %
self.stopAfterFailedMatches)
return S_OK(jobRequest['Message'])
# Reset the Counter
self.matchFailedCount = 0
matcherInfo = jobRequest['Value']
if not self.pilotInfoReportedFlag:
# Check the flag after the first access to the Matcher
self.pilotInfoReportedFlag = matcherInfo.get(
'PilotInfoReportedFlag', False)
jobID = matcherInfo['JobID']
matcherParams = ['JDL', 'DN', 'Group']
for param in matcherParams:
if param not in matcherInfo:
self.__report(jobID, 'Failed',
'Matcher did not return %s' % (param))
return self.__finish('Matcher Failed')
elif not matcherInfo[param]:
self.__report(jobID, 'Failed',
'Matcher returned null %s' % (param))
return self.__finish('Matcher Failed')
else:
self.log.verbose('Matcher returned %s = %s ' %
(param, matcherInfo[param]))
jobJDL = matcherInfo['JDL']
jobGroup = matcherInfo['Group']
ownerDN = matcherInfo['DN']
optimizerParams = {}
for key in matcherInfo:
if key not in matcherParams:
optimizerParams[key] = matcherInfo[key]
parameters = self.__getJDLParameters(jobJDL)
if not parameters['OK']:
self.__report(jobID, 'Failed', 'Could Not Extract JDL Parameters')
self.log.warn(parameters['Message'])
return self.__finish('JDL Problem')
params = parameters['Value']
if 'JobID' not in params:
msg = 'Job has not JobID defined in JDL parameters'
self.__report(jobID, 'Failed', msg)
self.log.warn(msg)
return self.__finish('JDL Problem')
else:
jobID = params['JobID']
if 'JobType' not in params:
self.log.warn('Job has no JobType defined in JDL parameters')
jobType = 'Unknown'
else:
jobType = params['JobType']
if 'CPUTime' not in params:
self.log.warn(
'Job has no CPU requirement defined in JDL parameters')
# Job requirement for a number of processors
processors = int(params.get('NumberOfProcessors', 1))
wholeNode = 'WholeNode' in params
if self.extraOptions:
params['Arguments'] += ' ' + self.extraOptions
params['ExtraOptions'] = self.extraOptions
self.log.verbose('Job request successful: \n', jobRequest['Value'])
self.log.info('Received JobID=%s, JobType=%s' % (jobID, jobType))
self.log.info('OwnerDN: %s JobGroup: %s' % (ownerDN, jobGroup))
self.jobCount += 1
try:
jobReport = JobReport(jobID, 'JobAgent@%s' % self.siteName)
jobReport.setJobParameter('MatcherServiceTime',
str(matchTime),
sendFlag=False)
if 'BOINC_JOB_ID' in os.environ:
# Report BOINC environment
for thisp in ('BoincUserID', 'BoincHostID',
'BoincHostPlatform', 'BoincHostName'):
jobReport.setJobParameter(thisp,
gConfig.getValue(
'/LocalSite/%s' % thisp,
'Unknown'),
sendFlag=False)
jobReport.setJobStatus('Matched', 'Job Received by Agent')
result = self.__setupProxy(ownerDN, jobGroup)
if not result['OK']:
return self.__rescheduleFailedJob(
jobID, result['Message'], self.stopOnApplicationFailure)
proxyChain = result.get('Value')
# Save the job jdl for external monitoring
self.__saveJobJDLRequest(jobID, jobJDL)
software = self.__checkInstallSoftware(jobID, params, ceDict)
if not software['OK']:
self.log.error('Failed to install software for job',
'%s' % (jobID))
errorMsg = software['Message']
if not errorMsg:
errorMsg = 'Failed software installation'
return self.__rescheduleFailedJob(
jobID, errorMsg, self.stopOnApplicationFailure)
self.log.debug('Before %sCE submitJob()' % (self.ceName))
result = self.__submitJob(jobID, params, ceDict, optimizerParams,
proxyChain, processors, wholeNode)
if not result['OK']:
self.__report(jobID, 'Failed', result['Message'])
return self.__finish(result['Message'])
elif 'PayloadFailed' in result:
# Do not keep running and do not overwrite the Payload error
message = 'Payload execution failed with error code %s' % result[
'PayloadFailed']
if self.stopOnApplicationFailure:
return self.__finish(message,
self.stopOnApplicationFailure)
else:
self.log.info(message)
self.log.debug('After %sCE submitJob()' % (self.ceName))
except Exception as subExcept: # pylint: disable=broad-except
self.log.exception("Exception in submission",
"",
lException=subExcept,
lExcInfo=True)
return self.__rescheduleFailedJob(
jobID, 'Job processing failed with exception',
self.stopOnApplicationFailure)
# Sum all times but the last one (elapsed_time) and remove times at init (is this correct?)
cpuTime = sum(os.times()[:-1]) - sum(self.initTimes[:-1])
result = self.timeLeftUtil.getTimeLeft(cpuTime, processors)
if result['OK']:
self.timeLeft = result['Value']
else:
if result['Message'] != 'Current batch system is not supported':
self.timeLeftError = result['Message']
else:
# if the batch system is not defined, use the process time and the CPU normalization defined locally
self.timeLeft = self.__getCPUTimeLeft()
return S_OK('Job Agent cycle complete')
class JobDescription(object):
def __init__(self):
self.__description = CFG()
self.__dirty = False
def isDirty(self):
return self.__dirty
def loadDescription(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL(dataString)
else:
return self.loadDescriptionFromCFG(dataString)
def loadDescriptionFromJDL(self, jdlString):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__description = CFG()
return result
self.__description = result['Value'][0]
return S_OK()
def loadDescriptionFromCFG(self, cfgString):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load description from cfg: %s" % str(e))
return S_OK()
def dumpDescriptionAsCFG(self):
return str(self.__description)
def dumpDescriptionAsJDL(self):
return dumpCFGAsJDL(self.__description)
def __checkNumericalVarInDescription(self, varName, defaultVal, minVal,
maxVal):
"""
Check a numerical var
"""
initialVal = 0
if varName not in self.__description:
varValue = Operations().getValue(
"JobDescription/Default%s" % varName, defaultVal)
else:
varValue = self.__description[varName]
initialVal = varValue
try:
varValue = long(varValue)
except BaseException:
return S_ERROR("%s must be a number" % varName)
minVal = Operations().getValue("JobDescription/Min%s" % varName,
minVal)
maxVal = Operations().getValue("JobDescription/Max%s" % varName,
maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVarInDescription(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__description:
varValue = Operations().getValue(
"JobDescription/Default%s" % varName, defaultVal)
else:
varValue = self.__description[varName]
initialVal = varValue
if varValue not in Operations().getValue(
"JobDescription/Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" %
(varValue, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoiceInDescription(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__description:
return S_OK()
else:
varValue = self.__description[varName]
initialVal = varValue
choices = Operations().getValue("JobDescription/Choices%s" % varName,
choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__description:
return S_OK()
varValue = self.__description[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
'Number of Input Data Files (%s) greater than current limit: %s'
% (len(List.fromChar(varValue)), maxNumber))
return S_OK()
def setVarsFromDict(self, varDict):
for k in sorted(varDict):
self.setVar(k, varDict[k])
def checkDescription(self):
"""
Check that the description is OK
"""
for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
if k not in self.__description:
return S_ERROR("Missing var %s in description" % k)
# Check CPUTime
result = self.__checkNumericalVarInDescription("CPUTime", 86400, 0,
500000)
if not result['OK']:
return result
result = self.__checkNumericalVarInDescription("Priority", 1, 0, 10)
if not result['OK']:
return result
allowedSubmitPools = getSubmitPools(self.__description['OwnerGroup'])
result = self.__checkMultiChoiceInDescription(
"SubmitPools", list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoiceInDescription(
"SubmitPools", list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoiceInDescription("PilotTypes",
['private'])
if not result['OK']:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData",
500)
result = self.__checkMaxInputData(maxInputData)
if not result['OK']:
return result
transformationTypes = Operations().getValue(
"Transformations/DataProcessing", [])
result = self.__checkMultiChoiceInDescription(
"JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
return S_OK()
def setVar(self, varName, varValue):
"""
Set a var in job description
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__description
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def getVar(self, varName, defaultValue=None):
cfg = self.__description
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
cfg = self.__description.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listOptions()
def getSectionList(self, section=""):
cfg = self.__description.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listSections()
def __init__(self):
self.__description = CFG()
self.__dirty = False
def __addItem(self, params):
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip()
elif params["type"] == "voms":
sectionPath = sectionPath + "VOMS/Servers"
elif params["type"] == "servers":
sectionPath = sectionPath + "VOMS/Servers/" + params["vom"]
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["port"].strip() != "":
configText = configText + "Port = " + params["port"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip()
sectionPath = sectionPath + "/" + params["name"]
if self.__configData[ 'cfgData' ].createSection(sectionPath):
cfgData = self.__configData[ 'cfgData' ].getCFG()
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"success":1, "op": "addItem"}
else:
return {"success":0, "op":"addItem", "message":"Section can't be created. It already exists?"}
