def loadWebAppCFGFiles():
"""
Load WebApp/web.cfg definitions
"""
exts = []
for ext in CSGlobals.getCSExtensions():
if ext == "DIRAC":
continue
if ext[-5:] != "DIRAC":
ext = "%sDIRAC" % ext
if ext != "WebAppDIRAC":
exts.append(ext)
exts.append("DIRAC")
exts.append("WebAppDIRAC")
webCFG = CFG()
for modName in reversed(exts):
try:
modPath = imp.find_module(modName)[1]
except ImportError:
continue
gLogger.verbose("Found module %s at %s" % (modName, modPath))
cfgPath = os.path.join(modPath, "WebApp", "web.cfg")
if not os.path.isfile(cfgPath):
gLogger.verbose("Inexistant %s" % cfgPath)
continue
try:
modCFG = CFG().loadFromFile(cfgPath)
except Exception, excp:
gLogger.error("Could not load %s: %s" % (cfgPath, excp))
continue
gLogger.verbose("Loaded %s" % cfgPath)
expl = [BASECS]
while len(expl):
current = expl.pop(0)
if not modCFG.isSection(current):
continue
if modCFG.getOption("%s/AbsoluteDefinition" % current, False):
gLogger.verbose("%s:%s is an absolute definition" %
(modName, current))
try:
webCFG.deleteKey(current)
except:
pass
modCFG.deleteKey("%s/AbsoluteDefinition" % current)
else:
for sec in modCFG[current].listSections():
expl.append("%s/%s" % (current, sec))
#Add the modCFG
webCFG = webCFG.mergeWith(modCFG)
def __loadRemoteConfig(self):
if not 'cfgData' in session or not 'csName' in session:
if 'csFilename' in session:
del (session['csFilename'])
gLogger.info("Loading configuration from server...")
retVal = self.__getRemoteConfiguration()
if not retVal['OK']:
c.error = "Can't get configuration from server<br/>%s" % retVal[
'Message']
return S_ERROR()
else:
try:
gLogger.info("Recovering configuration")
c.cfgData = CFG()
c.cfgData.loadFromBuffer(session['cfgData'])
c.csName = session['csName']
except Exception, e:
c.error = "There was an error with your modifications. %s" % str(
e)
c.link = ('resetConfigurationToRemote',
"Click here to reset your configuration")
gLogger.error(
"There was an error with modified configuration %s" %
str(e))
S_ERROR()
def setUp(self):
self.authMgr = AuthManager('/Systems/Service/Authorization')
cfg = CFG()
cfg.loadFromBuffer(testSystemsCFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(testRegistryCFG)
gConfig.loadCFG(cfg)
self.noAuthCredDict = {'group': 'group_test'}
self.userCredDict = {
'DN': '/User/test/DN/CN=userA',
'group': 'group_test'
}
self.suspendedOtherVOUserCredDict = {
'DN': '/User/test/DN/CN=userS',
'group': 'group_test_other'
}
self.badUserCredDict = {
'DN': '/User/test/DN/CN=userB',
'group': 'group_bad'
}
self.suspendedUserCredDict = {
'DN': '/User/test/DN/CN=userS',
'group': 'group_test'
}
self.hostCredDict = {
'DN': '/User/test/DN/CN=test.hostA.ch',
'group': 'hosts'
}
self.badHostCredDict = {
'DN': '/User/test/DN/CN=test.hostB.ch',
'group': 'hosts'
}
def __init__( self, rpcClient = False, commiterId = "unknown" ):
self.commiterTag = "@@-"
self.commiterId = commiterId
self.cfgData = CFG()
self.rpcClient = None
if rpcClient:
self.setRPCClient( rpcClient )
def _getCurrentConfig(self):
"""Return the current system configuration."""
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfig.forceRefresh()
fullCfg = CFG()
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
for system, setup in serviceSetups.items():
if self.systems and system not in self.systems:
continue
systemCfg = gConfigurationData.remoteCFG.getAsCFG("/Systems/%s/%s" % (system, setup))
for section in systemCfg.listSections():
if section not in ('Agents', 'Services', 'Executors'):
systemCfg.deleteKey(section)
fullCfg.createNewSection("/%s" % system, contents=systemCfg)
return S_OK(fullCfg)
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
if manifest:
result = self.loadManifest(manifest)
if not result['OK']:
raise Exception(result['Message'])
def parseConfigTemplate(self, templatePath, cfg):
"""Parse the ConfigTemplate.cfg files.
:param str templatePath: path to the folder containing a ConfigTemplate.cfg file
:param CFG cfg: cfg to merge with the systems config
:returns: CFG object
"""
system = os.path.split(templatePath.rstrip('/'))[1]
if system.lower().endswith('system'):
system = system[:-len('System')]
templatePath = os.path.join(templatePath, 'ConfigTemplate.cfg')
if not os.path.exists(templatePath):
return S_ERROR('File not found: %s' % templatePath)
loadCfg = CFG()
try:
loadCfg.loadFromFile(templatePath)
except ValueError as err:
LOG.error('Failed loading file %r: %r', templatePath, err)
self.retVal = 1
return S_ERROR()
cfg.createNewSection('/Systems/%s' % system, contents=loadCfg)
return S_OK(cfg)
def appendToRepository(self, repoLocation):
if not os.path.exists(repoLocation):
gLogger.error("Secondary repository does not exist", repoLocation)
return S_ERROR("Secondary repository does not exist")
self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo)
self._writeRepository(self.location)
return S_OK()
def setUp(self):
cfg = CFG()
cfg.loadFromBuffer(diracTestCACFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(userCFG)
gConfig.loadCFG(cfg)
result = ProxyProviderFactory().getProxyProvider('DIRAC_TEST_CA')
self.assertTrue(result['OK'], '\n%s' % result.get('Message') or 'Error message is absent.')
self.pp = result['Value']
self.userDictClean = {
"FullName": "DIRAC test user",
"EMail": "*****@*****.**"
}
self.userDictCleanDN = {
"DN": "/C=FR/O=DIRAC/OU=DIRAC Consortium/CN=DIRAC test user/[email protected]",
"EMail": "*****@*****.**"
}
self.userDictGroup = {
"FullName": "DIRAC test user",
"EMail": "*****@*****.**",
"DiracGroup": "dirac_user"
}
self.userDictNoGroup = {
"FullName": "DIRAC test user",
"EMail": "*****@*****.**",
"DiracGroup": "dirac_no_user"
}
def checkAgentOptions(getOptionMock, systemName, agentName,
ignoreOptions=None, extension='DIRAC'):
"""Ensure that all the agent options are properly documented.
:param getOptionMock: Mock object for agentmodule.get_amOption function
:param str systemName: name of the **System**
:param str agentName: name of the **Agent**
:param list ignoreOptions: list of options to ignore
:param str extension: name of the DIRAC **Extension** where the Agent comes from
"""
if ignoreOptions is None:
ignoreOptions = []
# add some options that can be set, see the AgentModule for all of them
ignoreOptions.extend(['PollingTime', 'Status', 'Enabled', 'MaxCycles', 'LogOutputs', 'ControlDirectory'])
ignoreOptions = list(set(ignoreOptions))
config = CFG()
LOG.info("Testing %s/%s, ignoring options %s", systemName, agentName, ignoreOptions)
# get the location where DIRAC is in from basefolder/DIRAC/__ini__.py
configFilePath = os.path.join(os.path.dirname(os.path.dirname(DIRAC.__file__)),
extension, systemName, 'ConfigTemplate.cfg')
config.loadFromFile(configFilePath)
optionsDict = config.getAsDict('Agents/%s' % agentName)
outDict = {}
_parseOption(outDict, optionsDict)
optionsDict = outDict
LOG.info("Calls: %s", pformat(getOptionMock.call_args_list))
LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys()))
# check that values in ConfigTemplate are used
for option, value in optionsDict.iteritems():
if any(ignoreOp in option for ignoreOp in ignoreOptions):
LOG.info("From Agent: ignoring option %r with value %r, (%s)", option, value, type(value))
continue
LOG.info("Looking for call to option %r with value %r, (%s)", option, value, type(value))
if not isinstance(value, bool) and not value: # empty string, list, dict ...
assert any(call(option, null) in getOptionMock.call_args_list for null in ({}, set(), [], '', 0))
else:
assert call(option, value) in getOptionMock.call_args_list or \
call(option, [value]) in getOptionMock.call_args_list
# check that options used in the agent are in the ConfigTemplates
for opCall in getOptionMock.call_args_list:
optionArguments = opCall[0]
if len(optionArguments) != 2:
continue
optionName = optionArguments[0]
optionValue = optionArguments[1]
if optionName in ignoreOptions:
LOG.info("From Template: ignoring option %r with %r", optionName, optionValue)
continue
LOG.info("Checking Template option %r with %r", optionName, optionValue)
assert optionName in optionsDict
if not optionsDict[optionName]:
assert not optionValue
continue
assert optionsDict[optionName] == optionValue or [optionsDict[optionName]] == optionValue
def loadFile(self, fileName):
try:
fileCFG = CFG()
fileCFG.loadFromFile(fileName)
except IOError:
self.localCFG = self.localCFG.mergeWith(fileCFG)
return S_ERROR("Can't load a cfg file '%s'" % fileName)
return self.mergeWithLocal(fileCFG)
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def getComputingElementDefaults(ceName='',
ceType='',
cfg=None,
currentSectionPath=''):
"""
Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg
"""
cesCfg = CFG()
if cfg:
try:
cesCfg.loadFromFile(cfg)
cesPath = cfgInstallPath('ComputingElements')
if cesCfg.isSection(cesPath):
for section in cfgPathToList(cesPath):
cesCfg = cesCfg[section]
except:
return CFG()
# Overwrite the cfg with Command line arguments
if ceName:
if not cesCfg.isSection(ceName):
cesCfg.createNewSection(ceName)
if currentSectionPath:
# Add Options from Command Line
optionsDict = __getExtraOptions(currentSectionPath)
for name, value in optionsDict.items():
cesCfg[ceName].setOption(name, value) #pylint: disable=no-member
if ceType:
cesCfg[ceName].setOption('CEType', ceType) #pylint: disable=no-member
ceDefaultSection = cfgPath(defaultSection('ComputingElements'))
# Load Default for the given type from Central configuration is defined
ceDefaults = __gConfigDefaults(ceDefaultSection)
for ceName in cesCfg.listSections():
if 'CEType' in cesCfg[ceName]:
ceType = cesCfg[ceName]['CEType']
if ceType in ceDefaults:
for option in ceDefaults[ceType].listOptions():
if option not in cesCfg[ceName]:
cesCfg[ceName].setOption(option,
ceDefaults[ceType][option])
return cesCfg
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__manifest = CFG()
return result
self.__manifest = result['Value'][0]
return S_OK()
def __init__(self, location):
self.cfg = CFG()
self.location = location
self.goodProcessList = True
if os.path.exists(self.location):
self.cfg.loadFromFile(self.location)
if not self.cfg.existsKey('Processes'):
self.cfg.createNewSection('Processes')
else:
self.goodProcessList = False
def getSystemsCFG(self):
"""Find all the ConfigTemplates and collate them into one CFG object."""
cfg = CFG()
cfg.createNewSection('/Systems')
templateLocations = self.findConfigTemplates()
for templatePath in templateLocations:
cfgRes = self.parseConfigTemplate(templatePath, cfg)
if cfgRes['OK']:
cfg = cfgRes['Value']
return cfg
def loadDescriptionFromJDL(self, jdlString):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__description = CFG()
return result
self.__description = result['Value'][0]
return S_OK()
def __init__(self, loadDefaultCFG=True):
self.threadingEvent = threading.Event()
self.threadingEvent.set()
self.threadingLock = threading.Lock()
self.runningThreadsNumber = 0
self.compressedConfigurationData = ""
self.configurationPath = "/DIRAC/Configuration"
self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup")
self._isService = False
self.localCFG = CFG()
self.remoteCFG = CFG()
self.mergedCFG = CFG()
self.remoteServerList = []
if loadDefaultCFG:
defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg")
gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile)
retVal = self.loadFile(defaultCFGFile)
if not retVal['OK']:
gLogger.warn("Can't load %s file" % defaultCFGFile)
self.sync()
def updateCompleteDiracCFG(self):
"""Read the dirac.cfg and update the Systems sections from the ConfigTemplate.cfg files."""
compCfg = CFG()
mainDiracCfgPath = self.config.cfg_baseFile
if not os.path.exists(mainDiracCfgPath):
LOG.error('Failed to find Main Dirac cfg at %r', mainDiracCfgPath)
return 1
LOG.info('Extracting default configuration from %r', mainDiracCfgPath)
loadCFG = CFG()
loadCFG.loadFromFile(mainDiracCfgPath)
compCfg = loadCFG.mergeWith(compCfg)
cfg = self.getSystemsCFG()
compCfg = compCfg.mergeWith(cfg)
diracCfgOutput = self.config.cfg_targetFile
LOG.info('Writing output to %r', diracCfgOutput)
with open(diracCfgOutput, 'w') as rst:
rst.write(
textwrap.dedent("""
==========================
Full Configuration Example
==========================
.. This file is created by docs/Tools/UpdateDiracCFG.py
Below is a complete example configuration with anotations for some sections::
"""))
# indent the cfg text
cfgString = ''.join(' ' + line
for line in str(compCfg).splitlines(True))
# fix the links, add back the # for targets
# match .html with following character using positive look ahead
htmlMatch = re.compile(r'\.html(?=[a-zA-Z0-9])')
cfgString = re.sub(htmlMatch, '.html#', cfgString)
rst.write(cfgString)
return self.retVal
def _loadDefaultWebCFG(self):
""" This method reloads the web.cfg file from etc/web.cfg """
modCFG = None
cfgPath = os.path.join(DIRAC.rootPath, 'etc', 'web.cfg')
isLoaded = True
if not os.path.isfile(cfgPath):
isLoaded = False
else:
try:
modCFG = CFG().loadFromFile(cfgPath)
except Exception, excp:
isLoaded = False
gLogger.error("Could not load %s: %s" % (cfgPath, excp))
def setUp(self):
cfg = CFG()
cfg.loadFromBuffer(diracTestCACFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(userCFG)
gConfig.loadCFG(cfg)
result = ProxyProviderFactory().getProxyProvider('DIRAC_TEST_CA')
self.assertTrue(
result['OK'], '\n%s' % result.get('Message')
or 'Error message is absent.')
self.pp = result['Value']
def _loadWebAppCFGFiles(self, extension):
"""
Load WebApp/web.cfg definitions
:param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC
"""
exts = [extension, "WebAppDIRAC"]
webCFG = CFG()
for modName in reversed(exts):
cfgPath = os.path.join(self.__params.destination,
"%s/WebApp" % modName, "web.cfg")
if not os.path.isfile(cfgPath):
gLogger.verbose("Web configuration file %s does not exists!" %
cfgPath)
continue
try:
modCFG = CFG().loadFromFile(cfgPath)
except Exception, excp:
gLogger.error("Could not load %s: %s" % (cfgPath, excp))
continue
gLogger.verbose("Loaded %s" % cfgPath)
expl = ["/WebApp"]
while len(expl):
current = expl.pop(0)
if not modCFG.isSection(current):
continue
if modCFG.getOption("%s/AbsoluteDefinition" % current, False):
gLogger.verbose("%s:%s is an absolute definition" %
(modName, current))
try:
webCFG.deleteKey(current)
except:
pass
modCFG.deleteKey("%s/AbsoluteDefinition" % current)
else:
for sec in modCFG[current].listSections():
expl.append("%s/%s" % (current, sec))
# Add the modCFG
webCFG = webCFG.mergeWith(modCFG)
def toCFG(self):
""" Get the full description of the file in CFG format
"""
oCFG = CFG()
strippedLFN = self.lfn.replace('/', '&&')
oCFG.createNewSection(strippedLFN)
oCFG.setOption('%s/Status' % (strippedLFN), self.status)
oCFG.setOption('%s/Size' % (strippedLFN), self.size)
oCFG.setOption('%s/GUID' % (strippedLFN), self.guid)
oCFG.setOption('%s/Checksum' % (strippedLFN), self.checksum)
#TODO: still have to include the CFG from the replica objects
if self.catalogReplicas:
oCFG.createNewSection('%s/CatalogReplicas' % strippedLFN)
for replica in self.catalogReplicas:
pass
# rCFG.mergeWith(CFG().loadFromBuffer(replica.toCFG()['Value']))
return S_OK(str(oCFG))
def __init__(self, repository=None):
self.location = repository
if not self.location:
if "HOME" in os.environ:
self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
else:
self.location = '%s/.dirac.repo.rep' % os.getcwd()
self.repo = CFG()
if os.path.exists(self.location):
self.repo.loadFromFile(self.location)
if not self.repo.existsKey('Jobs'):
self.repo.createNewSection('Jobs')
else:
self.repo.createNewSection('Jobs')
self.OK = True
written = self._writeRepository(self.location)
if not written:
self.OK = False
def create_serverAndClient(request):
""" This function starts a server, and closes it after
The server will use the parametrized transport type
"""
# Reinitialize the configuration.
# We do it here rather than at the start of the module
# to accomodate for pytest when going through all the DIRAC tests
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
gConfigurationData.setOptionInCFG('/DIRAC/Security/CALocation', caLocation)
gConfigurationData.setOptionInCFG('/DIRAC/Security/CertFile',
hostCertLocation)
gConfigurationData.setOptionInCFG('/DIRAC/Security/KeyFile',
hostKeyLocation)
testStr = request.param
serverName, clientName = testStr.split("-")
serverClass = transportByName(serverName)
clientClass = transportByName(clientName)
sr = DummyServiceReactor(serverClass, PORT_NUMBER)
server_thread = threading.Thread(target=sr.serve)
sr.prepare()
server_thread.start()
# Create the client
clientOptions = {
'clientMode': True,
'proxyLocation': proxyFile,
}
clientTransport = clientClass(("localhost", PORT_NUMBER),
bServerMode=False,
**clientOptions)
res = clientTransport.initAsClient()
assert res['OK'], res
yield sr, clientTransport
clientTransport.close()
sr.closeListeningConnections()
server_thread.join()
# Clean the config
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
def showTextConfiguration(self):
response.headers['Content-type'] = 'text/plain'
if 'download' in request.params and request.params['download'] in (
'yes', 'true', '1'):
version = ""
try:
cfg = CFG()
cfg.loadFromBuffer(session['cfgData'])
cfg = cfg['DIRAC']['Configuration']
version = ".%s.%s" % (cfg['Name'], cfg['Version'].replace(
":", "").replace("-", ""))
except Exception, e:
print e
print 'attachment; filename="cs%s.cfg"' % version.replace(" ", "_")
response.headers[
'Content-Disposition'] = 'attachment; filename="cs%s.cfg"' % version.replace(
" ", "")
response.headers['Content-Length'] = len(session['cfgData'])
response.headers['Content-Transfer-Encoding'] = 'Binary'
def __gConfigDefaults(defaultPath):
"""
Build a cfg from a Default Section
"""
from DIRAC import gConfig
cfgDefaults = CFG()
result = gConfig.getSections(defaultPath)
if not result['OK']:
return cfgDefaults
for name in result['Value']:
typePath = cfgPath(defaultPath, name)
cfgDefaults.createNewSection(name)
result = gConfig.getOptionsDict(typePath)
if result['OK']:
optionsDict = result['Value']
for option, value in optionsDict.items():
cfgDefaults[name].setOption(option, value)
return cfgDefaults
def _check(self):
"""Obtain default configuration and current configuration and print the diff."""
cfg = CFG()
templateLocations = self._findConfigTemplates()
for templatePath in templateLocations:
cfgRes = self._parseConfigTemplate(templatePath, cfg)
if cfgRes['OK']:
cfg = cfgRes['Value']
currentCfg = self._getCurrentConfig()
if not currentCfg['OK']:
return
currentCfg = currentCfg['Value']
diff = currentCfg.getModifications(cfg, ignoreOrder=True, ignoreComments=True)
LOG.debug("*" * 80)
LOG.debug("Default Configuration: %s" % str(cfg))
LOG.debug("*" * 80)
LOG.debug("Current Configuration: %s " % str(currentCfg))
for entry in diff:
self._printDiff(entry)
def __saveVomsMapping(self, params):
sectionPath = "/Registry/VOMS/Mapping"
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
if not sectionCfg.isSection(opt):
self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)
configText = ""
for newOpt in params:
if newOpt != "op":
if configText == "":
configText = newOpt + "=" + params[newOpt] + "\n"
else:
configText = configText + newOpt + "=" + params[newOpt] + "\n"
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"op":"saveVomsMapping", "success": 1}
# $HeadURL$
__RCSID__ = "$Id$"
from dirac import DIRAC
from DIRAC.Core.Utilities.CFG import CFG
DIRAC.gLogger.initialize('test_gConfig', '/testSectionDebug')
testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath
dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath
cfg1 = CFG()
cfg1.loadFromFile(testconfig)
fd = file(testconfig)
cfg1String = fd.read()
fd.close()
cfg2 = CFG()
cfg2.loadFromBuffer(cfg1.serialize())
cfg3 = cfg1.mergeWith(cfg2)
testList = [{
'method': DIRAC.gConfig.loadFile,
'arguments': (testconfig, ),
'output': {
'OK': True,
'Value': ''
}
}, {
