def clearCFG():
"""SUPER UGLY: one must recreate the CFG objects of gConfigurationData
not to conflict with other tests that might be using a local dirac.cfg"""
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
def __init__(self, loadDefaultCFG=True):
envVar = os.environ.get("DIRAC_FEWER_CFG_LOCKS", "no").lower()
self.__locksEnabled = envVar not in ("y", "yes", "t", "true", "on",
"1")
if self.__locksEnabled:
lr = LockRing()
self.threadingEvent = lr.getEvent()
self.threadingEvent.set()
self.threadingLock = lr.getLock()
self.runningThreadsNumber = 0
self.__compressedConfigurationData = None
self.configurationPath = "/DIRAC/Configuration"
self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup")
self._isService = False
self.localCFG = CFG()
self.remoteCFG = CFG()
self.mergedCFG = CFG()
self.remoteServerList = []
if loadDefaultCFG:
defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg")
gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile)
retVal = self.loadFile(defaultCFGFile)
if not retVal["OK"]:
gLogger.warn("Can't load %s file" % defaultCFGFile)
self.sync()
def parseConfigTemplate(self, templatePath, cfg):
"""Parse the ConfigTemplate.cfg files.
:param str templatePath: path to the folder containing a ConfigTemplate.cfg file
:param CFG cfg: cfg to merge with the systems config
:returns: CFG object
"""
system = os.path.split(templatePath.rstrip("/"))[1]
if system.lower().endswith("system"):
system = system[: -len("System")]
templatePath = os.path.join(templatePath, "ConfigTemplate.cfg")
if not os.path.exists(templatePath):
return S_ERROR("File not found: %s" % templatePath)
loadCfg = CFG()
try:
loadCfg.loadFromFile(templatePath)
except ValueError as err:
LOG.error("Failed loading file %r: %r", templatePath, err)
self.retVal = 1
return S_ERROR()
cfg.createNewSection("/Systems/%s" % system, contents=loadCfg)
return S_OK(cfg)
def fts3db():
FTS3DB.utc_timestamp = func.datetime
FTS3DB.fts3FileTable.columns["lastUpdate"].onupdate = func.datetime
FTS3DB.fts3JobTable.columns["lastUpdate"].onupdate = func.datetime
FTS3DB.fts3OperationTable.columns["lastUpdate"].onupdate = func.datetime
db = FTS3DB.FTS3DB(url="sqlite+pysqlite:///:memory:")
@event.listens_for(engine.Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
"""Make sure that the foreign keys are checked
See https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#foreign-key-support
"""
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
db.createTables()
yield db
# SUPER UGLY: one must recreate the CFG objects of gConfigurationData
# not to conflict with other tests that might be using a local dirac.cfg
# Note that here we do not use it specifically, but the FTS3 objets
# are doing it
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
def __init__(self, rpcClient=False, commiterId="unknown"):
self.commiterTag = "@@-"
self.commiterId = commiterId
self.cfgData = CFG()
self.rpcClient = None
if rpcClient:
self.setRPCClient(rpcClient)
def checkAgentOptions(getOptionMock, systemName, agentName, agentLocation, ignoreOptions=None):
"""Ensure that all the agent options are properly documented.
:param getOptionMock: Mock object for agentmodule.get_amOption function
:param str systemName: name of the **System**
:param str agentName: name of the **Agent**
:param list ignoreOptions: list of options to ignore
"""
if ignoreOptions is None:
ignoreOptions = []
# add some options that can be set, see the AgentModule for all of them
ignoreOptions.extend(
["PollingTime", "Status", "Enabled", "MaxCycles", "LogOutputs", "ControlDirectory", "shifterProxy"]
)
ignoreOptions = list(set(ignoreOptions))
config = CFG()
LOG.info("Testing %s/%s, ignoring options %s", systemName, agentName, ignoreOptions)
# expect the ConfigTemplate one level above the agent module
configFilePath = os.path.join(agentLocation, "..", "ConfigTemplate.cfg")
config.loadFromFile(configFilePath)
optionsDict = config.getAsDict("Agents/%s" % agentName)
outDict = {}
_parseOption(outDict, optionsDict)
optionsDict = outDict
LOG.info("Calls: %s", pformat(getOptionMock.call_args_list))
LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys()))
# check that values in ConfigTemplate are used
for option, value in optionsDict.items():
if any(ignoreOp in option for ignoreOp in ignoreOptions):
LOG.info("From Agent: ignoring option %r with value %r, (%s)", option, value, type(value))
continue
LOG.info("Looking for call to option %r with value %r, (%s)", option, value, type(value))
if not isinstance(value, bool) and not value: # empty string, list, dict ...
assert any(call(option, null) in getOptionMock.call_args_list for null in ({}, set(), [], "", 0, None))
else:
assert (
call(option, value) in getOptionMock.call_args_list
or call(option, [value]) in getOptionMock.call_args_list
)
# check that options used in the agent are in the ConfigTemplates
for opCall in getOptionMock.call_args_list:
optionArguments = opCall[0]
if len(optionArguments) != 2:
continue
optionName = optionArguments[0]
optionValue = optionArguments[1]
if optionName in ignoreOptions:
LOG.info("From Template: ignoring option %r with %r", optionName, optionValue)
continue
LOG.info("Checking Template option %r with %r", optionName, optionValue)
assert optionName in optionsDict
if not optionsDict[optionName]:
assert not optionValue
continue
assert optionsDict[optionName] == optionValue or [optionsDict[optionName]] == optionValue
def appendToRepository(self, repoLocation):
if not os.path.exists(repoLocation):
gLogger.error("Secondary repository does not exist", repoLocation)
return S_ERROR("Secondary repository does not exist")
self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo)
self._writeRepository(self.location)
return S_OK()
def _getCurrentConfig(self):
"""Return the current system configuration."""
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfig.forceRefresh()
fullCfg = CFG()
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
for system, setup in serviceSetups.items():
if self.systems and system not in self.systems:
continue
systemCfg = gConfigurationData.remoteCFG.getAsCFG(
"/Systems/%s/%s" % (system, setup))
for section in systemCfg.listSections():
if section not in ('Agents', 'Services', 'Executors'):
systemCfg.deleteKey(section)
fullCfg.createNewSection("/%s" % system, contents=systemCfg)
return S_OK(fullCfg)
def __getCache(self):
Operations.__cacheLock.acquire()
try:
currentVersion = gConfigurationData.getVersion()
if currentVersion != Operations.__cacheVersion:
Operations.__cache = {}
Operations.__cacheVersion = currentVersion
cacheKey = (self.__vo, self.__setup)
if cacheKey in Operations.__cache:
return Operations.__cache[cacheKey]
mergedCFG = CFG()
for path in self.__getSearchPaths():
pathCFG = gConfigurationData.mergedCFG[path]
if pathCFG:
mergedCFG = mergedCFG.mergeWith(pathCFG)
Operations.__cache[cacheKey] = mergedCFG
return Operations.__cache[cacheKey]
finally:
try:
Operations.__cacheLock.release()
except thread.error:
pass
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result["OK"]:
raise Exception(result["Message"])
def loadFile(self, fileName):
try:
fileCFG = CFG()
fileCFG.loadFromFile(fileName)
except IOError:
self.localCFG = self.localCFG.mergeWith(fileCFG)
return S_ERROR("Can't load a cfg file '%s'" % fileName)
return self.mergeWithLocal(fileCFG)
def loadFromRemote(self):
retVal = self.rpcClient.getCompressedData()
if retVal["OK"]:
self.cfgData = CFG()
data = retVal["Value"]
if isinstance(data, str):
data = data.encode(errors="surrogateescape")
self.cfgData.loadFromBuffer(zlib.decompress(data).decode())
return retVal
def main():
Script.disableCS()
Script.registerSwitch("t:", "type=",
"Installation type. 'server' by default.",
setInstallType)
Script.parseCommandLine(ignoreErrors=True)
# Collect all the requested python modules to install
reqDict = {}
for entry in os.listdir(rootPath):
if len(entry) < 5 or entry.find("DIRAC") != len(entry) - 5:
continue
reqFile = os.path.join(rootPath, entry, "releases.cfg")
try:
with open(reqFile, "r") as extfd:
reqCFG = CFG().loadFromBuffer(extfd.read())
except BaseException:
gLogger.verbose("%s not found" % reqFile)
continue
reqList = reqCFG.getOption(
"/RequiredExternals/%s" % instType.capitalize(), [])
if not reqList:
gLogger.verbose(
"%s does not have requirements for %s installation" %
(entry, instType))
continue
for req in reqList:
reqName = False
reqCond = ""
for cond in ("==", ">="):
iP = cond.find(req)
if iP > 0:
reqName = req[:iP]
reqCond = req[iP:]
break
if not reqName:
reqName = req
if reqName not in reqDict:
reqDict[reqName] = (reqCond, entry)
else:
gLogger.notice("Skipping %s, it's already requested by %s" %
(reqName, reqDict[reqName][1]))
if not reqDict:
gLogger.notice("No extra python module requested to be installed")
sys.exit(0)
for reqName in reqDict:
package = "%s%s" % (reqName, reqDict[reqName][0])
gLogger.notice("Requesting installation of %s" % package)
status, output = pipInstall(package)
if status != 0:
gLogger.error(output)
else:
gLogger.notice("Successfully installed %s" % package)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result["OK"]:
self.__manifest = CFG()
return result
self.__manifest = result["Value"][0]
return S_OK()
def getSystemsCFG(self):
"""Find all the ConfigTemplates and collate them into one CFG object."""
cfg = CFG()
cfg.createNewSection("/Systems")
templateLocations = self.findConfigTemplates()
for templatePath in templateLocations:
cfgRes = self.parseConfigTemplate(templatePath, cfg)
if cfgRes["OK"]:
cfg = cfgRes["Value"]
return cfg
def tearDown(self):
try:
os.remove(self.testCfgFileName)
except OSError:
pass
# SUPER UGLY: one must recreate the CFG objects of gConfigurationData
# not to conflict with other tests that might be using a local dirac.cfg
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
def mergeWithServer(self):
retVal = self.rpcClient.getCompressedData()
if retVal["OK"]:
remoteCFG = CFG()
data = retVal["Value"]
if isinstance(data, str):
data = data.encode(errors="surrogateescape")
remoteCFG.loadFromBuffer(zlib.decompress(data).decode())
serverVersion = gConfigurationData.getVersion(remoteCFG)
self.cfgData = remoteCFG.mergeWith(self.cfgData)
gConfigurationData.setVersion(serverVersion, self.cfgData)
return retVal
def getComputingElementDefaults(ceName="",
ceType="",
cfg=None,
currentSectionPath=""):
"""
Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg
"""
cesCfg = CFG()
if cfg:
try:
cesCfg.loadFromFile(cfg)
cesPath = cfgInstallPath("ComputingElements")
if cesCfg.isSection(cesPath):
for section in cfgPathToList(cesPath):
cesCfg = cesCfg[section]
except Exception:
return CFG()
# Overwrite the cfg with Command line arguments
if ceName:
if not cesCfg.isSection(ceName):
cesCfg.createNewSection(ceName)
if currentSectionPath:
# Add Options from Command Line
optionsDict = __getExtraOptions(currentSectionPath)
for name, value in optionsDict.items():
cesCfg[ceName].setOption(name, value) # pylint: disable=no-member
if ceType:
cesCfg[ceName].setOption("CEType", ceType) # pylint: disable=no-member
ceDefaultSection = cfgPath(defaultSection("ComputingElements"))
# Load Default for the given type from Central configuration is defined
ceDefaults = __gConfigDefaults(ceDefaultSection)
for ceName in cesCfg.listSections():
if "CEType" in cesCfg[ceName]:
ceType = cesCfg[ceName]["CEType"]
if ceType in ceDefaults:
for option in ceDefaults[ceType].listOptions(): # pylint: disable=no-member
if option not in cesCfg[ceName]:
cesCfg[ceName].setOption(
option,
ceDefaults[ceType][option] # pylint: disable=unsubscriptable-object
)
return cesCfg
def __init__(self, repository=None):
self.location = repository
if not self.location:
if "HOME" in os.environ:
self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
else:
self.location = '%s/.dirac.repo.rep' % os.getcwd()
self.repo = CFG()
if os.path.exists(self.location):
self.repo.loadFromFile(self.location)
if not self.repo.existsKey('Jobs'):
self.repo.createNewSection('Jobs')
else:
self.repo.createNewSection('Jobs')
self.OK = True
written = self._writeRepository(self.location)
if not written:
self.OK = False
def __gConfigDefaults(defaultPath):
"""
Build a cfg from a Default Section
"""
from DIRAC import gConfig
cfgDefaults = CFG()
result = gConfig.getSections(defaultPath)
if not result['OK']:
return cfgDefaults
for name in result['Value']:
typePath = cfgPath(defaultPath, name)
cfgDefaults.createNewSection(name)
result = gConfig.getOptionsDict(typePath)
if result['OK']:
optionsDict = result['Value']
for option, value in optionsDict.items():
cfgDefaults[name].setOption(option, value)
return cfgDefaults
def loadDIRACCFG():
installPath = gConfig.getValue("/LocalInstallation/TargetPath", "")
if not installPath:
installPath = rootPath
cfgPath = os.path.join(installPath, "etc", "dirac.cfg")
try:
diracCFG = CFG().loadFromFile(cfgPath)
except Exception as excp:
return S_ERROR("Could not load dirac.cfg: %s" % repr(excp))
return S_OK((cfgPath, diracCFG))
def __init__(self, loadDefaultCFG=True):
lr = LockRing()
self.threadingEvent = lr.getEvent()
self.threadingEvent.set()
self.threadingLock = lr.getLock()
self.runningThreadsNumber = 0
self.__compressedConfigurationData = None
self.configurationPath = "/DIRAC/Configuration"
self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup")
self._isService = False
self.localCFG = CFG()
self.remoteCFG = CFG()
self.mergedCFG = CFG()
self.remoteServerList = []
if loadDefaultCFG:
defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg")
gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile)
retVal = self.loadFile(defaultCFGFile)
if not retVal['OK']:
gLogger.warn("Can't load %s file" % defaultCFGFile)
self.sync()
def loadDIRACCFG():
installPath = gConfig.getValue('/LocalInstallation/TargetPath',
gConfig.getValue('/LocalInstallation/RootPath', ''))
if not installPath:
installPath = rootPath
cfgPath = os.path.join(installPath, 'etc', 'dirac.cfg')
try:
diracCFG = CFG().loadFromFile(cfgPath)
except BaseException as excp:
return S_ERROR("Could not load dirac.cfg: %s" % repr(excp))
return S_OK((cfgPath, diracCFG))
def setUp(self):
self.authMgr = AuthManager("/Systems/Service/Authorization")
cfg = CFG()
cfg.loadFromBuffer(testSystemsCFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(testRegistryCFG)
gConfig.loadCFG(cfg)
self.noAuthCredDict = {"group": "group_test"}
self.userCredDict = {
"DN": "/User/test/DN/CN=userA",
"group": "group_test"
}
self.suspendedOtherVOUserCredDict = {
"DN": "/User/test/DN/CN=userS",
"group": "group_test_other"
}
self.badUserCredDict = {
"DN": "/User/test/DN/CN=userB",
"group": "group_bad"
}
self.suspendedUserCredDict = {
"DN": "/User/test/DN/CN=userS",
"group": "group_test"
}
self.hostCredDict = {
"DN": "/User/test/DN/CN=test.hostA.ch",
"group": "hosts"
}
self.badHostCredDict = {
"DN": "/User/test/DN/CN=test.hostB.ch",
"group": "hosts"
}
def updateCompleteDiracCFG(self):
"""Read the dirac.cfg and update the Systems sections from the ConfigTemplate.cfg files."""
compCfg = CFG()
mainDiracCfgPath = self.config.cfg_baseFile
if not os.path.exists(mainDiracCfgPath):
LOG.error("Failed to find Main Dirac cfg at %r", mainDiracCfgPath)
return 1
self.prepareDiracCFG()
LOG.info("Extracting default configuration from %r", mainDiracCfgPath)
loadCFG = CFG()
loadCFG.loadFromFile(mainDiracCfgPath)
compCfg = loadCFG.mergeWith(compCfg)
cfg = self.getSystemsCFG()
compCfg = compCfg.mergeWith(cfg)
diracCfgOutput = self.config.cfg_targetFile
LOG.info("Writing output to %r", diracCfgOutput)
with open(diracCfgOutput, "w") as rst:
rst.write(
textwrap.dedent(
"""
.. _full_configuration_example:
==========================
Full Configuration Example
==========================
.. This file is created by docs/Tools/UpdateDiracCFG.py
Below is a complete example configuration with anotations for some sections::
"""
)
)
# indent the cfg text
cfgString = "".join(" " + line for line in str(compCfg).splitlines(True))
# fix the links, add back the # for targets
# match .html with following character using positive look ahead
htmlMatch = re.compile(r"\.html(?=[a-zA-Z0-9])")
cfgString = re.sub(htmlMatch, ".html#", cfgString)
rst.write(cfgString)
return self.retVal
def setUp(self):
cfg = CFG()
cfg.loadFromBuffer(diracTestCACFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(userCFG)
gConfig.loadCFG(cfg)
result = ProxyProviderFactory().getProxyProvider('DIRAC_TEST_CA')
self.assertTrue(
result['OK'], '\n%s' % result.get('Message')
or 'Error message is absent.')
self.pp = result['Value']
def create_serverAndClient(request):
"""This function starts a server, and closes it after
The server will use the parametrized transport type
"""
# Reinitialize the configuration.
# We do it here rather than at the start of the module
# to accommodate for pytest when going through all the DIRAC tests
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
gConfigurationData.setOptionInCFG("/DIRAC/Security/CALocation", caLocation)
gConfigurationData.setOptionInCFG("/DIRAC/Security/CertFile",
hostCertLocation)
gConfigurationData.setOptionInCFG("/DIRAC/Security/KeyFile",
hostKeyLocation)
testStr = request.param
serverName, clientName = testStr.split("-")
serverClass = transportByName(serverName)
clientClass = transportByName(clientName)
sr = DummyServiceReactor(serverClass, PORT_NUMBER)
server_thread = threading.Thread(target=sr.serve)
sr.prepare()
server_thread.start()
# Create the client
clientOptions = {
"clientMode": True,
"proxyLocation": proxyFile,
}
clientTransport = clientClass(("localhost", PORT_NUMBER),
bServerMode=False,
**clientOptions)
res = clientTransport.initAsClient()
assert res["OK"], res
yield sr, clientTransport
clientTransport.close()
sr.closeListeningConnections()
server_thread.join()
# Clean the config
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
def main():
Script.registerSwitch("t", "test", "Only test. Don't commit changes")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"UserCfg: Cfg FileName with Users as sections containing"
"DN, Groups, and other properties as options")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getExtraCLICFGFiles()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
testOnly = False
errorList = []
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("t", "test"):
testOnly = True
try:
usersCFG = CFG().loadFromFile(args[0])
except Exception as e:
errorList.append("file open",
"Can't parse file %s: %s" % (args[0], str(e)))
errorCode = 1
else:
if not diracAdmin.csSyncUsersWithCFG(usersCFG):
errorList.append(("modify users", "Cannot sync with %s" % args[0]))
exitCode = 255
if not exitCode and not testOnly:
result = diracAdmin.csCommitChanges()
if not result["OK"]:
errorList.append(("commit", result["Message"]))
exitCode = 255
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def _check(self):
"""Obtain default configuration and current configuration and print the diff."""
cfg = CFG()
templateLocations = self._findConfigTemplates()
for templatePath in templateLocations:
cfgRes = self._parseConfigTemplate(templatePath, cfg)
if cfgRes["OK"]:
cfg = cfgRes["Value"]
currentCfg = self._getCurrentConfig()
if not currentCfg["OK"]:
return
currentCfg = currentCfg["Value"]
diff = currentCfg.getModifications(cfg, ignoreOrder=True, ignoreComments=True)
LOG.debug("*" * 80)
LOG.debug("Default Configuration: %s" % str(cfg))
LOG.debug("*" * 80)
LOG.debug("Current Configuration: %s " % str(currentCfg))
for entry in diff:
self._printDiff(entry)
def main():
Script.registerSwitch("t", "test", "Only test. Don't commit changes")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getExtraCLICFGFiles()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
testOnly = False
errorList = []
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("t", "test"):
testOnly = True
try:
usersCFG = CFG().loadFromFile(args[0])
except Exception as e:
errorList.append("file open", "Can't parse file %s: %s" % (args[0], str(e)))
errorCode = 1
else:
if not diracAdmin.csSyncUsersWithCFG(usersCFG):
errorList.append(("modify users", "Cannot sync with %s" % args[0]))
exitCode = 255
if not exitCode and not testOnly:
result = diracAdmin.csCommitChanges()
if not result['OK']:
errorList.append(("commit", result['Message']))
exitCode = 255
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
