def clearCFG(): """SUPER UGLY: one must recreate the CFG objects of gConfigurationData not to conflict with other tests that might be using a local dirac.cfg""" gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion()
def __init__(self, loadDefaultCFG=True): envVar = os.environ.get("DIRAC_FEWER_CFG_LOCKS", "no").lower() self.__locksEnabled = envVar not in ("y", "yes", "t", "true", "on", "1") if self.__locksEnabled: lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.__compressedConfigurationData = None self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup") self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg") gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile) retVal = self.loadFile(defaultCFGFile) if not retVal["OK"]: gLogger.warn("Can't load %s file" % defaultCFGFile) self.sync()
def parseConfigTemplate(self, templatePath, cfg): """Parse the ConfigTemplate.cfg files. :param str templatePath: path to the folder containing a ConfigTemplate.cfg file :param CFG cfg: cfg to merge with the systems config :returns: CFG object """ system = os.path.split(templatePath.rstrip("/"))[1] if system.lower().endswith("system"): system = system[: -len("System")] templatePath = os.path.join(templatePath, "ConfigTemplate.cfg") if not os.path.exists(templatePath): return S_ERROR("File not found: %s" % templatePath) loadCfg = CFG() try: loadCfg.loadFromFile(templatePath) except ValueError as err: LOG.error("Failed loading file %r: %r", templatePath, err) self.retVal = 1 return S_ERROR() cfg.createNewSection("/Systems/%s" % system, contents=loadCfg) return S_OK(cfg)
def fts3db(): FTS3DB.utc_timestamp = func.datetime FTS3DB.fts3FileTable.columns["lastUpdate"].onupdate = func.datetime FTS3DB.fts3JobTable.columns["lastUpdate"].onupdate = func.datetime FTS3DB.fts3OperationTable.columns["lastUpdate"].onupdate = func.datetime db = FTS3DB.FTS3DB(url="sqlite+pysqlite:///:memory:") @event.listens_for(engine.Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): """Make sure that the foreign keys are checked See https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#foreign-key-support """ cursor = dbapi_connection.cursor() cursor.execute("PRAGMA foreign_keys=ON") cursor.close() db.createTables() yield db # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg # Note that here we do not use it specifically, but the FTS3 objets # are doing it gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion()
def __init__(self, rpcClient=False, commiterId="unknown"): self.commiterTag = "@@-" self.commiterId = commiterId self.cfgData = CFG() self.rpcClient = None if rpcClient: self.setRPCClient(rpcClient)
def checkAgentOptions(getOptionMock, systemName, agentName, agentLocation, ignoreOptions=None): """Ensure that all the agent options are properly documented. :param getOptionMock: Mock object for agentmodule.get_amOption function :param str systemName: name of the **System** :param str agentName: name of the **Agent** :param list ignoreOptions: list of options to ignore """ if ignoreOptions is None: ignoreOptions = [] # add some options that can be set, see the AgentModule for all of them ignoreOptions.extend( ["PollingTime", "Status", "Enabled", "MaxCycles", "LogOutputs", "ControlDirectory", "shifterProxy"] ) ignoreOptions = list(set(ignoreOptions)) config = CFG() LOG.info("Testing %s/%s, ignoring options %s", systemName, agentName, ignoreOptions) # expect the ConfigTemplate one level above the agent module configFilePath = os.path.join(agentLocation, "..", "ConfigTemplate.cfg") config.loadFromFile(configFilePath) optionsDict = config.getAsDict("Agents/%s" % agentName) outDict = {} _parseOption(outDict, optionsDict) optionsDict = outDict LOG.info("Calls: %s", pformat(getOptionMock.call_args_list)) LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys())) # check that values in ConfigTemplate are used for option, value in optionsDict.items(): if any(ignoreOp in option for ignoreOp in ignoreOptions): LOG.info("From Agent: ignoring option %r with value %r, (%s)", option, value, type(value)) continue LOG.info("Looking for call to option %r with value %r, (%s)", option, value, type(value)) if not isinstance(value, bool) and not value: # empty string, list, dict ... assert any(call(option, null) in getOptionMock.call_args_list for null in ({}, set(), [], "", 0, None)) else: assert ( call(option, value) in getOptionMock.call_args_list or call(option, [value]) in getOptionMock.call_args_list ) # check that options used in the agent are in the ConfigTemplates for opCall in getOptionMock.call_args_list: optionArguments = opCall[0] if len(optionArguments) != 2: continue optionName = optionArguments[0] optionValue = optionArguments[1] if optionName in ignoreOptions: LOG.info("From Template: ignoring option %r with %r", optionName, optionValue) continue LOG.info("Checking Template option %r with %r", optionName, optionValue) assert optionName in optionsDict if not optionsDict[optionName]: assert not optionValue continue assert optionsDict[optionName] == optionValue or [optionsDict[optionName]] == optionValue
def appendToRepository(self, repoLocation): if not os.path.exists(repoLocation): gLogger.error("Secondary repository does not exist", repoLocation) return S_ERROR("Secondary repository does not exist") self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo) self._writeRepository(self.location) return S_OK()
def _getCurrentConfig(self): """Return the current system configuration.""" from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData gConfig.forceRefresh() fullCfg = CFG() setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if setup not in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict for system, setup in serviceSetups.items(): if self.systems and system not in self.systems: continue systemCfg = gConfigurationData.remoteCFG.getAsCFG( "/Systems/%s/%s" % (system, setup)) for section in systemCfg.listSections(): if section not in ('Agents', 'Services', 'Executors'): systemCfg.deleteKey(section) fullCfg.createNewSection("/%s" % system, contents=systemCfg) return S_OK(fullCfg)
def __getCache(self): Operations.__cacheLock.acquire() try: currentVersion = gConfigurationData.getVersion() if currentVersion != Operations.__cacheVersion: Operations.__cache = {} Operations.__cacheVersion = currentVersion cacheKey = (self.__vo, self.__setup) if cacheKey in Operations.__cache: return Operations.__cache[cacheKey] mergedCFG = CFG() for path in self.__getSearchPaths(): pathCFG = gConfigurationData.mergedCFG[path] if pathCFG: mergedCFG = mergedCFG.mergeWith(pathCFG) Operations.__cache[cacheKey] = mergedCFG return Operations.__cache[cacheKey] finally: try: Operations.__cacheLock.release() except thread.error: pass
def __init__(self, manifest=""): self.__manifest = CFG() self.__dirty = False self.__ops = False if manifest: result = self.load(manifest) if not result["OK"]: raise Exception(result["Message"])
def loadFile(self, fileName): try: fileCFG = CFG() fileCFG.loadFromFile(fileName) except IOError: self.localCFG = self.localCFG.mergeWith(fileCFG) return S_ERROR("Can't load a cfg file '%s'" % fileName) return self.mergeWithLocal(fileCFG)
def loadFromRemote(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: self.cfgData = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") self.cfgData.loadFromBuffer(zlib.decompress(data).decode()) return retVal
def main(): Script.disableCS() Script.registerSwitch("t:", "type=", "Installation type. 'server' by default.", setInstallType) Script.parseCommandLine(ignoreErrors=True) # Collect all the requested python modules to install reqDict = {} for entry in os.listdir(rootPath): if len(entry) < 5 or entry.find("DIRAC") != len(entry) - 5: continue reqFile = os.path.join(rootPath, entry, "releases.cfg") try: with open(reqFile, "r") as extfd: reqCFG = CFG().loadFromBuffer(extfd.read()) except BaseException: gLogger.verbose("%s not found" % reqFile) continue reqList = reqCFG.getOption( "/RequiredExternals/%s" % instType.capitalize(), []) if not reqList: gLogger.verbose( "%s does not have requirements for %s installation" % (entry, instType)) continue for req in reqList: reqName = False reqCond = "" for cond in ("==", ">="): iP = cond.find(req) if iP > 0: reqName = req[:iP] reqCond = req[iP:] break if not reqName: reqName = req if reqName not in reqDict: reqDict[reqName] = (reqCond, entry) else: gLogger.notice("Skipping %s, it's already requested by %s" % (reqName, reqDict[reqName][1])) if not reqDict: gLogger.notice("No extra python module requested to be installed") sys.exit(0) for reqName in reqDict: package = "%s%s" % (reqName, reqDict[reqName][0]) gLogger.notice("Requesting installation of %s" % package) status, output = pipInstall(package) if status != 0: gLogger.error(output) else: gLogger.notice("Successfully installed %s" % package)
def loadJDL(self, jdlString): """ Load job manifest from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result["OK"]: self.__manifest = CFG() return result self.__manifest = result["Value"][0] return S_OK()
def getSystemsCFG(self): """Find all the ConfigTemplates and collate them into one CFG object.""" cfg = CFG() cfg.createNewSection("/Systems") templateLocations = self.findConfigTemplates() for templatePath in templateLocations: cfgRes = self.parseConfigTemplate(templatePath, cfg) if cfgRes["OK"]: cfg = cfgRes["Value"] return cfg
def tearDown(self): try: os.remove(self.testCfgFileName) except OSError: pass # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion()
def mergeWithServer(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: remoteCFG = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") remoteCFG.loadFromBuffer(zlib.decompress(data).decode()) serverVersion = gConfigurationData.getVersion(remoteCFG) self.cfgData = remoteCFG.mergeWith(self.cfgData) gConfigurationData.setVersion(serverVersion, self.cfgData) return retVal
def getComputingElementDefaults(ceName="", ceType="", cfg=None, currentSectionPath=""): """ Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg """ cesCfg = CFG() if cfg: try: cesCfg.loadFromFile(cfg) cesPath = cfgInstallPath("ComputingElements") if cesCfg.isSection(cesPath): for section in cfgPathToList(cesPath): cesCfg = cesCfg[section] except Exception: return CFG() # Overwrite the cfg with Command line arguments if ceName: if not cesCfg.isSection(ceName): cesCfg.createNewSection(ceName) if currentSectionPath: # Add Options from Command Line optionsDict = __getExtraOptions(currentSectionPath) for name, value in optionsDict.items(): cesCfg[ceName].setOption(name, value) # pylint: disable=no-member if ceType: cesCfg[ceName].setOption("CEType", ceType) # pylint: disable=no-member ceDefaultSection = cfgPath(defaultSection("ComputingElements")) # Load Default for the given type from Central configuration is defined ceDefaults = __gConfigDefaults(ceDefaultSection) for ceName in cesCfg.listSections(): if "CEType" in cesCfg[ceName]: ceType = cesCfg[ceName]["CEType"] if ceType in ceDefaults: for option in ceDefaults[ceType].listOptions(): # pylint: disable=no-member if option not in cesCfg[ceName]: cesCfg[ceName].setOption( option, ceDefaults[ceType][option] # pylint: disable=unsubscriptable-object ) return cesCfg
def __init__(self, repository=None): self.location = repository if not self.location: if "HOME" in os.environ: self.location = '%s/.dirac.repo.rep' % os.environ['HOME'] else: self.location = '%s/.dirac.repo.rep' % os.getcwd() self.repo = CFG() if os.path.exists(self.location): self.repo.loadFromFile(self.location) if not self.repo.existsKey('Jobs'): self.repo.createNewSection('Jobs') else: self.repo.createNewSection('Jobs') self.OK = True written = self._writeRepository(self.location) if not written: self.OK = False
def __gConfigDefaults(defaultPath): """ Build a cfg from a Default Section """ from DIRAC import gConfig cfgDefaults = CFG() result = gConfig.getSections(defaultPath) if not result['OK']: return cfgDefaults for name in result['Value']: typePath = cfgPath(defaultPath, name) cfgDefaults.createNewSection(name) result = gConfig.getOptionsDict(typePath) if result['OK']: optionsDict = result['Value'] for option, value in optionsDict.items(): cfgDefaults[name].setOption(option, value) return cfgDefaults
def loadDIRACCFG(): installPath = gConfig.getValue("/LocalInstallation/TargetPath", "") if not installPath: installPath = rootPath cfgPath = os.path.join(installPath, "etc", "dirac.cfg") try: diracCFG = CFG().loadFromFile(cfgPath) except Exception as excp: return S_ERROR("Could not load dirac.cfg: %s" % repr(excp)) return S_OK((cfgPath, diracCFG))
def __init__(self, loadDefaultCFG=True): lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.__compressedConfigurationData = None self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup") self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg") gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile) retVal = self.loadFile(defaultCFGFile) if not retVal['OK']: gLogger.warn("Can't load %s file" % defaultCFGFile) self.sync()
def loadDIRACCFG(): installPath = gConfig.getValue('/LocalInstallation/TargetPath', gConfig.getValue('/LocalInstallation/RootPath', '')) if not installPath: installPath = rootPath cfgPath = os.path.join(installPath, 'etc', 'dirac.cfg') try: diracCFG = CFG().loadFromFile(cfgPath) except BaseException as excp: return S_ERROR("Could not load dirac.cfg: %s" % repr(excp)) return S_OK((cfgPath, diracCFG))
def setUp(self): self.authMgr = AuthManager("/Systems/Service/Authorization") cfg = CFG() cfg.loadFromBuffer(testSystemsCFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(testRegistryCFG) gConfig.loadCFG(cfg) self.noAuthCredDict = {"group": "group_test"} self.userCredDict = { "DN": "/User/test/DN/CN=userA", "group": "group_test" } self.suspendedOtherVOUserCredDict = { "DN": "/User/test/DN/CN=userS", "group": "group_test_other" } self.badUserCredDict = { "DN": "/User/test/DN/CN=userB", "group": "group_bad" } self.suspendedUserCredDict = { "DN": "/User/test/DN/CN=userS", "group": "group_test" } self.hostCredDict = { "DN": "/User/test/DN/CN=test.hostA.ch", "group": "hosts" } self.badHostCredDict = { "DN": "/User/test/DN/CN=test.hostB.ch", "group": "hosts" }
def updateCompleteDiracCFG(self): """Read the dirac.cfg and update the Systems sections from the ConfigTemplate.cfg files.""" compCfg = CFG() mainDiracCfgPath = self.config.cfg_baseFile if not os.path.exists(mainDiracCfgPath): LOG.error("Failed to find Main Dirac cfg at %r", mainDiracCfgPath) return 1 self.prepareDiracCFG() LOG.info("Extracting default configuration from %r", mainDiracCfgPath) loadCFG = CFG() loadCFG.loadFromFile(mainDiracCfgPath) compCfg = loadCFG.mergeWith(compCfg) cfg = self.getSystemsCFG() compCfg = compCfg.mergeWith(cfg) diracCfgOutput = self.config.cfg_targetFile LOG.info("Writing output to %r", diracCfgOutput) with open(diracCfgOutput, "w") as rst: rst.write( textwrap.dedent( """ .. _full_configuration_example: ========================== Full Configuration Example ========================== .. This file is created by docs/Tools/UpdateDiracCFG.py Below is a complete example configuration with anotations for some sections:: """ ) ) # indent the cfg text cfgString = "".join(" " + line for line in str(compCfg).splitlines(True)) # fix the links, add back the # for targets # match .html with following character using positive look ahead htmlMatch = re.compile(r"\.html(?=[a-zA-Z0-9])") cfgString = re.sub(htmlMatch, ".html#", cfgString) rst.write(cfgString) return self.retVal
def setUp(self): cfg = CFG() cfg.loadFromBuffer(diracTestCACFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(userCFG) gConfig.loadCFG(cfg) result = ProxyProviderFactory().getProxyProvider('DIRAC_TEST_CA') self.assertTrue( result['OK'], '\n%s' % result.get('Message') or 'Error message is absent.') self.pp = result['Value']
def create_serverAndClient(request): """This function starts a server, and closes it after The server will use the parametrized transport type """ # Reinitialize the configuration. # We do it here rather than at the start of the module # to accommodate for pytest when going through all the DIRAC tests gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion() gConfigurationData.setOptionInCFG("/DIRAC/Security/CALocation", caLocation) gConfigurationData.setOptionInCFG("/DIRAC/Security/CertFile", hostCertLocation) gConfigurationData.setOptionInCFG("/DIRAC/Security/KeyFile", hostKeyLocation) testStr = request.param serverName, clientName = testStr.split("-") serverClass = transportByName(serverName) clientClass = transportByName(clientName) sr = DummyServiceReactor(serverClass, PORT_NUMBER) server_thread = threading.Thread(target=sr.serve) sr.prepare() server_thread.start() # Create the client clientOptions = { "clientMode": True, "proxyLocation": proxyFile, } clientTransport = clientClass(("localhost", PORT_NUMBER), bServerMode=False, **clientOptions) res = clientTransport.initAsClient() assert res["OK"], res yield sr, clientTransport clientTransport.close() sr.closeListeningConnections() server_thread.join() # Clean the config gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion()
def main(): Script.registerSwitch("t", "test", "Only test. Don't commit changes") # Registering arguments will automatically add their description to the help menu Script.registerArgument( "UserCfg: Cfg FileName with Users as sections containing" "DN, Groups, and other properties as options") Script.parseCommandLine(ignoreErrors=True) args = Script.getExtraCLICFGFiles() from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 testOnly = False errorList = [] for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ("t", "test"): testOnly = True try: usersCFG = CFG().loadFromFile(args[0]) except Exception as e: errorList.append("file open", "Can't parse file %s: %s" % (args[0], str(e))) errorCode = 1 else: if not diracAdmin.csSyncUsersWithCFG(usersCFG): errorList.append(("modify users", "Cannot sync with %s" % args[0])) exitCode = 255 if not exitCode and not testOnly: result = diracAdmin.csCommitChanges() if not result["OK"]: errorList.append(("commit", result["Message"])) exitCode = 255 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def _check(self): """Obtain default configuration and current configuration and print the diff.""" cfg = CFG() templateLocations = self._findConfigTemplates() for templatePath in templateLocations: cfgRes = self._parseConfigTemplate(templatePath, cfg) if cfgRes["OK"]: cfg = cfgRes["Value"] currentCfg = self._getCurrentConfig() if not currentCfg["OK"]: return currentCfg = currentCfg["Value"] diff = currentCfg.getModifications(cfg, ignoreOrder=True, ignoreComments=True) LOG.debug("*" * 80) LOG.debug("Default Configuration: %s" % str(cfg)) LOG.debug("*" * 80) LOG.debug("Current Configuration: %s " % str(currentCfg)) for entry in diff: self._printDiff(entry)
def main(): Script.registerSwitch("t", "test", "Only test. Don't commit changes") Script.parseCommandLine(ignoreErrors=True) args = Script.getExtraCLICFGFiles() if len(args) < 1: Script.showHelp() from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 testOnly = False errorList = [] for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ("t", "test"): testOnly = True try: usersCFG = CFG().loadFromFile(args[0]) except Exception as e: errorList.append("file open", "Can't parse file %s: %s" % (args[0], str(e))) errorCode = 1 else: if not diracAdmin.csSyncUsersWithCFG(usersCFG): errorList.append(("modify users", "Cannot sync with %s" % args[0])) exitCode = 255 if not exitCode and not testOnly: result = diracAdmin.csCommitChanges() if not result['OK']: errorList.append(("commit", result['Message'])) exitCode = 255 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)