def setUp(self): self.authMgr = AuthManager("/Systems/Service/Authorization") cfg = CFG() cfg.loadFromBuffer(testSystemsCFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(testRegistryCFG) gConfig.loadCFG(cfg) self.noAuthCredDict = {"group": "group_test"} self.userCredDict = { "DN": "/User/test/DN/CN=userA", "group": "group_test" } self.suspendedOtherVOUserCredDict = { "DN": "/User/test/DN/CN=userS", "group": "group_test_other" } self.badUserCredDict = { "DN": "/User/test/DN/CN=userB", "group": "group_bad" } self.suspendedUserCredDict = { "DN": "/User/test/DN/CN=userS", "group": "group_test" } self.hostCredDict = { "DN": "/User/test/DN/CN=test.hostA.ch", "group": "hosts" } self.badHostCredDict = { "DN": "/User/test/DN/CN=test.hostB.ch", "group": "hosts" }
def mergeWithServer(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: remoteCFG = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") remoteCFG.loadFromBuffer(zlib.decompress(data).decode()) serverVersion = gConfigurationData.getVersion(remoteCFG) self.cfgData = remoteCFG.mergeWith(self.cfgData) gConfigurationData.setVersion(serverVersion, self.cfgData) return retVal
def setUp(self): cfg = CFG() cfg.loadFromBuffer(diracTestCACFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(userCFG) gConfig.loadCFG(cfg) result = ProxyProviderFactory().getProxyProvider('DIRAC_TEST_CA') self.assertTrue( result['OK'], '\n%s' % result.get('Message') or 'Error message is absent.') self.pp = result['Value']
def setUp(self): self.authMgr = AuthManager('/Systems/Service/Authorization') cfg = CFG() cfg.loadFromBuffer(testSystemsCFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(testRegistryCFG) gConfig.loadCFG(cfg) self.noAuthCredDict = {'group': 'group_test'} self.userCredDict = {'DN': '/User/test/DN/CN=userA', 'group': 'group_test'} self.suspendedOtherVOUserCredDict = {'DN': '/User/test/DN/CN=userS', 'group': 'group_test_other'} self.badUserCredDict = {'DN': '/User/test/DN/CN=userB', 'group': 'group_bad'} self.suspendedUserCredDict = {'DN': '/User/test/DN/CN=userS', 'group': 'group_test'} self.hostCredDict = {'DN': '/User/test/DN/CN=test.hostA.ch', 'group': 'hosts'} self.badHostCredDict = {'DN': '/User/test/DN/CN=test.hostB.ch', 'group': 'hosts'}
class Modificator(object): def __init__(self, rpcClient=False, commiterId="unknown"): self.commiterTag = "@@-" self.commiterId = commiterId self.cfgData = CFG() self.rpcClient = None if rpcClient: self.setRPCClient(rpcClient) def loadCredentials(self): retVal = getProxyInfo() if retVal["OK"]: credDict = retVal["Value"] self.commiterId = "%s@%s - %s" % ( credDict["username"], credDict["group"], datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), ) return retVal return retVal def setRPCClient(self, rpcClient): self.rpcClient = rpcClient def loadFromRemote(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: self.cfgData = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") self.cfgData.loadFromBuffer(zlib.decompress(data).decode()) return retVal def getCFG(self): return self.cfgData def getSections(self, sectionPath): return gConfigurationData.getSectionsFromCFG(sectionPath, self.cfgData) def getComment(self, sectionPath): return gConfigurationData.getCommentFromCFG(sectionPath, self.cfgData) def getOptions(self, sectionPath): return gConfigurationData.getOptionsFromCFG(sectionPath, self.cfgData) def getOptionsDict(self, sectionPath): """Gives the options of a CS section in a Python dict with values as lists""" opts = self.getOptions(sectionPath) pathDict = dict( (o, self.getValue("%s/%s" % (sectionPath, o))) for o in opts) return pathDict def getDictRootedAt(self, relpath="", root=""): """Gives the configuration rooted at path in a Python dict. The result is a Python dictionary that reflects the structure of the config file.""" def getDictRootedAt(path): retval = {} opts = self.getOptionsDict(path) secs = self.getSections(path) for k in opts: retval[k] = opts[k] for i in secs: retval[i] = getDictRootedAt(path + "/" + i) return retval return getDictRootedAt(root + "/" + relpath) def getValue(self, optionPath): return gConfigurationData.extractOptionFromCFG(optionPath, self.cfgData) def sortAlphabetically(self, path, ascending=True): cfg = self.__getParentCFG(path, parentLevel=0) if cfg: if cfg.sortAlphabetically(ascending): self.__setCommiter(path) def __getParentCFG(self, path, parentLevel=1): sectionList = List.fromChar(path, "/") cfg = self.cfgData try: if parentLevel > 0: sectionList = sectionList[:-parentLevel] for section in sectionList: cfg = cfg[section] return cfg except Exception: return False def __setCommiter(self, entryPath, cfg=False): if not cfg: cfg = self.__getParentCFG(entryPath) entry = List.fromChar(entryPath, "/")[-1] comment = cfg.getComment(entry) filteredComment = [ line.strip() for line in comment.split("\n") if line.find(self.commiterTag) != 0 ] filteredComment.append("%s%s" % (self.commiterTag, self.commiterId)) cfg.setComment(entry, "\n".join(filteredComment)) def setOptionValue(self, optionPath, value): levelList = [ level.strip() for level in optionPath.split("/") if level.strip() != "" ] parentPath = "/%s" % "/".join(levelList[:-1]) optionName = List.fromChar(optionPath, "/")[-1] self.createSection(parentPath) cfg = self.__getParentCFG(optionPath) if not cfg: return cfg.setOption(optionName, value) self.__setCommiter(optionPath, cfg) def createSection(self, sectionPath): levelList = [ level.strip() for level in sectionPath.split("/") if level.strip() != "" ] currentPath = "" cfg = self.cfgData createdSection = False for section in levelList: currentPath += "/%s" % section if section not in cfg.listSections(): cfg.createNewSection(section) self.__setCommiter(currentPath) createdSection = True cfg = cfg[section] return createdSection def setComment(self, entryPath, value): cfg = self.__getParentCFG(entryPath) entry = List.fromChar(entryPath, "/")[-1] if cfg.setComment(entry, value): self.__setCommiter(entryPath) return True return False def existsSection(self, sectionPath): sectionList = List.fromChar(sectionPath, "/") cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[section] return len( sectionList) == 0 or sectionList[-1] in cfg.listSections() except Exception: return False def existsOption(self, optionPath): sectionList = List.fromChar(optionPath, "/") cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[section] return sectionList[-1] in cfg.listOptions() except Exception: return False def renameKey(self, path, newName): parentCfg = self.cfgData.getRecursive(path, -1) if not parentCfg: return False pathList = List.fromChar(path, "/") oldName = pathList[-1] if parentCfg["value"].renameKey(oldName, newName): pathList[-1] = newName self.__setCommiter("/%s" % "/".join(pathList)) return True else: return False def copyKey(self, originalKeyPath, newKey): parentCfg = self.cfgData.getRecursive(originalKeyPath, -1) if not parentCfg: return False pathList = List.fromChar(originalKeyPath, "/") originalKey = pathList[-1] if parentCfg["value"].copyKey(originalKey, newKey): self.__setCommiter("/%s/%s" % ("/".join(pathList[:-1]), newKey)) return True return False def removeOption(self, optionPath): if not self.existsOption(optionPath): return False cfg = self.__getParentCFG(optionPath) optionName = List.fromChar(optionPath, "/")[-1] return cfg.deleteKey(optionName) def removeSection(self, sectionPath): if not self.existsSection(sectionPath): return False cfg = self.__getParentCFG(sectionPath) sectionName = List.fromChar(sectionPath, "/")[-1] return cfg.deleteKey(sectionName) def loadFromBuffer(self, data): self.cfgData = CFG() self.cfgData.loadFromBuffer(data) def loadFromFile(self, filename): self.cfgData = CFG() self.mergeFromFile(filename) def dumpToFile(self, filename): with open(filename, "wt") as fd: fd.write(str(self.cfgData)) def mergeFromFile(self, filename): cfg = CFG() cfg.loadFromFile(filename) self.cfgData = self.cfgData.mergeWith(cfg) def mergeFromCFG(self, cfg): self.cfgData = self.cfgData.mergeWith(cfg) def mergeSectionFromCFG(self, sectionPath, cfg): parentDict = self.cfgData.getRecursive(sectionPath, -1) parentCFG = parentDict["value"] secName = [ lev.strip() for lev in sectionPath.split("/") if lev.strip() ][-1] secCFG = parentCFG[secName] if not secCFG: return False mergedCFG = secCFG.mergeWith(cfg) parentCFG.deleteKey(secName) parentCFG.createNewSection(secName, parentDict["comment"], mergedCFG) self.__setCommiter(sectionPath) return True def __str__(self): return str(self.cfgData) def commit(self): compressedData = zlib.compress(str(self.cfgData).encode(), 9) return self.rpcClient.commitNewData(compressedData) def getHistory(self, limit=0): retVal = self.rpcClient.getCommitHistory(limit) if retVal["OK"]: return retVal["Value"] return [] def showCurrentDiff(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") remoteData = zlib.decompress(data).decode().splitlines() localData = str(self.cfgData).splitlines() return difflib.ndiff(remoteData, localData) return [] def getVersionDiff(self, fromDate, toDate): retVal = self.rpcClient.getVersionContents([fromDate, toDate]) if retVal["OK"]: fromData = retVal["Value"][0] if isinstance(fromData, str): fromData = fromData.encode(errors="surrogateescape") fromData = zlib.decompress(fromData).decode() toData = retVal["Value"][1] if isinstance(toData, str): toData = toData.encode(errors="surrogateescape") toData = zlib.decompress(toData).decode() return difflib.ndiff(fromData.split("\n"), toData.split("\n")) return [] def mergeWithServer(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: remoteCFG = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") remoteCFG.loadFromBuffer(zlib.decompress(data).decode()) serverVersion = gConfigurationData.getVersion(remoteCFG) self.cfgData = remoteCFG.mergeWith(self.cfgData) gConfigurationData.setVersion(serverVersion, self.cfgData) return retVal def rollbackToVersion(self, version): return self.rpcClient.rollbackToVersion(version) def updateGConfigurationData(self): gConfigurationData.setRemoteCFG(self.cfgData)
class ConfigurationData(object): def __init__(self, loadDefaultCFG=True): envVar = os.environ.get("DIRAC_FEWER_CFG_LOCKS", "no").lower() self.__locksEnabled = envVar not in ("y", "yes", "t", "true", "on", "1") if self.__locksEnabled: lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.__compressedConfigurationData = None self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup") self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg") gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile) retVal = self.loadFile(defaultCFGFile) if not retVal["OK"]: gLogger.warn("Can't load %s file" % defaultCFGFile) self.sync() def getBackupDir(self): return self.backupsDir def sync(self): gLogger.debug("Updating configuration internals") self.mergedCFG = self.remoteCFG.mergeWith(self.localCFG) self.remoteServerList = [] localServers = self.extractOptionFromCFG("%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones=True) if localServers: self.remoteServerList.extend(List.fromChar(localServers, ",")) remoteServers = self.extractOptionFromCFG("%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones=True) if remoteServers: self.remoteServerList.extend(List.fromChar(remoteServers, ",")) self.remoteServerList = List.uniqueElements(self.remoteServerList) self.__compressedConfigurationData = None def loadFile(self, fileName): try: fileCFG = CFG() fileCFG.loadFromFile(fileName) except IOError: self.localCFG = self.localCFG.mergeWith(fileCFG) return S_ERROR("Can't load a cfg file '%s'" % fileName) return self.mergeWithLocal(fileCFG) def mergeWithLocal(self, extraCFG): self.lock() try: self.localCFG = self.localCFG.mergeWith(extraCFG) self.unlock() gLogger.debug("CFG merged") except Exception as e: self.unlock() return S_ERROR("Cannot merge with new cfg: %s" % str(e)) self.sync() return S_OK() def loadRemoteCFGFromCompressedMem(self, data): if six.PY3 and isinstance(data, str): data = data.encode(errors="surrogateescape") sUncompressedData = zlib.decompress(data).decode() self.loadRemoteCFGFromMem(sUncompressedData) def loadRemoteCFGFromMem(self, data): self.lock() self.remoteCFG.loadFromBuffer(data) self.unlock() self.sync() def loadConfigurationData(self, fileName=False): name = self.getName() self.lock() try: if not fileName: fileName = "%s.cfg" % name if fileName[0] != "/": fileName = os.path.join(DIRAC.rootPath, "etc", fileName) self.remoteCFG.loadFromFile(fileName) except Exception as e: print(e) self.unlock() self.sync() def getCommentFromCFG(self, path, cfg=False): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: cfg = cfg[section] return self.dangerZoneEnd(cfg.getComment(levelList[-1])) except Exception: pass return self.dangerZoneEnd(None) def getSectionsFromCFG(self, path, cfg=False, ordered=False): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList: cfg = cfg[section] return self.dangerZoneEnd(cfg.listSections(ordered)) except Exception: pass return self.dangerZoneEnd(None) def getOptionsFromCFG(self, path, cfg=False, ordered=False): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList: cfg = cfg[section] return self.dangerZoneEnd(cfg.listOptions(ordered)) except Exception: pass return self.dangerZoneEnd(None) def extractOptionFromCFG(self, path, cfg=False, disableDangerZones=False): if not cfg: cfg = self.mergedCFG if not disableDangerZones: self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: cfg = cfg[section] if levelList[-1] in cfg.listOptions(): return self.dangerZoneEnd(cfg[levelList[-1]]) except Exception: pass if not disableDangerZones: self.dangerZoneEnd() def setOptionInCFG(self, path, value, cfg=False, disableDangerZones=False): if not cfg: cfg = self.localCFG if not disableDangerZones: self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: if section not in cfg.listSections(): cfg.createNewSection(section) cfg = cfg[section] cfg.setOption(levelList[-1], value) finally: if not disableDangerZones: self.dangerZoneEnd() self.sync() def deleteOptionInCFG(self, path, cfg=False): if not cfg: cfg = self.localCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: if section not in cfg.listSections(): return cfg = cfg[section] cfg.deleteKey(levelList[-1]) finally: self.dangerZoneEnd() self.sync() def generateNewVersion(self): self.setVersion(Time.toString()) self.sync() gLogger.info("Generated new version %s" % self.getVersion()) def setVersion(self, version, cfg=False): if not cfg: cfg = self.remoteCFG self.setOptionInCFG("%s/Version" % self.configurationPath, version, cfg) def getVersion(self, cfg=False): if not cfg: cfg = self.remoteCFG value = self.extractOptionFromCFG( "%s/Version" % self.configurationPath, cfg) if value: return value return "0" def getName(self): return self.extractOptionFromCFG("%s/Name" % self.configurationPath, self.mergedCFG) def exportName(self): return self.setOptionInCFG("%s/Name" % self.configurationPath, self.getName(), self.remoteCFG) def getRefreshTime(self): try: return int( self.extractOptionFromCFG( "%s/RefreshTime" % self.configurationPath, self.mergedCFG)) except Exception: return 300 def getPropagationTime(self): try: return int( self.extractOptionFromCFG( "%s/PropagationTime" % self.configurationPath, self.mergedCFG)) except Exception: return 300 def getSlavesGraceTime(self): try: return int( self.extractOptionFromCFG( "%s/SlavesGraceTime" % self.configurationPath, self.mergedCFG)) except Exception: return 600 def mergingEnabled(self): try: val = self.extractOptionFromCFG( "%s/EnableAutoMerge" % self.configurationPath, self.mergedCFG) return val.lower() in ("yes", "true", "y") except Exception: return False def getAutoPublish(self): value = self.extractOptionFromCFG( "%s/AutoPublish" % self.configurationPath, self.localCFG) if value and value.lower() in ("no", "false", "n"): return False else: return True def getAutoSlaveSync(self): value = self.extractOptionFromCFG( "%s/AutoSlaveSync" % self.configurationPath, self.localCFG) if value and value.lower() in ("no", "false", "n"): return False else: return True def getServers(self): return list(self.remoteServerList) def getConfigurationGateway(self): return self.extractOptionFromCFG("/DIRAC/Gateway", self.localCFG) def setServers(self, sServers): self.setOptionInCFG("%s/Servers" % self.configurationPath, sServers, self.remoteCFG) self.sync() def deleteLocalOption(self, optionPath): self.deleteOptionInCFG(optionPath, self.localCFG) def getMasterServer(self): return self.extractOptionFromCFG( "%s/MasterServer" % self.configurationPath, self.remoteCFG) def setMasterServer(self, sURL): self.setOptionInCFG("%s/MasterServer" % self.configurationPath, sURL, self.remoteCFG) self.sync() def getCompressedData(self): if self.__compressedConfigurationData is None: self.__compressedConfigurationData = zlib.compress( str(self.remoteCFG).encode(), 9) return self.__compressedConfigurationData def isMaster(self): value = self.extractOptionFromCFG("%s/Master" % self.configurationPath, self.localCFG) if value and value.lower() in ("yes", "true", "y"): return True else: return False def getServicesPath(self): return "/Services" def setAsService(self): self._isService = True def isService(self): return self._isService def useServerCertificate(self): value = self.extractOptionFromCFG( "/DIRAC/Security/UseServerCertificate") if value and value.lower() in ("y", "yes", "true"): return True return False def skipCACheck(self): value = self.extractOptionFromCFG("/DIRAC/Security/SkipCAChecks") if value and value.lower() in ("y", "yes", "true"): return True return False def dumpLocalCFGToFile(self, fileName): try: with open(fileName, "w") as fd: fd.write(str(self.localCFG)) gLogger.verbose("Configuration file dumped", "'%s'" % fileName) except IOError: gLogger.error("Can't dump cfg file", "'%s'" % fileName) return S_ERROR("Can't dump cfg file '%s'" % fileName) return S_OK() def getRemoteCFG(self): return self.remoteCFG def getMergedCFGAsString(self): return str(self.mergedCFG) def dumpRemoteCFGToFile(self, fileName): with open(fileName, "w") as fd: fd.write(str(self.remoteCFG)) def __backupCurrentConfiguration(self, backupName): configurationFilename = "%s.cfg" % self.getName() configurationFile = os.path.join(DIRAC.rootPath, "etc", configurationFilename) today = Time.date() backupPath = os.path.join(self.getBackupDir(), str(today.year), "%02d" % today.month) mkDir(backupPath) backupFile = os.path.join( backupPath, configurationFilename.replace(".cfg", ".%s.zip" % backupName)) if os.path.isfile(configurationFile): gLogger.info("Making a backup of configuration in %s" % backupFile) try: with zipfile.ZipFile(backupFile, "w", zipfile.ZIP_DEFLATED) as zf: zf.write( configurationFile, "%s.backup.%s" % (os.path.split(configurationFile)[1], backupName)) except Exception: gLogger.exception() gLogger.error("Cannot backup configuration data file", "file %s" % backupFile) else: gLogger.warn("CS data file does not exist", configurationFile) def writeRemoteConfigurationToDisk(self, backupName=False): configurationFile = os.path.join(DIRAC.rootPath, "etc", "%s.cfg" % self.getName()) try: with open(configurationFile, "w") as fd: fd.write(str(self.remoteCFG)) except Exception as e: gLogger.fatal( "Cannot write new configuration to disk!", "file %s exception %s" % (configurationFile, repr(e))) return S_ERROR("Can't write cs file %s!: %s" % (configurationFile, repr(e).replace(",)", ")"))) if backupName: self.__backupCurrentConfiguration(backupName) return S_OK() def setRemoteCFG(self, cfg, disableSync=False): self.remoteCFG = cfg.clone() if not disableSync: self.sync() def lock(self): """ Locks Event to prevent further threads from reading. Stops current thread until no other thread is accessing. PRIVATE USE """ if not self.__locksEnabled: return self.threadingEvent.clear() while self.runningThreadsNumber > 0: time.sleep(0.1) def unlock(self): """ Unlocks Event. PRIVATE USE """ if not self.__locksEnabled: return self.threadingEvent.set() def dangerZoneStart(self): """ Start of danger zone. This danger zone may be or may not be a mutual exclusion zone. Counter is maintained to know how many threads are inside and be able to enable and disable mutual exclusion. PRIVATE USE """ if not self.__locksEnabled: return self.threadingEvent.wait() self.threadingLock.acquire() self.runningThreadsNumber += 1 try: self.threadingLock.release() except thread.error: pass def dangerZoneEnd(self, returnValue=None): """ End of danger zone. PRIVATE USE """ if not self.__locksEnabled: return returnValue self.threadingLock.acquire() self.runningThreadsNumber -= 1 try: self.threadingLock.release() except thread.error: pass return returnValue
cfg.loadFromBuffer(""" DIRAC { Security { Authorization { issuer = https://issuer.url/ Clients { DIRACWeb { client_id = client_identificator client_secret = client_secret_key redirect_uri = https://redirect.url/ } } } } } Resources { IdProviders { SomeIdP { ProviderType = OAuth2 issuer = https://idp.url/ client_id = IdP_client_id client_secret = IdP_client_secret redirect_uri = https://dirac/redirect jwks_uri = https://idp.url/jwk scope = openid+profile+offline_access+eduperson_entitlement } } } """)
mergedCFG.loadFromBuffer(""" DIRAC { Setup=TestSetup Setups { TestSetup { WorkloadManagement=MyWM } } } Systems { WorkloadManagement { MyWM { URLs { Service1 = dips://server1:1234/WorkloadManagement/Service1 Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2 } FailoverURLs { Service2 = dips://failover1:5678/WorkloadManagement/Service2 Service2 += dips://failover2:5678/WorkloadManagement/Service2 } } } } Operations{ Defaults { MainServers = gw1, gw2 } } """)
class JobManifest(object): def __init__(self, manifest=""): self.__manifest = CFG() self.__dirty = False self.__ops = False if manifest: result = self.load(manifest) if not result["OK"]: raise Exception(result["Message"]) def isDirty(self): return self.__dirty def setDirty(self): self.__dirty = True def clearDirty(self): self.__dirty = False def load(self, dataString): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadJDL(dataString) else: return self.loadCFG(dataString) def loadJDL(self, jdlString): """ Load job manifest from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result["OK"]: self.__manifest = CFG() return result self.__manifest = result["Value"][0] return S_OK() def loadCFG(self, cfgString): """ Load job manifest from CFG format """ try: self.__manifest.loadFromBuffer(cfgString) except Exception as e: return S_ERROR("Can't load manifest from cfg: %s" % str(e)) return S_OK() def dumpAsCFG(self): return str(self.__manifest) def getAsCFG(self): return self.__manifest.clone() def dumpAsJDL(self): return dumpCFGAsJDL(self.__manifest) def __getCSValue(self, varName, defaultVal=None): if not self.__ops: self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"]) if varName[0] != "/": varName = "JobDescription/%s" % varName return self.__ops.getValue(varName, defaultVal) def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal): """ Check a numerical var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue try: varValue = int(varValue) except ValueError: return S_ERROR("%s must be a number" % varName) minVal = self.__getCSValue("Min%s" % varName, minVal) maxVal = self.__getCSValue("Max%s" % varName, maxVal) varValue = max(minVal, min(varValue, maxVal)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkChoiceVar(self, varName, defaultVal, choices): """ Check a choice var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue if varValue not in self.__getCSValue("Choices%s" % varName, choices): return S_ERROR("%s is not a valid value for %s" % (varValue, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMultiChoice(self, varName, choices): """ Check a multi choice var """ initialVal = False if varName not in self.__manifest: return S_OK() else: varValue = self.__manifest[varName] initialVal = varValue choices = self.__getCSValue("Choices%s" % varName, choices) for v in List.fromChar(varValue): if v not in choices: return S_ERROR("%s is not a valid value for %s" % (v, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMaxInputData(self, maxNumber): """ Check Maximum Number of Input Data files allowed """ varName = "InputData" if varName not in self.__manifest: return S_OK() varValue = self.__manifest[varName] if len(List.fromChar(varValue)) > maxNumber: return S_ERROR( "Number of Input Data Files (%s) greater than current limit: %s" % (len(List.fromChar(varValue)), maxNumber)) return S_OK() def __contains__(self, key): """Check if the manifest has the required key""" return key in self.__manifest def setOptionsFromDict(self, varDict): for k in sorted(varDict): self.setOption(k, varDict[k]) def check(self): """ Check that the manifest is OK """ for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]: if k not in self.__manifest: return S_ERROR("Missing var %s in manifest" % k) # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) if not result["OK"]: return result result = self.__checkNumericalVar("Priority", 1, 0, 10) if not result["OK"]: return result maxInputData = Operations().getValue("JobDescription/MaxInputData", 500) result = self.__checkMaxInputData(maxInputData) if not result["OK"]: return result operation = Operations(group=self.__manifest["OwnerGroup"]) allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"]) transformationTypes = operation.getValue( "Transformations/DataProcessing", []) result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes) if not result["OK"]: return result return S_OK() def createSection(self, secName, contents=False): if secName not in self.__manifest: if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True return S_OK( self.__manifest.createNewSection(secName, contents=contents)) return S_ERROR("Section %s already exists" % secName) def getSection(self, secName): self.__dirty = True if secName not in self.__manifest: return S_ERROR("%s does not exist" % secName) sec = self.__manifest[secName] if not sec: return S_ERROR("%s section empty" % secName) return S_OK(sec) def setSectionContents(self, secName, contents): if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True if secName in self.__manifest: self.__manifest[secName].reset() self.__manifest[secName].mergeWith(contents) else: self.__manifest.createNewSection(secName, contents=contents) def setOption(self, varName, varValue): """ Set a var in job manifest """ self.__dirty = True levels = List.fromChar(varName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: cfg.createNewSection(l) cfg = cfg[l] cfg.setOption(levels[-1], varValue) def remove(self, opName): levels = List.fromChar(opName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: return S_ERROR("%s does not exist" % opName) cfg = cfg[l] if cfg.deleteKey(levels[-1]): self.__dirty = True return S_OK() return S_ERROR("%s does not exist" % opName) def getOption(self, varName, defaultValue=None): """ Get a variable from the job manifest """ cfg = self.__manifest return cfg.getOption(varName, defaultValue) def getOptionList(self, section=""): """ Get a list of variables in a section of the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listOptions() def isOption(self, opName): """ Check if it is a valid option """ return self.__manifest.isOption(opName) def getSectionList(self, section=""): """ Get a list of sections in the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listSections()
def setUpClass(cls): cls.failed = False # Add configuration cfg = CFG() cfg.loadFromBuffer(diracTestCACFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(userCFG) gConfig.loadCFG(cfg) # Prepare CA lines = [] cfgDict = {} cls.caPath = os.path.join(certsPath, "ca") cls.caConfigFile = os.path.join(cls.caPath, "openssl_config_ca.cnf") # Save original configuration file shutil.copyfile(cls.caConfigFile, cls.caConfigFile + "bak") # Parse fields = [ "dir", "database", "serial", "new_certs_dir", "private_key", "certificate" ] with open(cls.caConfigFile, "r") as caCFG: for line in caCFG: if re.findall("=", re.sub(r"#.*", "", line)): field = re.sub(r"#.*", "", line).replace(" ", "").rstrip().split("=")[0] line = "dir = %s #PUT THE RIGHT DIR HERE!\n" % ( cls.caPath) if field == "dir" else line val = re.sub(r"#.*", "", line).replace(" ", "").rstrip().split("=")[1] if field in fields: for i in fields: if cfgDict.get(i): val = val.replace("$%s" % i, cfgDict[i]) cfgDict[field] = val if not cfgDict[field]: cls.failed = "%s have empty value in %s" % ( field, cls.caConfigFile) lines.append(line) with open(cls.caConfigFile, "w") as caCFG: caCFG.writelines(lines) for field in fields: if field not in cfgDict.keys(): cls.failed = "%s value is absent in %s" % (field, cls.caConfigFile) cls.hostCert = os.path.join(certsPath, "host/hostcert.pem") cls.hostKey = os.path.join(certsPath, "host/hostkey.pem") cls.caCert = cfgDict["certificate"] cls.caKey = cfgDict["private_key"] os.chmod(cls.caKey, stat.S_IREAD) # Check directory for new certificates cls.newCertDir = cfgDict["new_certs_dir"] if not os.path.exists(cls.newCertDir): os.makedirs(cls.newCertDir) for f in os.listdir(cls.newCertDir): os.remove(os.path.join(cls.newCertDir, f)) # Empty the certificate database cls.index = cfgDict["database"] with open(cls.index, "w") as indx: indx.write("") # Write down serial cls.serial = cfgDict["serial"] with open(cls.serial, "w") as serialFile: serialFile.write("1000") # Create temporaly directory for users certificates cls.userDir = tempfile.mkdtemp(dir=certsPath) # Create user certificates for userName in ["no_user", "user", "user_1", "user_2", "user_3"]: userConf = """[ req ] default_bits = 4096 encrypt_key = yes distinguished_name = req_dn prompt = no req_extensions = v3_req [ req_dn ] C = CC O = DN 0.O = DIRAC CN = %s [ v3_req ] # Extensions for client certificates (`man x509v3_config`). nsComment = "OpenSSL Generated Client Certificate" keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth """ % (userName) userConfFile = os.path.join(cls.userDir, userName + ".cnf") userReqFile = os.path.join(cls.userDir, userName + ".req") userKeyFile = os.path.join(cls.userDir, userName + ".key.pem") userCertFile = os.path.join(cls.userDir, userName + ".cert.pem") with open(userConfFile, "w") as f: f.write(userConf) status, output = commands.getstatusoutput( "openssl genrsa -out %s" % userKeyFile) if status: gLogger.error(output) exit() gLogger.debug(output) os.chmod(userKeyFile, stat.S_IREAD) status, output = commands.getstatusoutput( "openssl req -config %s -key %s -new -out %s" % (userConfFile, userKeyFile, userReqFile)) if status: gLogger.error(output) exit() gLogger.debug(output) cmd = "openssl ca -config %s -extensions usr_cert -batch -days 375 -in %s -out %s" cmd = cmd % (cls.caConfigFile, userReqFile, userCertFile) status, output = commands.getstatusoutput(cmd) if status: gLogger.error(output) exit() gLogger.debug(output) # Result status, output = commands.getstatusoutput("ls -al %s" % cls.userDir) if status: gLogger.error(output) exit() gLogger.debug("User certificates:\n", output)
mergedCFG.loadFromBuffer( """ Resources { ProxyProviders { DIRAC_TEST_CA { ProviderType = DIRACCA CertFile = %s KeyFile = %s Match = Supplied = C, O, OU, CN Optional = emailAddress DNOrder = C, O, OU, CN, emailAddress C = FR O = DIRAC OU = DIRAC TEST } } } Registry { Users { testuser { DN = /C=FR/O=DIRAC/OU=DIRAC TEST/CN=DIRAC test user/[email protected] } } Groups { dirac_user { Users = testuser } dirac_no_user { Users = nouser } } } """ % (os.path.join(certsPath, "ca/ca.cert.pem"), os.path.join(certsPath, "ca/ca.key.pem")) )