class Modificator(object): def __init__(self, rpcClient=False, commiterId="unknown"): self.commiterTag = "@@-" self.commiterId = commiterId self.cfgData = CFG() self.rpcClient = None if rpcClient: self.setRPCClient(rpcClient) def loadCredentials(self): retVal = getProxyInfo() if retVal["OK"]: credDict = retVal["Value"] self.commiterId = "%s@%s - %s" % ( credDict["username"], credDict["group"], datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), ) return retVal return retVal def setRPCClient(self, rpcClient): self.rpcClient = rpcClient def loadFromRemote(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: self.cfgData = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") self.cfgData.loadFromBuffer(zlib.decompress(data).decode()) return retVal def getCFG(self): return self.cfgData def getSections(self, sectionPath): return gConfigurationData.getSectionsFromCFG(sectionPath, self.cfgData) def getComment(self, sectionPath): return gConfigurationData.getCommentFromCFG(sectionPath, self.cfgData) def getOptions(self, sectionPath): return gConfigurationData.getOptionsFromCFG(sectionPath, self.cfgData) def getOptionsDict(self, sectionPath): """Gives the options of a CS section in a Python dict with values as lists""" opts = self.getOptions(sectionPath) pathDict = dict( (o, self.getValue("%s/%s" % (sectionPath, o))) for o in opts) return pathDict def getDictRootedAt(self, relpath="", root=""): """Gives the configuration rooted at path in a Python dict. The result is a Python dictionary that reflects the structure of the config file.""" def getDictRootedAt(path): retval = {} opts = self.getOptionsDict(path) secs = self.getSections(path) for k in opts: retval[k] = opts[k] for i in secs: retval[i] = getDictRootedAt(path + "/" + i) return retval return getDictRootedAt(root + "/" + relpath) def getValue(self, optionPath): return gConfigurationData.extractOptionFromCFG(optionPath, self.cfgData) def sortAlphabetically(self, path, ascending=True): cfg = self.__getParentCFG(path, parentLevel=0) if cfg: if cfg.sortAlphabetically(ascending): self.__setCommiter(path) def __getParentCFG(self, path, parentLevel=1): sectionList = List.fromChar(path, "/") cfg = self.cfgData try: if parentLevel > 0: sectionList = sectionList[:-parentLevel] for section in sectionList: cfg = cfg[section] return cfg except Exception: return False def __setCommiter(self, entryPath, cfg=False): if not cfg: cfg = self.__getParentCFG(entryPath) entry = List.fromChar(entryPath, "/")[-1] comment = cfg.getComment(entry) filteredComment = [ line.strip() for line in comment.split("\n") if line.find(self.commiterTag) != 0 ] filteredComment.append("%s%s" % (self.commiterTag, self.commiterId)) cfg.setComment(entry, "\n".join(filteredComment)) def setOptionValue(self, optionPath, value): levelList = [ level.strip() for level in optionPath.split("/") if level.strip() != "" ] parentPath = "/%s" % "/".join(levelList[:-1]) optionName = List.fromChar(optionPath, "/")[-1] self.createSection(parentPath) cfg = self.__getParentCFG(optionPath) if not cfg: return cfg.setOption(optionName, value) self.__setCommiter(optionPath, cfg) def createSection(self, sectionPath): levelList = [ level.strip() for level in sectionPath.split("/") if level.strip() != "" ] currentPath = "" cfg = self.cfgData createdSection = False for section in levelList: currentPath += "/%s" % section if section not in cfg.listSections(): cfg.createNewSection(section) self.__setCommiter(currentPath) createdSection = True cfg = cfg[section] return createdSection def setComment(self, entryPath, value): cfg = self.__getParentCFG(entryPath) entry = List.fromChar(entryPath, "/")[-1] if cfg.setComment(entry, value): self.__setCommiter(entryPath) return True return False def existsSection(self, sectionPath): sectionList = List.fromChar(sectionPath, "/") cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[section] return len( sectionList) == 0 or sectionList[-1] in cfg.listSections() except Exception: return False def existsOption(self, optionPath): sectionList = List.fromChar(optionPath, "/") cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[section] return sectionList[-1] in cfg.listOptions() except Exception: return False def renameKey(self, path, newName): parentCfg = self.cfgData.getRecursive(path, -1) if not parentCfg: return False pathList = List.fromChar(path, "/") oldName = pathList[-1] if parentCfg["value"].renameKey(oldName, newName): pathList[-1] = newName self.__setCommiter("/%s" % "/".join(pathList)) return True else: return False def copyKey(self, originalKeyPath, newKey): parentCfg = self.cfgData.getRecursive(originalKeyPath, -1) if not parentCfg: return False pathList = List.fromChar(originalKeyPath, "/") originalKey = pathList[-1] if parentCfg["value"].copyKey(originalKey, newKey): self.__setCommiter("/%s/%s" % ("/".join(pathList[:-1]), newKey)) return True return False def removeOption(self, optionPath): if not self.existsOption(optionPath): return False cfg = self.__getParentCFG(optionPath) optionName = List.fromChar(optionPath, "/")[-1] return cfg.deleteKey(optionName) def removeSection(self, sectionPath): if not self.existsSection(sectionPath): return False cfg = self.__getParentCFG(sectionPath) sectionName = List.fromChar(sectionPath, "/")[-1] return cfg.deleteKey(sectionName) def loadFromBuffer(self, data): self.cfgData = CFG() self.cfgData.loadFromBuffer(data) def loadFromFile(self, filename): self.cfgData = CFG() self.mergeFromFile(filename) def dumpToFile(self, filename): with open(filename, "wt") as fd: fd.write(str(self.cfgData)) def mergeFromFile(self, filename): cfg = CFG() cfg.loadFromFile(filename) self.cfgData = self.cfgData.mergeWith(cfg) def mergeFromCFG(self, cfg): self.cfgData = self.cfgData.mergeWith(cfg) def mergeSectionFromCFG(self, sectionPath, cfg): parentDict = self.cfgData.getRecursive(sectionPath, -1) parentCFG = parentDict["value"] secName = [ lev.strip() for lev in sectionPath.split("/") if lev.strip() ][-1] secCFG = parentCFG[secName] if not secCFG: return False mergedCFG = secCFG.mergeWith(cfg) parentCFG.deleteKey(secName) parentCFG.createNewSection(secName, parentDict["comment"], mergedCFG) self.__setCommiter(sectionPath) return True def __str__(self): return str(self.cfgData) def commit(self): compressedData = zlib.compress(str(self.cfgData).encode(), 9) return self.rpcClient.commitNewData(compressedData) def getHistory(self, limit=0): retVal = self.rpcClient.getCommitHistory(limit) if retVal["OK"]: return retVal["Value"] return [] def showCurrentDiff(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") remoteData = zlib.decompress(data).decode().splitlines() localData = str(self.cfgData).splitlines() return difflib.ndiff(remoteData, localData) return [] def getVersionDiff(self, fromDate, toDate): retVal = self.rpcClient.getVersionContents([fromDate, toDate]) if retVal["OK"]: fromData = retVal["Value"][0] if isinstance(fromData, str): fromData = fromData.encode(errors="surrogateescape") fromData = zlib.decompress(fromData).decode() toData = retVal["Value"][1] if isinstance(toData, str): toData = toData.encode(errors="surrogateescape") toData = zlib.decompress(toData).decode() return difflib.ndiff(fromData.split("\n"), toData.split("\n")) return [] def mergeWithServer(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: remoteCFG = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") remoteCFG.loadFromBuffer(zlib.decompress(data).decode()) serverVersion = gConfigurationData.getVersion(remoteCFG) self.cfgData = remoteCFG.mergeWith(self.cfgData) gConfigurationData.setVersion(serverVersion, self.cfgData) return retVal def rollbackToVersion(self, version): return self.rpcClient.rollbackToVersion(version) def updateGConfigurationData(self): gConfigurationData.setRemoteCFG(self.cfgData)
class JobManifest(object): def __init__(self, manifest=""): self.__manifest = CFG() self.__dirty = False self.__ops = False if manifest: result = self.load(manifest) if not result["OK"]: raise Exception(result["Message"]) def isDirty(self): return self.__dirty def setDirty(self): self.__dirty = True def clearDirty(self): self.__dirty = False def load(self, dataString): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadJDL(dataString) else: return self.loadCFG(dataString) def loadJDL(self, jdlString): """ Load job manifest from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result["OK"]: self.__manifest = CFG() return result self.__manifest = result["Value"][0] return S_OK() def loadCFG(self, cfgString): """ Load job manifest from CFG format """ try: self.__manifest.loadFromBuffer(cfgString) except Exception as e: return S_ERROR("Can't load manifest from cfg: %s" % str(e)) return S_OK() def dumpAsCFG(self): return str(self.__manifest) def getAsCFG(self): return self.__manifest.clone() def dumpAsJDL(self): return dumpCFGAsJDL(self.__manifest) def __getCSValue(self, varName, defaultVal=None): if not self.__ops: self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"]) if varName[0] != "/": varName = "JobDescription/%s" % varName return self.__ops.getValue(varName, defaultVal) def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal): """ Check a numerical var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue try: varValue = int(varValue) except ValueError: return S_ERROR("%s must be a number" % varName) minVal = self.__getCSValue("Min%s" % varName, minVal) maxVal = self.__getCSValue("Max%s" % varName, maxVal) varValue = max(minVal, min(varValue, maxVal)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkChoiceVar(self, varName, defaultVal, choices): """ Check a choice var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue if varValue not in self.__getCSValue("Choices%s" % varName, choices): return S_ERROR("%s is not a valid value for %s" % (varValue, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMultiChoice(self, varName, choices): """ Check a multi choice var """ initialVal = False if varName not in self.__manifest: return S_OK() else: varValue = self.__manifest[varName] initialVal = varValue choices = self.__getCSValue("Choices%s" % varName, choices) for v in List.fromChar(varValue): if v not in choices: return S_ERROR("%s is not a valid value for %s" % (v, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMaxInputData(self, maxNumber): """ Check Maximum Number of Input Data files allowed """ varName = "InputData" if varName not in self.__manifest: return S_OK() varValue = self.__manifest[varName] if len(List.fromChar(varValue)) > maxNumber: return S_ERROR( "Number of Input Data Files (%s) greater than current limit: %s" % (len(List.fromChar(varValue)), maxNumber)) return S_OK() def __contains__(self, key): """Check if the manifest has the required key""" return key in self.__manifest def setOptionsFromDict(self, varDict): for k in sorted(varDict): self.setOption(k, varDict[k]) def check(self): """ Check that the manifest is OK """ for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]: if k not in self.__manifest: return S_ERROR("Missing var %s in manifest" % k) # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) if not result["OK"]: return result result = self.__checkNumericalVar("Priority", 1, 0, 10) if not result["OK"]: return result maxInputData = Operations().getValue("JobDescription/MaxInputData", 500) result = self.__checkMaxInputData(maxInputData) if not result["OK"]: return result operation = Operations(group=self.__manifest["OwnerGroup"]) allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"]) transformationTypes = operation.getValue( "Transformations/DataProcessing", []) result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes) if not result["OK"]: return result return S_OK() def createSection(self, secName, contents=False): if secName not in self.__manifest: if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True return S_OK( self.__manifest.createNewSection(secName, contents=contents)) return S_ERROR("Section %s already exists" % secName) def getSection(self, secName): self.__dirty = True if secName not in self.__manifest: return S_ERROR("%s does not exist" % secName) sec = self.__manifest[secName] if not sec: return S_ERROR("%s section empty" % secName) return S_OK(sec) def setSectionContents(self, secName, contents): if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True if secName in self.__manifest: self.__manifest[secName].reset() self.__manifest[secName].mergeWith(contents) else: self.__manifest.createNewSection(secName, contents=contents) def setOption(self, varName, varValue): """ Set a var in job manifest """ self.__dirty = True levels = List.fromChar(varName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: cfg.createNewSection(l) cfg = cfg[l] cfg.setOption(levels[-1], varValue) def remove(self, opName): levels = List.fromChar(opName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: return S_ERROR("%s does not exist" % opName) cfg = cfg[l] if cfg.deleteKey(levels[-1]): self.__dirty = True return S_OK() return S_ERROR("%s does not exist" % opName) def getOption(self, varName, defaultValue=None): """ Get a variable from the job manifest """ cfg = self.__manifest return cfg.getOption(varName, defaultValue) def getOptionList(self, section=""): """ Get a list of variables in a section of the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listOptions() def isOption(self, opName): """ Check if it is a valid option """ return self.__manifest.isOption(opName) def getSectionList(self, section=""): """ Get a list of sections in the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listSections()