def execute( self ):
""" execute
"""
stopAgents = self.findStopAgents()[ 'Value' ]
if stopAgents:
self.log.info( 'Aborting, there are stop_agents to be picked' )
return S_OK()
pilotVersion = self.opHelper.getValue( 'Pilot/Version', '' )
if not pilotVersion:
self.log.error( 'There is no pilot version on the CS' )
return S_OK()
pilotVersion = self.getNewestPilotVersion()
if not pilotVersion[ 'OK' ]:
self.log.error( pilotVersion[ 'Message' ] )
return S_ERROR( pilotVersion[ 'Message' ] )
pilotVersion = pilotVersion[ 'Value' ]
localCFG = CFG()
#load local CFG
localCFG.loadFromFile( self.cfgToUpdate )
releaseVersion = localCFG.getRecursive( 'LocalSite/ReleaseVersion' )[ 'value' ]
self.log.info( 'PilotVersion : %s' % pilotVersion )
self.log.info( 'ReleaseVersion : %s' % releaseVersion )
if LooseVersion( pilotVersion ) > LooseVersion( releaseVersion ):
self.log.info( 'UPDATING %s > %s' % ( pilotVersion, releaseVersion ) )
localCFG.setOption( 'LocalSite/ReleaseVersion', pilotVersion )
localCFG.writeToFile( self.cfgToUpdate )
self.touchStopAgents()
else:
self.log.info( 'Nothing to do' )
return S_OK()
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result["OK"]:
raise Exception(result["Message"])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result["OK"]:
self.__manifest = CFG()
return result
self.__manifest = result["Value"][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def dumpAsCFG(self):
return str(self.__manifest)
def getAsCFG(self):
return self.__manifest.clone()
def dumpAsJDL(self):
return dumpCFGAsJDL(self.__manifest)
def __getCSValue(self, varName, defaultVal=None):
if not self.__ops:
self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"])
if varName[0] != "/":
varName = "JobDescription/%s" % varName
return self.__ops.getValue(varName, defaultVal)
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
"""
Check a numerical var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
try:
varValue = long(varValue)
except:
return S_ERROR("%s must be a number" % varName)
minVal = self.__getCSValue("Min%s" % varName, minVal)
maxVal = self.__getCSValue("Max%s" % varName, maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVar(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
if varValue not in self.__getCSValue("Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoice(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__manifest:
return S_OK()
else:
varValue = self.__manifest[varName]
initialVal = varValue
choices = self.__getCSValue("Choices%s" % varName, choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
"Number of Input Data Files (%s) greater than current limit: %s"
% (len(List.fromChar(varValue)), maxNumber)
)
return S_OK()
def __contains__(self, key):
""" Check if the manifest has the required key
"""
return key in self.__manifest
def setOptionsFromDict(self, varDict):
for k in sorted(varDict):
self.setOption(k, varDict[k])
def check(self):
"""
Check that the manifest is OK
"""
for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result["OK"]:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result["OK"]:
return result
allowedSubmitPools = getSubmitPools(self.__manifest["OwnerGroup"])
result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools)))
if not result["OK"]:
return result
result = self.__checkMultiChoice("PilotTypes", ["private"])
if not result["OK"]:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
result = self.__checkMaxInputData(maxInputData)
if not result["OK"]:
return result
transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
result = self.__checkMultiChoice("JobType", ["User", "Test", "Hospital"] + transformationTypes)
if not result["OK"]:
return result
return S_OK()
def createSection(self, secName, contents=False):
if secName not in self.__manifest:
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
return S_OK(self.__manifest.createNewSection(secName, contents=contents))
return S_ERROR("Section %s already exists" % secName)
def getSection(self, secName):
self.__dirty = True
sec = self.__manifest[secName]
if not sec:
return S_ERROR("%s does not exist")
return S_OK(sec)
def setSectionContents(self, secName, contents):
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
if secName in self.__manifest:
self.__manifest[secName].reset()
self.__manifest[secName].mergeWith(contents)
else:
self.__manifest.createNewSection(secName, contents=contents)
def setOption(self, varName, varValue):
"""
Set a var in job manifest
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def remove(self, opName):
levels = List.fromChar(opName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
return S_ERROR("%s does not exist" % opName)
cfg = cfg[l]
if cfg.deleteKey(levels[-1]):
self.__dirty = True
return S_OK()
return S_ERROR("%s does not exist" % opName)
def getOption(self, varName, defaultValue=None):
"""
Get a variable from the job manifest
"""
cfg = self.__manifest
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
"""
Get a list of variables in a section of the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listOptions()
def isOption(self, opName):
"""
Check if it is a valid option
"""
return self.__manifest.isOption(opName)
def getSectionList(self, section=""):
"""
Get a list of sections in the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listSections()
class Modificator:
def __init__( self, rpcClient = False, commiterId = "unknown" ):
self.commiterTag = "@@-"
self.commiterId = commiterId
self.cfgData = CFG()
self.rpcClient = None
if rpcClient:
self.setRPCClient( rpcClient )
def loadCredentials( self ):
retVal = getProxyInfo()
if retVal[ 'OK' ]:
credDict = retVal[ 'Value' ]
self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ],
credDict[ 'group' ],
Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) )
return retVal
return retVal
def setRPCClient( self, rpcClient ):
self.rpcClient = rpcClient
def loadFromRemote( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
self.cfgData = CFG()
self.cfgData.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
return retVal
def getCFG( self ):
return self.cfgData
def getSections( self, sectionPath ):
return gConfigurationData.getSectionsFromCFG( sectionPath, self.cfgData )
def getComment( self, sectionPath ):
return gConfigurationData.getCommentFromCFG( sectionPath, self.cfgData )
def getOptions( self, sectionPath ):
return gConfigurationData.getOptionsFromCFG( sectionPath, self.cfgData )
def getOptionsDict(self, sectionPath):
"""Gives the options of a CS section in a Python dict with values as
lists"""
opts = self.getOptions(sectionPath)
pathDict = dict( ( o, self.getValue( "%s/%s" % ( sectionPath, o ) ) ) for o in opts )
return pathDict
def getDictRootedAt(self, relpath = "", root = ""):
"""Gives the configuration rooted at path in a Python dict. The
result is a Python dictionary that reflects the structure of the
config file."""
def getDictRootedAt(path):
retval = {}
opts = self.getOptionsDict(path)
secs = self.getSections(path)
for k in opts:
retval[k] = opts[k]
for i in secs:
retval[i] = getDictRootedAt(path + "/" + i)
return retval
return getDictRootedAt(root + "/" + relpath)
def getValue( self, optionPath ):
return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData )
def sortAlphabetically( self, path, ascending = True ):
cfg = self.__getParentCFG( path, parentLevel = 0 )
if cfg:
if cfg.sortAlphabetically( ascending ):
self.__setCommiter( path )
def __getParentCFG( self, path, parentLevel = 1 ):
sectionList = List.fromChar( path, "/" )
cfg = self.cfgData
try:
if parentLevel > 0:
sectionList = sectionList[:-parentLevel]
for section in sectionList:
cfg = cfg[ section ]
return cfg
except:
return False
def __setCommiter( self, entryPath, cfg = False ):
if not cfg:
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
comment = cfg.getComment( entry )
filteredComment = [ line.strip() for line in comment.split( "\n" ) if line.find( self.commiterTag ) != 0 ]
filteredComment.append( "%s%s" % ( self.commiterTag, self.commiterId ) )
cfg.setComment( entry, "\n".join( filteredComment ) )
def setOptionValue( self, optionPath, value ):
levelList = [ level.strip() for level in optionPath.split( "/" ) if level.strip() != "" ]
parentPath = "/%s" % "/".join( levelList[:-1] )
optionName = List.fromChar( optionPath, "/" )[-1]
self.createSection( parentPath )
cfg = self.__getParentCFG( optionPath )
if not cfg:
return
cfg.setOption( optionName, value )
self.__setCommiter( optionPath, cfg )
def createSection( self, sectionPath ):
levelList = [ level.strip() for level in sectionPath.split( "/" ) if level.strip() != "" ]
currentPath = ""
cfg = self.cfgData
createdSection = False
for section in levelList:
currentPath += "/%s" % section
if section not in cfg.listSections():
cfg.createNewSection( section )
self.__setCommiter( currentPath )
createdSection = True
cfg = cfg[ section ]
return createdSection
def setComment( self, entryPath, value ):
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
if cfg.setComment( entry, value ):
self.__setCommiter( entryPath )
return True
return False
def existsSection( self, sectionPath ):
sectionList = List.fromChar( sectionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return len( sectionList ) == 0 or sectionList[-1] in cfg.listSections()
except:
return False
def existsOption( self, optionPath ):
sectionList = List.fromChar( optionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return sectionList[-1] in cfg.listOptions()
except:
return False
def renameKey( self, path, newName ):
parentCfg = self.cfgData.getRecursive( path, -1 )
if not parentCfg:
return False
pathList = List.fromChar( path, "/" )
oldName = pathList[-1]
if parentCfg[ 'value' ].renameKey( oldName, newName ):
pathList[-1] = newName
self.__setCommiter( "/%s" % "/".join( pathList ) )
return True
else:
return False
def copyKey( self, originalKeyPath, newKey ):
parentCfg = self.cfgData.getRecursive( originalKeyPath, -1 )
if not parentCfg:
return False
pathList = List.fromChar( originalKeyPath, "/" )
originalKey = pathList[-1]
if parentCfg[ 'value' ].copyKey( originalKey, newKey ):
self.__setCommiter( "/%s/%s" % ( "/".join( pathList[:-1] ), newKey ) )
return True
return False
def removeOption( self, optionPath ):
if not self.existsOption( optionPath ):
return False
cfg = self.__getParentCFG( optionPath )
optionName = List.fromChar( optionPath, "/" )[-1]
return cfg.deleteKey( optionName )
def removeSection( self, sectionPath ):
if not self.existsSection( sectionPath ):
return False
cfg = self.__getParentCFG( sectionPath )
sectionName = List.fromChar( sectionPath, "/" )[-1]
return cfg.deleteKey( sectionName )
def loadFromBuffer( self, data ):
self.cfgData = CFG()
self.cfgData.loadFromBuffer( data )
def loadFromFile( self, filename ):
self.cfgData = CFG()
self.mergeFromFile( filename )
def dumpToFile( self, filename ):
fd = file( filename, "w" )
fd.write( str( self.cfgData ) )
fd.close()
def mergeFromFile( self, filename ):
cfg = CFG()
cfg.loadFromFile( filename )
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeFromCFG( self, cfg ):
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeSectionFromCFG( self, sectionPath, cfg ):
parentDict = self.cfgData.getRecursive( sectionPath, -1 )
parentCFG = parentDict[ 'value' ]
secName = [ lev.strip() for lev in sectionPath.split( "/" ) if lev.strip() ][-1]
secCFG = parentCFG[ secName ]
if not secCFG:
return False
mergedCFG = secCFG.mergeWith( cfg )
parentCFG.deleteKey( secName )
parentCFG.createNewSection( secName, parentDict[ 'comment' ], mergedCFG )
self.__setCommiter( sectionPath )
return True
def __str__( self ):
return str( self.cfgData )
def commit( self ):
compressedData = zlib.compress( str( self.cfgData ), 9 )
return self.rpcClient.commitNewData( compressedData )
def getHistory( self, limit = 0 ):
retVal = self.rpcClient.getCommitHistory( limit )
if retVal[ 'OK' ]:
return retVal[ 'Value' ]
return []
def showCurrentDiff( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteData = zlib.decompress( retVal[ 'Value' ] ).splitlines()
localData = str( self.cfgData ).splitlines()
return difflib.ndiff( remoteData, localData )
return []
def getVersionDiff( self, fromDate, toDate ):
retVal = self.rpcClient.getVersionContents( [ fromDate, toDate ] )
if retVal[ 'OK' ]:
fromData = zlib.decompress( retVal[ 'Value' ][0] )
toData = zlib.decompress( retVal[ 'Value' ][1] )
return difflib.ndiff( fromData.split( "\n" ), toData.split( "\n" ) )
return []
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def rollbackToVersion( self, version ):
return self.rpcClient.rollbackToVersion( version )
def updateGConfigurationData( self ):
gConfigurationData.setRemoteCFG( self.cfgData )
class Modificator( object ):
def __init__( self, rpcClient = False, commiterId = "unknown" ):
self.commiterTag = "@@-"
self.commiterId = commiterId
self.cfgData = CFG()
self.rpcClient = None
if rpcClient:
self.setRPCClient( rpcClient )
def loadCredentials( self ):
retVal = getProxyInfo()
if retVal[ 'OK' ]:
credDict = retVal[ 'Value' ]
self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ],
credDict[ 'group' ],
Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) )
return retVal
return retVal
def setRPCClient( self, rpcClient ):
self.rpcClient = rpcClient
def loadFromRemote( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
self.cfgData = CFG()
self.cfgData.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
return retVal
def getCFG( self ):
return self.cfgData
def getSections( self, sectionPath ):
return gConfigurationData.getSectionsFromCFG( sectionPath, self.cfgData )
def getComment( self, sectionPath ):
return gConfigurationData.getCommentFromCFG( sectionPath, self.cfgData )
def getOptions( self, sectionPath ):
return gConfigurationData.getOptionsFromCFG( sectionPath, self.cfgData )
def getOptionsDict(self, sectionPath):
"""Gives the options of a CS section in a Python dict with values as
lists"""
opts = self.getOptions(sectionPath)
pathDict = dict( ( o, self.getValue( "%s/%s" % ( sectionPath, o ) ) ) for o in opts )
return pathDict
def getDictRootedAt(self, relpath = "", root = ""):
"""Gives the configuration rooted at path in a Python dict. The
result is a Python dictionary that reflects the structure of the
config file."""
def getDictRootedAt(path):
retval = {}
opts = self.getOptionsDict(path)
secs = self.getSections(path)
for k in opts:
retval[k] = opts[k]
for i in secs:
retval[i] = getDictRootedAt(path + "/" + i)
return retval
return getDictRootedAt(root + "/" + relpath)
def getValue( self, optionPath ):
return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData )
def sortAlphabetically( self, path, ascending = True ):
cfg = self.__getParentCFG( path, parentLevel = 0 )
if cfg:
if cfg.sortAlphabetically( ascending ):
self.__setCommiter( path )
def __getParentCFG( self, path, parentLevel = 1 ):
sectionList = List.fromChar( path, "/" )
cfg = self.cfgData
try:
if parentLevel > 0:
sectionList = sectionList[:-parentLevel]
for section in sectionList:
cfg = cfg[ section ]
return cfg
except:
return False
def __setCommiter( self, entryPath, cfg = False ):
if not cfg:
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
comment = cfg.getComment( entry )
filteredComment = [ line.strip() for line in comment.split( "\n" ) if line.find( self.commiterTag ) != 0 ]
filteredComment.append( "%s%s" % ( self.commiterTag, self.commiterId ) )
cfg.setComment( entry, "\n".join( filteredComment ) )
def setOptionValue( self, optionPath, value ):
levelList = [ level.strip() for level in optionPath.split( "/" ) if level.strip() != "" ]
parentPath = "/%s" % "/".join( levelList[:-1] )
optionName = List.fromChar( optionPath, "/" )[-1]
self.createSection( parentPath )
cfg = self.__getParentCFG( optionPath )
if not cfg:
return
cfg.setOption( optionName, value )
self.__setCommiter( optionPath, cfg )
def createSection( self, sectionPath ):
levelList = [ level.strip() for level in sectionPath.split( "/" ) if level.strip() != "" ]
currentPath = ""
cfg = self.cfgData
createdSection = False
for section in levelList:
currentPath += "/%s" % section
if section not in cfg.listSections():
cfg.createNewSection( section )
self.__setCommiter( currentPath )
createdSection = True
cfg = cfg[ section ]
return createdSection
def setComment( self, entryPath, value ):
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
if cfg.setComment( entry, value ):
self.__setCommiter( entryPath )
return True
return False
def existsSection( self, sectionPath ):
sectionList = List.fromChar( sectionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return len( sectionList ) == 0 or sectionList[-1] in cfg.listSections()
except:
return False
def existsOption( self, optionPath ):
sectionList = List.fromChar( optionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return sectionList[-1] in cfg.listOptions()
except:
return False
def renameKey( self, path, newName ):
parentCfg = self.cfgData.getRecursive( path, -1 )
if not parentCfg:
return False
pathList = List.fromChar( path, "/" )
oldName = pathList[-1]
if parentCfg[ 'value' ].renameKey( oldName, newName ):
pathList[-1] = newName
self.__setCommiter( "/%s" % "/".join( pathList ) )
return True
else:
return False
def copyKey( self, originalKeyPath, newKey ):
parentCfg = self.cfgData.getRecursive( originalKeyPath, -1 )
if not parentCfg:
return False
pathList = List.fromChar( originalKeyPath, "/" )
originalKey = pathList[-1]
if parentCfg[ 'value' ].copyKey( originalKey, newKey ):
self.__setCommiter( "/%s/%s" % ( "/".join( pathList[:-1] ), newKey ) )
return True
return False
def removeOption( self, optionPath ):
if not self.existsOption( optionPath ):
return False
cfg = self.__getParentCFG( optionPath )
optionName = List.fromChar( optionPath, "/" )[-1]
return cfg.deleteKey( optionName )
def removeSection( self, sectionPath ):
if not self.existsSection( sectionPath ):
return False
cfg = self.__getParentCFG( sectionPath )
sectionName = List.fromChar( sectionPath, "/" )[-1]
return cfg.deleteKey( sectionName )
def loadFromBuffer( self, data ):
self.cfgData = CFG()
self.cfgData.loadFromBuffer( data )
def loadFromFile( self, filename ):
self.cfgData = CFG()
self.mergeFromFile( filename )
def dumpToFile( self, filename ):
fd = file( filename, "w" )
fd.write( str( self.cfgData ) )
fd.close()
def mergeFromFile( self, filename ):
cfg = CFG()
cfg.loadFromFile( filename )
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeFromCFG( self, cfg ):
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeSectionFromCFG( self, sectionPath, cfg ):
parentDict = self.cfgData.getRecursive( sectionPath, -1 )
parentCFG = parentDict[ 'value' ]
secName = [ lev.strip() for lev in sectionPath.split( "/" ) if lev.strip() ][-1]
secCFG = parentCFG[ secName ]
if not secCFG:
return False
mergedCFG = secCFG.mergeWith( cfg )
parentCFG.deleteKey( secName )
parentCFG.createNewSection( secName, parentDict[ 'comment' ], mergedCFG )
self.__setCommiter( sectionPath )
return True
def __str__( self ):
return str( self.cfgData )
def commit( self ):
compressedData = zlib.compress( str( self.cfgData ), 9 )
return self.rpcClient.commitNewData( compressedData )
def getHistory( self, limit = 0 ):
retVal = self.rpcClient.getCommitHistory( limit )
if retVal[ 'OK' ]:
return retVal[ 'Value' ]
return []
def showCurrentDiff( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteData = zlib.decompress( retVal[ 'Value' ] ).splitlines()
localData = str( self.cfgData ).splitlines()
return difflib.ndiff( remoteData, localData )
return []
def getVersionDiff( self, fromDate, toDate ):
retVal = self.rpcClient.getVersionContents( [ fromDate, toDate ] )
if retVal[ 'OK' ]:
fromData = zlib.decompress( retVal[ 'Value' ][0] )
toData = zlib.decompress( retVal[ 'Value' ][1] )
return difflib.ndiff( fromData.split( "\n" ), toData.split( "\n" ) )
return []
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def rollbackToVersion( self, version ):
return self.rpcClient.rollbackToVersion( version )
def updateGConfigurationData( self ):
gConfigurationData.setRemoteCFG( self.cfgData )
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result['OK']:
raise Exception(result['Message'])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__manifest = CFG()
return result
self.__manifest = result['Value'][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def dumpAsCFG(self):
return str(self.__manifest)
def getAsCFG(self):
return self.__manifest.clone()
def dumpAsJDL(self):
return dumpCFGAsJDL(self.__manifest)
def __getCSValue(self, varName, defaultVal=None):
if not self.__ops:
self.__ops = Operations(group=self.__manifest['OwnerGroup'],
setup=self.__manifest['DIRACSetup'])
if varName[0] != "/":
varName = "JobDescription/%s" % varName
return self.__ops.getValue(varName, defaultVal)
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
"""
Check a numerical var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
try:
varValue = long(varValue)
except BaseException:
return S_ERROR("%s must be a number" % varName)
minVal = self.__getCSValue("Min%s" % varName, minVal)
maxVal = self.__getCSValue("Max%s" % varName, maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVar(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
if varValue not in self.__getCSValue("Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" %
(varValue, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoice(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__manifest:
return S_OK()
else:
varValue = self.__manifest[varName]
initialVal = varValue
choices = self.__getCSValue("Choices%s" % varName, choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
'Number of Input Data Files (%s) greater than current limit: %s'
% (len(List.fromChar(varValue)), maxNumber))
return S_OK()
def __contains__(self, key):
""" Check if the manifest has the required key
"""
return key in self.__manifest
def setOptionsFromDict(self, varDict):
for k in sorted(varDict):
self.setOption(k, varDict[k])
def check(self):
"""
Check that the manifest is OK
"""
for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result['OK']:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result['OK']:
return result
allowedSubmitPools = getSubmitPools(self.__manifest['OwnerGroup'])
result = self.__checkMultiChoice("SubmitPools",
list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoice("PilotTypes", ['private'])
if not result['OK']:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData",
500)
result = self.__checkMaxInputData(maxInputData)
if not result['OK']:
return result
transformationTypes = Operations().getValue(
"Transformations/DataProcessing", [])
result = self.__checkMultiChoice(
"JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
if not result['OK']:
return result
return S_OK()
def createSection(self, secName, contents=False):
if secName not in self.__manifest:
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" %
secName)
self.__dirty = True
return S_OK(
self.__manifest.createNewSection(secName, contents=contents))
return S_ERROR("Section %s already exists" % secName)
def getSection(self, secName):
self.__dirty = True
if secName not in self.__manifest:
return S_ERROR("%s does not exist" % secName)
sec = self.__manifest[secName]
if not sec:
return S_ERROR("%s section empty" % secName)
return S_OK(sec)
def setSectionContents(self, secName, contents):
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" %
secName)
self.__dirty = True
if secName in self.__manifest:
self.__manifest[secName].reset()
self.__manifest[secName].mergeWith(contents)
else:
self.__manifest.createNewSection(secName, contents=contents)
def setOption(self, varName, varValue):
"""
Set a var in job manifest
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def remove(self, opName):
levels = List.fromChar(opName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
return S_ERROR("%s does not exist" % opName)
cfg = cfg[l]
if cfg.deleteKey(levels[-1]):
self.__dirty = True
return S_OK()
return S_ERROR("%s does not exist" % opName)
def getOption(self, varName, defaultValue=None):
"""
Get a variable from the job manifest
"""
cfg = self.__manifest
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
"""
Get a list of variables in a section of the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listOptions()
def isOption(self, opName):
"""
Check if it is a valid option
"""
return self.__manifest.isOption(opName)
def getSectionList(self, section=""):
"""
Get a list of sections in the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listSections()
class JobDescription(object):
def __init__(self):
self.__description = CFG()
self.__dirty = False
def isDirty(self):
return self.__dirty
def loadDescription(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL(dataString)
else:
return self.loadDescriptionFromCFG(dataString)
def loadDescriptionFromJDL(self, jdlString):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__description = CFG()
return result
self.__description = result['Value'][0]
return S_OK()
def loadDescriptionFromCFG(self, cfgString):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load description from cfg: %s" % str(e))
return S_OK()
def dumpDescriptionAsCFG(self):
return str(self.__description)
def dumpDescriptionAsJDL(self):
return dumpCFGAsJDL(self.__description)
def __checkNumericalVarInDescription(self, varName, defaultVal, minVal,
maxVal):
"""
Check a numerical var
"""
initialVal = 0
if varName not in self.__description:
varValue = Operations().getValue(
"JobDescription/Default%s" % varName, defaultVal)
else:
varValue = self.__description[varName]
initialVal = varValue
try:
varValue = long(varValue)
except BaseException:
return S_ERROR("%s must be a number" % varName)
minVal = Operations().getValue("JobDescription/Min%s" % varName,
minVal)
maxVal = Operations().getValue("JobDescription/Max%s" % varName,
maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVarInDescription(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__description:
varValue = Operations().getValue(
"JobDescription/Default%s" % varName, defaultVal)
else:
varValue = self.__description[varName]
initialVal = varValue
if varValue not in Operations().getValue(
"JobDescription/Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" %
(varValue, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoiceInDescription(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__description:
return S_OK()
else:
varValue = self.__description[varName]
initialVal = varValue
choices = Operations().getValue("JobDescription/Choices%s" % varName,
choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__description:
return S_OK()
varValue = self.__description[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
'Number of Input Data Files (%s) greater than current limit: %s'
% (len(List.fromChar(varValue)), maxNumber))
return S_OK()
def setVarsFromDict(self, varDict):
for k in sorted(varDict):
self.setVar(k, varDict[k])
def checkDescription(self):
"""
Check that the description is OK
"""
for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
if k not in self.__description:
return S_ERROR("Missing var %s in description" % k)
# Check CPUTime
result = self.__checkNumericalVarInDescription("CPUTime", 86400, 0,
500000)
if not result['OK']:
return result
result = self.__checkNumericalVarInDescription("Priority", 1, 0, 10)
if not result['OK']:
return result
allowedSubmitPools = getSubmitPools(self.__description['OwnerGroup'])
result = self.__checkMultiChoiceInDescription(
"SubmitPools", list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoiceInDescription(
"SubmitPools", list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoiceInDescription("PilotTypes",
['private'])
if not result['OK']:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData",
500)
result = self.__checkMaxInputData(maxInputData)
if not result['OK']:
return result
transformationTypes = Operations().getValue(
"Transformations/DataProcessing", [])
result = self.__checkMultiChoiceInDescription(
"JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
return S_OK()
def setVar(self, varName, varValue):
"""
Set a var in job description
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__description
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def getVar(self, varName, defaultValue=None):
cfg = self.__description
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
cfg = self.__description.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listOptions()
def getSectionList(self, section=""):
cfg = self.__description.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listSections()
return S_OK(modCfg.getComment(path))
@jsonify
def moveNode(self):
try:
nodePath = request.params['nodePath']
destinationParentPath = request.params['parentPath']
beforeOfIndex = int(request.params['beforeOfIndex'])
except Exception, e:
return S_ERROR("Can't decode parameter: %s" % str(e))
gLogger.info("Moving %s under %s before pos %s" %
(nodePath, destinationParentPath, beforeOfIndex))
cfgData = CFG()
cfgData.loadFromBuffer(session['cfgData'])
nodeDict = cfgData.getRecursive(nodePath)
if not nodeDict:
return S_ERROR("Moving entity does not exist")
oldParentDict = cfgData.getRecursive(nodePath, -1)
newParentDict = cfgData.getRecursive(destinationParentPath)
if type(newParentDict) == types.StringType:
return S_ERROR("Destination is not a section")
if not newParentDict:
return S_ERROR("Destination does not exist")
#Calculate the old parent path
oldParentPath = "/%s" % "/".join(List.fromChar(nodePath, "/")[:-1])
if not oldParentPath == destinationParentPath and newParentDict[
'value'].existsKey(nodeDict['key']):
return S_ERROR("Another entry with the same name already exists")
try:
session.save()
return S_OK(modCfg.getComment(path))
@jsonify
def moveNode(self):
try:
nodePath = request.params["nodePath"]
destinationParentPath = request.params["parentPath"]
beforeOfIndex = int(request.params["beforeOfIndex"])
except Exception, e:
return S_ERROR("Can't decode parameter: %s" % str(e))
gLogger.info("Moving %s under %s before pos %s" % (nodePath, destinationParentPath, beforeOfIndex))
cfgData = CFG()
cfgData.loadFromBuffer(session["cfgData"])
nodeDict = cfgData.getRecursive(nodePath)
if not nodeDict:
return S_ERROR("Moving entity does not exist")
oldParentDict = cfgData.getRecursive(nodePath, -1)
newParentDict = cfgData.getRecursive(destinationParentPath)
if type(newParentDict) == types.StringType:
return S_ERROR("Destination is not a section")
if not newParentDict:
return S_ERROR("Destination does not exist")
# Calculate the old parent path
oldParentPath = "/%s" % "/".join(List.fromChar(nodePath, "/")[:-1])
if not oldParentPath == destinationParentPath and newParentDict["value"].existsKey(nodeDict["key"]):
return S_ERROR("Another entry with the same name already exists")
try:
brothers = newParentDict["value"].listAll()
class JobDescription( object ):
def __init__( self ):
self.__description = CFG()
self.__dirty = False
def isDirty( self ):
return self.__dirty
def loadDescription( self, dataString ):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL( dataString )
else:
return self.loadDescriptionFromCFG( dataString )
def loadDescriptionFromJDL( self, jdlString ):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__description = CFG()
return result
self.__description = result[ 'Value' ][0]
return S_OK()
def loadDescriptionFromCFG( self, cfgString ):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer( cfgString )
except Exception as e:
return S_ERROR( "Can't load description from cfg: %s" % str( e ) )
return S_OK()
def dumpDescriptionAsCFG( self ):
return str( self.__description )
def dumpDescriptionAsJDL( self ):
return dumpCFGAsJDL( self.__description )
def __checkNumericalVarInDescription( self, varName, defaultVal, minVal, maxVal ):
"""
Check a numerical var
"""
initialVal = 0
if varName not in self.__description:
varValue = Operations().getValue( "JobDescription/Default%s" % varName , defaultVal )
else:
varValue = self.__description[ varName ]
initialVal = varValue
try:
varValue = long( varValue )
except:
return S_ERROR( "%s must be a number" % varName )
minVal = Operations().getValue( "JobDescription/Min%s" % varName, minVal )
maxVal = Operations().getValue( "JobDescription/Max%s" % varName, maxVal )
varValue = max( minVal, min( varValue, maxVal ) )
if initialVal != varValue:
self.__description.setOption( varName, varValue )
return S_OK( varValue )
def __checkChoiceVarInDescription( self, varName, defaultVal, choices ):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__description:
varValue = Operations().getValue( "JobDescription/Default%s" % varName , defaultVal )
else:
varValue = self.__description[ varName ]
initialVal = varValue
if varValue not in Operations().getValue( "JobDescription/Choices%s" % varName , choices ):
return S_ERROR( "%s is not a valid value for %s" % ( varValue, varName ) )
if initialVal != varValue:
self.__description.setOption( varName, varValue )
return S_OK( varValue )
def __checkMultiChoiceInDescription( self, varName, choices ):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__description:
return S_OK()
else:
varValue = self.__description[ varName ]
initialVal = varValue
choices = Operations().getValue( "JobDescription/Choices%s" % varName , choices )
for v in List.fromChar( varValue ):
if v not in choices:
return S_ERROR( "%s is not a valid value for %s" % ( v, varName ) )
if initialVal != varValue:
self.__description.setOption( varName, varValue )
return S_OK( varValue )
def __checkMaxInputData( self, maxNumber ):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__description:
return S_OK()
varValue = self.__description[ varName ]
if len( List.fromChar( varValue ) ) > maxNumber:
return S_ERROR( 'Number of Input Data Files (%s) greater than current limit: %s' % ( len( List.fromChar( varValue ) ), maxNumber ) )
return S_OK()
def setVarsFromDict( self, varDict ):
for k in sorted( varDict ):
self.setVar( k, varDict[ k ] )
def checkDescription( self ):
"""
Check that the description is OK
"""
for k in [ 'OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup' ]:
if k not in self.__description:
return S_ERROR( "Missing var %s in description" % k )
# Check CPUTime
result = self.__checkNumericalVarInDescription( "CPUTime", 86400, 0, 500000 )
if not result[ 'OK' ]:
return result
result = self.__checkNumericalVarInDescription( "Priority", 1, 0, 10 )
if not result[ 'OK' ]:
return result
allowedSubmitPools = getSubmitPools( self.__description['OwnerGroup'] )
result = self.__checkMultiChoiceInDescription( "SubmitPools", list( set( allowedSubmitPools ) ) )
if not result[ 'OK' ]:
return result
result = self.__checkMultiChoiceInDescription( "SubmitPools", list( set( allowedSubmitPools ) ) )
if not result[ 'OK' ]:
return result
result = self.__checkMultiChoiceInDescription( "PilotTypes", [ 'private' ] )
if not result[ 'OK' ]:
return result
maxInputData = Operations().getValue( "JobDescription/MaxInputData", 500 )
result = self.__checkMaxInputData( maxInputData )
if not result[ 'OK' ]:
return result
transformationTypes = Operations().getValue( "Transformations/DataProcessing", [] )
result = self.__checkMultiChoiceInDescription( "JobType", ['User', 'Test', 'Hospital'] + transformationTypes )
return S_OK()
def setVar( self, varName, varValue ):
"""
Set a var in job description
"""
self.__dirty = True
levels = List.fromChar( varName, "/" )
cfg = self.__description
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection( l )
cfg = cfg[ l ]
cfg.setOption( levels[-1], varValue )
def getVar( self, varName, defaultValue = None ):
cfg = self.__description
return cfg.getOption( varName, defaultValue )
def getOptionList( self, section = "" ):
cfg = self.__description.getRecursive( section )
if not cfg or 'value' not in cfg:
return []
cfg = cfg[ 'value' ]
return cfg.listOptions()
def getSectionList( self, section = "" ):
cfg = self.__description.getRecursive( section )
if not cfg or 'value' not in cfg:
return []
cfg = cfg[ 'value' ]
return cfg.listSections()
