def setUp(self):
self.authMgr = AuthManager('/Systems/Service/Authorization')
cfg = CFG()
cfg.loadFromBuffer(testSystemsCFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(testRegistryCFG)
gConfig.loadCFG(cfg)
self.noAuthCredDict = {'group': 'group_test'}
self.userCredDict = {
'DN': '/User/test/DN/CN=userA',
'group': 'group_test'
}
self.suspendedOtherVOUserCredDict = {
'DN': '/User/test/DN/CN=userS',
'group': 'group_test_other'
}
self.badUserCredDict = {
'DN': '/User/test/DN/CN=userB',
'group': 'group_bad'
}
self.suspendedUserCredDict = {
'DN': '/User/test/DN/CN=userS',
'group': 'group_test'
}
self.hostCredDict = {
'DN': '/User/test/DN/CN=test.hostA.ch',
'group': 'hosts'
}
self.badHostCredDict = {
'DN': '/User/test/DN/CN=test.hostB.ch',
'group': 'hosts'
}
def setUp(self):
cfg = CFG()
cfg.loadFromBuffer(diracTestCACFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(userCFG)
gConfig.loadCFG(cfg)
result = ProxyProviderFactory().getProxyProvider('DIRAC_TEST_CA')
self.assertTrue(result['OK'], '\n%s' % result.get('Message') or 'Error message is absent.')
self.pp = result['Value']
self.userDictClean = {
"FullName": "DIRAC test user",
"EMail": "*****@*****.**"
}
self.userDictCleanDN = {
"DN": "/C=FR/O=DIRAC/OU=DIRAC Consortium/CN=DIRAC test user/[email protected]",
"EMail": "*****@*****.**"
}
self.userDictGroup = {
"FullName": "DIRAC test user",
"EMail": "*****@*****.**",
"DiracGroup": "dirac_user"
}
self.userDictNoGroup = {
"FullName": "DIRAC test user",
"EMail": "*****@*****.**",
"DiracGroup": "dirac_no_user"
}
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def setUp(self):
cfg = CFG()
cfg.loadFromBuffer(diracTestCACFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(userCFG)
gConfig.loadCFG(cfg)
result = ProxyProviderFactory().getProxyProvider('DIRAC_TEST_CA')
self.assertTrue(
result['OK'], '\n%s' % result.get('Message')
or 'Error message is absent.')
self.pp = result['Value']
class JobManifest( object ):
def __init__( self, manifest = "" ):
self.__manifest = CFG()
self.__dirty = False
if manifest:
result = self.loadManifest( manifest )
if not result[ 'OK' ]:
raise Exception( result[ 'Message' ] )
def isDirty( self ):
return self.__dirty
def setDirty( self ):
self.__dirty = True
def clearDirty( self ):
self.__dirty = False
def load( self, dataString ):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL( dataString )
else:
return self.loadCFG( dataString )
def loadJDL( self, jdlString ):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__manifest = CFG()
return result
self.__manifest = result[ 'Value' ][0]
return S_OK()
def loadCFG( self, cfgString ):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer( cfgString )
except Exception, e:
return S_ERROR( "Can't load manifest from cfg: %s" % str( e ) )
return S_OK()
def showTextConfiguration(self):
response.headers["Content-type"] = "text/plain"
if "download" in request.params and request.params["download"] in ("yes", "true", "1"):
version = ""
try:
cfg = CFG()
cfg.loadFromBuffer(session["cfgData"])
cfg = cfg["DIRAC"]["Configuration"]
version = ".%s.%s" % (cfg["Name"], cfg["Version"].replace(":", "").replace("-", ""))
except Exception, e:
print e
print 'attachment; filename="cs%s.cfg"' % version.replace(" ", "_")
response.headers["Content-Disposition"] = 'attachment; filename="cs%s.cfg"' % version.replace(" ", "")
response.headers["Content-Length"] = len(session["cfgData"])
response.headers["Content-Transfer-Encoding"] = "Binary"
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
if manifest:
result = self.loadManifest(manifest)
if not result['OK']:
raise Exception(result['Message'])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__manifest = CFG()
return result
self.__manifest = result['Value'][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception, e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def showTextConfiguration(self):
response.headers['Content-type'] = 'text/plain'
if 'download' in request.params and request.params['download'] in (
'yes', 'true', '1'):
version = ""
try:
cfg = CFG()
cfg.loadFromBuffer(session['cfgData'])
cfg = cfg['DIRAC']['Configuration']
version = ".%s.%s" % (cfg['Name'], cfg['Version'].replace(
":", "").replace("-", ""))
except Exception, e:
print e
print 'attachment; filename="cs%s.cfg"' % version.replace(" ", "_")
response.headers[
'Content-Disposition'] = 'attachment; filename="cs%s.cfg"' % version.replace(
" ", "")
response.headers['Content-Length'] = len(session['cfgData'])
response.headers['Content-Transfer-Encoding'] = 'Binary'
def setUp( self ):
self.authMgr = AuthManager( '/Systems/Service/Authorization' )
cfg = CFG()
cfg.loadFromBuffer( testSystemsCFG )
gConfig.loadCFG( cfg )
cfg.loadFromBuffer( testRegistryCFG )
gConfig.loadCFG( cfg )
self.noAuthCredDict = { 'group': 'group_test' }
self.userCredDict = { 'DN': '/User/test/DN/CN=userA',
'group': 'group_test' }
self.badUserCredDict = { 'DN': '/User/test/DN/CN=userB',
'group': 'group_bad' }
self.hostCredDict = { 'DN': '/User/test/DN/CN=test.hostA.ch',
'group': 'hosts' }
self.badHostCredDict = { 'DN': '/User/test/DN/CN=test.hostB.ch',
'group': 'hosts' }
def __saveVomsMapping(self, params):
sectionPath = "/Registry/VOMS/Mapping"
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
if not sectionCfg.isSection(opt):
self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)
configText = ""
for newOpt in params:
if newOpt != "op":
if configText == "":
configText = newOpt + "=" + params[newOpt] + "\n"
else:
configText = configText + newOpt + "=" + params[newOpt] + "\n"
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"op":"saveVomsMapping", "success": 1}
class JobDescription:
def __init__( self ):
self.__description = CFG()
self.__dirty = False
def isDirty( self ):
return self.__dirty
def loadDescription( self, dataString ):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL( dataString )
else:
return self.loadDescriptionFromCFG( dataString )
def loadDescriptionFromJDL( self, jdlString ):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__description = CFG()
return result
self.__description = result[ 'Value' ][0]
return S_OK()
def loadDescriptionFromCFG( self, cfgString ):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer( cfgString )
except Exception, e:
return S_ERROR( "Can't load description from cfg: %s" % str( e ) )
return S_OK()
def __saveVomsMapping(self, params):
sectionPath = "/Registry/VOMS/Mapping"
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
if not sectionCfg.isSection(opt):
self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)
configText = ""
for newOpt in params:
if newOpt != "op":
if configText == "":
configText = newOpt + "=" + params[newOpt] + "\n"
else:
configText = configText + newOpt + "=" + params[newOpt] + "\n"
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"op":"saveVomsMapping", "success": 1}
class JobDescription:
def __init__( self ):
self.__description = CFG()
self.__dirty = False
def isDirty(self):
return self.__dirty
def loadDescription( self, dataString ):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL( dataString )
else:
return self.loadDescriptionFromCFG( dataString )
def loadDescriptionFromJDL( self, jdlString ):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__description = CFG()
return result
self.__description = result[ 'Value' ][0]
return S_OK()
def loadDescriptionFromCFG( self, cfgString ):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer( cfgString )
except Exception, e:
return S_ERROR( "Can't load description from cfg: %s" % str(e) )
return S_OK()
def setUpClass(cls):
cls.failed = False
# Add configuration
cfg = CFG()
cfg.loadFromBuffer(diracTestCACFG)
gConfig.loadCFG(cfg)
cfg.loadFromBuffer(userCFG)
gConfig.loadCFG(cfg)
# Prepare CA
lines = []
cfgDict = {}
cls.caPath = os.path.join(certsPath, 'ca')
cls.caConfigFile = os.path.join(cls.caPath, 'openssl_config_ca.cnf')
# Save original configuration file
shutil.copyfile(cls.caConfigFile, cls.caConfigFile + 'bak')
# Parse
fields = [
'dir', 'database', 'serial', 'new_certs_dir', 'private_key',
'certificate'
]
with open(cls.caConfigFile, "r") as caCFG:
for line in caCFG:
if re.findall('=', re.sub(r'#.*', '', line)):
field = re.sub(r'#.*', '',
line).replace(' ',
'').rstrip().split('=')[0]
line = 'dir = %s #PUT THE RIGHT DIR HERE!\n' % (
cls.caPath) if field == 'dir' else line
val = re.sub(r'#.*', '',
line).replace(' ', '').rstrip().split('=')[1]
if field in fields:
for i in fields:
if cfgDict.get(i):
val = val.replace('$%s' % i, cfgDict[i])
cfgDict[field] = val
if not cfgDict[field]:
cls.failed = '%s have empty value in %s' % (
field, cls.caConfigFile)
lines.append(line)
with open(cls.caConfigFile, "w") as caCFG:
caCFG.writelines(lines)
for field in fields:
if field not in cfgDict.keys():
cls.failed = '%s value is absent in %s' % (field,
cls.caConfigFile)
cls.hostCert = os.path.join(certsPath, 'host/hostcert.pem')
cls.hostKey = os.path.join(certsPath, 'host/hostkey.pem')
cls.caCert = cfgDict['certificate']
cls.caKey = cfgDict['private_key']
os.chmod(cls.caKey, stat.S_IREAD)
# Check directory for new certificates
cls.newCertDir = cfgDict['new_certs_dir']
if not os.path.exists(cls.newCertDir):
os.makedirs(cls.newCertDir)
for f in os.listdir(cls.newCertDir):
os.remove(os.path.join(cls.newCertDir, f))
# Empty the certificate database
cls.index = cfgDict['database']
with open(cls.index, 'w') as indx:
indx.write('')
# Write down serial
cls.serial = cfgDict['serial']
with open(cls.serial, 'w') as serialFile:
serialFile.write('1000')
# Create temporaly directory for users certificates
cls.userDir = tempfile.mkdtemp(dir=certsPath)
# Create user certificates
for userName in ['no_user', 'user', 'user_1', 'user_2', 'user_3']:
userConf = """[ req ]
default_bits = 4096
encrypt_key = yes
distinguished_name = req_dn
prompt = no
req_extensions = v3_req
[ req_dn ]
C = CC
O = DN
0.O = DIRAC
CN = %s
[ v3_req ]
# Extensions for client certificates (`man x509v3_config`).
nsComment = "OpenSSL Generated Client Certificate"
keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth
""" % (userName)
userConfFile = os.path.join(cls.userDir, userName + '.cnf')
userReqFile = os.path.join(cls.userDir, userName + '.req')
userKeyFile = os.path.join(cls.userDir, userName + '.key.pem')
userCertFile = os.path.join(cls.userDir, userName + '.cert.pem')
with open(userConfFile, "w") as f:
f.write(userConf)
status, output = commands.getstatusoutput(
'openssl genrsa -out %s' % userKeyFile)
if status:
gLogger.error(output)
exit()
gLogger.debug(output)
os.chmod(userKeyFile, stat.S_IREAD)
status, output = commands.getstatusoutput(
'openssl req -config %s -key %s -new -out %s' %
(userConfFile, userKeyFile, userReqFile))
if status:
gLogger.error(output)
exit()
gLogger.debug(output)
cmd = 'openssl ca -config %s -extensions usr_cert -batch -days 375 -in %s -out %s'
cmd = cmd % (cls.caConfigFile, userReqFile, userCertFile)
status, output = commands.getstatusoutput(cmd)
if status:
gLogger.error(output)
exit()
gLogger.debug(output)
# Result
status, output = commands.getstatusoutput('ls -al %s' % cls.userDir)
if status:
gLogger.error(output)
exit()
gLogger.debug('User certificates:\n', output)
class Modificator:
def __init__( self, rpcClient = False, commiterId = "unknown" ):
self.commiterTag = "@@-"
self.commiterId = commiterId
self.cfgData = CFG()
self.rpcClient = None
if rpcClient:
self.setRPCClient( rpcClient )
def loadCredentials( self ):
retVal = getProxyInfo()
if retVal[ 'OK' ]:
credDict = retVal[ 'Value' ]
self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ],
credDict[ 'group' ],
Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) )
return retVal
return retVal
def setRPCClient( self, rpcClient ):
self.rpcClient = rpcClient
def loadFromRemote( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
self.cfgData = CFG()
self.cfgData.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
return retVal
def getCFG( self ):
return self.cfgData
def getSections( self, sectionPath ):
return gConfigurationData.getSectionsFromCFG( sectionPath, self.cfgData )
def getComment( self, sectionPath ):
return gConfigurationData.getCommentFromCFG( sectionPath, self.cfgData )
def getOptions( self, sectionPath ):
return gConfigurationData.getOptionsFromCFG( sectionPath, self.cfgData )
def getOptionsDict(self, sectionPath):
"""Gives the options of a CS section in a Python dict with values as
lists"""
opts = self.getOptions(sectionPath)
pathDict = dict( ( o, self.getValue( "%s/%s" % ( sectionPath, o ) ) ) for o in opts )
return pathDict
def getDictRootedAt(self, relpath = "", root = ""):
"""Gives the configuration rooted at path in a Python dict. The
result is a Python dictionary that reflects the structure of the
config file."""
def getDictRootedAt(path):
retval = {}
opts = self.getOptionsDict(path)
secs = self.getSections(path)
for k in opts:
retval[k] = opts[k]
for i in secs:
retval[i] = getDictRootedAt(path + "/" + i)
return retval
return getDictRootedAt(root + "/" + relpath)
def getValue( self, optionPath ):
return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData )
def sortAlphabetically( self, path, ascending = True ):
cfg = self.__getParentCFG( path, parentLevel = 0 )
if cfg:
if cfg.sortAlphabetically( ascending ):
self.__setCommiter( path )
def __getParentCFG( self, path, parentLevel = 1 ):
sectionList = List.fromChar( path, "/" )
cfg = self.cfgData
try:
if parentLevel > 0:
sectionList = sectionList[:-parentLevel]
for section in sectionList:
cfg = cfg[ section ]
return cfg
except:
return False
def __setCommiter( self, entryPath, cfg = False ):
if not cfg:
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
comment = cfg.getComment( entry )
filteredComment = [ line.strip() for line in comment.split( "\n" ) if line.find( self.commiterTag ) != 0 ]
filteredComment.append( "%s%s" % ( self.commiterTag, self.commiterId ) )
cfg.setComment( entry, "\n".join( filteredComment ) )
def setOptionValue( self, optionPath, value ):
levelList = [ level.strip() for level in optionPath.split( "/" ) if level.strip() != "" ]
parentPath = "/%s" % "/".join( levelList[:-1] )
optionName = List.fromChar( optionPath, "/" )[-1]
self.createSection( parentPath )
cfg = self.__getParentCFG( optionPath )
if not cfg:
return
cfg.setOption( optionName, value )
self.__setCommiter( optionPath, cfg )
def createSection( self, sectionPath ):
levelList = [ level.strip() for level in sectionPath.split( "/" ) if level.strip() != "" ]
currentPath = ""
cfg = self.cfgData
createdSection = False
for section in levelList:
currentPath += "/%s" % section
if section not in cfg.listSections():
cfg.createNewSection( section )
self.__setCommiter( currentPath )
createdSection = True
cfg = cfg[ section ]
return createdSection
def setComment( self, entryPath, value ):
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
if cfg.setComment( entry, value ):
self.__setCommiter( entryPath )
return True
return False
def existsSection( self, sectionPath ):
sectionList = List.fromChar( sectionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return len( sectionList ) == 0 or sectionList[-1] in cfg.listSections()
except:
return False
def existsOption( self, optionPath ):
sectionList = List.fromChar( optionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return sectionList[-1] in cfg.listOptions()
except:
return False
def renameKey( self, path, newName ):
parentCfg = self.cfgData.getRecursive( path, -1 )
if not parentCfg:
return False
pathList = List.fromChar( path, "/" )
oldName = pathList[-1]
if parentCfg[ 'value' ].renameKey( oldName, newName ):
pathList[-1] = newName
self.__setCommiter( "/%s" % "/".join( pathList ) )
return True
else:
return False
def copyKey( self, originalKeyPath, newKey ):
parentCfg = self.cfgData.getRecursive( originalKeyPath, -1 )
if not parentCfg:
return False
pathList = List.fromChar( originalKeyPath, "/" )
originalKey = pathList[-1]
if parentCfg[ 'value' ].copyKey( originalKey, newKey ):
self.__setCommiter( "/%s/%s" % ( "/".join( pathList[:-1] ), newKey ) )
return True
return False
def removeOption( self, optionPath ):
if not self.existsOption( optionPath ):
return False
cfg = self.__getParentCFG( optionPath )
optionName = List.fromChar( optionPath, "/" )[-1]
return cfg.deleteKey( optionName )
def removeSection( self, sectionPath ):
if not self.existsSection( sectionPath ):
return False
cfg = self.__getParentCFG( sectionPath )
sectionName = List.fromChar( sectionPath, "/" )[-1]
return cfg.deleteKey( sectionName )
def loadFromBuffer( self, data ):
self.cfgData = CFG()
self.cfgData.loadFromBuffer( data )
def loadFromFile( self, filename ):
self.cfgData = CFG()
self.mergeFromFile( filename )
def dumpToFile( self, filename ):
fd = file( filename, "w" )
fd.write( str( self.cfgData ) )
fd.close()
def mergeFromFile( self, filename ):
cfg = CFG()
cfg.loadFromFile( filename )
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeFromCFG( self, cfg ):
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeSectionFromCFG( self, sectionPath, cfg ):
parentDict = self.cfgData.getRecursive( sectionPath, -1 )
parentCFG = parentDict[ 'value' ]
secName = [ lev.strip() for lev in sectionPath.split( "/" ) if lev.strip() ][-1]
secCFG = parentCFG[ secName ]
if not secCFG:
return False
mergedCFG = secCFG.mergeWith( cfg )
parentCFG.deleteKey( secName )
parentCFG.createNewSection( secName, parentDict[ 'comment' ], mergedCFG )
self.__setCommiter( sectionPath )
return True
def __str__( self ):
return str( self.cfgData )
def commit( self ):
compressedData = zlib.compress( str( self.cfgData ), 9 )
return self.rpcClient.commitNewData( compressedData )
def getHistory( self, limit = 0 ):
retVal = self.rpcClient.getCommitHistory( limit )
if retVal[ 'OK' ]:
return retVal[ 'Value' ]
return []
def showCurrentDiff( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteData = zlib.decompress( retVal[ 'Value' ] ).splitlines()
localData = str( self.cfgData ).splitlines()
return difflib.ndiff( remoteData, localData )
return []
def getVersionDiff( self, fromDate, toDate ):
retVal = self.rpcClient.getVersionContents( [ fromDate, toDate ] )
if retVal[ 'OK' ]:
fromData = zlib.decompress( retVal[ 'Value' ][0] )
toData = zlib.decompress( retVal[ 'Value' ][1] )
return difflib.ndiff( fromData.split( "\n" ), toData.split( "\n" ) )
return []
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def rollbackToVersion( self, version ):
return self.rpcClient.rollbackToVersion( version )
def updateGConfigurationData( self ):
gConfigurationData.setRemoteCFG( self.cfgData )
'text':
sectionName,
'csName':
sectionName,
'csComment':
self.__configData['cfgData'].getComment(sectionPath)
}
htmlC = self.__htmlComment(nD['csComment'])
if htmlC:
qtipDict = {'text': htmlC}
nD['qtipCfg'] = qtipDict
# If config Text is provided then a section is created out of that text
if configText != "":
cfgData = self.__configData['cfgData'].getCFG()
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__setCommiter()
self.__configData['cfgData'].mergeSectionFromCFG(
sectionPath, newCFG)
return {
"success": 1,
"op": "createSection",
"parentNodeId": params["parentNodeId"],
"node": nD,
"sectionFromConfig": 1
}
else:
return {
"success": 1,
"op": "createSection",
"parentNodeId": params["parentNodeId"],
def __addItem(self, params):
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params[
"properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(
params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params[
"autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params[
"autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params[
"autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params[
"properties"].strip()
elif params["type"] == "voms":
sectionPath = sectionPath + "VOMS/Servers"
elif params["type"] == "servers":
sectionPath = sectionPath + "VOMS/Servers/" + params["vom"]
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["port"].strip() != "":
configText = configText + "Port = " + params["port"].strip(
) + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip()
sectionPath = sectionPath + "/" + params["name"]
if self.__configData['cfgData'].createSection(sectionPath):
cfgData = self.__configData['cfgData'].getCFG()
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData['cfgData'].mergeSectionFromCFG(
sectionPath, newCFG)
return {"success": 1, "op": "addItem"}
else:
return {
"success": 0,
"op": "addItem",
"message": "Section can't be created. It already exists?"
}
class Modificator( object ):
def __init__( self, rpcClient = False, commiterId = "unknown" ):
self.commiterTag = "@@-"
self.commiterId = commiterId
self.cfgData = CFG()
self.rpcClient = None
if rpcClient:
self.setRPCClient( rpcClient )
def loadCredentials( self ):
retVal = getProxyInfo()
if retVal[ 'OK' ]:
credDict = retVal[ 'Value' ]
self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ],
credDict[ 'group' ],
Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) )
return retVal
return retVal
def setRPCClient( self, rpcClient ):
self.rpcClient = rpcClient
def loadFromRemote( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
self.cfgData = CFG()
self.cfgData.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
return retVal
def getCFG( self ):
return self.cfgData
def getSections( self, sectionPath ):
return gConfigurationData.getSectionsFromCFG( sectionPath, self.cfgData )
def getComment( self, sectionPath ):
return gConfigurationData.getCommentFromCFG( sectionPath, self.cfgData )
def getOptions( self, sectionPath ):
return gConfigurationData.getOptionsFromCFG( sectionPath, self.cfgData )
def getOptionsDict(self, sectionPath):
"""Gives the options of a CS section in a Python dict with values as
lists"""
opts = self.getOptions(sectionPath)
pathDict = dict( ( o, self.getValue( "%s/%s" % ( sectionPath, o ) ) ) for o in opts )
return pathDict
def getDictRootedAt(self, relpath = "", root = ""):
"""Gives the configuration rooted at path in a Python dict. The
result is a Python dictionary that reflects the structure of the
config file."""
def getDictRootedAt(path):
retval = {}
opts = self.getOptionsDict(path)
secs = self.getSections(path)
for k in opts:
retval[k] = opts[k]
for i in secs:
retval[i] = getDictRootedAt(path + "/" + i)
return retval
return getDictRootedAt(root + "/" + relpath)
def getValue( self, optionPath ):
return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData )
def sortAlphabetically( self, path, ascending = True ):
cfg = self.__getParentCFG( path, parentLevel = 0 )
if cfg:
if cfg.sortAlphabetically( ascending ):
self.__setCommiter( path )
def __getParentCFG( self, path, parentLevel = 1 ):
sectionList = List.fromChar( path, "/" )
cfg = self.cfgData
try:
if parentLevel > 0:
sectionList = sectionList[:-parentLevel]
for section in sectionList:
cfg = cfg[ section ]
return cfg
except:
return False
def __setCommiter( self, entryPath, cfg = False ):
if not cfg:
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
comment = cfg.getComment( entry )
filteredComment = [ line.strip() for line in comment.split( "\n" ) if line.find( self.commiterTag ) != 0 ]
filteredComment.append( "%s%s" % ( self.commiterTag, self.commiterId ) )
cfg.setComment( entry, "\n".join( filteredComment ) )
def setOptionValue( self, optionPath, value ):
levelList = [ level.strip() for level in optionPath.split( "/" ) if level.strip() != "" ]
parentPath = "/%s" % "/".join( levelList[:-1] )
optionName = List.fromChar( optionPath, "/" )[-1]
self.createSection( parentPath )
cfg = self.__getParentCFG( optionPath )
if not cfg:
return
cfg.setOption( optionName, value )
self.__setCommiter( optionPath, cfg )
def createSection( self, sectionPath ):
levelList = [ level.strip() for level in sectionPath.split( "/" ) if level.strip() != "" ]
currentPath = ""
cfg = self.cfgData
createdSection = False
for section in levelList:
currentPath += "/%s" % section
if section not in cfg.listSections():
cfg.createNewSection( section )
self.__setCommiter( currentPath )
createdSection = True
cfg = cfg[ section ]
return createdSection
def setComment( self, entryPath, value ):
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
if cfg.setComment( entry, value ):
self.__setCommiter( entryPath )
return True
return False
def existsSection( self, sectionPath ):
sectionList = List.fromChar( sectionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return len( sectionList ) == 0 or sectionList[-1] in cfg.listSections()
except:
return False
def existsOption( self, optionPath ):
sectionList = List.fromChar( optionPath, "/" )
cfg = self.cfgData
try:
for section in sectionList[:-1]:
cfg = cfg[ section ]
return sectionList[-1] in cfg.listOptions()
except:
return False
def renameKey( self, path, newName ):
parentCfg = self.cfgData.getRecursive( path, -1 )
if not parentCfg:
return False
pathList = List.fromChar( path, "/" )
oldName = pathList[-1]
if parentCfg[ 'value' ].renameKey( oldName, newName ):
pathList[-1] = newName
self.__setCommiter( "/%s" % "/".join( pathList ) )
return True
else:
return False
def copyKey( self, originalKeyPath, newKey ):
parentCfg = self.cfgData.getRecursive( originalKeyPath, -1 )
if not parentCfg:
return False
pathList = List.fromChar( originalKeyPath, "/" )
originalKey = pathList[-1]
if parentCfg[ 'value' ].copyKey( originalKey, newKey ):
self.__setCommiter( "/%s/%s" % ( "/".join( pathList[:-1] ), newKey ) )
return True
return False
def removeOption( self, optionPath ):
if not self.existsOption( optionPath ):
return False
cfg = self.__getParentCFG( optionPath )
optionName = List.fromChar( optionPath, "/" )[-1]
return cfg.deleteKey( optionName )
def removeSection( self, sectionPath ):
if not self.existsSection( sectionPath ):
return False
cfg = self.__getParentCFG( sectionPath )
sectionName = List.fromChar( sectionPath, "/" )[-1]
return cfg.deleteKey( sectionName )
def loadFromBuffer( self, data ):
self.cfgData = CFG()
self.cfgData.loadFromBuffer( data )
def loadFromFile( self, filename ):
self.cfgData = CFG()
self.mergeFromFile( filename )
def dumpToFile( self, filename ):
fd = file( filename, "w" )
fd.write( str( self.cfgData ) )
fd.close()
def mergeFromFile( self, filename ):
cfg = CFG()
cfg.loadFromFile( filename )
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeFromCFG( self, cfg ):
self.cfgData = self.cfgData.mergeWith( cfg )
def mergeSectionFromCFG( self, sectionPath, cfg ):
parentDict = self.cfgData.getRecursive( sectionPath, -1 )
parentCFG = parentDict[ 'value' ]
secName = [ lev.strip() for lev in sectionPath.split( "/" ) if lev.strip() ][-1]
secCFG = parentCFG[ secName ]
if not secCFG:
return False
mergedCFG = secCFG.mergeWith( cfg )
parentCFG.deleteKey( secName )
parentCFG.createNewSection( secName, parentDict[ 'comment' ], mergedCFG )
self.__setCommiter( sectionPath )
return True
def __str__( self ):
return str( self.cfgData )
def commit( self ):
compressedData = zlib.compress( str( self.cfgData ), 9 )
return self.rpcClient.commitNewData( compressedData )
def getHistory( self, limit = 0 ):
retVal = self.rpcClient.getCommitHistory( limit )
if retVal[ 'OK' ]:
return retVal[ 'Value' ]
return []
def showCurrentDiff( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteData = zlib.decompress( retVal[ 'Value' ] ).splitlines()
localData = str( self.cfgData ).splitlines()
return difflib.ndiff( remoteData, localData )
return []
def getVersionDiff( self, fromDate, toDate ):
retVal = self.rpcClient.getVersionContents( [ fromDate, toDate ] )
if retVal[ 'OK' ]:
fromData = zlib.decompress( retVal[ 'Value' ][0] )
toData = zlib.decompress( retVal[ 'Value' ][1] )
return difflib.ndiff( fromData.split( "\n" ), toData.split( "\n" ) )
return []
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def rollbackToVersion( self, version ):
return self.rpcClient.rollbackToVersion( version )
def updateGConfigurationData( self ):
gConfigurationData.setRemoteCFG( self.cfgData )
class ConfigurationData( object ):
def __init__( self, loadDefaultCFG = True ):
lr = LockRing()
self.threadingEvent = lr.getEvent()
self.threadingEvent.set()
self.threadingLock = lr.getLock()
self.runningThreadsNumber = 0
self.__compressedConfigurationData = None
self.configurationPath = "/DIRAC/Configuration"
self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" )
self._isService = False
self.localCFG = CFG()
self.remoteCFG = CFG()
self.mergedCFG = CFG()
self.remoteServerList = []
if loadDefaultCFG:
defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" )
gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile )
retVal = self.loadFile( defaultCFGFile )
if not retVal[ 'OK' ]:
gLogger.warn( "Can't load %s file" % defaultCFGFile )
self.sync()
def getBackupDir( self ):
return self.backupsDir
def sync( self ):
gLogger.debug( "Updating configuration internals" )
self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG )
self.remoteServerList = []
localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.localCFG,
disableDangerZones = True )
if localServers:
self.remoteServerList.extend( List.fromChar( localServers, "," ) )
remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.remoteCFG,
disableDangerZones = True )
if remoteServers:
self.remoteServerList.extend( List.fromChar( remoteServers, "," ) )
self.remoteServerList = List.uniqueElements( self.remoteServerList )
self.__compressedConfigurationData = None
def loadFile( self, fileName ):
try:
fileCFG = CFG()
fileCFG.loadFromFile( fileName )
except IOError:
self.localCFG = self.localCFG.mergeWith( fileCFG )
return S_ERROR( "Can't load a cfg file '%s'" % fileName )
return self.mergeWithLocal( fileCFG )
def mergeWithLocal( self, extraCFG ):
self.lock()
try:
self.localCFG = self.localCFG.mergeWith( extraCFG )
self.unlock()
gLogger.debug( "CFG merged" )
except Exception as e:
self.unlock()
return S_ERROR( "Cannot merge with new cfg: %s" % str( e ) )
self.sync()
return S_OK()
def loadRemoteCFGFromCompressedMem( self, data ):
sUncompressedData = zlib.decompress( data )
self.loadRemoteCFGFromMem( sUncompressedData )
def loadRemoteCFGFromMem( self, data ):
self.lock()
self.remoteCFG.loadFromBuffer( data )
self.unlock()
self.sync()
def loadConfigurationData( self, fileName = False ):
name = self.getName()
self.lock()
try:
if not fileName:
fileName = "%s.cfg" % name
if fileName[0] != "/":
fileName = os.path.join( DIRAC.rootPath, "etc", fileName )
self.remoteCFG.loadFromFile( fileName )
except Exception as e:
print e
pass
self.unlock()
self.sync()
def getCommentFromCFG( self, path, cfg = False ):
if not cfg:
cfg = self.mergedCFG
self.dangerZoneStart()
try:
levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ]
for section in levelList[:-1]:
cfg = cfg[ section ]
return self.dangerZoneEnd( cfg.getComment( levelList[-1] ) )
except Exception:
pass
return self.dangerZoneEnd( None )
def getSectionsFromCFG( self, path, cfg = False, ordered = False ):
if not cfg:
cfg = self.mergedCFG
self.dangerZoneStart()
try:
levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ]
for section in levelList:
cfg = cfg[ section ]
return self.dangerZoneEnd( cfg.listSections( ordered ) )
except Exception:
pass
return self.dangerZoneEnd( None )
def getOptionsFromCFG( self, path, cfg = False, ordered = False ):
if not cfg:
cfg = self.mergedCFG
self.dangerZoneStart()
try:
levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ]
for section in levelList:
cfg = cfg[ section ]
return self.dangerZoneEnd( cfg.listOptions( ordered ) )
except Exception:
pass
return self.dangerZoneEnd( None )
def extractOptionFromCFG( self, path, cfg = False, disableDangerZones = False ):
if not cfg:
cfg = self.mergedCFG
if not disableDangerZones:
self.dangerZoneStart()
try:
levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ]
for section in levelList[:-1]:
cfg = cfg[ section ]
if levelList[-1] in cfg.listOptions():
return self.dangerZoneEnd( cfg[ levelList[ -1 ] ] )
except Exception:
pass
if not disableDangerZones:
self.dangerZoneEnd()
def setOptionInCFG( self, path, value, cfg = False, disableDangerZones = False ):
if not cfg:
cfg = self.localCFG
if not disableDangerZones:
self.dangerZoneStart()
try:
levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ]
for section in levelList[:-1]:
if section not in cfg.listSections():
cfg.createNewSection( section )
cfg = cfg[ section ]
cfg.setOption( levelList[ -1 ], value )
finally:
if not disableDangerZones:
self.dangerZoneEnd()
self.sync()
def deleteOptionInCFG( self, path, cfg = False ):
if not cfg:
cfg = self.localCFG
self.dangerZoneStart()
try:
levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ]
for section in levelList[:-1]:
if section not in cfg.listSections():
return
cfg = cfg[ section ]
cfg.deleteKey( levelList[ -1 ] )
finally:
self.dangerZoneEnd()
self.sync()
def generateNewVersion( self ):
self.setVersion( Time.toString() )
self.sync()
gLogger.info( "Generated new version %s" % self.getVersion() )
def setVersion( self, version, cfg = False ):
if not cfg:
cfg = self.remoteCFG
self.setOptionInCFG( "%s/Version" % self.configurationPath, version, cfg )
def getVersion( self, cfg = False ):
if not cfg:
cfg = self.remoteCFG
value = self.extractOptionFromCFG( "%s/Version" % self.configurationPath, cfg )
if value:
return value
return "0"
def getName( self ):
return self.extractOptionFromCFG( "%s/Name" % self.configurationPath, self.mergedCFG )
def exportName( self ):
return self.setOptionInCFG( "%s/Name" % self.configurationPath, self.getName(), self.remoteCFG )
def getRefreshTime( self ):
try:
return int( self.extractOptionFromCFG( "%s/RefreshTime" % self.configurationPath, self.mergedCFG ) )
except:
return 300
def getPropagationTime( self ):
try:
return int( self.extractOptionFromCFG( "%s/PropagationTime" % self.configurationPath, self.mergedCFG ) )
except:
return 300
def getSlavesGraceTime( self ):
try:
return int( self.extractOptionFromCFG( "%s/SlavesGraceTime" % self.configurationPath, self.mergedCFG ) )
except:
return 600
def mergingEnabled( self ):
try:
val = self.extractOptionFromCFG( "%s/EnableAutoMerge" % self.configurationPath, self.mergedCFG )
return val.lower() in ( "yes", "true", "y" )
except:
return False
def getAutoPublish( self ):
value = self.extractOptionFromCFG( "%s/AutoPublish" % self.configurationPath, self.localCFG )
if value and value.lower() in ( "no", "false", "n" ):
return False
else:
return True
def getServers( self ):
return list( self.remoteServerList )
def getConfigurationGateway( self ):
return self.extractOptionFromCFG( "/DIRAC/Gateway", self.localCFG )
def setServers( self, sServers ):
self.setOptionInCFG( "%s/Servers" % self.configurationPath, sServers, self.remoteCFG )
self.sync()
def deleteLocalOption( self, optionPath ):
self.deleteOptionInCFG( optionPath, self.localCFG )
def getMasterServer( self ):
return self.extractOptionFromCFG( "%s/MasterServer" % self.configurationPath, self.remoteCFG )
def setMasterServer( self, sURL ):
self.setOptionInCFG( "%s/MasterServer" % self.configurationPath, sURL, self.remoteCFG )
self.sync()
def getCompressedData( self ):
if self.__compressedConfigurationData is None:
self.__compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 )
return self.__compressedConfigurationData
def isMaster( self ):
value = self.extractOptionFromCFG( "%s/Master" % self.configurationPath, self.localCFG )
if value and value.lower() in ( "yes", "true", "y" ):
return True
else:
return False
def getServicesPath( self ):
return "/Services"
def setAsService( self ):
self._isService = True
def isService( self ):
return self._isService
def useServerCertificate( self ):
value = self.extractOptionFromCFG( "/DIRAC/Security/UseServerCertificate" )
if value and value.lower() in ( "y", "yes", "true" ):
return True
return False
def skipCACheck( self ):
value = self.extractOptionFromCFG( "/DIRAC/Security/SkipCAChecks" )
if value and value.lower() in ( "y", "yes", "true" ):
return True
return False
def dumpLocalCFGToFile( self, fileName ):
try:
with open( fileName, "w" ) as fd:
fd.write( str( self.localCFG ) )
gLogger.verbose( "Configuration file dumped", "'%s'" % fileName )
except IOError:
gLogger.error( "Can't dump cfg file", "'%s'" % fileName )
return S_ERROR( "Can't dump cfg file '%s'" % fileName )
return S_OK()
def getRemoteCFG( self ):
return self.remoteCFG
def getMergedCFGAsString( self ):
return str( self.mergedCFG )
def dumpRemoteCFGToFile( self, fileName ):
with open( fileName, "w" ) as fd:
fd.write( str( self.remoteCFG ) )
def __backupCurrentConfiguration( self, backupName ):
configurationFilename = "%s.cfg" % self.getName()
configurationFile = os.path.join( DIRAC.rootPath, "etc", configurationFilename )
today = Time.date()
backupPath = os.path.join( self.getBackupDir(), str( today.year ), "%02d" % today.month )
mkDir(backupPath)
backupFile = os.path.join( backupPath, configurationFilename.replace( ".cfg", ".%s.zip" % backupName ) )
if os.path.isfile( configurationFile ):
gLogger.info( "Making a backup of configuration in %s" % backupFile )
try:
zf = zipfile.ZipFile( backupFile, "w", zipfile.ZIP_DEFLATED )
zf.write( configurationFile, "%s.backup.%s" % ( os.path.split( configurationFile )[1], backupName ) )
zf.close()
except Exception:
gLogger.exception()
gLogger.error( "Cannot backup configuration data file", "file %s" % backupFile )
else:
gLogger.warn( "CS data file does not exist", configurationFile )
def writeRemoteConfigurationToDisk( self, backupName = False ):
configurationFile = os.path.join( DIRAC.rootPath, "etc", "%s.cfg" % self.getName() )
try:
with open( configurationFile, "w" ) as fd:
fd.write( str( self.remoteCFG ) )
except Exception as e:
gLogger.fatal( "Cannot write new configuration to disk!",
"file %s" % configurationFile )
return S_ERROR( "Can't write cs file %s!: %s" % ( configurationFile, repr( e ).replace( ',)', ')' ) ) )
if backupName:
self.__backupCurrentConfiguration( backupName )
return S_OK()
def setRemoteCFG( self, cfg, disableSync = False ):
self.remoteCFG = cfg.clone()
if not disableSync:
self.sync()
def lock( self ):
"""
Locks Event to prevent further threads from reading.
Stops current thread until no other thread is accessing.
PRIVATE USE
"""
self.threadingEvent.clear()
while self.runningThreadsNumber > 0:
time.sleep( 0.1 )
def unlock( self ):
"""
Unlocks Event.
PRIVATE USE
"""
self.threadingEvent.set()
def dangerZoneStart( self ):
"""
Start of danger zone. This danger zone may be or may not be a mutual exclusion zone.
Counter is maintained to know how many threads are inside and be able to enable and disable mutual exclusion.
PRIVATE USE
"""
self.threadingEvent.wait()
self.threadingLock.acquire()
self.runningThreadsNumber += 1
try:
self.threadingLock.release()
except thread.error:
pass
def dangerZoneEnd( self, returnValue = None ):
"""
End of danger zone.
PRIVATE USE
"""
self.threadingLock.acquire()
self.runningThreadsNumber -= 1
try:
self.threadingLock.release()
except thread.error:
pass
return returnValue
def __editItem(self, params):
ret = self.__deleteItem(params)
if ret["success"] == 1:
ret = self.__addItem(params)
ret["op"] = "editItem"
return ret
ret["op"] = "editItem"
return ret
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip()
sectionPath = sectionPath + "/" + params["name"]
# deleting the options underneath
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
print "deleting " + opt + "\n"
self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"success":1, "op": "editItem"}
if len(sectionName) == 0:
return {"success":0, "op":"createSection", "message":"Put any name for the section!"}
sectionPath = "%s/%s" % (parentPath, sectionName)
gLogger.info("Creating section", "%s" % sectionPath)
if self.__configData[ 'cfgData' ].createSection(sectionPath):
nD = { 'text' : sectionName, 'csName' : sectionName, 'csComment' : self.__configData[ 'cfgData' ].getComment(sectionPath) }
htmlC = self.__htmlComment(nD[ 'csComment' ])
if htmlC:
qtipDict = { 'text' : htmlC }
nD[ 'qtipCfg' ] = qtipDict
# If config Text is provided then a section is created out of that text
if configText != "":
cfgData = self.__configData[ 'cfgData' ].getCFG()
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
# newCreatedSection = cfgData.getRecursive(sectionPath)["value"]
# newCreatedSection.loadFromBuffer(configText)
return {"success":1, "op":"createSection", "parentNodeId":params["parentNodeId"], "node":nD, "sectionFromConfig": 1}
else:
return {"success":1, "op":"createSection", "parentNodeId":params["parentNodeId"], "node":nD, "sectionFromConfig": 0}
else:
return {"success":0, "op":"createSection", "message":"Section can't be created. It already exists?"}
except Exception, e:
return {"success":0, "op":"createSection", "message":"Can't create section: %s" % str(e)}
def __createOption(self, params):
try:
parentPath = str(params[ 'path' ]).strip()
optionName = str(params[ 'name' ]).strip()
def __editItem(self, params):
ret = self.__deleteItem(params)
if ret["success"] == 1:
ret = self.__addItem(params)
ret["op"] = "editItem"
return ret
ret["op"] = "editItem"
return ret
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params[
"properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(
params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params[
"autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params[
"autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params[
"autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params[
"properties"].strip()
sectionPath = sectionPath + "/" + params["name"]
# deleting the options underneath
sectionCfg = self.getSectionCfg(sectionPath)
for opt in sectionCfg.listAll():
print "deleting " + opt + "\n"
self.__configData['cfgData'].removeOption(sectionPath + "/" + opt)
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData['cfgData'].mergeSectionFromCFG(sectionPath, newCFG)
return {"success": 1, "op": "editItem"}
from DIRAC.Core.Utilities.CFG import CFG
DIRAC.gLogger.initialize('test_gConfig', '/testSectionDebug')
testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath
dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath
cfg1 = CFG()
cfg1.loadFromFile(testconfig)
fd = file(testconfig)
cfg1String = fd.read()
fd.close()
cfg2 = CFG()
cfg2.loadFromBuffer(cfg1.serialize())
cfg3 = cfg1.mergeWith(cfg2)
testList = [{
'method': DIRAC.gConfig.loadFile,
'arguments': (testconfig, ),
'output': {
'OK': True,
'Value': ''
}
}, {
'method': DIRAC.gConfig.dumpLocalCFGToFile,
'arguments': (dumpconfig, ),
'output': {
'OK': True,
class JobDescription( object ):
def __init__( self ):
self.__description = CFG()
self.__dirty = False
def isDirty( self ):
return self.__dirty
def loadDescription( self, dataString ):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL( dataString )
else:
return self.loadDescriptionFromCFG( dataString )
def loadDescriptionFromJDL( self, jdlString ):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG( jdlString.strip() )
if not result[ 'OK' ]:
self.__description = CFG()
return result
self.__description = result[ 'Value' ][0]
return S_OK()
def loadDescriptionFromCFG( self, cfgString ):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer( cfgString )
except Exception as e:
return S_ERROR( "Can't load description from cfg: %s" % str( e ) )
return S_OK()
def dumpDescriptionAsCFG( self ):
return str( self.__description )
def dumpDescriptionAsJDL( self ):
return dumpCFGAsJDL( self.__description )
def __checkNumericalVarInDescription( self, varName, defaultVal, minVal, maxVal ):
"""
Check a numerical var
"""
initialVal = 0
if varName not in self.__description:
varValue = Operations().getValue( "JobDescription/Default%s" % varName , defaultVal )
else:
varValue = self.__description[ varName ]
initialVal = varValue
try:
varValue = long( varValue )
except:
return S_ERROR( "%s must be a number" % varName )
minVal = Operations().getValue( "JobDescription/Min%s" % varName, minVal )
maxVal = Operations().getValue( "JobDescription/Max%s" % varName, maxVal )
varValue = max( minVal, min( varValue, maxVal ) )
if initialVal != varValue:
self.__description.setOption( varName, varValue )
return S_OK( varValue )
def __checkChoiceVarInDescription( self, varName, defaultVal, choices ):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__description:
varValue = Operations().getValue( "JobDescription/Default%s" % varName , defaultVal )
else:
varValue = self.__description[ varName ]
initialVal = varValue
if varValue not in Operations().getValue( "JobDescription/Choices%s" % varName , choices ):
return S_ERROR( "%s is not a valid value for %s" % ( varValue, varName ) )
if initialVal != varValue:
self.__description.setOption( varName, varValue )
return S_OK( varValue )
def __checkMultiChoiceInDescription( self, varName, choices ):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__description:
return S_OK()
else:
varValue = self.__description[ varName ]
initialVal = varValue
choices = Operations().getValue( "JobDescription/Choices%s" % varName , choices )
for v in List.fromChar( varValue ):
if v not in choices:
return S_ERROR( "%s is not a valid value for %s" % ( v, varName ) )
if initialVal != varValue:
self.__description.setOption( varName, varValue )
return S_OK( varValue )
def __checkMaxInputData( self, maxNumber ):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__description:
return S_OK()
varValue = self.__description[ varName ]
if len( List.fromChar( varValue ) ) > maxNumber:
return S_ERROR( 'Number of Input Data Files (%s) greater than current limit: %s' % ( len( List.fromChar( varValue ) ), maxNumber ) )
return S_OK()
def setVarsFromDict( self, varDict ):
for k in sorted( varDict ):
self.setVar( k, varDict[ k ] )
def checkDescription( self ):
"""
Check that the description is OK
"""
for k in [ 'OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup' ]:
if k not in self.__description:
return S_ERROR( "Missing var %s in description" % k )
# Check CPUTime
result = self.__checkNumericalVarInDescription( "CPUTime", 86400, 0, 500000 )
if not result[ 'OK' ]:
return result
result = self.__checkNumericalVarInDescription( "Priority", 1, 0, 10 )
if not result[ 'OK' ]:
return result
allowedSubmitPools = getSubmitPools( self.__description['OwnerGroup'] )
result = self.__checkMultiChoiceInDescription( "SubmitPools", list( set( allowedSubmitPools ) ) )
if not result[ 'OK' ]:
return result
result = self.__checkMultiChoiceInDescription( "SubmitPools", list( set( allowedSubmitPools ) ) )
if not result[ 'OK' ]:
return result
result = self.__checkMultiChoiceInDescription( "PilotTypes", [ 'private' ] )
if not result[ 'OK' ]:
return result
maxInputData = Operations().getValue( "JobDescription/MaxInputData", 500 )
result = self.__checkMaxInputData( maxInputData )
if not result[ 'OK' ]:
return result
transformationTypes = Operations().getValue( "Transformations/DataProcessing", [] )
result = self.__checkMultiChoiceInDescription( "JobType", ['User', 'Test', 'Hospital'] + transformationTypes )
return S_OK()
def setVar( self, varName, varValue ):
"""
Set a var in job description
"""
self.__dirty = True
levels = List.fromChar( varName, "/" )
cfg = self.__description
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection( l )
cfg = cfg[ l ]
cfg.setOption( levels[-1], varValue )
def getVar( self, varName, defaultValue = None ):
cfg = self.__description
return cfg.getOption( varName, defaultValue )
def getOptionList( self, section = "" ):
cfg = self.__description.getRecursive( section )
if not cfg or 'value' not in cfg:
return []
cfg = cfg[ 'value' ]
return cfg.listOptions()
def getSectionList( self, section = "" ):
cfg = self.__description.getRecursive( section )
if not cfg or 'value' not in cfg:
return []
cfg = cfg[ 'value' ]
return cfg.listSections()
modifier = self.__getModificator()
modifier.loadFromBuffer(session['cfgData'])
retDict = modifier.commit()
if not retDict['OK']:
return S_ERROR(retDict['Message'])
return S_OK()
@jsonify
def expandSection(self):
try:
parentNodeId = str(request.params['node'])
sectionPath = str(request.params['nodePath'])
except Exception, e:
return S_ERROR("Cannot expand section %s" % str(e))
cfgData = CFG()
cfgData.loadFromBuffer(session['cfgData'])
gLogger.info("Expanding section", "%s" % sectionPath)
try:
sectionCfg = cfgData
for section in [
section for section in sectionPath.split("/")
if not section.strip() == ""
]:
sectionCfg = sectionCfg[section]
except Exception, v:
gLogger.error("Section does not exist",
"%s -> %s" % (sectionPath, str(v)))
return S_ERROR("Section %s does not exist: %s" %
(sectionPath, str(v)))
gLogger.verbose("Section to expand %s" % sectionPath)
retData = []
from DIRAC.Core.Utilities.CFG import CFG
DIRAC.gLogger.initialize('test_gConfig','/testSectionDebug')
testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath
dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath
cfg1 = CFG()
cfg1.loadFromFile( testconfig )
fd = file( testconfig )
cfg1String = fd.read()
fd.close()
cfg2 = CFG()
cfg2.loadFromBuffer( cfg1.serialize() )
cfg3 = cfg1.mergeWith( cfg2 )
testList = [{ 'method' : DIRAC.gConfig.loadFile,
'arguments' : ( testconfig, ),
'output' : {'OK': True, 'Value': ''}
},
{ 'method' : DIRAC.gConfig.dumpLocalCFGToFile,
'arguments' : ( dumpconfig, ),
'output' : {'OK': True, 'Value': ''}
},
{ 'method' : cfg1.serialize,
'arguments' : ( ),
'output' : cfg1String
},
def __addItem(self, params):
sectionPath = "/Registry/"
configText = ""
if params["type"] == "users":
sectionPath = sectionPath + "Users"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip() + "\n"
if params["email"].strip() != "":
configText = configText + "Email = " + params["email"].strip()
elif params["type"] == "groups":
sectionPath = sectionPath + "Groups"
if params["users"].strip() != "":
configText = "Users = " + params["users"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip() + "\n"
if str(params["jobshare"]).strip() != "":
configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"
if params["autouploadproxy"].strip() != "":
configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"
if params["autouploadpilotproxy"].strip() != "":
configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"
if params["autoaddvoms"].strip() != "":
configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()
elif params["type"] == "hosts":
sectionPath = sectionPath + "Hosts"
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["properties"].strip() != "":
configText = configText + "Properties = " + params["properties"].strip()
elif params["type"] == "voms":
sectionPath = sectionPath + "VOMS/Servers"
elif params["type"] == "servers":
sectionPath = sectionPath + "VOMS/Servers/" + params["vom"]
if params["dn"].strip() != "":
configText = "DN = " + params["dn"].strip() + "\n"
if params["port"].strip() != "":
configText = configText + "Port = " + params["port"].strip() + "\n"
if params["ca"].strip() != "":
configText = configText + "CA = " + params["ca"].strip()
sectionPath = sectionPath + "/" + params["name"]
if self.__configData[ 'cfgData' ].createSection(sectionPath):
cfgData = self.__configData[ 'cfgData' ].getCFG()
newCFG = CFG()
newCFG.loadFromBuffer(configText)
self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
return {"success":1, "op": "addItem"}
else:
return {"success":0, "op":"addItem", "message":"Section can't be created. It already exists?"}
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result['OK']:
raise Exception(result['Message'])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__manifest = CFG()
return result
self.__manifest = result['Value'][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def dumpAsCFG(self):
return str(self.__manifest)
def getAsCFG(self):
return self.__manifest.clone()
def dumpAsJDL(self):
return dumpCFGAsJDL(self.__manifest)
def __getCSValue(self, varName, defaultVal=None):
if not self.__ops:
self.__ops = Operations(group=self.__manifest['OwnerGroup'],
setup=self.__manifest['DIRACSetup'])
if varName[0] != "/":
varName = "JobDescription/%s" % varName
return self.__ops.getValue(varName, defaultVal)
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
"""
Check a numerical var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
try:
varValue = long(varValue)
except BaseException:
return S_ERROR("%s must be a number" % varName)
minVal = self.__getCSValue("Min%s" % varName, minVal)
maxVal = self.__getCSValue("Max%s" % varName, maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVar(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
if varValue not in self.__getCSValue("Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" %
(varValue, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoice(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__manifest:
return S_OK()
else:
varValue = self.__manifest[varName]
initialVal = varValue
choices = self.__getCSValue("Choices%s" % varName, choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
'Number of Input Data Files (%s) greater than current limit: %s'
% (len(List.fromChar(varValue)), maxNumber))
return S_OK()
def __contains__(self, key):
""" Check if the manifest has the required key
"""
return key in self.__manifest
def setOptionsFromDict(self, varDict):
for k in sorted(varDict):
self.setOption(k, varDict[k])
def check(self):
"""
Check that the manifest is OK
"""
for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result['OK']:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result['OK']:
return result
allowedSubmitPools = getSubmitPools(self.__manifest['OwnerGroup'])
result = self.__checkMultiChoice("SubmitPools",
list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoice("PilotTypes", ['private'])
if not result['OK']:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData",
500)
result = self.__checkMaxInputData(maxInputData)
if not result['OK']:
return result
transformationTypes = Operations().getValue(
"Transformations/DataProcessing", [])
result = self.__checkMultiChoice(
"JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
if not result['OK']:
return result
return S_OK()
def createSection(self, secName, contents=False):
if secName not in self.__manifest:
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" %
secName)
self.__dirty = True
return S_OK(
self.__manifest.createNewSection(secName, contents=contents))
return S_ERROR("Section %s already exists" % secName)
def getSection(self, secName):
self.__dirty = True
if secName not in self.__manifest:
return S_ERROR("%s does not exist" % secName)
sec = self.__manifest[secName]
if not sec:
return S_ERROR("%s section empty" % secName)
return S_OK(sec)
def setSectionContents(self, secName, contents):
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" %
secName)
self.__dirty = True
if secName in self.__manifest:
self.__manifest[secName].reset()
self.__manifest[secName].mergeWith(contents)
else:
self.__manifest.createNewSection(secName, contents=contents)
def setOption(self, varName, varValue):
"""
Set a var in job manifest
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def remove(self, opName):
levels = List.fromChar(opName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
return S_ERROR("%s does not exist" % opName)
cfg = cfg[l]
if cfg.deleteKey(levels[-1]):
self.__dirty = True
return S_OK()
return S_ERROR("%s does not exist" % opName)
def getOption(self, varName, defaultValue=None):
"""
Get a variable from the job manifest
"""
cfg = self.__manifest
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
"""
Get a list of variables in a section of the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listOptions()
def isOption(self, opName):
"""
Check if it is a valid option
"""
return self.__manifest.isOption(opName)
def getSectionList(self, section=""):
"""
Get a list of sections in the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listSections()
class ConfigurationData(object):
def __init__(self, loadDefaultCFG=True):
lr = LockRing()
self.threadingEvent = lr.getEvent()
self.threadingEvent.set()
self.threadingLock = lr.getLock()
self.runningThreadsNumber = 0
self.__compressedConfigurationData = None
self.configurationPath = "/DIRAC/Configuration"
self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup")
self._isService = False
self.localCFG = CFG()
self.remoteCFG = CFG()
self.mergedCFG = CFG()
self.remoteServerList = []
if loadDefaultCFG:
defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg")
gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile)
retVal = self.loadFile(defaultCFGFile)
if not retVal['OK']:
gLogger.warn("Can't load %s file" % defaultCFGFile)
self.sync()
def getBackupDir(self):
return self.backupsDir
def sync(self):
gLogger.debug("Updating configuration internals")
self.mergedCFG = self.remoteCFG.mergeWith(self.localCFG)
self.remoteServerList = []
localServers = self.extractOptionFromCFG("%s/Servers" %
self.configurationPath,
self.localCFG,
disableDangerZones=True)
if localServers:
self.remoteServerList.extend(List.fromChar(localServers, ","))
remoteServers = self.extractOptionFromCFG("%s/Servers" %
self.configurationPath,
self.remoteCFG,
disableDangerZones=True)
if remoteServers:
self.remoteServerList.extend(List.fromChar(remoteServers, ","))
self.remoteServerList = List.uniqueElements(self.remoteServerList)
self.__compressedConfigurationData = None
def loadFile(self, fileName):
try:
fileCFG = CFG()
fileCFG.loadFromFile(fileName)
except IOError:
self.localCFG = self.localCFG.mergeWith(fileCFG)
return S_ERROR("Can't load a cfg file '%s'" % fileName)
return self.mergeWithLocal(fileCFG)
def mergeWithLocal(self, extraCFG):
self.lock()
try:
self.localCFG = self.localCFG.mergeWith(extraCFG)
self.unlock()
gLogger.debug("CFG merged")
except Exception as e:
self.unlock()
return S_ERROR("Cannot merge with new cfg: %s" % str(e))
self.sync()
return S_OK()
def loadRemoteCFGFromCompressedMem(self, data):
sUncompressedData = zlib.decompress(data)
self.loadRemoteCFGFromMem(sUncompressedData)
def loadRemoteCFGFromMem(self, data):
self.lock()
self.remoteCFG.loadFromBuffer(data)
self.unlock()
self.sync()
def loadConfigurationData(self, fileName=False):
name = self.getName()
self.lock()
try:
if not fileName:
fileName = "%s.cfg" % name
if fileName[0] != "/":
fileName = os.path.join(DIRAC.rootPath, "etc", fileName)
self.remoteCFG.loadFromFile(fileName)
except Exception as e:
print e
self.unlock()
self.sync()
def getCommentFromCFG(self, path, cfg=False):
if not cfg:
cfg = self.mergedCFG
self.dangerZoneStart()
try:
levelList = [
level.strip() for level in path.split("/")
if level.strip() != ""
]
for section in levelList[:-1]:
cfg = cfg[section]
return self.dangerZoneEnd(cfg.getComment(levelList[-1]))
except Exception:
pass
return self.dangerZoneEnd(None)
def getSectionsFromCFG(self, path, cfg=False, ordered=False):
if not cfg:
cfg = self.mergedCFG
self.dangerZoneStart()
try:
levelList = [
level.strip() for level in path.split("/")
if level.strip() != ""
]
for section in levelList:
cfg = cfg[section]
return self.dangerZoneEnd(cfg.listSections(ordered))
except Exception:
pass
return self.dangerZoneEnd(None)
def getOptionsFromCFG(self, path, cfg=False, ordered=False):
if not cfg:
cfg = self.mergedCFG
self.dangerZoneStart()
try:
levelList = [
level.strip() for level in path.split("/")
if level.strip() != ""
]
for section in levelList:
cfg = cfg[section]
return self.dangerZoneEnd(cfg.listOptions(ordered))
except Exception:
pass
return self.dangerZoneEnd(None)
def extractOptionFromCFG(self, path, cfg=False, disableDangerZones=False):
if not cfg:
cfg = self.mergedCFG
if not disableDangerZones:
self.dangerZoneStart()
try:
levelList = [
level.strip() for level in path.split("/")
if level.strip() != ""
]
for section in levelList[:-1]:
cfg = cfg[section]
if levelList[-1] in cfg.listOptions():
return self.dangerZoneEnd(cfg[levelList[-1]])
except Exception:
pass
if not disableDangerZones:
self.dangerZoneEnd()
def setOptionInCFG(self, path, value, cfg=False, disableDangerZones=False):
if not cfg:
cfg = self.localCFG
if not disableDangerZones:
self.dangerZoneStart()
try:
levelList = [
level.strip() for level in path.split("/")
if level.strip() != ""
]
for section in levelList[:-1]:
if section not in cfg.listSections():
cfg.createNewSection(section)
cfg = cfg[section]
cfg.setOption(levelList[-1], value)
finally:
if not disableDangerZones:
self.dangerZoneEnd()
self.sync()
def deleteOptionInCFG(self, path, cfg=False):
if not cfg:
cfg = self.localCFG
self.dangerZoneStart()
try:
levelList = [
level.strip() for level in path.split("/")
if level.strip() != ""
]
for section in levelList[:-1]:
if section not in cfg.listSections():
return
cfg = cfg[section]
cfg.deleteKey(levelList[-1])
finally:
self.dangerZoneEnd()
self.sync()
def generateNewVersion(self):
self.setVersion(Time.toString())
self.sync()
gLogger.info("Generated new version %s" % self.getVersion())
def setVersion(self, version, cfg=False):
if not cfg:
cfg = self.remoteCFG
self.setOptionInCFG("%s/Version" % self.configurationPath, version,
cfg)
def getVersion(self, cfg=False):
if not cfg:
cfg = self.remoteCFG
value = self.extractOptionFromCFG(
"%s/Version" % self.configurationPath, cfg)
if value:
return value
return "0"
def getName(self):
return self.extractOptionFromCFG("%s/Name" % self.configurationPath,
self.mergedCFG)
def exportName(self):
return self.setOptionInCFG("%s/Name" % self.configurationPath,
self.getName(), self.remoteCFG)
def getRefreshTime(self):
try:
return int(
self.extractOptionFromCFG(
"%s/RefreshTime" % self.configurationPath, self.mergedCFG))
except:
return 300
def getPropagationTime(self):
try:
return int(
self.extractOptionFromCFG(
"%s/PropagationTime" % self.configurationPath,
self.mergedCFG))
except:
return 300
def getSlavesGraceTime(self):
try:
return int(
self.extractOptionFromCFG(
"%s/SlavesGraceTime" % self.configurationPath,
self.mergedCFG))
except:
return 600
def mergingEnabled(self):
try:
val = self.extractOptionFromCFG(
"%s/EnableAutoMerge" % self.configurationPath, self.mergedCFG)
return val.lower() in ("yes", "true", "y")
except:
return False
def getAutoPublish(self):
value = self.extractOptionFromCFG(
"%s/AutoPublish" % self.configurationPath, self.localCFG)
if value and value.lower() in ("no", "false", "n"):
return False
else:
return True
def getServers(self):
return list(self.remoteServerList)
def getConfigurationGateway(self):
return self.extractOptionFromCFG("/DIRAC/Gateway", self.localCFG)
def setServers(self, sServers):
self.setOptionInCFG("%s/Servers" % self.configurationPath, sServers,
self.remoteCFG)
self.sync()
def deleteLocalOption(self, optionPath):
self.deleteOptionInCFG(optionPath, self.localCFG)
def getMasterServer(self):
return self.extractOptionFromCFG(
"%s/MasterServer" % self.configurationPath, self.remoteCFG)
def setMasterServer(self, sURL):
self.setOptionInCFG("%s/MasterServer" % self.configurationPath, sURL,
self.remoteCFG)
self.sync()
def getCompressedData(self):
if self.__compressedConfigurationData is None:
self.__compressedConfigurationData = zlib.compress(
str(self.remoteCFG), 9)
return self.__compressedConfigurationData
def isMaster(self):
value = self.extractOptionFromCFG("%s/Master" % self.configurationPath,
self.localCFG)
if value and value.lower() in ("yes", "true", "y"):
return True
else:
return False
def getServicesPath(self):
return "/Services"
def setAsService(self):
self._isService = True
def isService(self):
return self._isService
def useServerCertificate(self):
value = self.extractOptionFromCFG(
"/DIRAC/Security/UseServerCertificate")
if value and value.lower() in ("y", "yes", "true"):
return True
return False
def skipCACheck(self):
value = self.extractOptionFromCFG("/DIRAC/Security/SkipCAChecks")
if value and value.lower() in ("y", "yes", "true"):
return True
return False
def dumpLocalCFGToFile(self, fileName):
try:
with open(fileName, "w") as fd:
fd.write(str(self.localCFG))
gLogger.verbose("Configuration file dumped", "'%s'" % fileName)
except IOError:
gLogger.error("Can't dump cfg file", "'%s'" % fileName)
return S_ERROR("Can't dump cfg file '%s'" % fileName)
return S_OK()
def getRemoteCFG(self):
return self.remoteCFG
def getMergedCFGAsString(self):
return str(self.mergedCFG)
def dumpRemoteCFGToFile(self, fileName):
with open(fileName, "w") as fd:
fd.write(str(self.remoteCFG))
def __backupCurrentConfiguration(self, backupName):
configurationFilename = "%s.cfg" % self.getName()
configurationFile = os.path.join(DIRAC.rootPath, "etc",
configurationFilename)
today = Time.date()
backupPath = os.path.join(self.getBackupDir(), str(today.year),
"%02d" % today.month)
mkDir(backupPath)
backupFile = os.path.join(
backupPath,
configurationFilename.replace(".cfg", ".%s.zip" % backupName))
if os.path.isfile(configurationFile):
gLogger.info("Making a backup of configuration in %s" % backupFile)
try:
with zipfile.ZipFile(backupFile, "w",
zipfile.ZIP_DEFLATED) as zf:
zf.write(
configurationFile, "%s.backup.%s" %
(os.path.split(configurationFile)[1], backupName))
except Exception:
gLogger.exception()
gLogger.error("Cannot backup configuration data file",
"file %s" % backupFile)
else:
gLogger.warn("CS data file does not exist", configurationFile)
def writeRemoteConfigurationToDisk(self, backupName=False):
configurationFile = os.path.join(DIRAC.rootPath, "etc",
"%s.cfg" % self.getName())
try:
with open(configurationFile, "w") as fd:
fd.write(str(self.remoteCFG))
except Exception as e:
gLogger.fatal("Cannot write new configuration to disk!",
"file %s" % configurationFile)
return S_ERROR("Can't write cs file %s!: %s" %
(configurationFile, repr(e).replace(',)', ')')))
if backupName:
self.__backupCurrentConfiguration(backupName)
return S_OK()
def setRemoteCFG(self, cfg, disableSync=False):
self.remoteCFG = cfg.clone()
if not disableSync:
self.sync()
def lock(self):
"""
Locks Event to prevent further threads from reading.
Stops current thread until no other thread is accessing.
PRIVATE USE
"""
self.threadingEvent.clear()
while self.runningThreadsNumber > 0:
time.sleep(0.1)
def unlock(self):
"""
Unlocks Event.
PRIVATE USE
"""
self.threadingEvent.set()
def dangerZoneStart(self):
"""
Start of danger zone. This danger zone may be or may not be a mutual exclusion zone.
Counter is maintained to know how many threads are inside and be able to enable and disable mutual exclusion.
PRIVATE USE
"""
self.threadingEvent.wait()
self.threadingLock.acquire()
self.runningThreadsNumber += 1
try:
self.threadingLock.release()
except thread.error:
pass
def dangerZoneEnd(self, returnValue=None):
"""
End of danger zone.
PRIVATE USE
"""
self.threadingLock.acquire()
self.runningThreadsNumber -= 1
try:
self.threadingLock.release()
except thread.error:
pass
return returnValue
modifier = self.__getModificator()
modifier.loadFromBuffer(session["cfgData"])
retDict = modifier.commit()
if not retDict["OK"]:
return S_ERROR(retDict["Message"])
return S_OK()
@jsonify
def expandSection(self):
try:
parentNodeId = str(request.params["node"])
sectionPath = str(request.params["nodePath"])
except Exception, e:
return S_ERROR("Cannot expand section %s" % str(e))
cfgData = CFG()
cfgData.loadFromBuffer(session["cfgData"])
gLogger.info("Expanding section", "%s" % sectionPath)
try:
sectionCfg = cfgData
for section in [section for section in sectionPath.split("/") if not section.strip() == ""]:
sectionCfg = sectionCfg[section]
except Exception, v:
gLogger.error("Section does not exist", "%s -> %s" % (sectionPath, str(v)))
return S_ERROR("Section %s does not exist: %s" % (sectionPath, str(v)))
gLogger.verbose("Section to expand %s" % sectionPath)
retData = []
for entryName in sectionCfg.listAll():
id = "%s/%s" % (parentNodeId, entryName)
comment = sectionCfg.getComment(entryName)
nodeDef = {"text": entryName, "csName": entryName, "csComment": comment}
if not sectionCfg.isSection(entryName):
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result["OK"]:
raise Exception(result["Message"])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result["OK"]:
self.__manifest = CFG()
return result
self.__manifest = result["Value"][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def dumpAsCFG(self):
return str(self.__manifest)
def getAsCFG(self):
return self.__manifest.clone()
def dumpAsJDL(self):
return dumpCFGAsJDL(self.__manifest)
def __getCSValue(self, varName, defaultVal=None):
if not self.__ops:
self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"])
if varName[0] != "/":
varName = "JobDescription/%s" % varName
return self.__ops.getValue(varName, defaultVal)
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
"""
Check a numerical var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
try:
varValue = long(varValue)
except:
return S_ERROR("%s must be a number" % varName)
minVal = self.__getCSValue("Min%s" % varName, minVal)
maxVal = self.__getCSValue("Max%s" % varName, maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVar(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
if varValue not in self.__getCSValue("Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoice(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__manifest:
return S_OK()
else:
varValue = self.__manifest[varName]
initialVal = varValue
choices = self.__getCSValue("Choices%s" % varName, choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
"Number of Input Data Files (%s) greater than current limit: %s"
% (len(List.fromChar(varValue)), maxNumber)
)
return S_OK()
def __contains__(self, key):
""" Check if the manifest has the required key
"""
return key in self.__manifest
def setOptionsFromDict(self, varDict):
for k in sorted(varDict):
self.setOption(k, varDict[k])
def check(self):
"""
Check that the manifest is OK
"""
for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result["OK"]:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result["OK"]:
return result
allowedSubmitPools = getSubmitPools(self.__manifest["OwnerGroup"])
result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools)))
if not result["OK"]:
return result
result = self.__checkMultiChoice("PilotTypes", ["private"])
if not result["OK"]:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
result = self.__checkMaxInputData(maxInputData)
if not result["OK"]:
return result
transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
result = self.__checkMultiChoice("JobType", ["User", "Test", "Hospital"] + transformationTypes)
if not result["OK"]:
return result
return S_OK()
def createSection(self, secName, contents=False):
if secName not in self.__manifest:
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
return S_OK(self.__manifest.createNewSection(secName, contents=contents))
return S_ERROR("Section %s already exists" % secName)
def getSection(self, secName):
self.__dirty = True
sec = self.__manifest[secName]
if not sec:
return S_ERROR("%s does not exist")
return S_OK(sec)
def setSectionContents(self, secName, contents):
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
if secName in self.__manifest:
self.__manifest[secName].reset()
self.__manifest[secName].mergeWith(contents)
else:
self.__manifest.createNewSection(secName, contents=contents)
def setOption(self, varName, varValue):
"""
Set a var in job manifest
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def remove(self, opName):
levels = List.fromChar(opName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
return S_ERROR("%s does not exist" % opName)
cfg = cfg[l]
if cfg.deleteKey(levels[-1]):
self.__dirty = True
return S_OK()
return S_ERROR("%s does not exist" % opName)
def getOption(self, varName, defaultValue=None):
"""
Get a variable from the job manifest
"""
cfg = self.__manifest
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
"""
Get a list of variables in a section of the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listOptions()
def isOption(self, opName):
"""
Check if it is a valid option
"""
return self.__manifest.isOption(opName)
def getSectionList(self, section=""):
"""
Get a list of sections in the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listSections()
class JobDescription(object):
def __init__(self):
self.__description = CFG()
self.__dirty = False
def isDirty(self):
return self.__dirty
def loadDescription(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadDescriptionFromJDL(dataString)
else:
return self.loadDescriptionFromCFG(dataString)
def loadDescriptionFromJDL(self, jdlString):
"""
Load job description from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result['OK']:
self.__description = CFG()
return result
self.__description = result['Value'][0]
return S_OK()
def loadDescriptionFromCFG(self, cfgString):
"""
Load job description from CFG format
"""
try:
self.__description.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load description from cfg: %s" % str(e))
return S_OK()
def dumpDescriptionAsCFG(self):
return str(self.__description)
def dumpDescriptionAsJDL(self):
return dumpCFGAsJDL(self.__description)
def __checkNumericalVarInDescription(self, varName, defaultVal, minVal,
maxVal):
"""
Check a numerical var
"""
initialVal = 0
if varName not in self.__description:
varValue = Operations().getValue(
"JobDescription/Default%s" % varName, defaultVal)
else:
varValue = self.__description[varName]
initialVal = varValue
try:
varValue = long(varValue)
except BaseException:
return S_ERROR("%s must be a number" % varName)
minVal = Operations().getValue("JobDescription/Min%s" % varName,
minVal)
maxVal = Operations().getValue("JobDescription/Max%s" % varName,
maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVarInDescription(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__description:
varValue = Operations().getValue(
"JobDescription/Default%s" % varName, defaultVal)
else:
varValue = self.__description[varName]
initialVal = varValue
if varValue not in Operations().getValue(
"JobDescription/Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" %
(varValue, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoiceInDescription(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__description:
return S_OK()
else:
varValue = self.__description[varName]
initialVal = varValue
choices = Operations().getValue("JobDescription/Choices%s" % varName,
choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__description:
return S_OK()
varValue = self.__description[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
'Number of Input Data Files (%s) greater than current limit: %s'
% (len(List.fromChar(varValue)), maxNumber))
return S_OK()
def setVarsFromDict(self, varDict):
for k in sorted(varDict):
self.setVar(k, varDict[k])
def checkDescription(self):
"""
Check that the description is OK
"""
for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
if k not in self.__description:
return S_ERROR("Missing var %s in description" % k)
# Check CPUTime
result = self.__checkNumericalVarInDescription("CPUTime", 86400, 0,
500000)
if not result['OK']:
return result
result = self.__checkNumericalVarInDescription("Priority", 1, 0, 10)
if not result['OK']:
return result
allowedSubmitPools = getSubmitPools(self.__description['OwnerGroup'])
result = self.__checkMultiChoiceInDescription(
"SubmitPools", list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoiceInDescription(
"SubmitPools", list(set(allowedSubmitPools)))
if not result['OK']:
return result
result = self.__checkMultiChoiceInDescription("PilotTypes",
['private'])
if not result['OK']:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData",
500)
result = self.__checkMaxInputData(maxInputData)
if not result['OK']:
return result
transformationTypes = Operations().getValue(
"Transformations/DataProcessing", [])
result = self.__checkMultiChoiceInDescription(
"JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
return S_OK()
def setVar(self, varName, varValue):
"""
Set a var in job description
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__description
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def getVar(self, varName, defaultValue=None):
cfg = self.__description
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
cfg = self.__description.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listOptions()
def getSectionList(self, section=""):
cfg = self.__description.getRecursive(section)
if not cfg or 'value' not in cfg:
return []
cfg = cfg['value']
return cfg.listSections()
