def serialize(self, tabLevelString=""):
"""
Generate a human readable serialization of a CFG
@type tabLevelString: string
@param tabLevelString: Tab string to apply to entries before representing them
@return: String with the contents of the CFG
"""
indentation = " "
cfgString = ""
for entryName in self.__orderedList:
if entryName in self.__commentDict:
for commentLine in List.fromChar(self.__commentDict[entryName], "\n"):
cfgString += "%s#%s\n" % (tabLevelString, commentLine)
if entryName in self.listSections():
cfgString += "%s%s\n%s{\n" % (tabLevelString, entryName, tabLevelString)
cfgString += self.__dataDict[entryName].serialize("%s%s" % (tabLevelString, indentation))
cfgString += "%s}\n" % tabLevelString
elif entryName in self.listOptions():
valueList = List.fromChar(self.__dataDict[entryName])
if len(valueList) == 0:
cfgString += "%s%s = \n" % (tabLevelString, entryName)
else:
cfgString += "%s%s = %s\n" % (tabLevelString, entryName, valueList[0])
for value in valueList[1:]:
cfgString += "%s%s += %s\n" % (tabLevelString, entryName, value)
else:
raise ValueError("Oops. There is an entry in the order which is not a section nor an option")
return cfgString
def copyKey( self, oldName, newName ):
"""
Copy an option/section
@type oldName: string
@param oldName: Name of the option / section to copy
@type newName: string
@param newName: Destination name
@return: Boolean with the result
"""
if oldName == newName:
return True
result = self.getRecursive( oldName, -1 )
if not result:
raise KeyError( "%s does not exist" % "/".join( List.fromChar( oldName, "/" )[:-1] ) )
oldCfg = result[ 'value' ]
oldEnd = result[ 'levelsBelow' ]
if oldEnd in oldCfg.__dataDict:
result = self.getRecursive( newName, -1 )
if not result:
raise KeyError( "%s does not exist" % "/".join( List.fromChar( newName, "/" )[:-1] ) )
newCfg = result[ 'value' ]
newEnd = result[ 'levelsBelow' ]
newCfg.__dataDict[ newEnd ] = oldCfg.__dataDict[ oldEnd ]
newCfg.__commentDict[ newEnd ] = oldCfg.__commentDict[ oldEnd ]
refKeyPos = oldCfg.__orderedList.index( oldEnd )
newCfg.__orderedList.insert( refKeyPos + 1, newEnd )
return True
else:
return False
def renameKey(self, oldName, newName):
"""
Rename a option/section
@type oldName: string
@param oldName: Name of the option/section to change
@type newName: string
@param newName: New name of the option/section
@return: Boolean with the result of the rename
"""
if oldName == newName:
return True
result = self.getRecursive(oldName, -1)
if not result:
raise KeyError("%s does not exist" % "/".join(List.fromChar(oldName, "/")[:-1]))
oldCfg = result["value"]
oldEnd = result["levelsBelow"]
if oldEnd in oldCfg.__dataDict:
result = self.getRecursive(newName, -1)
if not result:
raise KeyError("%s does not exist" % "/".join(List.fromChar(newName, "/")[:-1]))
newCfg = result["value"]
newEnd = result["levelsBelow"]
newCfg.__dataDict[newEnd] = oldCfg.__dataDict[oldEnd]
newCfg.__commentDict[newEnd] = oldCfg.__commentDict[oldEnd]
refKeyPos = oldCfg.__orderedList.index(oldEnd)
oldCfg.__orderedList.remove(oldEnd)
newCfg.__orderedList.insert(refKeyPos, newEnd)
del (oldCfg.__dataDict[oldEnd])
del (oldCfg.__commentDict[oldEnd])
return True
else:
return False
def __findServiceURL( self ):
if not self.__initStatus[ 'OK' ]:
return self.__initStatus
gatewayURL = False
if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[ self.KW_IGNORE_GATEWAYS ]:
dRetVal = gConfig.getOption( "/DIRAC/Gateways/%s" % DIRAC.siteName() )
if dRetVal[ 'OK' ]:
rawGatewayURL = List.randomize( List.fromChar( dRetVal[ 'Value'], "," ) )[0]
gatewayURL = "/".join( rawGatewayURL.split( "/" )[:3] )
for protocol in gProtocolDict.keys():
if self._destinationSrv.find( "%s://" % protocol ) == 0:
gLogger.debug( "Already given a valid url", self._destinationSrv )
if not gatewayURL:
return S_OK( self._destinationSrv )
gLogger.debug( "Reconstructing given URL to pass through gateway" )
path = "/".join( self._destinationSrv.split( "/" )[3:] )
finalURL = "%s/%s" % ( gatewayURL, path )
gLogger.debug( "Gateway URL conversion:\n %s -> %s" % ( self._destinationSrv, finalURL ) )
return S_OK( finalURL )
if gatewayURL:
gLogger.debug( "Using gateway", gatewayURL )
return S_OK( "%s/%s" % ( gatewayURL, self._destinationSrv ) )
try:
urls = getServiceURL( self._destinationSrv, setup = self.setup )
except Exception, e:
return S_ERROR( "Cannot get URL for %s in setup %s: %s" % ( self._destinationSrv, self.setup, str( e ) ) )
def __executeVOPlugin(self, voPlugin, jobState):
if voPlugin not in self.__voPlugins:
modName = List.fromChar(voPlugin, ".")[-1]
try:
module = __import__(voPlugin, globals(), locals(), [modName])
except ImportError as excp:
self.jobLog.exception("Could not import VO plugin %s" % voPlugin)
return S_ERROR("Could not import VO plugin %s: %s" % (voPlugin, excp))
try:
self.__voPlugins[voPlugin] = getattr(module, modName)
except AttributeError as excp:
return S_ERROR("Could not get plugin %s from module %s: %s" % (modName, voPlugin, str(excp)))
argsDict = {'JobID': jobState.jid,
'JobState': jobState,
'ConfigPath': self.ex_getProperty("section")}
try:
modInstance = self.__voPlugins[voPlugin](argsDict)
result = modInstance.execute()
except Exception as excp:
self.jobLog.exception("Excp while executing %s" % voPlugin)
return S_ERROR("Could not execute VO plugin %s: %s" % (voPlugin, excp))
if not result['OK']:
return result
extraPath = result['Value']
if isinstance(extraPath, basestring):
extraPath = List.fromChar(result['Value'])
return S_OK(extraPath)
def __refresh( self ):
self.__lastUpdateTime = time.time()
gLogger.debug( "Refreshing configuration..." )
gatewayList = getGatewayURLs( "Configuration/Server" )
updatingErrorsList = []
if gatewayList:
lInitialListOfServers = gatewayList
gLogger.debug( "Using configuration gateway", str( lInitialListOfServers[0] ) )
else:
lInitialListOfServers = gConfigurationData.getServers()
gLogger.debug( "Refreshing from list %s" % str( lInitialListOfServers ) )
lRandomListOfServers = List.randomize( lInitialListOfServers )
gLogger.debug( "Randomized server list is %s" % ", ".join( lRandomListOfServers ) )
for sServer in lRandomListOfServers:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient( sServer,
useCertificates = gConfigurationData.useServerCertificate(),
skipCACheck = gConfigurationData.skipCACheck() )
dRetVal = _updateFromRemoteLocation( oClient )
if dRetVal[ 'OK' ]:
return dRetVal
else:
updatingErrorsList.append( dRetVal[ 'Message' ] )
gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % ( sServer, dRetVal[ 'Message' ] ) )
if dRetVal[ 'Message' ].find( "Insane environment" ) > -1:
break
return S_ERROR( "Reason(s):\n\t%s" % "\n\t".join( List.uniqueElements( updatingErrorsList ) ) )
def checkJob( self, job, classAdJob ):
"""This method controls the checking of the job.
"""
jobDesc = JobDescription()
result = jobDesc.loadDescription( classAdJob.asJDL() )
if not result[ 'OK' ]:
self.setFailedJob( job, result['Message'], classAdJob )
return result
self.__syncJobDesc( job, jobDesc, classAdJob )
#Check if job defines a path itself
# FIXME: only some group might be able to overwrite the jobPath
jobPath = classAdJob.get_expression( 'JobPath' ).replace( '"', '' ).replace( 'Unknown', '' )
#jobPath = jobDesc.getVarWithDefault( 'JobPath' ).replace( 'Unknown', '' )
if jobPath:
# HACK: Remove the { and } to ensure we have a simple string
jobPath = jobPath.replace( "{", "" ).replace( "}", "" )
self.log.info( 'Job %s defines its own optimizer chain %s' % ( job, jobPath ) )
return self.processJob( job, List.fromChar( jobPath ) )
#If no path, construct based on JDL and VO path module if present
path = list( self.basePath )
if self.voPlugin:
argumentsDict = {'JobID':job, 'ClassAd':classAdJob, 'ConfigPath':self.am_getModuleParam( "section" )}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule( self.voPlugin, argumentsDict )
if not moduleInstance['OK']:
self.log.error( 'Could not instantiate module:', '%s' % ( self.voPlugin ) )
self.setFailedJob( job, 'Could not instantiate module: %s' % ( self.voPlugin ), classAdJob )
return S_ERROR( 'Holding pending jobs' )
module = moduleInstance['Value']
result = module.execute()
if not result['OK']:
self.log.warn( 'Execution of %s failed' % ( self.voPlugin ) )
return result
extraPath = List.fromChar( result['Value'] )
if extraPath:
path.extend( extraPath )
self.log.verbose( 'Adding extra VO specific optimizers to path: %s' % ( extraPath ) )
else:
self.log.verbose( 'No VO specific plugin module specified' )
#Should only rely on an input data setting in absence of VO plugin
result = self.jobDB.getInputData( job )
if not result['OK']:
self.log.error( 'Failed to get input data from JobDB', job )
self.log.warn( result['Message'] )
return result
if result['Value']:
# if the returned tuple is not empty it will evaluate true
self.log.info( 'Job %s has an input data requirement' % ( job ) )
path.extend( self.inputData )
else:
self.log.info( 'Job %s has no input data requirement' % ( job ) )
path.extend( self.endPath )
self.log.info( 'Constructed path for job %s is: %s' % ( job, path ) )
return self.processJob( job, path )
def testUniqueElements(self):
""" uniqueElements tests """
# empty list
aList = []
self.assertEqual(List.uniqueElements(aList), [])
# redundant elements
aList = [1, 1, 2, 3]
self.assertEqual(List.uniqueElements(aList), [1, 2, 3])
def testRemoveEmptyElements(self):
""" removeEmptyElements tests """
# empty list
aList = []
self.assertEqual(List.removeEmptyElements(aList), [])
# None or "" (empty string) in
aList = ["", None, 1]
self.assertEqual(List.removeEmptyElements(aList), [1])
def getOption(self, opName, defaultValue=None):
"""
Get option value with default applied
@type opName: string
@param opName: Path to the option to retrieve
@type defaultValue: optional (any python type)
@param defaultValue: Default value for the option if the option is not defined.
If the option is defined, the value will be returned casted to
the type of defaultValue if it is defined.
@return: Value of the option casted to defaultValue type, or defaultValue
"""
levels = List.fromChar(opName, "/")
dataD = self.__dataDict
while len(levels) > 0:
try:
dataV = dataD[levels.pop(0)]
except KeyError:
return defaultValue
dataD = dataV
if type(dataV) != types.StringType:
optionValue = defaultValue
else:
optionValue = dataV
# Return value if existing, defaultValue if not
if optionValue == defaultValue:
if defaultValue == None or type(defaultValue) == types.TypeType:
return defaultValue
return optionValue
# Value has been returned from the configuration
if defaultValue == None:
return optionValue
# Casting to defaultValue's type
defaultType = defaultValue
if not type(defaultValue) == types.TypeType:
defaultType = type(defaultValue)
if defaultType == types.ListType:
try:
return List.fromChar(optionValue, ",")
except Exception:
return defaultValue
elif defaultType == types.BooleanType:
try:
return optionValue.lower() in ("y", "yes", "true", "1")
except Exception:
return defaultValue
else:
try:
return defaultType(optionValue)
except Exception:
return defaultValue
def getGatewayURLs( serviceName = "" ):
siteName = gConfigurationData.extractOptionFromCFG( "/LocalSite/Site" )
if not siteName:
return False
gatewayList = gConfigurationData.extractOptionFromCFG( "/DIRAC/Gateways/%s" % siteName )
if not gatewayList:
return False
if serviceName:
gatewayList = [ "%s/%s" % ( gw, serviceName ) for gw in List.fromChar( gatewayList, "," ) ]
return List.randomize( gatewayList )
def testBreakListIntoChunks(self):
""" breakListIntoChunks tests """
# empty list
aList = []
self.assertEqual(List.breakListIntoChunks(aList, 5), [])
# negative number of chunks
try:
List.breakListIntoChunks([], -2)
except Exception, val:
self.assertEqual(isinstance(val, RuntimeError), True)
self.assertEqual(str(val), "chunkSize cannot be less than 1")
def testPop(self):
""" pop tests """
# empty list
aList = []
x = List.pop(aList, 1)
self.assertEqual(aList, [])
self.assertEqual(x, None)
# pop
aList = [1, 2, 3]
x = List.pop(aList, 2)
self.assertEqual(x, 2)
self.assertEqual(aList, [1, 3])
def uniquePath( path = None ):
"""
Utility to squeeze the string containing a PATH-like value to
leave only unique elements preserving the original order
"""
if not isinstance( path, basestring ):
return None
try:
elements = List.uniqueElements( List.fromChar( path, ":" ) )
return ':'.join( elements )
except Exception:
return None
def testRandomize(self):
""" randomize tests """
# empty list
aList = []
randList = List.randomize(aList)
self.assertEqual(randList, [])
# non empty
aList = [1, 2, 3]
randList = List.randomize(aList)
self.assertEqual(len(aList), len(randList))
for x in aList:
self.assertEqual(x in randList, True)
for x in randList:
self.assertEqual(x in aList, True)
def testIntListToString(self):
""" intListToString """
# empty list
aList = []
aStr = List.intListToString(aList)
self.assertEqual(aStr, "")
# int list
aList = [1, 2, 3]
aStr = List.intListToString(aList)
self.assertEqual(aStr, "1,2,3")
# mixture elements (should it raise an exception???)
aList = ["1", 2, 3]
aStr = List.intListToString(aList)
self.assertEqual(aStr, "1,2,3")
def testStringListToString(self):
""" stringListToString tests """
# empty list
aList = []
aStr = List.stringListToString(aList)
self.assertEqual(aStr, "")
# not string elements (should it raise an exception???)
aList = ["a", 1]
aStr = List.stringListToString(aList)
self.assertEqual(aStr, "'a','1'")
# normal list
aList = ["a", "b", "c"]
aStr = List.stringListToString(aList)
self.assertEqual(aStr, "'a','b','c'")
def uniquePath(path=None):
"""
Utility to squeeze the string containing a PATH-like value to
leave only unique elements preserving the original order
"""
if not StringTypes.__contains__(type(path)):
return None
try:
elements = List.uniqueElements(List.fromChar(path, ":"))
return ":".join(elements)
except Exception:
return None
def testSorted(self):
""" sortList tests """
# empty list
aList = []
self.assertEqual(List.sortList(aList), [])
# already sorted
aList = ["a", "b", "c"]
self.assertEqual(List.sortList(aList), ["a", "b", "c"])
# unsorted
aList = ["a", "c", "b"]
self.assertEqual(List.sortList(aList), ["a", "b", "c"])
# invert
aList = ["a", "b", "c"]
self.assertEqual(List.sortList(aList, invert=True), ["c", "b", "a"])
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
"Number of Input Data Files (%s) greater than current limit: %s"
% (len(List.fromChar(varValue)), maxNumber)
)
return S_OK()
def sync( self ):
gLogger.debug( "Updating configuration internals" )
self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG )
self.remoteServerList = []
localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.localCFG,
disableDangerZones = True )
if localServers:
self.remoteServerList.extend( List.fromChar( localServers, "," ) )
remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.remoteCFG,
disableDangerZones = True )
if remoteServers:
self.remoteServerList.extend( List.fromChar( remoteServers, "," ) )
self.remoteServerList = List.uniqueElements( self.remoteServerList )
self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 )
def getSocket( self, hostAddress, **kwargs ):
hostName = hostAddress[0]
retVal = self.generateClientInfo( hostName, kwargs )
if not retVal[ 'OK' ]:
return retVal
socketInfo = retVal[ 'Value' ]
retVal = Network.getIPsForHostName( hostName )
if not retVal[ 'OK' ]:
return S_ERROR( "Could not resolve %s: %s" % ( hostName, retVal[ 'Message' ] ) )
ipList = List.randomize( retVal[ 'Value' ] )
for i in range( 3 ):
connected = False
errorsList = []
for ip in ipList :
ipAddress = ( ip, hostAddress[1] )
retVal = self.__connect( socketInfo, ipAddress )
if retVal[ 'OK' ]:
sslSocket = retVal[ 'Value' ]
connected = True
break
errorsList.append( "%s: %s" % ( ipAddress, retVal[ 'Message' ] ) )
if not connected:
return S_ERROR( "Could not connect to %s: %s" % ( hostAddress, "," .join( [ e for e in errorsList ] ) ) )
retVal = socketInfo.doClientHandshake()
if retVal[ 'OK' ]:
#Everything went ok. Don't need to retry
break
#Did the auth or the connection fail?
if not retVal['OK']:
return retVal
if 'enableSessions' in kwargs and kwargs[ 'enableSessions' ]:
sessionId = hash( hostAddress )
gSessionManager.set( sessionId, sslSocket.get_session() )
return S_OK( socketInfo )
def do_set( self, args ):
"""
Set options
usage:
set host <hostname> - Set the hostname to work with
set project <project> - Set the project to install/upgrade in the host
"""
cmds = { 'host' : ( 1, self.__do_set_host ),
'project' : ( 1, self.__do_set_project ) }
args = List.fromChar( args, " " )
found = False
for cmd in cmds:
if cmd == args[0]:
if len( args ) != 1 + cmds[ cmd ][0]:
self.__errMsg( "Missing arguments" )
print self.do_set.__doc__
return
return cmds[ cmd ][1]( args[1:] )
self.__errMsg( "Invalid command" )
print self.do_set.__doc__
return
def getComponentsStatus( self, conditionDict = {} ):
"""
Get the status of the defined components in the CS compared to the ones that are known in the DB
"""
result = self.__getComponents( conditionDict )
if not result[ 'OK' ]:
return result
statusSet = result[ 'Value' ]
requiredComponents = {}
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
for setup in result[ 'Value' ]:
if not self.__checkCondition( conditionDict, "Setup", setup ):
continue
#Iterate through systems
result = gConfig.getOptionsDict( "/DIRAC/Setups/%s" % setup )
if not result[ 'OK' ]:
return result
systems = result[ 'Value' ]
for system in systems:
instance = systems[ system ]
#Check defined agents and serviecs
for type in ( 'agent' , 'service' ):
#Get entries for the instance of a system
result = gConfig.getSections( "/Systems/%s/%s/%s" % ( system, instance, "%ss" % type.capitalize() ) )
if not result[ 'OK' ]:
self.log.warn( "Opps, sytem seems to be defined wrong\n", "System %s at %s: %s" % ( system, instance, result[ 'Message' ] ) )
continue
components = result[ 'Value' ]
for component in components:
componentName = "%s/%s" % ( system, component )
compDict = self.__getComponentDefinitionFromCS( system, setup, instance, type, component )
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#Walk the URLs
result = gConfig.getOptionsDict( "/Systems/%s/%s/URLs" % ( system, instance ) )
if not result[ 'OK' ]:
self.log.warn ( "There doesn't to be defined the URLs section for %s in %s instance" % ( system, instance ) )
else:
serviceURLs = result[ 'Value' ]
for service in serviceURLs:
for url in List.fromChar( serviceURLs[ service ] ):
loc = url[ url.find( "://" ) + 3: ]
iS = loc.find( "/" )
componentName = loc[ iS + 1: ]
loc = loc[ :iS ]
hostname, port = loc.split( ":" )
compDict = { 'ComponentName' : componentName,
'Type' : 'service',
'Setup' : setup,
'Host' : hostname,
'Port' : int( port )
}
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#WALK THE DICT
statusSet.setComponentsAsRequired( requiredComponents )
return S_OK( ( statusSet.getRequiredComponents(),
self.__mainFields[1:] + self.__versionFields + ( 'Status', 'Message' ) ) )
def setNextOptimizer(self, jobState=None):
if not jobState:
jobState = self.__jobData.jobState
result = jobState.getOptParameter('OptimizerChain')
if not result['OK']:
return result
opChain = List.fromChar(result['Value'], ",")
opName = self.__optimizerName
try:
opIndex = opChain.index(opName)
except ValueError:
return S_ERROR("Optimizer %s is not in the chain!" % opName)
chainLength = len(opChain)
if chainLength - 1 == opIndex:
# This is the last optimizer in the chain!
result = jobState.setStatus(self.ex_getOption('WaitingStatus', 'Waiting'),
minorStatus=self.ex_getOption('WaitingMinorStatus', 'Pilot Agent Submission'),
appStatus="Unknown",
source=opName)
if not result['OK']:
return result
result = jobState.insertIntoTQ()
if not result['OK']:
return result
return S_OK()
# Keep optimizing!
nextOp = opChain[opIndex + 1]
self.jobLog.info("Set to Checking/%s" % nextOp)
return jobState.setStatus("Checking", nextOp, source=opName)
def getOption( self, optionPath, typeValue = None ):
gRefresher.refreshConfigurationIfNeeded()
optionValue = gConfigurationData.extractOptionFromCFG( optionPath )
if optionValue == None:
return S_ERROR( "Path %s does not exist or it's not an option" % optionPath )
#Value has been returned from the configuration
if typeValue == None:
return S_OK( optionValue )
#Casting to typeValue's type
requestedType = typeValue
if not type( typeValue ) == types.TypeType:
requestedType = type( typeValue )
if requestedType == types.ListType:
try:
return S_OK( List.fromChar( optionValue, ',' ) )
except Exception:
return S_ERROR( "Can't convert value (%s) to comma separated list" % str( optionValue ) )
elif requestedType == types.BooleanType:
try:
return S_OK( optionValue.lower() in ( "y", "yes", "true", "1" ) )
except Exception:
return S_ERROR( "Can't convert value (%s) to Boolean" % str( optionValue ) )
else:
try:
return S_OK( requestedType( optionValue ) )
except:
return S_ERROR( "Type mismatch between default (%s) and configured value (%s) " % ( str( typeValue ), optionValue ) )
def __loadOptimizer(self):
# Need to load an optimizer
gLogger.info("Loading optimizer %s" % self.optimizerName)
optList = List.fromChar(self.optimizerName, "/")
optList[1] = "/".join(optList[1:])
systemName = optList[0]
agentName = "%sAgent" % optList[1]
rootModulesToLook = gConfig.getValue("/LocalSite/Extensions", []) + ["DIRAC"]
for rootModule in rootModulesToLook:
try:
gLogger.info("Trying to load from root module %s" % rootModule)
opPyPath = "%s.%sSystem.Agent.%s" % (rootModule, systemName, agentName)
optimizerModule = __import__(opPyPath, globals(), locals(), agentName)
except ImportError, e:
gLogger.info("Can't load %s: %s" % (opPyPath, str(e)))
continue
try:
optimizerClass = getattr(optimizerModule, agentName)
optimizer = optimizerClass("%sAgent" % self.optimizerName, self.containerName)
result = optimizer.am_initialize(self.jobDB, self.jobLoggingDB)
if not result["OK"]:
return S_ERROR("Can't initialize optimizer %s: %s" % (self.optimizerName, result["Message"]))
return S_OK(optimizer)
except Exception, e:
gLogger.exception("Can't load optimizer %s with root module %s" % (self.optimizerName, rootModule))
def getCSTreeAsDict( treePath ):
'''
Function to recursively iterate over a CS tree
'''
csTreeDict = {}
opts = opHelper.getOptionsDict( treePath )
if opts[ 'OK' ]:
opts = opts[ 'Value' ]
for optKey, optValue in opts.items():
if optValue.find( ',' ) > -1:
optValue = List.fromChar( optValue )
else:
optValue = [ optValue ]
csTreeDict[ optKey ] = optValue
secs = opHelper.getSections( treePath )
if secs[ 'OK' ]:
secs = secs[ 'Value' ]
for sec in secs:
secTree = getCSTreeAsDict( '%s/%s' % ( treePath, sec ) )
if not secTree[ 'OK' ]:
return secTree
csTreeDict[ sec ] = secTree[ 'Value' ]
return S_OK( csTreeDict )
def getJobs():
result = gOAManager.authorize()
if not result[ 'OK' ]:
bottle.abort( 401, result[ 'Message' ] )
selDict = {}
startJob = 0
maxJobs = 100
for convList in ( attrConv, flagConv ):
for attrPair in convList:
jAtt = attrPair[0]
if jAtt in bottle.request.params:
selDict[ attrPair[1] ] = List.fromChar( bottle.request.params[ attrPair[0] ] )
if 'allOwners' not in bottle.request.params:
selDict[ 'Owner' ] = gOAData.userName
if 'startJob' in bottle.request.params:
try:
startJob = max( 0, int( bottle.request.params[ 'startJob' ] ) )
except:
bottle.abort( 400, "startJob has to be a positive integer!" )
if 'maxJobs' in bottle.request.params:
try:
maxJobs = min( 1000, int( bottle.request.params[ 'maxJobs' ] ) )
except:
bottle.abort( 400, "maxJobs has to be a positive integer no greater than 1000!" )
return __getJobs( selDict, startJob, maxJobs )
def setComment( self, entryPath, value ):
cfg = self.__getParentCFG( entryPath )
entry = List.fromChar( entryPath, "/" )[-1]
if cfg.setComment( entry, value ):
self.__setCommiter( entryPath )
return True
return False
def appendToOption(self, optionName, value):
"""
Append a value to an option prepending a comma
:type optionName: string
:param optionName: Name of the option to append the value
:type value: string
:param value: Value to append to the option
"""
result = self.getRecursive(optionName, -1)
if not result:
raise KeyError("%s does not exist" %
"/".join(List.fromChar(optionName, "/")[:-1]))
cfg = result['value']
end = result['levelsBelow']
if end not in cfg.__dataDict:
raise KeyError("Option %s has not been declared" % end)
cfg.__dataDict[end] += str(value)
def __checkMultiChoiceInDescription(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__description:
return S_OK()
else:
varValue = self.__description[varName]
initialVal = varValue
choices = Operations().getValue("JobDescription/Choices%s" % varName,
choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def loadObject( self, importString, objName = False, hideExceptions = False ):
""" Load an object from inside a module
"""
result = self.loadModule( importString, hideExceptions = hideExceptions )
if not result[ 'OK' ]:
return result
modObj = result[ 'Value' ]
modFile = modObj.__file__
if not objName:
objName = List.fromChar( importString, "." )[-1]
try:
result = S_OK( getattr( modObj, objName ) )
result['ModuleFile'] = modFile
return result
except AttributeError:
return S_ERROR( "%s does not contain a %s object" % ( importString, objName ) )
def loadObjects( path, reFilter = None, parentClass = None ):
if not reFilter:
reFilter = re.compile( r".*[a-z1-9]\.py$" )
pathList = List.fromChar( path, "/" )
parentModuleList = [ "%sDIRAC" % ext for ext in CSGlobals.getCSExtensions() ] + [ 'DIRAC' ]
objectsToLoad = {}
#Find which object files match
for parentModule in parentModuleList:
objDir = os.path.join( DIRAC.rootPath, parentModule, *pathList )
if not os.path.isdir( objDir ):
continue
for objFile in os.listdir( objDir ):
if reFilter.match( objFile ):
pythonClassName = objFile[:-3]
if pythonClassName not in objectsToLoad:
gLogger.info( "Adding to message load queue %s/%s/%s" % ( parentModule, path, pythonClassName ) )
objectsToLoad[ pythonClassName ] = parentModule
#Load them!
loadedObjects = {}
for pythonClassName in objectsToLoad:
parentModule = objectsToLoad[ pythonClassName ]
try:
#Where parentModule can be DIRAC, pathList is something like [ "AccountingSystem", "Client", "Types" ]
#And the python class name is.. well, the python class name
objPythonPath = "%s.%s.%s" % ( parentModule, ".".join( pathList ), pythonClassName )
objModule = __import__( objPythonPath,
globals(),
locals(), pythonClassName )
objClass = getattr( objModule, pythonClassName )
except Exception as e:
gLogger.exception( "Can't load type %s/%s: %s" % ( parentModule, pythonClassName, str( e ) ) )
continue
if parentClass == objClass:
continue
if parentClass and not issubclass( objClass, parentClass ):
gLogger.warn( "%s is not a subclass of %s. Skipping" % ( objClass, parentClass ) )
continue
gLogger.info( "Loaded %s" % objPythonPath )
loadedObjects[ pythonClassName ] = objClass
return loadedObjects
def parseListMatchStdout(self, proxy, cmd, taskQueueID, rb):
"""
Parse List Match stdout to return list of matched CE's
"""
self.log.verbose('Executing List Match for TaskQueue', taskQueueID)
start = time.time()
ret = executeGridCommand(proxy, cmd, self.gridEnv)
if not ret['OK']:
self.log.error('Failed to execute List Match:', ret['Message'])
self.__sendErrorMail(rb, 'List Match', cmd, ret, proxy)
return False
if ret['Value'][0] != 0:
self.log.error('Error executing List Match:',
str(ret['Value'][0]) + '\n'.join(ret['Value'][1:3]))
self.__sendErrorMail(rb, 'List Match', cmd, ret, proxy)
return False
self.log.info('List Match Execution Time: %.2f for TaskQueue %d' %
((time.time() - start), taskQueueID))
stdout = ret['Value'][1]
stderr = ret['Value'][2]
availableCEs = []
# Parse std.out
for line in List.fromChar(stdout, '\n'):
if re.search('/jobmanager-', line) or re.search('/cream-', line):
# TODO: the line has to be stripped from extra info
availableCEs.append(line)
if not availableCEs:
self.log.info('List-Match failed to find CEs for TaskQueue',
taskQueueID)
self.log.info(stdout)
self.log.info(stderr)
else:
self.log.debug('List-Match returns:',
str(ret['Value'][0]) + '\n'.join(ret['Value'][1:3]))
self.log.info(
'List-Match found %s CEs for TaskQueue' % len(availableCEs),
taskQueueID)
self.log.verbose(', '.join(availableCEs))
return availableCEs
def __removeEmptyDir(self, dirPath, useOwnerProxy=True):
''' unlink empty folder :dirPath: '''
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
if len(List.fromChar(dirPath, "/")) < self.__keepDirLevels:
return S_OK()
if useOwnerProxy:
result = self.__getOwnerProxy(dirPath)
if not result['OK']:
if 'Proxy not available' not in result['Message']:
self.log.error(result['Message'])
return result
upFile = result['Value']
prevProxyEnv = os.environ['X509_USER_PROXY']
os.environ['X509_USER_PROXY'] = upFile
try:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
# res = self.catalog.removeDirectory( dirPath )
res = self.catalog.writeCatalogs[0][1].removeDirectory(dirPath)
if not res['OK']:
self.log.error(
"Error removing empty directory from File Catalog.",
res['Message'])
return res
elif dirPath in res['Value']['Failed']:
self.log.error(
"Failed to remove empty directory from File Catalog.",
res['Value']['Failed'][dirPath])
self.log.debug(str(res))
return S_ERROR(res['Value']['Failed'][dirPath])
res = self.storageUsage.removeDirectory(dirPath)
if not res['OK']:
self.log.error(
"Failed to remove empty directory from Storage Usage database.",
res['Message'])
return res
return S_OK()
finally:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if useOwnerProxy:
os.environ['X509_USER_PROXY'] = prevProxyEnv
def __getLoadAvg(self):
result = self.__getCSConfig()
if not result['OK']:
return result
with open("/proc/loadavg", "r") as fd:
data = [float(v) for v in List.fromChar(fd.read(), " ")[:3]]
self.__loadHistory.append(data)
numRequiredSamples = max(
self.vmLoadAvgTimespan / self.am_getPollingTime(), 1)
while len(self.__loadHistory) > numRequiredSamples:
self.__loadHistory.pop(0)
self.log.info("Load averaged over %d seconds" % self.vmLoadAvgTimespan)
self.log.info(" %d/%s required samples to average load" %
(len(self.__loadHistory), numRequiredSamples))
avgLoad = 0
for f in self.__loadHistory:
avgLoad += f[0]
return avgLoad / len(self.__loadHistory), len(
self.__loadHistory) == numRequiredSamples
def retrieveReleaseNotes( packages ):
if type( packages ) in ( types.StringType, types.UnicodeType ):
packages = [ str( packages ) ]
packageCFGDict = {}
#Get the versions.cfg
for package in packages:
packageCFGDict[ package ] = Distribution( package ).getVersionsCFG()
#Parse the release notes
pkgNotesDict = {}
for package in packageCFGDict:
versionsCFG = packageCFGDict[ package ][ 'Versions' ]
pkgNotesDict[ package ] = []
for mainVersion in versionsCFG.listSections( ordered = True ):
vCFG = versionsCFG[ mainVersion ]
versionNotes = {}
for subsys in vCFG.listOptions():
comment = vCFG.getComment( subsys )
if not comment:
continue
versionNotes[ subsys ] = {}
lines = List.fromChar( comment, "\n" )
lastCommentType = False
for line in lines:
processedLine = False
for typeComment in gAllowedNoteTypes:
if line.find( "%s:" % typeComment ) == 0:
if typeComment in gNoteTypeAlias:
effectiveType = gNoteTypeAlias[ typeComment ]
else:
effectiveType = typeComment
if effectiveType not in versionNotes[ subsys ]:
versionNotes[ subsys ][ effectiveType ] = []
versionNotes[ subsys ][ effectiveType ].append( line[ len( typeComment ) + 1: ].strip() )
lastCommentType = effectiveType
processedLine = True
if not processedLine and lastCommentType:
versionNotes[ subsys ][ effectiveType ][-1] += " %s" % line.strip()
if versionNotes:
pkgNotesDict[ package ].append( { 'version' : mainVersion, 'notes' : versionNotes } )
versionComment = versionsCFG.getComment( mainVersion )
if versionComment:
pkgNotesDict[ package ][-1][ 'comment' ] = "\n".join( [ l.strip() for l in versionComment.split( "\n" ) ] )
return pkgNotesDict
def getOption(self, optionPath, defaultValue=None):
gRefresher.refreshConfigurationIfNeeded()
optionValue = gConfigurationData.extractOptionFromCFG(optionPath)
if not optionValue:
optionValue = defaultValue
#Return value if existing, defaultValue if not
if optionValue == defaultValue:
if defaultValue == None or type(defaultValue) == types.TypeType:
return S_ERROR("Path %s does not exist or it's not an option" %
optionPath)
return S_OK(optionValue)
#Value has been returned from the configuration
if defaultValue == None:
return S_OK(optionValue)
#Casting to defaultValue's type
defaultType = defaultValue
if not type(defaultValue) == types.TypeType:
defaultType = type(defaultValue)
if defaultType == types.ListType:
try:
return S_OK(List.fromChar(optionValue, ','))
except Exception:
return S_ERROR(
"Can't convert value (%s) to comma separated list" %
str(optionValue))
elif defaultType == types.BooleanType:
try:
return S_OK(optionValue.lower() in ("y", "yes", "true", "1"))
except Exception:
return S_ERROR(
"Can't convert value (%s) to comma separated list" %
str(optionValue))
else:
try:
return S_OK(defaultType(optionValue))
except:
return S_ERROR(
"Type mismatch between default (%s) and configured value (%s) "
% (str(defaultValue), optionValue))
def __recurseImport( self, modName, parentModule = False, hideExceptions = False ):
if type( modName ) in types.StringTypes:
modName = List.fromChar( modName, "." )
try:
if parentModule:
impData = imp.find_module( modName[0], parentModule.__path__ )
else:
impData = imp.find_module( modName[0] )
impModule = imp.load_module( modName[0], *impData )
if impData[0]:
impData[0].close()
except ImportError, excp:
strExcp = str( excp )
if strExcp.find( "No module named" ) == 0 and strExcp.find( modName[0] ) == len( strExcp ) - len( modName[0] ):
return S_OK()
errMsg = "Can't load %s" % ".".join( modName )
if not hideExceptions:
gLogger.exception( errMsg )
return S_ERROR( errMsg )
def syncUsersWithCFG(self, usersCFG):
"""
Sync users with the cfg contents. Usernames have to be sections containing
DN, Groups, and extra properties as parameters
"""
if not self.__initialized['OK']:
return self.__initialized
done = True
for user in usersCFG.listSections():
properties = {}
propList = usersCFG[user].listOptions()
for prop in propList:
if prop == "Groups":
properties[prop] = List.fromChar(usersCFG[user][prop])
else:
properties[prop] = usersCFG[user][prop]
if not self.modifyUser(user, properties, createIfNonExistant=True):
done = False
return S_OK(done)
def getServiceFailoverURL(system, service=None, setup=False):
"""Get failover URLs for service
:param str system: system name or full name, like 'Framework/Service'.
:param str service: service name, like 'ProxyManager'.
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:return: str -- complete list of urls
"""
system, service = divideFullName(system, service)
systemSection = getSystemSection(system, setup=setup)
failovers = gConfigurationData.extractOptionFromCFG(
"%s/FailoverURLs/%s" % (systemSection, service))
if not failovers:
return ""
return ",".join([
checkComponentURL(u, system, service)
for u in List.fromChar(failovers, ",") if u
])
def testAppendUnique(self):
""" appendUnique tests """
# empty
aList = []
List.appendUnique(aList, None)
self.assertEqual(aList, [None])
# redundant element
aList = [1, 2, 3]
List.appendUnique(aList, 1)
self.assertEqual(aList, [1, 2, 3])
# all unique
aList = [1, 2]
List.appendUnique(aList, 3)
self.assertEqual(aList, [1, 2, 3])
def parseJobSubmitStdout(self, proxy, cmd, taskQueueID, rb):
"""
Parse Job Submit stdout to return pilot reference
"""
start = time.time()
self.log.verbose('Executing Job Submit for TaskQueue', taskQueueID)
ret = executeGridCommand(proxy, cmd, self.gridEnv)
if not ret['OK']:
self.log.error('Failed to execute Job Submit:', ret['Message'])
self.__sendErrorMail(rb, 'Job Submit', cmd, ret, proxy)
return False
if ret['Value'][0] != 0:
self.log.error('Error executing Job Submit:',
str(ret['Value'][0]) + '\n'.join(ret['Value'][1:3]))
self.__sendErrorMail(rb, 'Job Submit', cmd, ret, proxy)
return False
self.log.info('Job Submit Execution Time: %.2f for TaskQueue %d' %
((time.time() - start), taskQueueID))
stdout = ret['Value'][1]
failed = 1
rb = ''
for line in List.fromChar(stdout, '\n'):
m = re.search("(https:\S+)", line)
if (m):
glite_id = m.group(1)
if not rb:
m = re.search("https://(.+):.+", glite_id)
rb = m.group(1)
failed = 0
if failed:
self.log.error('Job Submit returns no Reference:',
str(ret['Value'][0]) + '\n'.join(ret['Value'][1:3]))
return False
self.log.info('Reference %s for TaskQueue %s' %
(glite_id, taskQueueID))
return glite_id, rb
def __decodeMetadataQuery(self):
""" Decode metadata query
:return: dict
"""
cond = {}
for k in self.request.arguments:
for val in self.request.arguments[k]:
if val.find("|") == -1:
continue
val = val.split("|")
op = val[0]
val = "|".join(val[1:])
if 'in' == op:
val = List.fromChar(val, ",")
if k not in cond:
cond[k] = {}
cond[k][op] = val
self.log.info("Metadata condition is %s" % cond)
return cond
def deleteKey( self, key ):
"""
Delete an option/section
:type key: string
:param key: Name of the option/section to delete
:return: Boolean with the result
"""
result = self.getRecursive( key, -1 )
if not result:
raise KeyError( "%s does not exist" % "/".join( List.fromChar( key, "/" )[:-1] ) )
cfg = result[ 'value' ]
end = result[ 'levelsBelow' ]
if end in cfg.__orderedList:
del( cfg.__commentDict[ end ] )
del( cfg.__dataDict[ end ] )
cfg.__orderedList.remove( end )
return True
return False
def export_assignSandboxesToEntities(self,
enDict,
ownerName="",
ownerGroup="",
entitySetup=False):
"""
Assign sandboxes to jobs.
Expects a dict of { entityId : [ ( SB, SBType ), ... ] }
"""
if not entitySetup:
entitySetup = self.serviceInfoDict['clientSetup']
assignList = []
for entityId in enDict:
for sbTuple in enDict[entityId]:
if type(sbTuple) not in (types.TupleType, types.ListType):
return S_ERROR(
"Entry for entity %s is not a itterable of tuples/lists"
% entityId)
if len(sbTuple) != 2:
return S_ERROR(
"SB definition is not ( SBLocation, Type )! It's '%s'"
% str(sbTuple))
SBLocation = sbTuple[0]
if SBLocation.find("SB:") != 0:
return S_ERROR("%s doesn't seem to be a sandbox" %
SBLocation)
SBLocation = SBLocation[3:]
splitted = List.fromChar(SBLocation, "|")
if len(splitted) < 2:
return S_ERROR("SB Location has to have SEName|SEPFN form")
SEName = splitted[0]
SEPFN = ":".join(splitted[1:])
assignList.append(
(entityId, entitySetup, sbTuple[1], SEName, SEPFN))
if not assignList:
return S_OK()
credDict = self.getRemoteCredentials()
return sandboxDB.assignSandboxesToEntities(assignList,
credDict['username'],
credDict['group'],
ownerName, ownerGroup)
def describeGroups(self, mask=None):
"""
List all groups that are in the mask (or all if no mask) with their properties
"""
if mask is None:
mask = []
if not self.__initialized['OK']:
return self.__initialized
groups = [
group for group in self.__csMod.getSections(
"%s/Groups" %
self.__baseSecurity) if not mask or (
mask and group in mask)]
groupsDict = {}
for group in groups:
groupsDict[group] = {}
for option in self.__csMod.getOptions("%s/Groups/%s" % (self.__baseSecurity, group)):
groupsDict[group][option] = self.__csMod.getValue("%s/Groups/%s/%s" % (self.__baseSecurity, group, option))
if option in ("Users", "Properties"):
groupsDict[group][option] = List.fromChar(groupsDict[group][option])
return S_OK(groupsDict)
def _getJobManifest( self, jid ):
result = RPCClient( "WorkloadManagement/JobMonitoring" ).getJobJDL( int( jid ) )
if not result[ 'OK' ]:
return WErr( 500, result[ 'Message' ] )
result = loadJDLAsCFG( result[ 'Value' ] )
if not result[ 'OK' ]:
return WErr( 500, result[ 'Message' ] )
cfg = result[ 'Value' ][0]
jobData = {}
stack = [ ( cfg, jobData ) ]
while stack:
cfg, level = stack.pop( 0 )
for op in cfg.listOptions():
val = List.fromChar( cfg[ op ] )
if len( val ) == 1:
val = val[0]
level[ op ] = val
for sec in cfg.listSections():
level[ sec ] = {}
stack.append( ( cfg[ sec ], level[ sec ] ) )
return WOK( jobData )
def _getChildrenReferences(self, proxy, parentReference, taskQueueID):
"""
Get reference for all Children
"""
cmd = ['glite-wms-job-status', parentReference]
start = time.time()
self.log.verbose('Executing Job Status for TaskQueue', taskQueueID)
ret = executeGridCommand(proxy, cmd, self.gridEnv)
if not ret['OK']:
self.log.error('Failed to execute Job Status', ret['Message'])
return []
if ret['Value'][0] != 0:
self.log.error('Error executing Job Status:',
str(ret['Value'][0]) + '\n'.join(ret['Value'][1:3]))
return []
self.log.info('Job Status Execution Time: %.2f' %
(time.time() - start))
stdout = ret['Value'][1]
# stderr = ret['Value'][2]
references = []
failed = 1
for line in List.fromChar(stdout, '\n'):
match = re.search("Status info for the Job : (https:\S+)", line)
if (match):
glite_id = match.group(1)
if glite_id not in references and glite_id != parentReference:
references.append(glite_id)
failed = 0
if failed:
error = str(ret['Value'][0]) + '\n'.join(ret['Value'][1:3])
self.log.error('Job Status returns no Child Reference:', error)
return [parentReference]
return references
def do_plotView( self, args ):
"""
Gets a summary
Usage : getSummary <Summary name> <startdate YYYYMMDDHHMM> <enddate YYYYMMDDHHMM> <destLocation> (<field name> <field value>)*
"""
try:
argList = List.fromChar( args, " " )
if len( argList ) < 4:
gLogger.error( "Missing arguments!" )
return
startDT = self.__getDatetimeFromArg( argList[1] )
if not startDT:
gLogger.error( "Start time has invalid format" )
endDT = self.__getDatetimeFromArg( argList[2] )
if not endDT:
gLogger.error( "End time has invalid format" )
gLogger.info( "Start time is %s" % startDT )
gLogger.info( "End time is %s" % endDT )
sumArgs = {}
for iP in range( 4, len( argList ), 2 ):
key = argList[ iP ]
if key in sumArgs:
sumArgs[ key ].append( argList[ iP + 1 ] )
else:
sumArgs[ key ] = [ argList[ iP + 1 ] ]
repClient = ReportsClient()
retVal = repClient.plotView( argList[ 0 ], startDT, endDT, sumArgs )
if not retVal[ 'OK' ]:
gLogger.error( "Error: %s" % retVal[ 'Message' ] )
return
destDir = argList[3]
plotImg = retVal[ 'Value' ]
print "Downloading %s plot to %s.." % ( plotImg, destDir )
retVal = repClient.getPlotToDirectory( plotImg, destDir )
if not retVal[ 'OK' ]:
print " Error: %s" % retVal[ 'Message' ]
else:
print " done (%s/%s)!" % ( destDir, plotImg )
except:
self.showTraceback()
def initialize(self):
self.SystemLoggingDB = SystemLoggingDB()
self.agentName = self.am_getModuleParam('fullName')
self.notification = NotificationClient()
mailList = self.am_getOption("MailList", [])
userString = self.am_getOption("Reviewer", 'mseco')
userList = List.fromChar(userString, ",")
self.log.debug("Users to be notified", ": " + userString)
for user in userList:
retval = gConfig.getOption("/Registry/Users/" + user + "/email")
if not retval['OK']:
self.log.warn("Could not get user's mail", retval['Message'])
else:
mailList.append(retval['Value'])
if not mailList:
mailList = gConfig.getValue('/Operations/EMail/Logging', [])
if not len(mailList):
errString = "There are no valid users in the list"
varString = "[" + ','.join(userList) + "]"
self.log.error(errString, varString)
return S_ERROR(errString + varString)
self.log.info("List of mails to be notified", ','.join(mailList))
self._mailAddress = mailList
self._threshold = int(self.am_getOption('Threshold', 10))
self.__days = self.am_getOption('QueryPeriod', 7)
self._period = int(self.__days) * day
self._limit = int(self.am_getOption('NumberOfErrors', 10))
string = "The %i most common errors in the SystemLoggingDB" % self._limit
self._subject = string + " for the last %s days" % self.__days
return S_OK()
def getCEsFromCS():
""" Get all the CEs defined in the CS
"""
knownCEs = []
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return result
grids = result['Value']
for grid in grids:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return result
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
ces = List.fromChar( opt.get( 'CE', '' ) )
knownCEs += ces
return S_OK( knownCEs )
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all Grid PilotDirectors
"""
PilotDirector.configure( self, csSection, submitPool )
self.reloadConfiguration( csSection, submitPool )
self.__failingWMSCache.purgeExpired()
self.__ticketsWMSCache.purgeExpired()
for rb in self.__failingWMSCache.getKeys():
if rb in self.resourceBrokers:
try:
self.resourceBrokers.remove( rb )
except:
pass
self.resourceBrokers = List.randomize( self.resourceBrokers )
if self.gridEnv:
self.log.info( ' GridEnv: ', self.gridEnv )
if self.resourceBrokers:
self.log.info( ' ResourceBrokers:', ', '.join( self.resourceBrokers ) )
def __isGroupAuthApp(self, appLoc):
"""
The method checks if the application is authorized for a certain user group
:param str appLoc It is the application name for example: DIRAC.JobMonitor
:return bool if the handler is authorized to the user returns True otherwise False
"""
handlerLoc = "/".join(List.fromChar(appLoc, ".")[1:])
if not handlerLoc:
gLogger.error("Application handler does not exists:", appLoc)
return False
if handlerLoc not in self.__handlers:
gLogger.error("Handler %s required by %s does not exist!" %
(handlerLoc, appLoc))
return False
handler = self.__handlers[handlerLoc]
auth = AuthManager(Conf.getAuthSectionForHandler(handlerLoc))
gLogger.info("Authorization: %s -> %s" %
(dict(self.__credDict), handler.AUTH_PROPS))
return auth.authQuery("", dict(self.__credDict), handler.AUTH_PROPS)
def __init__(self, *args, **kwargs):
''' c'tor
'''
AgentModule.__init__(self, *args, **kwargs)
self.__baseDir = '/lhcb'
self.__baseDirLabel = "_".join(List.fromChar(self.__baseDir, "/"))
self.__ignoreDirsList = []
self.__keepDirLevels = 4
self.__startExecutionTime = long(time.time())
self.__dirExplorer = DirectoryExplorer(reverse=True)
self.__processedDirs = 0
self.__directoryOwners = {}
self.catalog = FileCatalog()
self.__maxToPublish = self.am_getOption('MaxDirectories', 5000)
if self.am_getOption('DirectDB', False):
self.storageUsage = StorageUsageDB()
else:
# Set a timeout of 0.1 seconds per directory (factor 5 margin)
self.storageUsage = RPCClient('DataManagement/StorageUsage',
timeout=self.am_getOption(
'Timeout',
int(self.__maxToPublish * 0.1)))
self.activePeriod = self.am_getOption('ActivePeriod',
self.activePeriod)
self.dataLock = threading.Lock()
self.replicaListLock = threading.Lock()
self.proxyCache = DictCache(removeProxy)
self.__noProxy = set()
self.__catalogType = None
self.__recalculateUsage = Operations().getValue(
'DataManagement/RecalculateDirSize', False)
self.enableStartupSleep = self.am_getOption('EnableStartupSleep',
self.enableStartupSleep)
self.__publishDirQueue = {}
self.__dirsToPublish = {}
self.__replicaFilesUsed = set()
self.__replicaListFilesDir = ""
def getValidPropertiesForMethod( self, method, defaultProperties = False ):
"""
Get all authorized groups for calling a method
:type method: string
:param method: Method to test
:return: List containing the allowed groups
"""
authProps = gConfig.getValue( "%s/%s" % ( self.authSection, method ), [] )
if authProps:
return authProps
if defaultProperties:
self.__authLogger.verbose( "Using hardcoded properties for method %s : %s" % ( method, defaultProperties ) )
if type( defaultProperties ) not in ( types.ListType, types.TupleType ):
return List.fromChar( defaultProperties )
return defaultProperties
defaultPath = "%s/Default" % "/".join( method.split( "/" )[:-1] )
authProps = gConfig.getValue( "%s/%s" % ( self.authSection, defaultPath ), [] )
if authProps:
self.__authLogger.verbose( "Method %s has no properties defined using %s" % ( method, defaultPath ) )
return authProps
self.__authLogger.verbose( "Method %s has no authorization rules defined. Allowing no properties" % method )
return []
def getSocket(self, hostAddress, **kwargs):
hostName = hostAddress[0]
retVal = self.generateClientInfo(hostName, kwargs)
if not retVal['OK']:
return retVal
socketInfo = retVal['Value']
retVal = Network.getIPsForHostName(hostName)
if not retVal['OK']:
return S_ERROR("Could not resolve %s: %s" %
(hostName, retVal['Message']))
ipList = List.randomize(retVal['Value'])
for i in range(3):
connected = False
errorsList = []
for ip in ipList:
ipAddress = (ip, hostAddress[1])
retVal = self.__connect(socketInfo, ipAddress)
if retVal['OK']:
sslSocket = retVal['Value']
connected = True
break
errorsList.append("%s: %s" % (ipAddress, retVal['Message']))
if not connected:
return S_ERROR("Could not connect to %s: %s" %
(hostAddress, ",".join([e
for e in errorsList])))
retVal = socketInfo.doClientHandshake()
if retVal['OK']:
#Everything went ok. Don't need to retry
break
#Did the auth or the connection fail?
if not retVal['OK']:
return retVal
if 'enableSessions' in kwargs and kwargs['enableSessions']:
sessionId = hash(hostAddress)
gSessionManager.set(sessionId, sslSocket.get_session())
return S_OK(socketInfo)
def __recurseImport(self,
modName,
parentModule=None,
hideExceptions=False,
fullName=False):
""" Internal function to load modules
"""
if isinstance(modName, basestring):
modName = List.fromChar(modName, ".")
if not fullName:
fullName = ".".join(modName)
if fullName in self.__objs:
return S_OK(self.__objs[fullName])
try:
if parentModule:
impData = imp.find_module(modName[0], parentModule.__path__)
else:
impData = imp.find_module(modName[0])
impModule = imp.load_module(modName[0], *impData)
if impData[0]:
impData[0].close()
except ImportError as excp:
if str(excp).find("No module named %s" % modName[0]) == 0:
return S_OK(None)
errMsg = "Can't load %s in %s" % (".".join(modName),
parentModule.__path__[0])
if not hideExceptions:
gLogger.exception(errMsg)
return S_ERROR(DErrno.EIMPERR, errMsg)
if len(modName) == 1:
self.__objs[fullName] = impModule
return S_OK(impModule)
return self.__recurseImport(modName[1:],
impModule,
hideExceptions=hideExceptions,
fullName=fullName)
def getTypedDictRootedAt(path):
retval = {}
opts = gConfig.getOptionsDict(path)
secs = gConfig.getSections(path)
if not opts['OK']:
raise CSError, opts['Message']
if not secs['OK']:
raise CSError, secs['Message']
opts = opts['Value']
secs = secs['Value']
for k in opts:
if opts[k].find(",") > -1:
retval[k] = [
Utils.typedobj_of_string(e) for e in List.fromChar(opts[k])
]
else:
retval[k] = Utils.typedobj_of_string(opts[k])
for i in secs:
retval[i] = getTypedDictRootedAt(path + "/" + i)
return retval
