def _curlDownload( self, granularity, site, tests ):
""" Download SAM status for entity using the SAM DB programmatic interface
"""
samdbpi_url = "http://lcg-sam.cern.ch:8080/same-pi/"
# Set your method
if granularity in ( 'Site', 'Sites' ):
samdbpi_method = "site_status.jsp?"
elif granularity in ( 'Resource', 'Resources' ):
samdbpi_method = "service_endpoint_status.jsp?"
# Set your site
samdbpi_site = site
# set test
samdbpi_test = ""
if tests is None:
samdbpi_test = "&only_ss"
extension = CSGlobals.getCSExtensions()[0]
samdb_ep = samdbpi_url + samdbpi_method + "VO_name=" + extension + "&Site_name=" + samdbpi_site + samdbpi_test
req = urllib2.Request( samdb_ep )
samPage = urllib2.urlopen( req )
sam = samPage.read()
return sam
def getPaths(self, dirName):
"""
Get lists of paths for all installed and enabled extensions
"""
pathList = []
for extName in CSGlobals.getCSExtensions():
if extName.rfind("DIRAC") != len(extName) - 5:
extName = "%sDIRAC" % extName
if extName == "WebAppDIRAC":
continue
if extName == "EiscatWebDIRAC":
continue
try:
modFile, modPath, desc = imp.find_module(extName)
#to match in the real root path to enabling module web extensions (static, templates...)
realModPath = os.path.realpath(modPath)
except ImportError:
continue
staticPath = os.path.join(realModPath, "WebApp", dirName)
if os.path.isdir(staticPath):
pathList.append(staticPath)
#Add WebAppDirac to the end
pathList.append(
os.path.join(EiscatWebDIRAC.rootPath, "WebApp", dirName))
return pathList
def __getInstallFlags(self):
""" Get the flags to pass to dirac-install.py inside the container.
Returns a string containing the command line flags.
"""
instOpts = []
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
opsHelper = Operations.Operations(setup=setup)
installationName = opsHelper.getValue("Pilot/Installation", "")
if installationName:
instOpts.append('-V %s' % installationName)
diracVersions = opsHelper.getValue("Pilot/Version", [])
instOpts.append("-r '%s'" % diracVersions[0])
pyVer = "%u%u" % (sys.version_info.major, sys.version_info.minor)
instOpts.append("-i %s" % pyVer)
pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
instOpts.append("-e '%s'" % ','.join([ext for ext in extensionsList if 'Web' not in ext]))
if 'ContainerExtraOpts' in self.ceParameters:
instOpts.append(self.ceParameters['ContainerExtraOpts'])
return ' '.join(instOpts)
def __getInstallFlags(self):
""" Get the flags to pass to dirac-install.py inside the container.
Returns a string containing the command line flags.
"""
instOpts = []
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
opsHelper = Operations.Operations(setup=setup)
installationName = opsHelper.getValue("Pilot/Installation", "")
if installationName:
instOpts.append('-V %s' % installationName)
diracVersions = opsHelper.getValue("Pilot/Version", [])
instOpts.append("-r '%s'" % diracVersions[0])
pyVer = "%u%u" % (sys.version_info.major, sys.version_info.minor)
instOpts.append("-i %s" % pyVer)
pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
instOpts.append(
"-e '%s'" %
','.join([ext for ext in extensionsList if 'Web' not in ext]))
lcgVer = opsHelper.getValue("Pilot/LCGBundleVersion", None)
if lcgVer:
instOpts.append("-g %s" % lcgVer)
if 'ContainerExtraOpts' in self.ceParameters:
instOpts.append(self.ceParameters['ContainerExtraOpts'])
return ' '.join(instOpts)
def __generateRootModules( self, baseModules ):
""" Iterate over all the possible root modules
"""
self.__rootModules = baseModules
for rootModule in reversed( CSGlobals.getCSExtensions() ):
if rootModule[-5:] != "DIRAC" and rootModule not in self.__rootModules:
self.__rootModules.append( "%sDIRAC" % rootModule )
self.__rootModules.append( "" )
def includeExtensionErrors():
""" Merge all the errors of all the extensions into the errors of these modules
Should be called only at the initialization of DIRAC, so by the parseCommandLine,
dirac-agent.py, dirac-service.py, dirac-executor.py
"""
def __recurseImport(modName, parentModule=None, fullName=False):
""" Internal function to load modules
"""
if isinstance(modName, basestring):
modName = modName.split(".")
if not fullName:
fullName = ".".join(modName)
try:
if parentModule:
impData = imp.find_module(modName[0], parentModule.__path__)
else:
impData = imp.find_module(modName[0])
impModule = imp.load_module(modName[0], *impData)
if impData[0]:
impData[0].close()
except ImportError:
return None
if len(modName) == 1:
return impModule
return __recurseImport(modName[1:], impModule, fullName=fullName)
from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals
allExtensions = CSGlobals.getCSExtensions()
for extension in allExtensions:
ext_derrno = None
try:
ext_derrno = __recurseImport('%sDIRAC.Core.Utilities.DErrno' %
extension)
if ext_derrno:
# The next 3 dictionary MUST be present for consistency
# Global name of errors
sys.modules[__name__].__dict__.update(
ext_derrno.extra_dErrName)
# Dictionary with the error codes
sys.modules[__name__].dErrorCode.update(
ext_derrno.extra_dErrorCode)
# Error description string
sys.modules[__name__].dStrError.update(
ext_derrno.extra_dStrError)
# extra_compatErrorString is optional
for err in getattr(ext_derrno, 'extra_compatErrorString', []):
sys.modules[__name__].compatErrorString.setdefault(
err,
[]).extend(ext_derrno.extra_compatErrorString[err])
except BaseException:
pass
def __generateRootModules(self, baseModules):
""" Iterate over all the possible root modules
"""
self.__rootModules = baseModules
for rootModule in reversed(CSGlobals.getCSExtensions()):
if rootModule[
-5:] != "DIRAC" and rootModule not in self.__rootModules:
self.__rootModules.append("%sDIRAC" % rootModule)
self.__rootModules.append("")
def _loadWebAppCFGFiles(self):
"""
Load EiscatWeb/web.cfg definitions
"""
exts = []
for ext in CSGlobals.getCSExtensions():
if ext == "DIRAC":
continue
if ext[-5:] != "DIRAC":
ext = "%sDIRAC" % ext
if ext == "WebAppDIRAC":
continue
if ext != "EiscatWebDIRAC":
exts.append(ext)
exts.append("DIRAC")
exts.append("EiscatWebDIRAC")
print "exts in loadWebAppCFGFiles of App.py"
print exts
webCFG = CFG()
for modName in reversed(exts):
try:
modPath = imp.find_module(modName)[1]
except ImportError:
continue
gLogger.verbose("Found module %s at %s" % (modName, modPath))
cfgPath = os.path.join(modPath, "WebApp", "web.cfg")
print "cfgPath"
print cfgPath
if not os.path.isfile(cfgPath):
gLogger.verbose("Inexistant %s" % cfgPath)
continue
try:
modCFG = CFG().loadFromFile(cfgPath)
except Exception, excp:
gLogger.error("Could not load %s: %s" % (cfgPath, excp))
continue
gLogger.verbose("Loaded %s" % cfgPath)
expl = [Conf.BASECS]
while len(expl):
current = expl.pop(0)
if not modCFG.isSection(current):
continue
if modCFG.getOption("%s/AbsoluteDefinition" % current, False):
gLogger.verbose("%s:%s is an absolute definition" %
(modName, current))
try:
webCFG.deleteKey(current)
except:
pass
modCFG.deleteKey("%s/AbsoluteDefinition" % current)
else:
for sec in modCFG[current].listSections():
expl.append("%s/%s" % (current, sec))
# Add the modCFG
webCFG = webCFG.mergeWith(modCFG)
print "al final webCFG"
print webCFG
def __rootModules( self ):
""" Iterate over all the possible root modules
"""
for rootModule in CSGlobals.getCSExtensions():
if rootModule[-5:] != "DIRAC":
rootModule = "%sDIRAC" % rootModule
yield rootModule
yield 'DIRAC'
yield ''
def loadObjects(path, reFilter=None, parentClass=None):
"""
:param str path the path to the syetem for example: DIRAC/AccountingSystem
:param object reFilter regular expression used to found the class
:param object parentClass class instance
:return dict it return the name of the clase and the instance of the class.
"""
if not reFilter:
reFilter = re.compile(".*[a-z1-9]\.py$")
pathList = List.fromChar(path, "/")
parentModuleList = [
"%sDIRAC" % ext for ext in CSGlobals.getCSExtensions()
] + ['DIRAC']
objectsToLoad = {}
#Find which object files match
for parentModule in parentModuleList:
objDir = os.path.join(DIRAC.rootPath, parentModule, *pathList)
if not os.path.isdir(objDir):
continue
for objFile in os.listdir(objDir):
if reFilter.match(objFile):
pythonClassName = objFile[:-3]
if pythonClassName not in objectsToLoad:
gLogger.info("Adding to load queue %s/%s/%s" %
(parentModule, path, pythonClassName))
objectsToLoad[pythonClassName] = parentModule
#Load them!
loadedObjects = {}
for pythonClassName in objectsToLoad:
parentModule = objectsToLoad[pythonClassName]
try:
#Where parentModule can be DIRAC, pathList is something like [ "AccountingSystem", "Client", "Types" ]
#And the python class name is.. well, the python class name
objPythonPath = "%s.%s.%s" % (parentModule, ".".join(pathList),
pythonClassName)
objModule = __import__(objPythonPath, globals(), locals(),
pythonClassName)
objClass = getattr(objModule, pythonClassName)
except Exception as e:
gLogger.error(
"Can't load type",
"%s/%s: %s" % (parentModule, pythonClassName, str(e)))
continue
if parentClass == objClass:
continue
if parentClass and not issubclass(objClass, parentClass):
gLogger.warn("%s is not a subclass of %s. Skipping" %
(objClass, parentClass))
continue
gLogger.info("Loaded %s" % objPythonPath)
loadedObjects[pythonClassName] = objClass
return loadedObjects
def __generateRootModules( self, baseModules ):
""" Iterate over all the possible root modules
"""
self.__rootModules = baseModules
for rootModule in reversed( CSGlobals.getCSExtensions() ):
if rootModule[-5:] != "DIRAC" and rootModule not in self.__rootModules:
self.__rootModules.append( "%sDIRAC" % rootModule )
self.__rootModules.append( "" )
# Reversing the order because we want first to look in the extension(s)
self.__rootModules.reverse()
def __getInstallFlags(self, infoDict=None):
""" Get the flags to pass to dirac-install.py inside the container.
Returns a string containing the command line flags.
"""
if not infoDict:
infoDict = {}
instOpts = []
setup = infoDict.get('DefaultSetup')
if not setup:
setup = list(infoDict.get('Setups'))[0]
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
setup = str(setup)
installationName = str(infoDict.get('Installation'))
if not installationName or installationName == 'None':
installationName = Operations.Operations(setup=setup).getValue(
"Pilot/Installation", "")
if installationName:
instOpts.append('-V %s' % installationName)
diracVersions = str(
infoDict['Setups'][setup].get('Version')).split(',')
if not diracVersions:
diracVersions = str(
infoDict['Setups']['Defaults'].get('Version')).split(',')
if not diracVersions:
diracVersions = Operations.Operations(setup=setup).getValue(
"Pilot/Version", [])
instOpts.append("-r '%s'" % diracVersions[0].strip())
pilotExtensionsList = str(
infoDict['Setups'][setup].get('CommandExtensions')).split(',')
if not pilotExtensionsList:
pilotExtensionsList = str(infoDict['Setups']['Defaults'].get(
'CommandExtensions')).split(',')
if not pilotExtensionsList:
pilotExtensionsList = Operations.Operations(setup=setup).getValue(
"Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
instOpts.append(
"-e '%s'" %
','.join([ext for ext in extensionsList if 'Web' not in ext]))
if 'ContainerExtraOpts' in self.ceParameters:
instOpts.append(self.ceParameters['ContainerExtraOpts'])
return ' '.join(instOpts)
def __generateRootModules(self, baseModules):
""" Iterate over all the possible root modules
"""
self.__rootModules = baseModules
for rootModule in reversed(CSGlobals.getCSExtensions()):
if rootModule[-5:] != "DIRAC" and rootModule not in self.__rootModules:
self.__rootModules.append("%sDIRAC" % rootModule)
self.__rootModules.append("")
# Reversing the order because we want first to look in the extension(s)
self.__rootModules.reverse()
def includeExtensionErrors():
""" Merge all the errors of all the extensions into the errors of these modules
Should be called only at the initialization of DIRAC, so by the parseCommandLine,
dirac-agent.py, dirac-service.py, dirac-executor.py
"""
def __recurseImport( modName, parentModule = None, fullName = False ):
""" Internal function to load modules
"""
if isinstance( modName, basestring ):
modName = modName.split( "." )
if not fullName:
fullName = ".".join( modName )
try:
if parentModule:
impData = imp.find_module( modName[0], parentModule.__path__ )
else:
impData = imp.find_module( modName[0] )
impModule = imp.load_module( modName[0], *impData )
if impData[0]:
impData[0].close()
except ImportError:
return None
if len( modName ) == 1:
return impModule
return __recurseImport( modName[1:], impModule, fullName = fullName )
from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals
allExtensions = CSGlobals.getCSExtensions()
for extension in allExtensions:
ext_derrno = None
try:
ext_derrno = __recurseImport( '%sDIRAC.Core.Utilities.DErrno' % extension )
if ext_derrno:
# The next 3 dictionary MUST be present for consistency
# Global name of errors
sys.modules[__name__].__dict__.update( ext_derrno.extra_dErrName )
# Dictionary with the error codes
sys.modules[__name__].dErrorCode.update( ext_derrno.extra_dErrorCode )
# Error description string
sys.modules[__name__].dStrError.update( ext_derrno.extra_dStrError )
# extra_compatErrorString is optional
for err in getattr( ext_derrno, 'extra_compatErrorString', [] ) :
sys.modules[__name__].compatErrorString.setdefault( err, [] ).extend( ext_derrno.extra_compatErrorString[err] )
except:
pass
def loadObjects( path, reFilter = None, parentClass = None ):
"""
:param str path the path to the syetem for example: DIRAC/AccountingSystem
:param object reFilter regular expression used to found the class
:param object parentClass class instance
:return dict it return the name of the clase and the instance of the class.
"""
if not reFilter:
reFilter = re.compile( ".*[a-z1-9]\.py$" )
pathList = List.fromChar( path, "/" )
parentModuleList = [ "%sDIRAC" % ext for ext in CSGlobals.getCSExtensions() ] + [ 'DIRAC' ]
objectsToLoad = {}
#Find which object files match
for parentModule in parentModuleList:
objDir = os.path.join( DIRAC.rootPath, parentModule, *pathList )
if not os.path.isdir( objDir ):
continue
for objFile in os.listdir( objDir ):
if reFilter.match( objFile ):
pythonClassName = objFile[:-3]
if pythonClassName not in objectsToLoad:
gLogger.info( "Adding to load queue %s/%s/%s" % ( parentModule, path, pythonClassName ) )
objectsToLoad[ pythonClassName ] = parentModule
#Load them!
loadedObjects = {}
for pythonClassName in objectsToLoad:
parentModule = objectsToLoad[ pythonClassName ]
try:
#Where parentModule can be DIRAC, pathList is something like [ "AccountingSystem", "Client", "Types" ]
#And the python class name is.. well, the python class name
objPythonPath = "%s.%s.%s" % ( parentModule, ".".join( pathList ), pythonClassName )
objModule = __import__( objPythonPath,
globals(),
locals(), pythonClassName )
objClass = getattr( objModule, pythonClassName )
except Exception as e:
gLogger.error( "Can't load type", "%s/%s: %s" % ( parentModule, pythonClassName, str( e ) ) )
continue
if parentClass == objClass:
continue
if parentClass and not issubclass( objClass, parentClass ):
gLogger.warn( "%s is not a subclass of %s. Skipping" % ( objClass, parentClass ) )
continue
gLogger.info( "Loaded %s" % objPythonPath )
loadedObjects[ pythonClassName ] = objClass
return loadedObjects
def loadWebAppCFGFiles():
"""
Load WebApp/web.cfg definitions
"""
exts = []
for ext in CSGlobals.getCSExtensions():
if ext == "DIRAC":
continue
if ext[-5:] != "DIRAC":
ext = "%sDIRAC" % ext
if ext != "WebAppDIRAC":
exts.append( ext )
exts.append( "DIRAC" )
exts.append( "WebAppDIRAC" )
webCFG = CFG()
for modName in reversed( exts ):
try:
modPath = imp.find_module( modName )[1]
except ImportError:
continue
gLogger.verbose( "Found module %s at %s" % ( modName, modPath ) )
cfgPath = os.path.join( modPath, "WebApp", "web.cfg" )
if not os.path.isfile( cfgPath ):
gLogger.verbose( "Inexistant %s" % cfgPath )
continue
try:
modCFG = CFG().loadFromFile( cfgPath )
except Exception, excp:
gLogger.error( "Could not load %s: %s" % ( cfgPath, excp ) )
continue
gLogger.verbose( "Loaded %s" % cfgPath )
expl = [ BASECS ]
while len( expl ):
current = expl.pop( 0 )
if not modCFG.isSection( current ):
continue
if modCFG.getOption( "%s/AbsoluteDefinition" % current, False ):
gLogger.verbose( "%s:%s is an absolute definition" % ( modName, current ) )
try:
webCFG.deleteKey( current )
except:
pass
modCFG.deleteKey( "%s/AbsoluteDefinition" % current )
else:
for sec in modCFG[ current ].listSections():
expl.append( "%s/%s" % ( current, sec ) )
#Add the modCFG
webCFG = webCFG.mergeWith( modCFG )
def getPaths( self, dirName ):
"""
Get lists of paths for all installed and enabled extensions
"""
pathList = []
for extName in CSGlobals.getCSExtensions():
if extName.rfind( "DIRAC" ) != len( extName ) - 5:
extName = "%sDIRAC" % extName
if extName == "WebAppDIRAC":
continue
try:
modFile, modPath, desc = imp.find_module( extName )
except ImportError:
continue
staticPath = os.path.join( modPath, "WebApp", dirName )
if os.path.isdir( staticPath ):
pathList.append( staticPath )
#Add WebAppDirac to the end
pathList.append( os.path.join( WebAppDIRAC.rootPath, "WebApp", dirName ) )
return pathList
def __getPilotOptions(self, queue, pilotsToSubmit):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]["ParametersDict"]
pilotOptions = []
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == "unknown":
self.log.error("Setup is not defined in the configuration")
return [None, None]
pilotOptions.append("-S %s" % setup)
opsHelper = Operations.Operations(group=self.pilotGroup, setup=setup)
# Installation defined?
installationName = opsHelper.getValue("Pilot/Installation", "")
if installationName:
pilotOptions.append("-V %s" % installationName)
# Project defined?
projectName = opsHelper.getValue("Pilot/Project", "")
if projectName:
pilotOptions.append("-l %s" % projectName)
else:
self.log.info("DIRAC project will be installed by pilots")
# Request a release
diracVersion = opsHelper.getValue("Pilot/Version", [])
if not diracVersion:
self.log.error("Pilot/Version is not defined in the configuration")
return [None, None]
# diracVersion is a list of accepted releases. Just take the first one
pilotOptions.append("-r %s" % diracVersion[0])
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken(ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode)
if not result["OK"]:
self.log.error("Invalid proxy token request", result["Message"])
return [None, None]
(token, numberOfUses) = result["Value"]
pilotOptions.append("-o /Security/ProxyToken=%s" % token)
# Use Filling mode
pilotOptions.append("-M %s" % min(numberOfUses, self.maxJobsInFillMode))
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min(numberOfUses, self.maxJobsInFillMode)
if newPilotsToSubmit != pilotsToSubmit:
self.log.info(
"Number of pilots to submit is changed to %d after getting the proxy token" % newPilotsToSubmit
)
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == "debug":
pilotOptions.append("-d")
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append("-C %s" % ",".join(csServers))
# DIRAC Extensions to be used in pilots
pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != "None":
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append("-e %s" % ",".join(extensionsList))
# Requested CPU time
pilotOptions.append("-T %s" % queueDict["CPUTime"])
# CEName
pilotOptions.append("-N %s" % self.queueDict[queue]["CEName"])
# SiteName
pilotOptions.append("-n %s" % queueDict["Site"])
if "ClientPlatform" in queueDict:
pilotOptions.append("-p '%s'" % queueDict["ClientPlatform"])
if "SharedArea" in queueDict:
pilotOptions.append("-o '/LocalSite/SharedArea=%s'" % queueDict["SharedArea"])
if "SI00" in queueDict:
factor = float(queueDict["SI00"]) / 250.0
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor)
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % factor)
else:
if "CPUScalingFactor" in queueDict:
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % queueDict["CPUScalingFactor"])
if "CPUNormalizationFactor" in queueDict:
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict["CPUNormalizationFactor"])
# Hack
if self.defaultSubmitPools:
pilotOptions.append("-o /Resources/Computing/CEDefaults/SubmitPool=%s" % self.defaultSubmitPools)
if self.group:
pilotOptions.append("-G %s" % self.group)
self.log.verbose("pilotOptions: ", " ".join(pilotOptions))
return [pilotOptions, pilotsToSubmit]
def _getPilotOptions( self, queue, pilotsToSubmit ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
pilotOptions = []
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return [ None, None ]
pilotOptions.append( '-S %s' % setup )
opsHelper = Operations.Operations( group = self.pilotGroup, setup = setup )
#Installation defined?
installationName = opsHelper.getValue( "Pilot/Installation", "" )
if installationName:
pilotOptions.append( '-V %s' % installationName )
#Project defined?
projectName = opsHelper.getValue( "Pilot/Project", "" )
if projectName:
pilotOptions.append( '-l %s' % projectName )
else:
self.log.info( 'DIRAC project will be installed by pilots' )
#Request a release
diracVersion = opsHelper.getValue( "Pilot/Version", [] )
if not diracVersion:
self.log.error( 'Pilot/Version is not defined in the configuration' )
return [ None, None ]
# diracVersion is a list of accepted releases
pilotOptions.append( '-r %s' % ','.join( str( it ) for it in diracVersion ) )
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode )
if not result[ 'OK' ]:
self.log.error( 'Invalid proxy token request', result['Message'] )
return [ None, None ]
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % min( numberOfUses, self.maxJobsInFillMode ) )
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
if newPilotsToSubmit != pilotsToSubmit:
self.log.info( 'Number of pilots to submit is changed to %d after getting the proxy token' % newPilotsToSubmit )
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == 'debug':
pilotOptions.append( '-d' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions to be used in pilots
pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", [] )
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# CEName
pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] )
# Queue
pilotOptions.append( '-Q %s' % self.queueDict[queue]['QueueName'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
# if 'SI00' in queueDict:
# factor = float( queueDict['SI00'] ) / 250.
# pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor )
# pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor )
# else:
# if 'CPUScalingFactor' in queueDict:
# pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] )
# if 'CPUNormalizationFactor' in queueDict:
# pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] )
if "ExtraPilotOptions" in queueDict:
pilotOptions.append( queueDict['ExtraPilotOptions'] )
# Hack
if self.defaultSubmitPools:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools )
if "Tag" in queueDict:
tagString = ','.join( queueDict['Tag'] )
pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=%s' % tagString )
if self.group:
pilotOptions.append( '-G %s' % self.group )
return [ pilotOptions, pilotsToSubmit ]
def getTicketsList(self, name, startDate=None, endDate=None):
""" Return tickets of entity in name
@param name: should be the name of the site
@param startDate: starting date (optional)
@param endDate: end date (optional)
"""
self.statusCount = {}
self.shortDescription = {}
# create client instance using GGUS wsdl:
gclient = Client("https://prod-ars.ggus.eu/arsys/WSDL/public/prod-ars/GGUS")
authInfo = gclient.factory.create("AuthenticationInfo")
authInfo.userName = "******"
authInfo.password = "******"
gclient.set_options(soapheaders=authInfo)
# prepare the query string:
extension = CSGlobals.getCSExtensions()[0].lower()
query = "'GHD_Affected Site'=\"" + name + '" AND \'GHD_Affected VO\'="%s"' % extension
if startDate is not None:
query = query + " AND 'GHD_Date Of Creation'>" + str(startDate)
if endDate is not None:
query = query + " AND 'GHD_Date Of Creation'<" + str(endDate)
# create the URL to get tickets relative to the site:
# Updated from https://gus.fzk.de to https://ggus.eu
ggusURL = (
"https://ggus.eu/ws/ticket_search.php?show_columns_check[]=REQUEST_ID&"
"show_columns_check[]=TICKET_TYPE&"
"show_columns_check[]=AFFECTED_VO&"
"show_columns_check[]=AFFECTED_SITE&"
"show_columns_check[]=RESPONSIBLE_UNIT&"
"show_columns_check[]=STATUS&"
"show_columns_check[]=DATE_OF_CREATION&"
"show_columns_check[]=LAST_UPDATE&"
"show_columns_check[]=SHORT_DESCRIPTION&"
"ticket=&"
"supportunit=all&"
"vo=%s&"
"user=&"
"keyword=&"
"involvedsupporter=&"
"assignto=&"
"affectedsite=" + name + "&"
"specattrib=0&"
"status=open&"
"priority=all&"
"typeofproblem=all&"
"mouarea=&"
"radiotf=1&"
"timeframe=any&"
"tf_date_day_s=&"
"tf_date_month_s=&"
"tf_date_year_s=&"
"tf_date_day_e=&"
"tf_date_month_e=&"
"tf_date_year_e=&"
"lm_date_day=12&"
"lm_date_month=2&"
"lm_date_year=2010&"
"orderticketsby=GHD_INT_REQUEST_ID&"
"orderhow=descending" % (extension, extension)
)
# the query must be into a try block. Empty queries, though formally correct, raise an exception
try:
self.ticketList = gclient.service.TicketGetList(query)
self.globalStatistics()
except WebFault:
self.statusCount["terminal"] = 0
self.statusCount["open"] = 0
return S_OK((self.statusCount, ggusURL, self.shortDescription))
def __getPilotOptions(self, queue, pilotsToSubmit):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
pilotOptions = []
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == 'unknown':
self.log.error('Setup is not defined in the configuration')
return [None, None]
pilotOptions.append('-S %s' % setup)
opsHelper = Operations.Operations(group=self.pilotGroup, setup=setup)
#Installation defined?
installationName = opsHelper.getValue("Pilot/Installation", "")
if installationName:
pilotOptions.append('-V %s' % installationName)
#Project defined?
projectName = opsHelper.getValue("Pilot/Project", "")
if projectName:
pilotOptions.append('-l %s' % projectName)
else:
self.log.info('DIRAC project will be installed by pilots')
#Request a release
diracVersion = opsHelper.getValue("Pilot/Version", [])
if not diracVersion:
self.log.error('Pilot/Version is not defined in the configuration')
return [None, None]
#diracVersion is a list of accepted releases. Just take the first one
pilotOptions.append('-r %s' % diracVersion[0])
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken(
ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode)
if not result['OK']:
self.log.error('Invalid proxy token request', result['Message'])
return [None, None]
(token, numberOfUses) = result['Value']
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
# Use Filling mode
pilotOptions.append('-M %s' %
min(numberOfUses, self.maxJobsInFillMode))
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min(numberOfUses,
self.maxJobsInFillMode)
if newPilotsToSubmit != pilotsToSubmit:
self.log.info(
'Number of pilots to submit is changed to %d after getting the proxy token'
% newPilotsToSubmit)
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == 'debug':
pilotOptions.append('-d')
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
# Requested CPU time
pilotOptions.append('-T %s' % queueDict['CPUTime'])
# CEName
pilotOptions.append('-N %s' % self.queueDict[queue]['CEName'])
# SiteName
pilotOptions.append('-n %s' % queueDict['Site'])
if 'ClientPlatform' in queueDict:
pilotOptions.append("-p '%s'" % queueDict['ClientPlatform'])
if 'SharedArea' in queueDict:
pilotOptions.append("-o '/LocalSite/SharedArea=%s'" %
queueDict['SharedArea'])
if 'SI00' in queueDict:
factor = float(queueDict['SI00']) / 250.
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor)
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" %
factor)
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" %
queueDict['CPUScalingFactor'])
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append(
"-o '/LocalSite/CPUNormalizationFactor=%s'" %
queueDict['CPUNormalizationFactor'])
# Hack
if self.defaultSubmitPools:
pilotOptions.append(
'-o /Resources/Computing/CEDefaults/SubmitPool=%s' %
self.defaultSubmitPools)
if self.group:
pilotOptions.append('-G %s' % self.group)
self.log.verbose("pilotOptions: ", ' '.join(pilotOptions))
return [pilotOptions, pilotsToSubmit]
