def __init__(self, vo=None):
"""c'tor
:param str vo: name of the virtual organization (community)
"""
if vo is None:
vo = getVO()
if not vo:
raise Exception("No VO name given")
self.vo = vo
self.vomsVO = getVOOption(vo, "VOMSName")
if not self.vomsVO:
raise Exception("Can not get VOMS name for VO %s" % vo)
self.urls = []
result = gConfig.getSections("/Registry/VO/%s/VOMSServers" % self.vo)
if result["OK"]:
for server in result["Value"]:
gLogger.verbose("Adding 'https://%s:8443/voms/%s/apiv2/users'" % (server, self.vomsVO))
self.urls.append("https://%s:8443/voms/%s/apiv2/users" % (server, self.vomsVO))
else:
gLogger.error("Section '/Registry/VO/%s/VOMSServers' not found" % self.vo)
self.userDict = None
def initialize(self):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', [])
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/SAMManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'SAMManager')
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal("VO option not defined for agent")
return S_ERROR()
return S_OK()
def getVOs():
""" Get all the configured VOs
"""
voName = getVO()
if voName:
return S_OK([voName])
return gConfig.getSections('%s/VO' % gBaseRegistrySection)
def __configurePilot(basepath, vo):
"""
Configures the pilot.
This method was created specifically for LHCb pilots, more info
about othe VOs is needed to make it more general.
"""
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO, getSetup
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
vo = getVO()
currentSetup = getSetup()
masterCS = gConfigurationData.getMasterServer()
os.system(
"python " + basepath +
"dirac-pilot.py -S %s -l %s -C %s -N ce.debug.ch -Q default -n DIRAC.JobDebugger.ch --pythonVersion=3 -dd"
% (currentSetup, vo, masterCS))
diracdir = os.path.expanduser("~") + os.path.sep
try:
os.rename(diracdir + ".dirac.cfg", diracdir + ".dirac.cfg.old")
except OSError:
pass
shutil.copyfile(diracdir + "pilot.cfg", diracdir + ".dirac.cfg")
def getVOs():
""" Get all the configured VOs
"""
voName = getVO()
if voName:
return S_OK([voName])
return gConfig.getSections('%s/VO' % gBaseRegistrySection)
def initialize( self ):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'TestManager' )
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal( "VO option not defined for agent" )
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def getDefaultVOMSVO():
""" Get default VOMS VO
:return: str
"""
return gConfig.getValue("%s/DefaultVOMSVO" % gBaseRegistrySection,
"") or getVO()
def __init__(self, vo=None):
""" c'tor
:param str vo: name of the virtual organization (community)
"""
if vo is None:
vo = getVO()
if not vo:
raise Exception('No VO name given')
self.vo = vo
self.vomsVO = getVOOption(vo, "VOMSName")
if not self.vomsVO:
raise Exception("Can not get VOMS name for VO %s" % vo)
self.urls = []
result = gConfig.getSections('/Registry/VO/%s/VOMSServers' % self.vo)
if result['OK']:
for server in result['Value']:
gLogger.verbose("Adding 'https://%s:8443/voms/%s/apiv2/users'" % (server, self.vomsVO))
self.urls.append('https://%s:8443/voms/%s/apiv2/users' % (server, self.vomsVO))
else:
gLogger.error("Section '/Registry/VO/%s/VOMSServers' not found" % self.vo)
self.userDict = None
def getVOs():
""" Get all the configured VOs
:return: S_OK(list)/S_ERROR()
"""
voName = getVO()
return S_OK([voName]) if voName else gConfig.getSections(
'%s/VO' % gBaseRegistrySection)
def getVOForGroup(group):
""" Search VO name for group
:param str group: group name
:return: str
"""
return getVO() or gConfig.getValue(
"%s/Groups/%s/VO" % (gBaseRegistrySection, group), "")
def getStepDefinition(stepName,
modulesNameList=None,
importLine="""""",
parametersList=None):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition(moduleName)
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (importLine, moduleName), globals(),
locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(importLine, moduleName, moduleName))
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (alternativeImportLine, moduleName),
globals(), locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(alternativeImportLine, moduleName, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def getGroupsForVO(vo):
""" Get groups for VO
:param str vo: DIRAC VO name
:return: S_OK(list)/S_ERROR()
"""
if getVO():
return gConfig.getSections("%s/Groups" % gBaseRegistrySection)
return __getGroupsWithAttr('VO', vo)
def getSiteSEMapping(gridName=''):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
gridTypes = gConfig.getSections('Resources/Sites/')
if not gridTypes['OK']:
gLogger.warn('Problem retrieving sections in /Resources/Sites')
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR('Could not get sections for /Resources/Sites/%s' %
gridName)
gridTypes = [gridName]
gLogger.debug('Grid Types are: %s' % (', '.join(gridTypes)))
for grid in gridTypes:
sites = gConfig.getSections('/Resources/Sites/%s' % grid)
if not sites['OK']:
gLogger.warn('Problem retrieving /Resources/Sites/%s section' %
grid)
return sites
for candidate in sites['Value']:
candidateSEs = gConfig.getValue(
'/Resources/Sites/%s/%s/SE' % (grid, candidate), [])
if candidateSEs:
siteSEMapping[candidate] = candidateSEs
else:
gLogger.debug('No SEs defined for site %s' % candidate)
# Add Sites from the SiteLocalSEMapping in the CS
vo = getVO()
setup = getDIRACSetup()
cfgLocalSEPath = cfgPath('Operations', vo, setup, 'SiteLocalSEMapping')
result = gConfig.getOptionsDict(cfgLocalSEPath)
if result['OK']:
mapping = result['Value']
for site in mapping:
ses = gConfig.getValue(cfgPath(cfgLocalSEPath, site), [])
if not ses:
continue
if gridName and site not in sites:
continue
if site not in siteSEMapping:
siteSEMapping[site] = []
for se in ses:
if se not in siteSEMapping[site]:
siteSEMapping[site].append(se)
return S_OK(siteSEMapping)
def getGroupsForVO(vo):
""" Get groups for VO
:param str vo: DIRAC VO name
:return: S_OK(list)/S_ERROR()
"""
if getVO(): # tries to get default VO in /DIRAC/VirtualOrganization
return gConfig.getSections("%s/Groups" % gBaseRegistrySection)
if not vo:
return S_ERROR("No VO requested")
return __getGroupsWithAttr('VO', vo)
def __init__(self,
vo=None,
adminUrl=False,
attributesUrl=False,
certificatesUrl=False):
if vo is None:
vo = getVO()
if not vo:
raise Exception('No VO name given')
self.vo = vo
self.vomsVO = getVOOption(vo, "VOMSName")
if not self.vomsVO:
raise Exception("Can not get VOMS name for VO %s" % vo)
self.__soapClients = {}
for key, url in (('Admin', adminUrl), ('Attributes', attributesUrl),
('Certificates', certificatesUrl)):
urls = []
if not url:
url = gConfig.getValue(
"/Registry/VO/%s/VOMSServices/VOMS%s" % (self.vo, key), "")
if not url:
result = gConfig.getSections('/Registry/VO/%s/VOMSServers' %
self.vo)
if result['OK']:
vomsServers = result['Value']
for server in vomsServers:
urls.append('https://%s:8443/voms/%s/services/VOMS%s' %
(server, self.vomsVO, key))
else:
urls = [url]
gotURL = False
for url in urls:
retries = 3
while retries:
retries -= 1
try:
client = getSOAPClient("%s?wsdl" % url)
client.set_options(headers={"X-VOMS-CSRF-GUARD": "1"})
self.__soapClients[key] = client
gotURL = True
break
except Exception:
pass
if gotURL:
break
if not gotURL:
raise Exception(
'Could not connect to the %s service for VO %s' %
(key, self.vo))
def getSiteSEMapping( gridName = '' ):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
gridTypes = gConfig.getSections( 'Resources/Sites/' )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName )
gridTypes = [gridName]
gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not sites['OK']:
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
for candidate in sites['Value']:
candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] )
if candidateSEs:
siteSEMapping[candidate] = candidateSEs
else:
gLogger.debug( 'No SEs defined for site %s' % candidate )
# Add Sites from the SiteLocalSEMapping in the CS
vo = getVO()
setup = getDIRACSetup()
cfgLocalSEPath = cfgPath( 'Operations', vo, setup, 'SiteLocalSEMapping' )
result = gConfig.getOptionsDict( cfgLocalSEPath )
if result['OK']:
mapping = result['Value']
for site in mapping:
ses = gConfig.getValue( cfgPath( cfgLocalSEPath, site ), [] )
if not ses:
continue
if gridName and site not in sites:
continue
if site not in siteSEMapping:
siteSEMapping[site] = []
for se in ses:
if se not in siteSEMapping[site]:
siteSEMapping[site].append( se )
return S_OK( siteSEMapping )
def getUserRootDir(self):
'''get user's initial root directory'''
username = self.getUserName()
initial = username[:1]
vo = getVO()
if not vo:
vo = 'bes'
ops = Operations(vo=vo)
user_prefix = ops.getValue('LFNUserPrefix', 'user')
basePath = '/' + vo + '/' + user_prefix + '/' + initial + '/' + username
return basePath
def getUserRootDir(self):
'''get user's initial root directory'''
username = self.getUserName()
initial = username[:1]
vo = getVO()
if not vo:
vo = 'bes'
ops = Operations(vo = vo)
user_prefix = ops.getValue('LFNUserPrefix', 'user')
basePath = '/' + vo + '/' + user_prefix + '/' + initial + '/' + username
return basePath
def getStepDefinition( stepName, modulesNameList = [], importLine = """""", parametersList = [] ):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition( stepName )
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition( moduleName )
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( importLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( importLine, moduleName, moduleName ) )
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( alternativeImportLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( alternativeImportLine, moduleName, moduleName ) )
# add the module to the step, and instance it
stepDef.addModule( moduleDef )
stepDef.createModuleInstance( module_type = moduleName, name = moduleName )
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter( pName, pValue, pType, "", "", True, False, pDesc )
stepDef.addParameter( Parameter( parameter = p ) )
return stepDef
def __init__( self, vo = None, adminUrl = False, attributesUrl = False, certificatesUrl = False ):
if vo is None:
vo = getVO()
if not vo:
raise Exception( 'No VO name given' )
self.vo = vo
self.vomsVO = getVOOption( vo, "VOMSName" )
if not self.vomsVO:
raise Exception( "Can not get VOMS name for VO %s" % vo )
self.__soapClients = {}
for key, url in ( ( 'Admin', adminUrl ), ( 'Attributes', attributesUrl ), ( 'Certificates', certificatesUrl ) ):
urls = []
if not url:
url = gConfig.getValue( "/Registry/VO/%s/VOMSServices/VOMS%s" % ( self.vo, key ), "" )
if not url:
result = gConfig.getSections( '/Registry/VO/%s/VOMSServers' % self.vo )
if result['OK']:
vomsServers = result['Value']
for server in vomsServers:
urls.append( 'https://%s:8443/voms/%s/services/VOMS%s' % ( server, self.vomsVO, key ) )
else:
urls = [url]
gotURL = False
for url in urls:
retries = 3
while retries:
retries -= 1
try:
client = getSOAPClient( "%s?wsdl" % url )
client.set_options(headers={"X-VOMS-CSRF-GUARD":"1"})
self.__soapClients[ key ] = client
gotURL = True
break
except Exception:
pass
if gotURL:
break
if not gotURL:
raise Exception( 'Could not connect to the %s service for VO %s' % ( key, self.vo ) )
def __init__(self, vo=None):
if vo is None:
vo = getVO()
if not vo:
raise Exception('No VO name given')
self.vo = vo
self.vomsVO = getVOOption(vo, "VOMSName")
if not self.vomsVO:
raise Exception("Can not get VOMS name for VO %s" % vo)
self.urls = []
result = gConfig.getSections('/Registry/VO/%s/VOMSServers' % self.vo)
if result['OK']:
vomsServers = result['Value']
for server in vomsServers:
self.urls.append('https://%s:8443/voms/%s/apiv2/users' % (server, self.vomsVO))
self.userDict = None
def main():
Script.registerSwitch(
"D:", "Download=",
"Set jobID here. Defines data acquisition as DownloadInputData")
Script.registerSwitch(
"I:", "Protocol=",
"Set jobID here. Defines data acquisition as InputDataByProtocol")
switches, args = Script.parseCommandLine(ignoreErrors=False)
_downloadinputdata = False
_jobID = None
for switch in switches:
if switch[0] in ("D", "Download"):
_downloadinputdata = True
_jobID = switch[1]
if switch[0] in ("I", "Protocol"):
_downloadinputdata = False
_jobID = switch[1]
_vo = getVO()
_dir = os.path.expanduser("~") + os.path.sep
try:
_path = __runSystemDefaults(_jobID, _vo)
__downloadJobDescriptionXML(_jobID, _path)
__modifyJobDescription(_jobID, _path, _downloadinputdata)
__downloadPilotScripts()
__configurePilot(_path, _vo)
__runJobLocally(_jobID, _path, _vo)
finally:
os.chdir(_dir)
os.rename(_dir + ".dirac.cfg.old", _dir + ".dirac.cfg")
def __configurePilot(basepath, vo):
"""
Configures the pilot.
This method was created specifically for LHCb pilots, more info
about othe VOs is needed to make it more general.
"""
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO, getSetup
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
vo = getVO()
currentSetup = getSetup()
masterCS = gConfigurationData.getMasterServer()
os.system("python " + basepath + "dirac-pilot.py -S %s -l %s -C %s -N ce.debug.ch -Q default -n DIRAC.JobDebugger.ch -dd" %(currentSetup, vo, masterCS))
dir = os.path.expanduser('~') + os.path.sep
try:
os.rename(dir + '.dirac.cfg', dir + '.dirac.cfg.old')
except OSError:
pass
shutil.copyfile(dir + 'pilot.cfg', dir + '.dirac.cfg')
class CE2CSAgent(AgentModule):
addressTo = ''
addressFrom = ''
vo = getVO()
csAPI = CSAPI()
subject = "CE2CSAgent"
def initialize(self):
# TODO: Have no default and if no mail is found then use the diracAdmin group and resolve all associated mail addresses.
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# create a list of alternative bdii urls
self.alternativeBDIIs = map(
lambda x: x.strip(),
self.am_getOption('AlternativeBDIIs', '').split(','))
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/SAMManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'SAMManager')
if not self.vo:
self.log.fatal("VO option not defined for agent")
return S_ERROR()
return S_OK()
def execute(self):
self.log.info("Start Execution")
result = getProxyInfo()
if not result['OK']:
return result
infoDict = result['Value']
self.log.info(formatProxyInfoAsString(infoDict))
self._lookForCE()
self._infoFromCE()
self.log.info("End Execution")
return S_OK()
def _checkAlternativeBDIISite(self, fun, *args):
if self.alternativeBDIIs:
self.log.warn("Trying to use alternative bdii sites")
for site in self.alternativeBDIIs:
self.log.info("Trying to contact alternative bdii ", site)
if len(args) == 1: result = fun(args[0], host=site)
elif len(args) == 2:
result = fun(args[0], vo=args[1], host=site)
if not result['OK']:
self.log.error("Problem contacting alternative bddii",
result['Message'])
elif result['OK']:
return result
self.log.warn("Also checking alternative BDII sites failed")
return result
def _lookForCE(self):
knownces = self.am_getOption('BannedCEs', [])
result = gConfig.getSections('/Resources/Sites')
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections('/Resources/Sites/%s' % grid)
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(grid, site))['Value']
ces = List.fromChar(opt.get('CE', ''))
knownces += ces
response = ldapCEState('', vo=self.vo)
if not response['OK']:
self.log.error("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(ldapCEState, '', self.vo)
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split(":")[0]
if not cename in knownces:
newces[cename] = None
self.log.debug("newce", cename)
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster(ce)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(ldapCluster, ce)
continue
clusters = response['Value']
if len(clusters) != 1:
self.log.warn("Error in cluster leng",
" CE %s Leng %d" % (ce, len(clusters)))
if len(clusters) == 0:
continue
cluster = clusters[0]
fkey = cluster.get('GlueForeignKey', [])
if type(fkey) == type(''):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count('GlueSiteUniqueID'):
nameBDII = entry.split('=')[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % (ce, nameBDII)
self.log.info(cestring)
response = ldapCE(ce)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(ldapCE, ce)
continue
ceinfos = response['Value']
if len(ceinfos):
ceinfo = ceinfos[0]
systemName = ceinfo.get('GlueHostOperatingSystemName',
'Unknown')
systemVersion = ceinfo.get('GlueHostOperatingSystemVersion',
'Unknown')
systemRelease = ceinfo.get('GlueHostOperatingSystemRelease',
'Unknown')
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % (
systemName, systemVersion, systemRelease)
self.log.info(osstring)
response = ldapCEState(ce, vo=self.vo)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(
ldapCEState, ce, self.vo)
continue
newcestring = "\n\n%s\n%s" % (cestring, osstring)
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get('GlueCEUniqueID', 'UnknownName')
queuestatus = cestate.get('GlueCEStateStatus', 'UnknownStatus')
queuestring = "%s %s" % (queuename, queuestatus)
self.log.info(queuestring)
newcestring += "\n%s" % queuestring
if queuestatus.count('Production'):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append(
'dirac-admin-add-site DIRACSiteName %s %s' %
(nameBDII, ce))
if body:
body = "We are glade to inform You about new CE(s) possibly suitable for %s:\n" % self.vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % (body, possibleNewSite)
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return S_OK()
def _infoFromCE(self):
sitesSection = cfgPath('Resources', 'Sites')
result = gConfig.getSections(sitesSection)
if not result['OK']:
return
grids = result['Value']
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result['OK']:
return
sites = result['Value']
changed = False
body = ""
for site in sites:
# if site[-2:]!='ru':
# continue
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)['Value']
name = opt.get('Name', '')
if name:
coor = opt.get('Coordinates', 'Unknown')
mail = opt.get('Mail', 'Unknown')
result = ldapSite(name)
if not result['OK']:
self.log.warn("BDII site %s: %s" %
(name, result['Message']))
result = self._checkAlternativeBDIISite(ldapSite, name)
if result['OK']:
bdiisites = result['Value']
if len(bdiisites) == 0:
self.log.warn(name, "Error in bdii: leng = 0")
else:
if not len(bdiisites) == 1:
self.log.warn(
name, "Warning in bdii: leng = %d" %
len(bdiisites))
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in bdii coor")
newcoor = "Unknown"
try:
newmail = bdiisite[
'GlueSiteSysAdminContact'].split(
":")[-1].strip()
except:
self.log.warn("Error in bdii mail")
newmail = "Unknown"
self.log.debug("%s %s %s" %
(name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name),
"%s -> %s" % (coor, newcoor))
if coor == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Coordinates'),
newcoor)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name),
"%s -> %s" % (mail, newmail))
if mail == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Mail'), newmail)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newmail)
changed = True
celist = List.fromChar(opt.get('CE', ''))
if not celist:
self.log.warn(site, 'Empty site list')
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath(siteSection, 'CEs', ce)
result = gConfig.getOptionsDict(ceSection)
if not result['OK']:
self.log.debug("Section CE", result['Message'])
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get('wnTmpDir', 'Unknown')
arch = ceopt.get('architecture', 'Unknown')
os = ceopt.get('OS', 'Unknown')
si00 = ceopt.get('SI00', 'Unknown')
pilot = ceopt.get('Pilot', 'Unknown')
cetype = ceopt.get('CEType', 'Unknown')
result = ldapCE(ce)
if not result['OK']:
self.log.warn('Error in bdii for %s' % ce,
result['Message'])
result = self._checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn('Error in bdii for %s' % ce, result)
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath(ceSection, 'wnTmpDir')
self.log.info(section, " -> ".join(
(wnTmpDir, newwnTmpDir)))
if wnTmpDir == 'Unknown':
self.csAPI.setOption(section, newwnTmpDir)
else:
self.csAPI.modifyValue(section, newwnTmpDir)
changed = True
try:
newarch = bdiice[
'GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath(ceSection, 'architecture')
self.log.info(section, " -> ".join(
(arch, newarch)))
if arch == 'Unknown':
self.csAPI.setOption(section, newarch)
else:
self.csAPI.modifyValue(section, newarch)
changed = True
try:
newos = '_'.join(
(bdiice['GlueHostOperatingSystemName'],
bdiice['GlueHostOperatingSystemVersion'],
bdiice['GlueHostOperatingSystemRelease']))
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath(ceSection, 'OS')
self.log.info(section, " -> ".join((os, newos)))
if os == 'Unknown':
self.csAPI.setOption(section, newos)
else:
self.csAPI.modifyValue(section, newos)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (
os, newos, ce, site)
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath(ceSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newsi00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
try:
rte = bdiice[
'GlueHostApplicationSoftwareRunTimeEnvironment']
if self.vo == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath(ceSection, 'Pilot')
self.log.info(section, " -> ".join(
(pilot, newpilot)))
if pilot == 'Unknown':
self.csAPI.setOption(section, newpilot)
else:
self.csAPI.modifyValue(section, newpilot)
changed = True
result = ldapService(ce)
if not result['OK']:
result = self._checkAlternativeBDIISite(
ldapService, ce)
if result['OK']:
services = result['Value']
newcetype = 'LCG'
for service in services:
if service['GlueServiceType'].count('CREAM'):
newcetype = "CREAM"
else:
newcetype = 'Unknown'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath(ceSection, 'CEType')
self.log.info(section, " -> ".join(
(cetype, newcetype)))
if cetype == 'Unknown':
self.csAPI.setOption(section, newcetype)
else:
self.csAPI.modifyValue(section, newcetype)
changed = True
result = ldapCEState(ce, vo=self.vo) #getBDIICEVOView
if not result['OK']:
self.log.warn('Error in bdii for queue %s' % ce,
result['Message'])
result = self._checkAlternativeBDIISite(
ldapCEState, ce, self.vo)
continue
try:
queues = result['Value']
except:
self.log.warn('Error in bdii for queue %s' % ce,
result['Massage'])
continue
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split('/')[-1]
except:
self.log.warn('error in queuename ', queue)
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type(caps) == type(''):
caps = [caps]
for cap in caps:
if cap.count('CPUScalingReferenceSI00'):
newsi00 = cap.split('=')[-1]
except:
newsi00 = None
queueSection = cfgPath(ceSection, 'Queues', queueName)
result = gConfig.getOptionsDict(queueSection)
if not result['OK']:
self.log.warn("Section Queues", result['Message'])
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get('maxCPUTime', 'Unknown')
si00 = queueopt.get('SI00', 'Unknown')
if newmaxCPUTime and (maxCPUTime != newmaxCPUTime):
section = cfgPath(queueSection, 'maxCPUTime')
self.log.info(
section, " -> ".join(
(maxCPUTime, newmaxCPUTime)))
if maxCPUTime == 'Unknown':
self.csAPI.setOption(section, newmaxCPUTime)
else:
self.csAPI.modifyValue(section, newmaxCPUTime)
changed = True
if newsi00 and (si00 != newsi00):
section = cfgPath(queueSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newsi00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
if False and changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return self.csAPI.commitChanges(sortUsers=False)
else:
self.log.info("No changes found")
return S_OK()
def getGroupsForVO(vo):
if getVO():
return gConfig.getSections("%s/Groups" % gBaseRegistrySection)
return __getGroupsWithAttr('VO', vo)
def getDefaultVOMSVO():
vomsVO = gConfig.getValue("%s/DefaultVOMSVO" % gBaseRegistrySection, "")
if vomsVO:
return vomsVO
return getVO()
def getVOForGroup(group):
voName = getVO()
if voName:
return voName
return gConfig.getValue("%s/Groups/%s/VO" % (gBaseRegistrySection, group),
"")
def getVOForGroup( group ):
voName = getVO()
if voName:
return voName
return gConfig.getValue( "%s/Groups/%s/VO" % ( gBaseSecuritySection, group ), "" )
def getGroupsForVO( vo ):
if getVO():
return gConfig.getSections( "%s/Groups" % gBaseSecuritySection )
return __getGroupsWithAttr( 'VO', vo )
def getDefaultVOMSVO():
vomsVO = gConfig.getValue( "%s/DefaultVOMSVO" % gBaseSecuritySection, "" )
if vomsVO:
return vomsVO
return getVO()
