def updateFlag():
# Here now setting the flag as the following inside /Operations/Defaults:
# in Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
res = csAPI.createSection('Operations/Defaults/Services/')
if not res['OK']:
print(res['Message'])
exit(1)
res = csAPI.createSection('Operations/Defaults/Services/JobMonitoring/')
if not res['OK']:
print(res['Message'])
exit(1)
csAPI.setOption(
'Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag',
True)
csAPI.commit()
# Now we need to restart the services for the new configuration to be picked up
time.sleep(2)
os.system("dirac-restart-component WorkloadManagement JobMonitoring")
os.system("dirac-restart-component WorkloadManagement JobStateUpdate")
time.sleep(5)
def updatePilot(version, vo):
"""
Update in the CS the pilot version used,
If only one version present in CS it's overwritten.
If two versions present, the new one is added and the last removed
:param version: version vArBpC of pilot you want to use
:param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version
"""
setup = vo
if not vo:
setup = gConfig.getValue('/DIRAC/DefaultSetup')
if not setup:
return S_ERROR("No value set for /DIRAC/DefaultSetup in CS")
pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup, [])
if not pilotVersion:
return S_ERROR(
"No pilot version set under Operations/%s/Pilot/Version in CS" %
setup)
pilotVersion.pop()
pilotVersion.insert(0, version)
api = CSAPI()
api.setOption('Operations/%s/Pilot/Version' % setup,
", ".join(pilotVersion))
result = api.commit()
if not result['OK']:
gLogger.fatal('Could not commit new version of pilot!')
return result
newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup)
return S_OK("New version of pilot set to %s" % newVersion)
def updateCS(changeSet):
global vo, dry, ceBdiiDict
changeList = sorted(changeSet)
if dry:
gLogger.notice('The following needed changes are detected:\n')
else:
gLogger.notice('We are about to make the following changes to CS:\n')
for entry in changeList:
gLogger.notice("%s/%s %s -> %s" % entry)
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object',
result['Message'])
DIRACExit(-1)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption(cfgPath(section, option), new_value)
else:
csAPI.modifyValue(cfgPath(section, option), new_value)
yn = six.moves.input(
'Do you want to commit changes to CS ? [default yes] [yes|no]: ')
if yn == '' or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" %
len(changeSet))
def updateCS( changeSet ):
global vo, dry, ceBdiiDict
changeList = list( changeSet )
changeList.sort()
if dry:
gLogger.notice( 'The following needed changes are detected:\n' )
else:
gLogger.notice( 'We are about to make the following changes to CS:\n' )
for entry in changeList:
gLogger.notice( "%s/%s %s -> %s" % entry )
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption( cfgPath( section, option ), new_value )
else:
csAPI.modifyValue( cfgPath( section, option ), new_value )
yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' )
if yn == '' or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
csAPI = CSAPI()
if len(args) < 3:
Script.showHelp(exitCode=1)
shifterRole = args[0]
userName = args[1]
diracGroup = args[2]
res = csAPI.addShifter(
{shifterRole: {
'User': userName,
'Group': diracGroup
}})
if not res['OK']:
gLogger.error("Could not add shifter", ": " + res['Message'])
DIRACExit(1)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Could not add shifter", ": " + res['Message'])
DIRACExit(1)
gLogger.notice("Added shifter %s as user %s with group %s" %
(shifterRole, userName, diracGroup))
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"ShifterRole: Name of the shifter role, e.g. DataManager")
Script.registerArgument(
"UserName: A user name, as registered in Registry section")
Script.registerArgument(
"DIRACGroup: DIRAC Group, e.g. diracAdmin (the user has to have this role)"
)
Script.parseCommandLine(ignoreErrors=True)
csAPI = CSAPI()
shifterRole, userName, diracGroup = Script.getPositionalArgs(group=True)
res = csAPI.addShifter(
{shifterRole: {
"User": userName,
"Group": diracGroup
}})
if not res["OK"]:
gLogger.error("Could not add shifter", ": " + res["Message"])
DIRACExit(1)
res = csAPI.commit()
if not res["OK"]:
gLogger.error("Could not add shifter", ": " + res["Message"])
DIRACExit(1)
gLogger.notice("Added shifter %s as user %s with group %s" %
(shifterRole, userName, diracGroup))
def commit(self):
"""Commit the changes to the configuration system."""
# Perform all the appending operations at the end to only get from current config once.
for path, value in chain(self._append_dict.iteritems(),
self._append_unique_dict.iteritems()):
section, option = path.rsplit('/', 1)
self.add(section, option, value)
self._num_changes += 1
result = CSAPI.commit(self)
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
raise RuntimeError("Error while commit to CS")
if self._num_changes:
gLogger.notice("Successfully committed %d changes to CS\n"
% self._num_changes)
self._num_changes = 0
self._append_dict.clear()
self._append_unique_dict.clear()
else:
gLogger.notice("No changes to commit")
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = "Bdii2CSAgent"
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = None
# What to get
self.processCEs = True
self.processSEs = False
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs',
self.alternativeBDIIs)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data",
result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set(result['Value'])
knownCEs = knownCEs.union(set(bannedCEs))
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get(
'GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get(
'GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (
queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii",
mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii,
resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult[
'Value'] and message: ## Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiSEInfo(self, vo):
if vo in self.voBdiiSEDict:
return S_OK(self.voBdiiSEDict[vo])
self.log.info("Check for available SEs for VO", vo)
result = getBdiiSEInfo(vo)
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs:
result = getBdiiSEInfo(vo, host=bdii)
if result['OK']:
break
if not result['OK']:
if message:
self.log.error("Error during BDII request", message)
else:
self.log.error("Error during BDII request", result['Message'])
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(
set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = list(changeSet)
changeList.sort()
body = '\n'.join(
["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if body:
self.log.info(
'The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS",
result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" %
len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get(
'GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get(
'GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def __updateSEs(self):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates(vo, bdiiInfo=seBdiiDict)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
class CE2CSAgent( AgentModule ):
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize( self ):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'TestManager' )
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal( "VO option not defined for agent" )
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute( self ):
self.log.info( "Start Execution" )
result = getProxyInfo()
if not result[ 'OK' ]:
return result
infoDict = result[ 'Value' ]
self.log.info( formatProxyInfoAsString( infoDict ) )
#Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result[ 'OK' ]:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
self.__lookForCE()
self.__infoFromCE()
self.log.info( "End Execution" )
return S_OK()
def __checkAlternativeBDIISite( self, fun, *args ):
if self.alternativeBDIIs:
self.log.warn( "Trying to use alternative bdii sites" )
for site in self.alternativeBDIIs :
self.log.info( "Trying to contact alternative bdii ", site )
if len( args ) == 1 :
result = fun( args[0], host = site )
elif len( args ) == 2 :
result = fun( args[0], vo = args[1], host = site )
if not result['OK'] :
self.log.error ( "Problem contacting alternative bddii", result['Message'] )
elif result['OK'] :
return result
self.log.warn( "Also checking alternative BDII sites failed" )
return result
def __lookForCE( self ):
knownces = self.am_getOption( 'BannedCEs', [] )
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
ces = List.fromChar( opt.get( 'CE', '' ) )
knownces += ces
response = ldapCEState( '', vo = self.voName )
if not response['OK']:
self.log.error( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName )
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split( ":" )[0]
if not cename in knownces:
newces[cename] = None
self.log.debug( "newce", cename )
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCluster, ce )
continue
clusters = response['Value']
if len( clusters ) != 1:
self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) )
if len( clusters ) == 0:
continue
cluster = clusters[0]
fkey = cluster.get( 'GlueForeignKey', [] )
if type( fkey ) == type( '' ):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count( 'GlueSiteUniqueID' ):
nameBDII = entry.split( '=' )[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII )
self.log.info( cestring )
response = ldapCE( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
ceinfos = response['Value']
if len( ceinfos ):
ceinfo = ceinfos[0]
systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' )
systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' )
systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' )
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease )
self.log.info( osstring )
response = ldapCEState( ce, vo = self.voName )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
newcestring = "\n\n%s\n%s" % ( cestring, osstring )
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' )
queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' )
queuestring = "%s %s" % ( queuename, queuestatus )
self.log.info( queuestring )
newcestring += "\n%s" % queuestring
if queuestatus.count( 'Production' ):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) )
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % ( body, possibleNewSite )
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return S_OK()
def __infoFromCE( self ):
sitesSection = cfgPath( 'Resources', 'Sites' )
result = gConfig.getSections( sitesSection )
if not result['OK']:
return
grids = result['Value']
changed = False
body = ""
for grid in grids:
gridSection = cfgPath( sitesSection, grid )
result = gConfig.getSections( gridSection )
if not result['OK']:
return
sites = result['Value']
for site in sites:
siteSection = cfgPath( gridSection, site )
opt = gConfig.getOptionsDict( siteSection )['Value']
name = opt.get( 'Name', '' )
if name:
coor = opt.get( 'Coordinates', 'Unknown' )
mail = opt.get( 'Mail', 'Unknown' )
result = ldapSite( name )
if not result['OK']:
self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) )
result = self.__checkAlternativeBDIISite( ldapSite, name )
if result['OK']:
bdiisites = result['Value']
if len( bdiisites ) == 0:
self.log.warn( name, "Error in bdii: leng = 0" )
else:
if not len( bdiisites ) == 1:
self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) )
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % ( longitude, latitude )
except:
self.log.warn( "Error in bdii coor" )
newcoor = "Unknown"
try:
newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip()
except:
self.log.warn( "Error in bdii mail" )
newmail = "Unknown"
self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) )
if newcoor != coor:
self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) )
if coor == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor )
changed = True
if newmail != mail:
self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) )
if mail == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Mail' ), newmail )
changed = True
celist = List.fromChar( opt.get( 'CE', '' ) )
if not celist:
self.log.warn( site, 'Empty site list' )
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath( siteSection, 'CEs', ce )
result = gConfig.getOptionsDict( ceSection )
if not result['OK']:
self.log.debug( "Section CE", result['Message'] )
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' )
arch = ceopt.get( 'architecture', 'Unknown' )
os = ceopt.get( 'OS', 'Unknown' )
si00 = ceopt.get( 'SI00', 'Unknown' )
pilot = ceopt.get( 'Pilot', 'Unknown' )
cetype = ceopt.get( 'CEType', 'Unknown' )
result = ldapCE( ce )
if not result['OK']:
self.log.warn( 'Error in bdii for %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn( 'Error in bdii for %s' % ce, result )
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath( ceSection, 'wnTmpDir' )
self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) )
if wnTmpDir == 'Unknown':
self.csAPI.setOption( section, newwnTmpDir )
else:
self.csAPI.modifyValue( section, newwnTmpDir )
changed = True
try:
newarch = bdiice['GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath( ceSection, 'architecture' )
self.log.info( section, " -> ".join( ( arch, newarch ) ) )
if arch == 'Unknown':
self.csAPI.setOption( section, newarch )
else:
self.csAPI.modifyValue( section, newarch )
changed = True
try:
newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'],
bdiice['GlueHostOperatingSystemVersion'],
bdiice['GlueHostOperatingSystemRelease'] ) )
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath( ceSection, 'OS' )
self.log.info( section, " -> ".join( ( os, newos ) ) )
if os == 'Unknown':
self.csAPI.setOption( section, newos )
else:
self.csAPI.modifyValue( section, newos )
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site )
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath( ceSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
try:
rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment']
if self.voName.lower() == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath( ceSection, 'Pilot' )
self.log.info( section, " -> ".join( ( pilot, newpilot ) ) )
if pilot == 'Unknown':
self.csAPI.setOption( section, newpilot )
else:
self.csAPI.modifyValue( section, newpilot )
changed = True
result = ldapCEState( ce, vo = self.voName ) #getBDIICEVOView
if not result['OK']:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
try:
queues = result['Value']
except:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] )
continue
newcetype = 'Unknown'
for queue in queues:
try:
queuetype = queue['GlueCEImplementationName']
except:
queuetype = 'Unknown'
if newcetype == 'Unknown':
newcetype = queuetype
else:
if queuetype != newcetype:
self.log.warn( 'Error in bdii for ce %s ' % ce, 'different cetypes %s %s' % ( newcetype, queuetype ) )
if newcetype=='ARC-CE':
newcetype = 'ARC'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath( ceSection, 'CEType' )
self.log.info( section, " -> ".join( ( cetype, newcetype ) ) )
if cetype == 'Unknown':
self.csAPI.setOption( section, newcetype )
else:
self.csAPI.modifyValue( section, newcetype )
changed = True
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split( '/' )[-1]
except:
self.log.warn( 'error in queuename ', queue )
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type( caps ) == type( '' ):
caps = [caps]
for cap in caps:
if cap.count( 'CPUScalingReferenceSI00' ):
newsi00 = cap.split( '=' )[-1]
except:
newsi00 = None
queueSection = cfgPath( ceSection, 'Queues', queueName )
result = gConfig.getOptionsDict( queueSection )
if not result['OK']:
self.log.warn( "Section Queues", result['Message'] )
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' )
si00 = queueopt.get( 'SI00', 'Unknown' )
if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ):
section = cfgPath( queueSection, 'maxCPUTime' )
self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) )
if maxCPUTime == 'Unknown':
self.csAPI.setOption( section, newmaxCPUTime )
else:
self.csAPI.modifyValue( section, newmaxCPUTime )
changed = True
if newsi00 and ( si00 != newsi00 ):
section = cfgPath( queueSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
if changed:
self.log.info( body )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return self.csAPI.commit()
else:
self.log.info( "No changes found" )
return S_OK()
# }
# }
# }
res = csAPI.createSection("Registry")
if not res["OK"]:
print(res["Message"])
exit(1)
res = csAPI.createSection("Registry/VO/")
if not res["OK"]:
print(res["Message"])
exit(1)
res = csAPI.createSection("Registry/VO/Jenkins")
if not res["OK"]:
print(res["Message"])
exit(1)
res = csAPI.createSection("Registry/VO/Jenkins/VOMSName")
if not res["OK"]:
print(res["Message"])
exit(1)
csAPI.setOption("Registry/VO/Jenkins/VOMSName", "myVOMS")
csAPI.setOption("Registry/Groups/jenkins_fcadmin/VO", "Jenkins")
csAPI.setOption("Registry/Groups/jenkins_user/VO", "Jenkins")
# Final action: commit in CS
res = csAPI.commit()
if not res["OK"]:
print(res["Message"])
exit(1)
class CVMFSAdder(object):
"""Container for all the objects and functions to add software to ILCDirac"""
def __init__(self, cliParams ):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.modifiedCS = False
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.mailadress = '*****@*****.**'
self.cliParams = cliParams
self.parameter = dict( softSec = self.softSec,
platform = cliParams.platform,
version = cliParams.version,
basepath = cliParams.basePath,
initsctipt = cliParams.initScriptLocation
)
self.applications = cliParams.applicationSet
self.detmodels = {}
self.csAPI = CSAPI()
def findDDSimDetectorModels( self ):
""" find all detector models in lcgeo and
fill the self.detmodels dictionary with Detmodel as key and path as value
:returns: None
"""
for root,dirs,_files in os.walk( os.path.join( self.parameter["basepath"], "lcgeo" ) ):
for direct in dirs:
if root.endswith("compact"):
## the main xml file must have the same name as the folder
xmlPath = os.path.join( root, direct, direct+".xml")
if os.path.exists( xmlPath ):
self.detmodels[direct] = xmlPath
def checkConsistency(self):
"""checks if platform is defined, application exists, etc."""
gLogger.notice("Checking consistency")
av_platforms = gConfig.getSections(self.softSec, [])
if av_platforms['OK']:
if not self.parameter['platform'] in av_platforms['Value']:
gLogger.error("Platform %s unknown, available are %s." % (self.parameter['platform'], ", ".join(av_platforms['Value'])))
gLogger.error("If yours is missing, add it in CS")
return S_ERROR()
else:
gLogger.error("Could not find all platforms available in CS")
return S_ERROR()
for application in self.applications:
av_apps = gConfig.getSections("%(softSec)s/%(platform)s/" % self.parameter + str(application), [])
if not av_apps['OK']:
gLogger.error("Could not find this application in the CS: '%s'" % application)
gLogger.error("Add its section to the CS, if it is missing")
return S_ERROR()
gLogger.notice("All OK, continuing...")
return S_OK()
def commitToCS(self):
"""write changes to the CS to the server"""
if self.modifiedCS and not self.cliParams.dryRun:
gLogger.notice("Commiting changes to the CS")
result = self.csAPI.commit()
if not result[ 'OK' ]:
gLogger.error('Commit failed with message = %s' % (result[ 'Message' ]))
return S_ERROR("Failed to commit to CS")
gLogger.info('Successfully committed changes to CS')
else:
gLogger.info('No modifications to CS required')
return S_OK()
def addAllToCS(self):
"""add all the applications to the CS, take care of special cases (mokka, ildconfig, ddsim,...)"""
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import insertCSSection
for application in self.applications:
csParameter = dict( CVMFSEnvScript = self.cliParams.initScriptLocation,
CVMFSPath = self.parameter['basepath']
)
if application == 'mokka':
csParameter['CVMFSDBSlice'] = self.cliParams.dbSliceLocation
if application == 'ddsim':
self.findDDSimDetectorModels()
csPathModels = "Operations/Defaults/DDSimDetectorModels"
csModels = { self.parameter["version"] : self.detmodels }
insertCSSection( self.csAPI, csPathModels, csModels )
self.modifiedCS = True
elif application.endswith('config'):
del csParameter['CVMFSEnvScript']
csParameter['CVMFSPath'] = self.cliParams.configPath
if self.cliParams.dbSliceLocation:
csParameter['CVMFSDBSlice'] = self.cliParams.dbSliceLocation
resInsert = self.insertApplicationToCS(application, csParameter)
if not resInsert['OK']:
return resInsert
return S_OK()
def insertApplicationToCS(self, name, csParameter):
"""add given application found via CVMFS to the CS"""
pars = dict(self.parameter)
pars['name'] = name
gLogger.notice("%(name)s: Adding version %(version)s to the CS" % pars)
existingVersions = gConfig.getSections("%(softSec)s/%(platform)s/%(name)s" % pars, [])
if not existingVersions['OK']:
gLogger.error("Could not find all versions available in CS: %s" % existingVersions['Message'])
dexit(255)
if pars['version'] in existingVersions['Value']:
gLogger.always('Application %s %s for %s already in CS, nothing to do' % (name.lower(),
pars['version'],
pars['platform']))
return S_OK()
csPath = self.softSec + ("/%(platform)s/%(name)s/%(version)s/" % pars)
for par, val in csParameter.iteritems():
gLogger.notice("Add: %s = %s" %(csPath+par, val))
result = self.csAPI.setOption(csPath+par, val)
if result['OK']:
self.modifiedCS = True
else:
gLogger.error("Failure to add to CS", result['Message'])
return S_ERROR("")
return S_OK()
def addSoftware(self):
"""run all the steps to add software to grid and CS"""
resAdd = self.addAllToCS()
if not resAdd['OK']:
return resAdd
resCommit = self.commitToCS()
if not resCommit['OK']:
return resCommit
return S_OK()
class GOCDB2CSAgent (AgentModule):
""" Class to retrieve information about service endpoints
from GOCDB and update configuration stored by CS
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
super(GOCDB2CSAgent, self).__init__(*args, **kwargs)
self.GOCDBClient = None
self.csAPI = None
self.dryRun = False
def initialize(self):
""" Run at the agent initialization (normally every 500 cycles)
"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
"""
Execute GOCDB queries according to the function map
and user request (options in configuration).
"""
# __functionMap is at the end of the class definition
for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
optionValue = self.am_getOption(option, True)
if optionValue:
result = functionCall(self)
if not result['OK']:
self.log.error("%s() failed with message: %s" % (functionCall.__name__, result['Message']))
else:
self.log.info("Successfully executed %s" % functionCall.__name__)
return S_OK()
def updatePerfSONARConfiguration(self):
"""
Get current status of perfSONAR endpoints from GOCDB
and update CS configuration accordingly.
"""
log = self.log.getSubLogger('updatePerfSONAREndpoints')
log.debug('Begin function ...')
# get endpoints
result = self.__getPerfSONAREndpoints()
if not result['OK']:
log.error("__getPerfSONAREndpoints() failed with message: %s" % result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.')
endpointList = result['Value']
# add DIRAC site name
result = self.__addDIRACSiteName(endpointList)
if not result['OK']:
log.error("__addDIRACSiteName() failed with message: %s" % result['Message'])
return S_ERROR('Unable to extend the list with DIRAC site names.')
extendedEndpointList = result['Value']
# prepare dictionary with new configuration
result = self.__preparePerfSONARConfiguration(extendedEndpointList)
if not result['OK']:
log.error("__preparePerfSONARConfiguration() failed with message: %s" % result['Message'])
return S_ERROR('Unable to prepare a new perfSONAR configuration.')
finalConfiguration = result['Value']
# update configuration according to the final status of endpoints
self.__updateConfiguration(finalConfiguration)
log.debug("Configuration updated succesfully")
log.debug('End function.')
return S_OK()
def __getPerfSONAREndpoints(self):
"""
Retrieve perfSONAR endpoint information directly from GOCDB.
:return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB.
"""
log = self.log.getSubLogger('__getPerfSONAREndpoints')
log.debug('Begin function ...')
# get perfSONAR endpoints (latency and bandwidth) form GOCDB
endpointList = []
for endpointType in ['Latency', 'Bandwidth']:
result = self.GOCDBClient.getServiceEndpointInfo('service_type', 'net.perfSONAR.%s' % endpointType)
if not result['OK']:
log.error("getServiceEndpointInfo() failed with message: %s" % result['Message'])
return S_ERROR('Could not fetch %s endpoints from GOCDB' % endpointType.lower())
log.debug('Number of %s endpoints: %s' % (endpointType.lower(), len(result['Value'])))
endpointList.extend(result['Value'])
log.debug('Number of perfSONAR endpoints: %s' % len(endpointList))
log.debug('End function.')
return S_OK(endpointList)
def __preparePerfSONARConfiguration(self, endpointList):
"""
Prepare a dictionary with a new CS configuration of perfSONAR endpoints.
:return: Dictionary where keys are configuration paths (options and sections)
and values are values of corresponding options
or None in case of a path pointing to a section.
"""
log = self.log.getSubLogger('__preparePerfSONARConfiguration')
log.debug('Begin function ...')
# static elements of a path
rootPath = '/Resources/Sites'
extPath = 'Network'
baseOptionName = 'Enabled'
options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'}
# enable GOCDB endpoints in configuration
newConfiguration = {}
for endpoint in endpointList:
if endpoint['DIRACSITENAME'] is None:
continue
split = endpoint['DIRACSITENAME'].split('.')
path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME'])
for name, defaultValue in options.iteritems():
newConfiguration[cfgPath(path, name)] = defaultValue
# get current configuration
currentConfiguration = {}
for option in options.iterkeys():
result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option)
if not result['OK']:
log.error("getConfigurationTree() failed with message: %s" % result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
currentConfiguration.update(result['Value'])
# disable endpoints that disappeared in GOCDB
removedElements = set(currentConfiguration) - set(newConfiguration)
newElements = set(newConfiguration) - set(currentConfiguration)
addedEndpoints = len(newElements) / len(options)
disabledEndpoints = 0
for path in removedElements:
if baseOptionName in path:
newConfiguration[path] = 'False'
if currentConfiguration[path] != 'False':
disabledEndpoints = disabledEndpoints + 1
# inform what will be changed
if addedEndpoints > 0:
self.log.info("%s new perfSONAR endpoints will be added to the configuration" % addedEndpoints)
if disabledEndpoints > 0:
self.log.info("%s old perfSONAR endpoints will be disable in the configuration" % disabledEndpoints)
if addedEndpoints == 0 and disabledEndpoints == 0:
self.log.info("perfSONAR configuration is up-to-date")
log.debug('End function.')
return S_OK(newConfiguration)
def __addDIRACSiteName(self, inputList):
"""
Extend given list of GOCDB endpoints with DIRAC site name, i.e.
add an entry "DIRACSITENAME" in dictionaries that describe endpoints.
If given site name could not be found "DIRACSITENAME" is set to 'None'.
:return: List of perfSONAR endpoints (dictionaries).
"""
log = self.log.getSubLogger('__addDIRACSiteName')
log.debug('Begin function ...')
# get site name dictionary
result = getDIRACGOCDictionary()
if not result['OK']:
log.error("getDIRACGOCDictionary() failed with message: %s" % result['Message'])
return S_ERROR('Could not get site name dictionary')
# reverse the dictionary (assume 1 to 1 relation)
DIRACGOCDict = result['Value']
GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys()))
# add DIRAC site names
outputList = []
for entry in inputList:
try:
entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']]
except KeyError:
self.log.warn("No dictionary entry for %s. " % entry['SITENAME'])
entry['DIRACSITENAME'] = None
outputList.append(entry)
log.debug('End function.')
return S_OK(outputList)
def __updateConfiguration(self, setElements=None, delElements=None):
"""
Update configuration stored by CS.
"""
if setElements is None:
setElements = {}
if delElements is None:
delElements = []
log = self.log.getSubLogger('__updateConfiguration')
log.debug('Begin function ...')
# assure existence and proper value of a section or an option
for path, value in setElements.iteritems():
if value is None:
section = path
else:
split = path.rsplit('/', 1)
section = split[0]
try:
result = self.csAPI.createSection(section)
if not result['OK']:
log.error("createSection() failed with message: %s" % result['Message'])
except Exception as e:
log.error("Exception in createSection(): %s" % repr(e).replace(',)', ')'))
if value is not None:
try:
result = self.csAPI.setOption(path, value)
if not result['OK']:
log.error("setOption() failed with message: %s" % result['Message'])
except Exception as e:
log.error("Exception in setOption(): %s" % repr(e).replace(',)', ')'))
# delete elements in the configuration
for path in delElements:
result = self.csAPI.delOption(path)
if not result['OK']:
log.warn("delOption() failed with message: %s" % result['Message'])
result = self.csAPI.delSection(path)
if not result['OK']:
log.warn("delSection() failed with message: %s" % result['Message'])
if self.dryRun:
log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
# update configuration stored by CS
result = self.csAPI.commit()
if not result['OK']:
log.error("commit() failed with message: %s" % result['Message'])
return S_ERROR("Could not commit changes to CS.")
else:
log.info("Committed changes to CS")
log.debug('End function.')
return S_OK()
# define mapping between an agent option in the configuration and a function call
__functionMap = {'UpdatePerfSONARS': updatePerfSONARConfiguration,
}
hasRun = True
if SORTBYNAME:
dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(
ascending=not REVERSE)
else:
dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country,
reverse=REVERSE)
if dirty:
isDirty = True
if not hasRun:
gLogger.notice(
"Failed to find suitable subsections with site names to sort")
DIRAC.exit(0)
if not isDirty:
gLogger.notice("Nothing to do, site names are already sorted")
DIRAC.exit(0)
timestamp = toString(dateTime())
stamp = "Site names are sorted by %s script at %s" % (Script.scriptName,
timestamp)
cs.setOptionComment("/Resources/Sites", stamp)
result = cs.commit()
if not result["OK"]:
gLogger.error("Failed to commit changes to CS", result["Message"])
DIRAC.exit(2)
gLogger.notice("Site names are sorted and committed to CS")
DIRAC.exit(0)
Script.setUsageMessage( '\n'.join( [__doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... ShifterRole UserName DIRACGroup ...' % Script.scriptName,
'Arguments:',
' ShifterRole: Name of the shifter role, e.g. DataManager',
' UserName: A user name, as registered in Registry section',
' DIRACGroup: DIRAC Group, e.g. diracAdmin (the user has to have this role)'] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs( )
csAPI = CSAPI( )
if len( args ) < 3:
Script.showHelp( )
DIRACExit( -1 )
shifterRole = args[0]
userName = args[1]
diracGroup = args[2]
res = csAPI.addShifter( {shifterRole: {'User': userName, 'Group': diracGroup}} )
if not res['OK']:
gLogger.error( "Could not add shifter", ": " + res['Message'] )
DIRACExit( 1 )
res = csAPI.commit()
if not res['OK']:
gLogger.error( "Could not add shifter", ": " + res['Message'] )
DIRACExit( 1 )
gLogger.notice( "Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup) )
class CE2CSAgent(AgentModule):
addressTo = ""
addressFrom = ""
voName = ""
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize(self):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption("MailTo", self.addressTo)
self.addressFrom = self.am_getOption("MailFrom", self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs", [])
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(":")[-1].isdigit():
self.alternativeBDIIs[index] += ":2170"
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption("shifterProxy", "TestManager")
self.voName = self.am_getOption("VirtualOrganization", [])
if not self.voName:
vo = getVO()
if vo:
self.voName = [vo]
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
self.log.info("Start Execution")
result = getProxyInfo()
if not result["OK"]:
return result
infoDict = result["Value"]
self.log.info(formatProxyInfoAsString(infoDict))
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result["OK"]:
self.log.warn("Could not download a fresh copy of the CS data", result["Message"])
self.__lookForCE()
self.__infoFromCE()
self.log.info("End Execution")
return S_OK()
def __checkAlternativeBDIISite(self, fun, *args):
if self.alternativeBDIIs:
self.log.warn("Trying to use alternative BDII sites")
for site in self.alternativeBDIIs:
self.log.info("Trying to contact alternative BDII", site)
if len(args) == 1:
result = fun(args[0], host=site)
elif len(args) == 2:
result = fun(args[0], vo=args[1], host=site)
if not result["OK"]:
self.log.error("Problem contacting alternative BDII", result["Message"])
elif result["OK"]:
return result
self.log.warn("Also checking alternative BDII sites failed")
return result
def __lookForCE(self):
knownCEs = self.am_getOption("BannedCEs", [])
result = gConfig.getSections("/Resources/Sites")
if not result["OK"]:
return
grids = result["Value"]
for grid in grids:
result = gConfig.getSections("/Resources/Sites/%s" % grid)
if not result["OK"]:
return
sites = result["Value"]
for site in sites:
opt = gConfig.getOptionsDict("/Resources/Sites/%s/%s" % (grid, site))["Value"]
ces = List.fromChar(opt.get("CE", ""))
knownCEs += ces
response = ""
for vo in self.voName:
self.log.info("Check for available CEs for VO", vo)
response = ldapCEState("", vo)
if not response["OK"]:
self.log.error("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCEState, "", vo)
return response
newCEs = {}
for queue in response["Value"]:
try:
queueName = queue["GlueCEUniqueID"]
except:
continue
ceName = queueName.split(":")[0]
if not ceName in knownCEs:
newCEs[ceName] = None
self.log.debug("New CE", ceName)
body = ""
possibleNewSites = []
for ce in newCEs.iterkeys():
response = ldapCluster(ce)
if not response["OK"]:
self.log.warn("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCluster, ce)
continue
clusters = response["Value"]
if len(clusters) != 1:
self.log.warn("Error in cluster length", " CE %s Length %d" % (ce, len(clusters)))
if len(clusters) == 0:
continue
cluster = clusters[0]
fkey = cluster.get("GlueForeignKey", [])
if type(fkey) == type(""):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count("GlueSiteUniqueID"):
nameBDII = entry.split("=")[1]
break
if not nameBDII:
continue
ceString = "CE: %s, GOCDB Name: %s" % (ce, nameBDII)
self.log.info(ceString)
response = ldapCE(ce)
if not response["OK"]:
self.log.warn("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
ceInfos = response["Value"]
if len(ceInfos):
ceInfo = ceInfos[0]
systemName = ceInfo.get("GlueHostOperatingSystemName", "Unknown")
systemVersion = ceInfo.get("GlueHostOperatingSystemVersion", "Unknown")
systemRelease = ceInfo.get("GlueHostOperatingSystemRelease", "Unknown")
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osString = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % (
systemName,
systemVersion,
systemRelease,
)
self.log.info(osString)
response = ldapCEState(ce, vo)
if not response["OK"]:
self.log.warn("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCEState, ce, vo)
continue
newCEString = "\n\n%s\n%s" % (ceString, osString)
usefull = False
ceStates = response["Value"]
for ceState in ceStates:
queueName = ceState.get("GlueCEUniqueID", "UnknownName")
queueStatus = ceState.get("GlueCEStateStatus", "UnknownStatus")
queueString = "%s %s" % (queueName, queueStatus)
self.log.info(queueString)
newCEString += "\n%s" % queueString
if queueStatus.count("Production"):
usefull = True
if usefull:
body += newCEString
possibleNewSites.append("dirac-admin-add-site DIRACSiteName %s %s" % (nameBDII, ce))
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % (body, possibleNewSite)
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(
self.addressTo, self.subject, body, self.addressFrom, localAttempt=False
)
return S_OK()
def __infoFromCE(self):
sitesSection = cfgPath("Resources", "Sites")
result = gConfig.getSections(sitesSection)
if not result["OK"]:
return
grids = result["Value"]
changed = False
body = ""
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result["OK"]:
return
sites = result["Value"]
for site in sites:
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)["Value"]
name = opt.get("Name", "")
if name:
coor = opt.get("Coordinates", "Unknown")
mail = opt.get("Mail", "Unknown")
result = ldapSite(name)
if not result["OK"]:
self.log.warn("BDII site %s: %s" % (name, result["Message"]))
result = self.__checkAlternativeBDIISite(ldapSite, name)
if result["OK"]:
bdiiSites = result["Value"]
if len(bdiiSites) == 0:
self.log.warn(name, "Error in BDII: leng = 0")
else:
if not len(bdiiSites) == 1:
self.log.warn(name, "Warning in BDII: leng = %d" % len(bdiiSites))
bdiiSite = bdiiSites[0]
try:
longitude = bdiiSite["GlueSiteLongitude"]
latitude = bdiiSite["GlueSiteLatitude"]
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in BDII coordinates")
newcoor = "Unknown"
try:
newmail = bdiiSite["GlueSiteSysAdminContact"].split(":")[-1].strip()
except:
self.log.warn("Error in BDII mail")
newmail = "Unknown"
self.log.debug("%s %s %s" % (name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor))
if coor == "Unknown":
self.csAPI.setOption(cfgPath(siteSection, "Coordinates"), newcoor)
else:
self.csAPI.modifyValue(cfgPath(siteSection, "Coordinates"), newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name), "%s -> %s" % (mail, newmail))
if mail == "Unknown":
self.csAPI.setOption(cfgPath(siteSection, "Mail"), newmail)
else:
self.csAPI.modifyValue(cfgPath(siteSection, "Mail"), newmail)
changed = True
ceList = List.fromChar(opt.get("CE", ""))
if not ceList:
self.log.warn(site, "Empty site list")
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in ceList:
ceSection = cfgPath(siteSection, "CEs", ce)
result = gConfig.getOptionsDict(ceSection)
if not result["OK"]:
self.log.debug("Section CE", result["Message"])
wnTmpDir = "Unknown"
arch = "Unknown"
os = "Unknown"
si00 = "Unknown"
pilot = "Unknown"
ceType = "Unknown"
else:
ceopt = result["Value"]
wnTmpDir = ceopt.get("wnTmpDir", "Unknown")
arch = ceopt.get("architecture", "Unknown")
os = ceopt.get("OS", "Unknown")
si00 = ceopt.get("SI00", "Unknown")
pilot = ceopt.get("Pilot", "Unknown")
ceType = ceopt.get("CEType", "Unknown")
result = ldapCE(ce)
if not result["OK"]:
self.log.warn("Error in BDII for %s" % ce, result["Message"])
result = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiiCE = result["Value"][0]
except:
self.log.warn("Error in BDII for %s" % ce, result)
bdiiCE = None
if bdiiCE:
try:
newWNTmpDir = bdiiCE["GlueSubClusterWNTmpDir"]
except:
newWNTmpDir = "Unknown"
if wnTmpDir != newWNTmpDir and newWNTmpDir != "Unknown":
section = cfgPath(ceSection, "wnTmpDir")
self.log.info(section, " -> ".join((wnTmpDir, newWNTmpDir)))
if wnTmpDir == "Unknown":
self.csAPI.setOption(section, newWNTmpDir)
else:
self.csAPI.modifyValue(section, newWNTmpDir)
changed = True
try:
newArch = bdiiCE["GlueHostArchitecturePlatformType"]
except:
newArch = "Unknown"
if arch != newArch and newArch != "Unknown":
section = cfgPath(ceSection, "architecture")
self.log.info(section, " -> ".join((arch, newArch)))
if arch == "Unknown":
self.csAPI.setOption(section, newArch)
else:
self.csAPI.modifyValue(section, newArch)
changed = True
try:
newOS = "_".join(
(
bdiiCE["GlueHostOperatingSystemName"],
bdiiCE["GlueHostOperatingSystemVersion"],
bdiiCE["GlueHostOperatingSystemRelease"],
)
)
except:
newOS = "Unknown"
if os != newOS and newOS != "Unknown":
section = cfgPath(ceSection, "OS")
self.log.info(section, " -> ".join((os, newOS)))
if os == "Unknown":
self.csAPI.setOption(section, newOS)
else:
self.csAPI.modifyValue(section, newOS)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (os, newOS, ce, site)
try:
newSI00 = bdiiCE["GlueHostBenchmarkSI00"]
except:
newSI00 = "Unknown"
if si00 != newSI00 and newSI00 != "Unknown":
section = cfgPath(ceSection, "SI00")
self.log.info(section, " -> ".join((si00, newSI00)))
if si00 == "Unknown":
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
try:
rte = bdiiCE["GlueHostApplicationSoftwareRunTimeEnvironment"]
for vo in self.voName:
if vo.lower() == "lhcb":
if "VO-lhcb-pilot" in rte:
newPilot = "True"
else:
newPilot = "False"
else:
newPilot = "Unknown"
except:
newPilot = "Unknown"
if pilot != newPilot and newPilot != "Unknown":
section = cfgPath(ceSection, "Pilot")
self.log.info(section, " -> ".join((pilot, newPilot)))
if pilot == "Unknown":
self.csAPI.setOption(section, newPilot)
else:
self.csAPI.modifyValue(section, newPilot)
changed = True
newVO = ""
for vo in self.voName:
result = ldapCEState(ce, vo) # getBDIICEVOView
if not result["OK"]:
self.log.warn("Error in BDII for queue %s" % ce, result["Message"])
result = self.__checkAlternativeBDIISite(ldapCEState, ce, vo)
continue
try:
queues = result["Value"]
except:
self.log.warn("Error in BDII for queue %s" % ce, result["Massage"])
continue
newCEType = "Unknown"
for queue in queues:
try:
queueType = queue["GlueCEImplementationName"]
except:
queueType = "Unknown"
if newCEType == "Unknown":
newCEType = queueType
else:
if queueType != newCEType:
self.log.warn(
"Error in BDII for CE %s " % ce,
"different CE types %s %s" % (newCEType, queueType),
)
if newCEType == "ARC-CE":
newCEType = "ARC"
if ceType != newCEType and newCEType != "Unknown":
section = cfgPath(ceSection, "CEType")
self.log.info(section, " -> ".join((ceType, newCEType)))
if ceType == "Unknown":
self.csAPI.setOption(section, newCEType)
else:
self.csAPI.modifyValue(section, newCEType)
changed = True
for queue in queues:
try:
queueName = queue["GlueCEUniqueID"].split("/")[-1]
except:
self.log.warn("Error in queueName ", queue)
continue
try:
newMaxCPUTime = queue["GlueCEPolicyMaxCPUTime"]
except:
newMaxCPUTime = None
newSI00 = None
try:
caps = queue["GlueCECapability"]
if type(caps) == type(""):
caps = [caps]
for cap in caps:
if cap.count("CPUScalingReferenceSI00"):
newSI00 = cap.split("=")[-1]
except:
newSI00 = None
queueSection = cfgPath(ceSection, "Queues", queueName)
result = gConfig.getOptionsDict(queueSection)
if not result["OK"]:
self.log.warn("Section Queues", result["Message"])
maxCPUTime = "Unknown"
si00 = "Unknown"
allowedVOs = [""]
else:
queueOpt = result["Value"]
maxCPUTime = queueOpt.get("maxCPUTime", "Unknown")
si00 = queueOpt.get("SI00", "Unknown")
if newVO == "": # Remember previous iteration, if none - read from conf
allowedVOs = queueOpt.get("VO", "").split(",")
else: # Else use newVO, as it can contain changes, which aren't in conf yet
allowedVOs = newVO.split(",")
if newMaxCPUTime and (maxCPUTime != newMaxCPUTime):
section = cfgPath(queueSection, "maxCPUTime")
self.log.info(section, " -> ".join((maxCPUTime, newMaxCPUTime)))
if maxCPUTime == "Unknown":
self.csAPI.setOption(section, newMaxCPUTime)
else:
self.csAPI.modifyValue(section, newMaxCPUTime)
changed = True
if newSI00 and (si00 != newSI00):
section = cfgPath(queueSection, "SI00")
self.log.info(section, " -> ".join((si00, newSI00)))
if si00 == "Unknown":
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
modifyVO = True # Flag saying if we need VO option to change
newVO = ""
if allowedVOs != [""]:
for allowedVO in allowedVOs:
allowedVO = allowedVO.strip() # Get rid of spaces
newVO += allowedVO
if allowedVO == vo: # Current VO has been already in list
newVO = ""
modifyVO = False # Don't change anything
break # Skip next 'if', proceed to next VO
newVO += ", "
if modifyVO:
section = cfgPath(queueSection, "VO")
newVO += vo
self.log.info(section, " -> ".join(("%s" % allowedVOs, newVO)))
if allowedVOs == [""]:
self.csAPI.setOption(section, newVO)
else:
self.csAPI.modifyValue(section, newVO)
changed = True
if changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
return self.csAPI.commit()
else:
self.log.info("No changes found")
return S_OK()
res = csAPI.createSection( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite' )
if not res['OK']:
print res['Message']
exit( 1 )
res = csAPI.createSection( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite/matchParams' )
csAPI.setOption( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite/policyType', 'AlwaysBanned' )
csAPI.setOption( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite/matchParams/element', 'Site' )
# Now setting the catalog list in Operations/Defults/Services/Catalogs/CatalogList
res = csAPI.createSection( 'Operations/Defaults/Services' )
if not res['OK']:
print res['Message']
exit( 1 )
res = csAPI.createSection( 'Operations/Defaults/Services/Catalogs' )
if not res['OK']:
print res['Message']
exit( 1 )
res = csAPI.createSection( 'Operations/Defaults/Services/Catalogs/CatalogList' )
if not res['OK']:
print res['Message']
exit( 1 )
csAPI.setOption('Operations/Defaults/Services/Catalogs/CatalogList', 'FileCatalog')
# Final action: commit in CS
csAPI.commit()
class CVMFSAdder(object):
"""Container for all the objects and functions to add software to ILCDirac"""
def __init__(self, cliParams):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.modifiedCS = False
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.mailadress = '*****@*****.**'
self.cliParams = cliParams
self.parameter = dict(softSec=self.softSec,
platform=cliParams.platform,
version=cliParams.version,
basepath=cliParams.basePath,
initsctipt=cliParams.initScriptLocation)
self.applications = cliParams.applicationSet
self.detmodels = {}
self.csAPI = CSAPI()
def findDDSimDetectorModels(self):
""" find all detector models in lcgeo and
fill the self.detmodels dictionary with Detmodel as key and path as value
:returns: None
"""
for root, dirs, _files in os.walk(
os.path.join(self.parameter["basepath"], "lcgeo")):
for direct in dirs:
if root.endswith("compact"):
## the main xml file must have the same name as the folder
xmlPath = os.path.join(root, direct, direct + ".xml")
if os.path.exists(xmlPath):
self.detmodels[direct] = xmlPath
def checkConsistency(self):
"""checks if platform is defined, application exists, etc."""
gLogger.notice("Checking consistency")
av_platforms = gConfig.getSections(self.softSec, [])
if av_platforms['OK']:
if not self.parameter['platform'] in av_platforms['Value']:
gLogger.error("Platform %s unknown, available are %s." %
(self.parameter['platform'], ", ".join(
av_platforms['Value'])))
gLogger.error("If yours is missing, add it in CS")
return S_ERROR()
else:
gLogger.error("Could not find all platforms available in CS")
return S_ERROR()
for application in self.applications:
av_apps = gConfig.getSections(
"%(softSec)s/%(platform)s/" % self.parameter +
str(application), [])
if not av_apps['OK']:
gLogger.error(
"Could not find this application in the CS: '%s'" %
application)
gLogger.error("Add its section to the CS, if it is missing")
return S_ERROR()
gLogger.notice("All OK, continuing...")
return S_OK()
def commitToCS(self):
"""write changes to the CS to the server"""
if self.modifiedCS and not self.cliParams.dryRun:
gLogger.notice("Commiting changes to the CS")
result = self.csAPI.commit()
if not result['OK']:
gLogger.error('Commit failed with message = %s' %
(result['Message']))
return S_ERROR("Failed to commit to CS")
gLogger.info('Successfully committed changes to CS')
else:
gLogger.info('No modifications to CS required')
return S_OK()
def addAllToCS(self):
"""add all the applications to the CS, take care of special cases (mokka, ildconfig, ddsim,...)"""
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import insertCSSection
for application in self.applications:
csParameter = dict(
CVMFSEnvScript=self.cliParams.initScriptLocation,
CVMFSPath=self.parameter['basepath'])
if application == 'mokka':
csParameter['CVMFSDBSlice'] = self.cliParams.dbSliceLocation
if application == 'ddsim':
self.findDDSimDetectorModels()
csPathModels = "Operations/Defaults/DDSimDetectorModels"
csModels = {self.parameter["version"]: self.detmodels}
insertCSSection(self.csAPI, csPathModels, csModels)
self.modifiedCS = True
elif application.endswith('config'):
del csParameter['CVMFSEnvScript']
csParameter['CVMFSPath'] = self.cliParams.configPath
if self.cliParams.dbSliceLocation:
csParameter[
'CVMFSDBSlice'] = self.cliParams.dbSliceLocation
resInsert = self.insertApplicationToCS(application, csParameter)
if not resInsert['OK']:
return resInsert
return S_OK()
def insertApplicationToCS(self, name, csParameter):
"""add given application found via CVMFS to the CS"""
pars = dict(self.parameter)
pars['name'] = name
gLogger.notice("%(name)s: Adding version %(version)s to the CS" % pars)
existingVersions = gConfig.getSections(
"%(softSec)s/%(platform)s/%(name)s" % pars, [])
if not existingVersions['OK']:
gLogger.error("Could not find all versions available in CS: %s" %
existingVersions['Message'])
dexit(255)
if pars['version'] in existingVersions['Value']:
gLogger.always(
'Application %s %s for %s already in CS, nothing to do' %
(name.lower(), pars['version'], pars['platform']))
return S_OK()
csPath = self.softSec + ("/%(platform)s/%(name)s/%(version)s/" % pars)
for par, val in csParameter.iteritems():
gLogger.notice("Add: %s = %s" % (csPath + par, val))
result = self.csAPI.setOption(csPath + par, val)
if result['OK']:
self.modifiedCS = True
else:
gLogger.error("Failure to add to CS", result['Message'])
return S_ERROR("")
return S_OK()
def addSoftware(self):
"""run all the steps to add software to grid and CS"""
resAdd = self.addAllToCS()
if not resAdd['OK']:
return resAdd
resCommit = self.commitToCS()
if not resCommit['OK']:
return resCommit
return S_OK()
class DDSimTarMaker(object):
""" create a tarball of the DDSim release """
def __init__(self):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.detmodels = {}
self.lcgeo_env = "lcgeo_DIR"
self.ddhep_env = "DD4HEP"
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.version = ''
self.platform = 'x86_64-slc5-gcc43-opt'
self.comment = ""
self.name = "ddsim"
self.csapi = CSAPI()
self.tarBallName = None
self.md5sum = None
def copyDetectorModels(self, basePath, folder, targetFolder):
"""copy the compact folders to the targetFolder """
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import copyFolder
for root, dirs, _files in os.walk(os.path.join(basePath, folder)):
for direct in dirs:
if root.endswith("compact"):
## the main xml file must have the same name as the folder, ILD and CLIC follow this convention already
xmlPath = os.path.join(root, direct, direct + ".xml")
if os.path.exists(xmlPath):
self.detmodels[
direct] = "detectors/" + direct + "/" + direct + ".xml"
copyFolder(os.path.join(root, direct), targetFolder)
def createTarBall(self, folder):
"""create a tar ball from the folder
tar zcf $TARBALLNAME $LIBFOLDER/*
"""
##Create the Tarball
if os.path.exists(self.tarBallName):
os.remove(self.tarBallName)
gLogger.notice("Creating Tarball...")
myappTar = tarfile.open(self.tarBallName, "w:gz")
myappTar.add(folder, self.tarBallName[:-4])
myappTar.close()
self.md5sum = md5.md5(open(self.tarBallName, 'r').read()).hexdigest()
gLogger.notice("...Done")
return S_OK("Created Tarball")
def parseArgs(self):
""" parse the command line arguments"""
if len(sys.argv) != 3:
raise RuntimeError("Wrong number of arguments in call: '%s'" %
" ".join(sys.argv))
self.name = sys.argv[1]
self.version = sys.argv[2]
self.tarBallName = "%s%s.tgz" % (self.name, self.version)
def checkEnvironment(self):
""" check if dd4hep and lcgeo are in the environment """
for var in [self.ddhep_env, self.lcgeo_env, 'ROOTSYS']:
if var not in os.environ:
raise RuntimeError("%s is not set" % var)
return os.environ[self.ddhep_env], os.environ[
self.lcgeo_env], os.environ['ROOTSYS']
def createCSEntry(self):
"""add the entries for this version into the Configuration System
.. code::
<version>
{
TarBall = ddsim<version>.tgz
AdditionalEnvVar
{
ROOTSYS = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/root/5.34.30
G4INSTALL = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01
G4DATA = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01/share/Geant4-10.1.0/data
}
Overwrite = True
}
Operations/DDSimDetectorModels/<Version>
{
CLIC_o2_v03 = detectors/CLIC_o2_v03/CLIC_o2_v03.xml
...
}
"""
from ILCDIRAC.Core.Utilities.CheckAndGetProdProxy import checkOrGetGroupProxy
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import insertCSSection
#FIXME: Get root and geant4 location from environment, make sure it is cvmfs
csParameter = {
"TarBall": self.tarBallName,
"AdditionalEnvVar": {
"ROOTSYS": os.path.realpath(os.environ.get("ROOTSYS")),
"G4INSTALL": os.path.realpath(os.environ.get("G4INSTALL")),
},
"Md5Sum": self.md5sum,
}
g4datavariables = [
"G4RADIOACTIVEDATA",
"G4NEUTRONHPDATA",
"G4LEDATA",
"G4LEVELGAMMADATA",
"G4RADIOACTIVEDATA",
"G4NEUTRONXSDATA",
"G4PIIDATA",
"G4REALSURFACEDATA",
"G4SAIDXSDATA",
"G4ABLADATA",
"G4ENSDFSTATEDATA",
]
for g4data in g4datavariables:
if os.environ.get(g4data):
csParameter["AdditionalEnvVar"][g4data] = os.path.realpath(
os.environ.get(g4data))
pars = dict(platform=self.platform, name="ddsim", version=self.version)
csPath = os.path.join(self.softSec,
"%(platform)s/%(name)s/%(version)s/" % pars)
pprint(csParameter)
result = insertCSSection(self.csapi, csPath, csParameter)
csPathModels = "Operations/Defaults/DDSimDetectorModels"
csModels = {self.version: self.detmodels}
pprint(csModels)
result = insertCSSection(self.csapi, csPathModels, csModels)
if self.csapi is not None:
resProxy = checkOrGetGroupProxy("diracAdmin")
if not resProxy['OK']:
gLogger.error("Failed to get AdminProxy", resProxy['Message'])
raise RuntimeError("Failed to get diracAdminProxy")
self.csapi.commit()
if not result['OK']:
gLogger.error("Failed to create CS Section", result['Message'])
raise RuntimeError("Failed to create CS Section")
def createDDSimTarBall(self):
""" do everything to create the DDSim tarball"""
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import copyFolder, getLibraryPath, getFiles, getDependentLibraries, copyLibraries, getPythonStuff, killRPath, resolveLinks, removeSystemLibraries
self.parseArgs()
ddBase, lcgeoBase, _rootsys = self.checkEnvironment()
realTargetFolder = os.path.join(os.getcwd(), self.name + self.version)
targetFolder = os.path.join(os.getcwd(), "temp",
self.name + self.version)
for folder in (targetFolder, targetFolder + "/lib"):
try:
os.makedirs(folder)
except OSError:
pass
libraries = set()
rootmaps = set()
dd4hepLibPath = getLibraryPath(ddBase)
lcgeoPath = getLibraryPath(lcgeoBase)
## FIXME: Automatically pick up folders with /compact/ in them
self.copyDetectorModels(lcgeoBase, "CLIC", targetFolder + "/detectors")
self.copyDetectorModels(lcgeoBase, "ILD", targetFolder + "/detectors")
self.copyDetectorModels(lcgeoBase, "SiD", targetFolder + "/detectors")
copyFolder(ddBase + "/DDDetectors/compact",
realTargetFolder.rstrip("/") + "/DDDetectors")
copyFolder(ddBase + "/include", realTargetFolder.rstrip("/") + "")
libraries.update(getFiles(dd4hepLibPath, ".so"))
libraries.update(getFiles(lcgeoPath, ".so"))
rootmaps.update(getFiles(dd4hepLibPath, ".rootmap"))
rootmaps.update(getFiles(lcgeoPath, ".rootmap"))
rootmaps.update(getFiles(dd4hepLibPath, ".pcm"))
rootmaps.update(getFiles(lcgeoPath, ".pcm"))
rootmaps.update(getFiles(dd4hepLibPath, ".components"))
rootmaps.update(getFiles(lcgeoPath, ".components"))
pprint(libraries)
pprint(rootmaps)
allLibs = set()
for lib in libraries:
allLibs.update(getDependentLibraries(lib))
### remote root and geant4 libraries, we pick them up from
allLibs = set([
lib for lib in allLibs
if not ("/geant4/" in lib.lower() or "/root/" in lib.lower())
])
print allLibs
copyLibraries(libraries, targetFolder + "/lib")
copyLibraries(allLibs, targetFolder + "/lib")
copyLibraries(rootmaps, targetFolder + "/lib")
getPythonStuff(ddBase + "/python", targetFolder + "/lib/")
getPythonStuff(lcgeoBase + "/lib/python", targetFolder + "/lib/")
getPythonStuff(lcgeoBase + "/bin/ddsim", targetFolder + "/bin/")
##Should get this from CVMFS
#getRootStuff( rootsys, targetFolder+"/ROOT" )
copyFolder(targetFolder + "/", realTargetFolder.rstrip("/"))
killRPath(realTargetFolder)
resolveLinks(realTargetFolder + "/lib")
removeSystemLibraries(realTargetFolder + "/lib")
#removeSystemLibraries( realTargetFolder+"/ROOT/lib" )
print self.detmodels
self.createTarBall(realTargetFolder)
self.createCSEntry()
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"DIRACSiteName: Name of the site for DIRAC in the form GRID.LOCATION.COUNTRY "
"(ie: LCG.CERN.ch)")
Script.registerArgument(
"GridSiteName: Name of the site in the Grid (ie: CERN-PROD)")
Script.registerArgument([
"CE: Name of the CE to be included in the site (ie: ce111.cern.ch)"
])
Script.parseCommandLine(ignoreErrors=True)
diracSiteName, gridSiteName, ces = Script.getPositionalArgs(group=True)
try:
diracGridType, place, country = diracSiteName.split(".")
except ValueError:
gLogger.error(
"The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch"
)
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result["OK"] and result["Value"]:
if len(result["Value"]) > 1:
gLogger.notice(
"%s GOC site name is associated with several DIRAC sites:" %
gridSiteName)
for i, dSite in enumerate(result["Value"]):
gLogger.notice("%d: %s" % (i, dSite))
inp = six.moves.input("Enter your choice number: ")
try:
inp = int(inp)
except ValueError:
gLogger.error("You should enter an integer number")
DIRACExit(-1)
if 0 <= inp < len(result["Value"]):
diracCSSite = result["Value"][inp]
else:
gLogger.error("Number out of range: %d" % inp)
DIRACExit(-1)
else:
diracCSSite = result["Value"][0]
if diracCSSite == diracSiteName:
gLogger.notice("Site with GOC name %s is already defined as %s" %
(gridSiteName, diracSiteName))
newSite = False
else:
gLogger.error(
"ERROR: Site with GOC name %s is already defined as %s" %
(gridSiteName, diracCSSite))
DIRACExit(-1)
else:
gLogger.error("ERROR getting DIRAC site name of %s" % gridSiteName,
result.get("Message"))
csAPI = CSAPI()
if newSite:
gLogger.notice("Site to CS: %s" % diracSiteName)
res = csAPI.addSite(diracSiteName, {"Name": gridSiteName})
if not res["OK"]:
gLogger.error("Failed adding site to CS", res["Message"])
DIRACExit(1)
res = csAPI.commit()
if not res["OK"]:
gLogger.error("Failure committing to CS", res["Message"])
DIRACExit(3)
for ce in ces:
gLogger.notice("Adding CE %s" % ce)
res = csAPI.addCEtoSite(diracSiteName, ce)
if not res["OK"]:
gLogger.error("Failed adding CE %s to CS" % ce, res["Message"])
DIRACExit(2)
res = csAPI.commit()
if not res["OK"]:
gLogger.error("Failure committing to CS", res["Message"])
DIRACExit(3)
class ComponentSupervisionAgent(AgentModule):
"""ComponentSupervisionAgent class."""
def __init__(self, *args, **kwargs):
"""Initialize the agent, clients, default values."""
AgentModule.__init__(self, *args, **kwargs)
self.name = "ComponentSupervisionAgent"
self.setup = "DIRAC-Production"
self.enabled = False
self.restartAgents = False
self.restartExecutors = False
self.restartServices = False
self.controlComponents = False
self.commitURLs = False
self.doNotRestartInstancePattern = ["RequestExecutingAgent"]
self.diracLocation = rootPath
self.sysAdminClient = SystemAdministratorClient(socket.getfqdn())
self.jobMonClient = JobMonitoringClient()
self.nClient = NotificationClient()
self.csAPI = None
self.agents = dict()
self.executors = dict()
self.services = dict()
self._tornadoPort = "8443"
self.errors = list()
self.accounting = defaultdict(dict)
self.addressTo = []
self.addressFrom = ""
self.emailSubject = "ComponentSupervisionAgent on %s" % socket.getfqdn(
)
def logError(self, errStr, varMsg=""):
"""Append errors to a list, which is sent in email notification."""
self.log.error(errStr, varMsg)
self.errors.append(errStr + " " + varMsg)
def beginExecution(self):
"""Reload the configurations before every cycle."""
self.setup = self.am_getOption("Setup", self.setup)
self.enabled = self.am_getOption("EnableFlag", self.enabled)
self.restartAgents = self.am_getOption("RestartAgents",
self.restartAgents)
self.restartExecutors = self.am_getOption("RestartExecutors",
self.restartExecutors)
self.restartServices = self.am_getOption("RestartServices",
self.restartServices)
self.addressTo = self.am_getOption("MailTo", self.addressTo)
self.addressFrom = self.am_getOption("MailFrom", self.addressFrom)
self.controlComponents = self.am_getOption("ControlComponents",
self.controlComponents)
self.commitURLs = self.am_getOption("CommitURLs", self.commitURLs)
self.doNotRestartInstancePattern = self.am_getOption(
"DoNotRestartInstancePattern", self.doNotRestartInstancePattern)
self.csAPI = CSAPI()
res = self.getRunningInstances(instanceType="Agents")
if not res["OK"]:
return S_ERROR("Failure to get running agents")
self.agents = res["Value"]
res = self.getRunningInstances(instanceType="Executors")
if not res["OK"]:
return S_ERROR("Failure to get running executors")
self.executors = res["Value"]
res = self.getRunningInstances(instanceType="Services")
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
self.accounting.clear()
return S_OK()
def sendNotification(self):
"""Send email notification about changes done in the last cycle."""
if not (self.errors or self.accounting):
return S_OK()
emailBody = ""
rows = []
for instanceName, val in self.accounting.items():
rows.append([[instanceName],
[val.get("Treatment", "No Treatment")],
[str(val.get("LogAge", "Not Relevant"))]])
if rows:
columns = ["Instance", "Treatment", "Log File Age (Minutes)"]
emailBody += printTable(columns,
rows,
printOut=False,
numbering=False,
columnSeparator=" | ")
if self.errors:
emailBody += "\n\nErrors:"
emailBody += "\n".join(self.errors)
self.log.notice("Sending Email:\n" + emailBody)
for address in self.addressTo:
res = self.nClient.sendMail(address,
self.emailSubject,
emailBody,
self.addressFrom,
localAttempt=False)
if not res["OK"]:
self.log.error("Failure to send Email notification to ",
address)
continue
self.errors = []
self.accounting.clear()
return S_OK()
def getRunningInstances(self, instanceType="Agents", runitStatus="Run"):
"""Return a dict of running agents, executors or services.
Key is component's name, value contains dict with PollingTime, PID, Port, Module, RunitStatus, LogFileLocation
:param str instanceType: 'Agents', 'Executors', 'Services'
:param str runitStatus: Return only those instances with given RunitStatus or 'All'
:returns: Dictionary of running instances
"""
res = self.sysAdminClient.getOverallStatus()
if not res["OK"]:
self.logError(
"Failure to get %s from system administrator client" %
instanceType, res["Message"])
return res
val = res["Value"][instanceType]
runningComponents = defaultdict(dict)
for system, components in val.items():
for componentName, componentInfo in components.items():
if componentInfo["Setup"] and componentInfo["Installed"]:
if runitStatus != "All" and componentInfo[
"RunitStatus"] != runitStatus:
continue
for option, default in (("PollingTime", HOUR),
("Port", None), ("Protocol",
None)):
runningComponents[componentName][
option] = self._getComponentOption(
instanceType, system, componentName, option,
default)
# remove empty values so we can use defaults in _getURL
if not runningComponents[componentName][option]:
runningComponents[componentName].pop(option)
runningComponents[componentName][
"LogFileLocation"] = os.path.join(
self.diracLocation, "runit", system, componentName,
"log", "current")
runningComponents[componentName]["PID"] = componentInfo[
"PID"]
runningComponents[componentName]["Module"] = componentInfo[
"Module"]
runningComponents[componentName][
"RunitStatus"] = componentInfo["RunitStatus"]
runningComponents[componentName]["System"] = system
return S_OK(runningComponents)
def _getComponentOption(self, instanceType, system, componentName, option,
default):
"""Get component option from DIRAC CS, using components' base classes methods."""
componentPath = PathFinder.getComponentSection(
system=system,
component=componentName,
setup=self.setup,
componentCategory=instanceType,
)
if instanceType != "Agents":
return gConfig.getValue(Path.cfgPath(componentPath, option),
default)
# deal with agent configuration
componentLoadModule = gConfig.getValue(
Path.cfgPath(componentPath, "Module"), componentName)
fullComponentName = Path.cfgPath(system, componentName)
fullComponentLoadName = Path.cfgPath(system, componentLoadModule)
return AgentModule(fullComponentName,
fullComponentLoadName).am_getOption(
option, default)
def on_terminate(self, componentName, process):
"""Execute callback when a process terminates gracefully."""
self.log.info(
"%s's process with ID: %s has been terminated successfully" %
(componentName, process.pid))
def execute(self):
"""Execute checks for agents, executors, services."""
for instanceType in ("executor", "agent", "service"):
for name, options in getattr(self, instanceType + "s").items():
# call checkAgent, checkExecutor, checkService
res = getattr(self,
"check" + instanceType.capitalize())(name,
options)
if not res["OK"]:
self.logError("Failure when checking %s" % instanceType,
"%s, %s" % (name, res["Message"]))
res = self.componentControl()
if not res["OK"]:
if "Stopped does not exist" not in res[
"Message"] and "Running does not exist" not in res[
"Message"]:
self.logError("Failure to control components", res["Message"])
if not self.errors:
res = self.checkURLs()
if not res["OK"]:
self.logError("Failure to check URLs", res["Message"])
else:
self.logError(
"Something was wrong before, not checking URLs this time")
self.sendNotification()
if self.errors:
return S_ERROR("Error during this cycle, check log")
return S_OK()
@staticmethod
def getLastAccessTime(logFileLocation):
"""Return the age of log file."""
lastAccessTime = 0
try:
lastAccessTime = os.path.getmtime(logFileLocation)
lastAccessTime = datetime.fromtimestamp(lastAccessTime)
except OSError as e:
return S_ERROR("Failed to access logfile %s: %r" %
(logFileLocation, e))
now = datetime.now()
age = now - lastAccessTime
return S_OK(age)
def restartInstance(self, pid, instanceName, enabled):
"""Kill a process which is then restarted automatically."""
if not (self.enabled and enabled):
self.log.info(
"Restarting is disabled, please restart %s manually" %
instanceName)
self.accounting[instanceName][
"Treatment"] = "Please restart it manually"
return S_OK(NO_RESTART)
if any(pattern in instanceName
for pattern in self.doNotRestartInstancePattern):
self.log.info(
"Restarting for %s is disabled, please restart it manually" %
instanceName)
self.accounting[instanceName][
"Treatment"] = "Please restart it manually"
return S_OK(NO_RESTART)
try:
componentProc = psutil.Process(int(pid))
processesToTerminate = componentProc.children(recursive=True)
processesToTerminate.append(componentProc)
for proc in processesToTerminate:
proc.terminate()
_gone, alive = psutil.wait_procs(processesToTerminate,
timeout=5,
callback=partial(
self.on_terminate,
instanceName))
for proc in alive:
self.log.info("Forcefully killing process %s" % proc.pid)
proc.kill()
return S_OK()
except psutil.Error as err:
self.logError("Exception occurred in terminating processes",
"%s" % err)
return S_ERROR()
def checkService(self, serviceName, options):
"""Ping the service, restart if the ping does not respond."""
url = self._getURL(serviceName, options)
self.log.info("Pinging service", url)
pingRes = Client().ping(url=url)
if not pingRes["OK"]:
self.log.info("Failure pinging service: %s: %s" %
(url, pingRes["Message"]))
res = self.restartInstance(int(options["PID"]), serviceName,
self.restartServices)
if not res["OK"]:
return res
if res["Value"] != NO_RESTART:
self.accounting[serviceName][
"Treatment"] = "Successfully Restarted"
self.log.info("Service %s has been successfully restarted" %
serviceName)
self.log.info("Service responded OK")
return S_OK()
def checkAgent(self, agentName, options):
"""Check the age of agent's log file, if it is too old then restart the agent."""
pollingTime, currentLogLocation, pid = (options["PollingTime"],
options["LogFileLocation"],
options["PID"])
self.log.info("Checking Agent: %s" % agentName)
self.log.info("Polling Time: %s" % pollingTime)
self.log.info("Current Log File location: %s" % currentLogLocation)
res = self.getLastAccessTime(currentLogLocation)
if not res["OK"]:
return res
age = res["Value"]
self.log.info("Current log file for %s is %d minutes old" %
(agentName, (age.seconds / MINUTES)))
maxLogAge = max(pollingTime + HOUR, 2 * HOUR)
if age.seconds < maxLogAge:
return S_OK()
self.log.info("Current log file is too old for Agent %s" % agentName)
self.accounting[agentName]["LogAge"] = age.seconds / MINUTES
res = self.restartInstance(int(pid), agentName, self.restartAgents)
if not res["OK"]:
return res
if res["Value"] != NO_RESTART:
self.accounting[agentName]["Treatment"] = "Successfully Restarted"
self.log.info("Agent %s has been successfully restarted" %
agentName)
return S_OK()
def checkExecutor(self, executor, options):
"""Check the age of executor log file, if too old check for jobs in checking status, then restart the executors."""
currentLogLocation = options["LogFileLocation"]
pid = options["PID"]
self.log.info("Checking executor: %s" % executor)
self.log.info("Current Log File location: %s" % currentLogLocation)
res = self.getLastAccessTime(currentLogLocation)
if not res["OK"]:
return res
age = res["Value"]
self.log.info("Current log file for %s is %d minutes old" %
(executor, (age.seconds / MINUTES)))
if age.seconds < 2 * HOUR:
return S_OK()
self.log.info("Current log file is too old for Executor %s" % executor)
self.accounting[executor]["LogAge"] = age.seconds / MINUTES
res = self.checkForCheckingJobs(executor)
if not res["OK"]:
return res
if res["OK"] and res["Value"] == NO_CHECKING_JOBS:
self.accounting.pop(executor, None)
return S_OK(NO_RESTART)
res = self.restartInstance(int(pid), executor, self.restartExecutors)
if not res["OK"]:
return res
elif res["OK"] and res["Value"] != NO_RESTART:
self.accounting[executor]["Treatment"] = "Successfully Restarted"
self.log.info("Executor %s has been successfully restarted" %
executor)
return S_OK()
def checkForCheckingJobs(self, executorName):
"""Check if there are checking jobs with the **executorName** as current MinorStatus."""
attrDict = {"Status": "Checking", "MinorStatus": executorName}
# returns list of jobs IDs
resJobs = self.jobMonClient.getJobs(attrDict)
if not resJobs["OK"]:
self.logError("Could not get jobs for this executor",
"%s: %s" % (executorName, resJobs["Message"]))
return resJobs
if resJobs["Value"]:
self.log.info('Found %d jobs in "Checking" status for %s' %
(len(resJobs["Value"]), executorName))
return S_OK(CHECKING_JOBS)
self.log.info('Found no jobs in "Checking" status for %s' %
executorName)
return S_OK(NO_CHECKING_JOBS)
def componentControl(self):
"""Monitor and control component status as defined in the CS.
Check for running and stopped components and ensure they have the proper status as defined in the CS
Registry/Hosts/_HOST_/[Running|Stopped] sections
:returns: :func:`~DIRAC:DIRAC.Core.Utilities.ReturnValues.S_OK`,
:func:`~DIRAC:DIRAC.Core.Utilities.ReturnValues.S_ERROR`
"""
# get the current status of the components
resCurrent = self._getCurrentComponentStatus()
if not resCurrent["OK"]:
return resCurrent
currentStatus = resCurrent["Value"]
resDefault = self._getDefaultComponentStatus()
if not resDefault["OK"]:
return resDefault
defaultStatus = resDefault["Value"]
# ensure instances are in the right state
shouldBe = {}
shouldBe["Run"] = defaultStatus["Run"].intersection(
currentStatus["Down"])
shouldBe["Down"] = defaultStatus["Down"].intersection(
currentStatus["Run"])
shouldBe["Unknown"] = defaultStatus["All"].symmetric_difference(
currentStatus["All"])
self._ensureComponentRunning(shouldBe["Run"])
self._ensureComponentDown(shouldBe["Down"])
for instance in shouldBe["Unknown"]:
self.logError("Unknown instance",
"%r, either uninstall or add to config" % instance)
return S_OK()
def _getCurrentComponentStatus(self):
"""Get current status for components."""
resOverall = self.sysAdminClient.getOverallStatus()
if not resOverall["OK"]:
return resOverall
currentStatus = {"Down": set(), "Run": set(), "All": set()}
informationDict = resOverall["Value"]
for systemsDict in informationDict.values():
for system, instancesDict in systemsDict.items():
for instanceName, instanceInfoDict in instancesDict.items():
identifier = "%s__%s" % (system, instanceName)
runitStatus = instanceInfoDict.get("RunitStatus")
if runitStatus in ("Run", "Down"):
currentStatus[runitStatus].add(identifier)
currentStatus["All"] = currentStatus["Run"] | currentStatus["Down"]
return S_OK(currentStatus)
def _getDefaultComponentStatus(self):
"""Get the configured status of the components."""
host = socket.getfqdn()
defaultStatus = {"Down": set(), "Run": set(), "All": set()}
resRunning = gConfig.getOptionsDict(
Path.cfgPath("/Registry/Hosts/", host, "Running"))
resStopped = gConfig.getOptionsDict(
Path.cfgPath("/Registry/Hosts/", host, "Stopped"))
if not resRunning["OK"]:
return resRunning
if not resStopped["OK"]:
return resStopped
defaultStatus["Run"] = set(resRunning["Value"])
defaultStatus["Down"] = set(resStopped["Value"])
defaultStatus["All"] = defaultStatus["Run"] | defaultStatus["Down"]
if defaultStatus["Run"].intersection(defaultStatus["Down"]):
self.logError(
"Overlap in configuration",
str(defaultStatus["Run"].intersection(defaultStatus["Down"])))
return S_ERROR("Bad host configuration")
return S_OK(defaultStatus)
def _ensureComponentRunning(self, shouldBeRunning):
"""Ensure the correct components are running."""
for instance in shouldBeRunning:
self.log.info("Starting instance %s" % instance)
system, name = instance.split("__")
if self.controlComponents:
res = self.sysAdminClient.startComponent(system, name)
if not res["OK"]:
self.logError("Failed to start component:",
"%s: %s" % (instance, res["Message"]))
else:
self.accounting[instance][
"Treatment"] = "Instance was down, started instance"
else:
self.accounting[instance][
"Treatment"] = "Instance is down, should be started"
def _ensureComponentDown(self, shouldBeDown):
"""Ensure the correct components are not running."""
for instance in shouldBeDown:
self.log.info("Stopping instance %s" % instance)
system, name = instance.split("__")
if self.controlComponents:
res = self.sysAdminClient.stopComponent(system, name)
if not res["OK"]:
self.logError("Failed to stop component:",
"%s: %s" % (instance, res["Message"]))
else:
self.accounting[instance][
"Treatment"] = "Instance was running, stopped instance"
else:
self.accounting[instance][
"Treatment"] = "Instance is running, should be stopped"
def checkURLs(self):
"""Ensure that the running services have their URL in the Config."""
self.log.info("Checking URLs")
# get services again, in case they were started/stop in controlComponents
gConfig.forceRefresh(fromMaster=True)
# get port used for https based services
try:
tornadoSystemInstance = PathFinder.getSystemInstance(
system="Tornado",
setup=self.setup,
)
self._tornadoPort = gConfig.getValue(
Path.cfgPath("/System/Tornado/", tornadoSystemInstance,
"Port"),
self._tornadoPort,
)
except RuntimeError:
pass
self.log.debug("Using Tornado Port:", self._tornadoPort)
res = self.getRunningInstances(instanceType="Services",
runitStatus="All")
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
for service, options in sorted(self.services.items()):
self.log.debug("Checking URL for %s with options %s" %
(service, options))
# ignore SystemAdministrator, does not have URLs
if "SystemAdministrator" in service:
continue
self._checkServiceURL(service, options)
if self.csAPI.csModified and self.commitURLs:
self.log.info("Commiting changes to the CS")
result = self.csAPI.commit()
if not result["OK"]:
self.logError("Commit to CS failed", result["Message"])
return S_ERROR("Failed to commit to CS")
return S_OK()
def _checkServiceURL(self, serviceName, options):
"""Ensure service URL is properly configured in the CS."""
url = self._getURL(serviceName, options)
system = options["System"]
module = options["Module"]
self.log.info("Checking URLs for %s/%s" % (system, module))
urlsConfigPath = Path.cfgPath(
PathFinder.getSystemURLSection(system=system, setup=self.setup),
module)
urls = gConfig.getValue(urlsConfigPath, [])
self.log.debug("Found configured URLs for %s: %s" % (module, urls))
self.log.debug("This URL is %s" % url)
runitStatus = options["RunitStatus"]
wouldHave = "Would have " if not self.commitURLs else ""
if runitStatus == "Run" and url not in urls:
urls.append(url)
message = "%sAdded URL %s to URLs for %s/%s" % (wouldHave, url,
system, module)
self.log.info(message)
self.accounting[serviceName + "/URL"]["Treatment"] = message
self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
if runitStatus == "Down" and url in urls:
urls.remove(url)
message = "%sRemoved URL %s from URLs for %s/%s" % (wouldHave, url,
system, module)
self.log.info(message)
self.accounting[serviceName + "/URL"]["Treatment"] = message
self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
def _getURL(self, serviceName, options):
"""Return URL for the service."""
system = options["System"]
port = options.get("Port", self._tornadoPort)
host = socket.getfqdn()
protocol = options.get("Protocol", "dips")
url = "%s://%s:%s/%s/%s" % (protocol, host, port, system, serviceName)
return url
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = "Bdii2CSAgent"
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.host = 'lcg-bdii.cern.ch:2170'
self.glue2URLs = []
self.glue2Only = False
self.csAPI = None
# What to get
self.processCEs = True
self.processSEs = False
self.selectedSites = []
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data", result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set(result['Value'])
knownCEs = knownCEs.union(set(bannedCEs))
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom,
localAttempt=False, avoidSpam=True)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo, host=self.host, glue2=self.glue2Only)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii", mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii, glue2=self.glue2Only)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii, resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
for glue2URL in self.glue2URLs:
if self.glue2Only:
break
resultGlue2 = getBdiiCEInfo(vo, host=glue2URL, glue2=True)
if resultGlue2['OK']:
totalResult['Value'].update(resultGlue2['Value'])
else:
self.log.error("Failed getting GLUE2 information for", "%s, %s: %s" %
(glue2URL, vo, resultGlue2['Message']))
message = (message + "\n" + resultGlue2['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult['Value'] and message: # Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiSEInfo(self, vo):
if vo in self.voBdiiSEDict:
return S_OK(self.voBdiiSEDict[vo])
self.log.info("Check for available SEs for VO", vo)
result = getBdiiSEInfo(vo)
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs:
result = getBdiiSEInfo(vo, host=bdii)
if result['OK']:
break
if not result['OK']:
if message:
self.log.error("Error during BDII request", message)
else:
self.log.error("Error during BDII request", result['Message'])
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
self.__purgeSites(ceBdiiDict)
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __purgeSites(self, ceBdiiDict):
"""Remove all sites that are not in self.selectedSites.
Modifies the ceBdiiDict!
"""
if not self.selectedSites:
return
for site in list(ceBdiiDict):
ces = list(ceBdiiDict[site]['CEs'])
if not ces:
self.log.error("No CE information for site:", site)
continue
diracSiteName = getSiteForCE(ces[0])
if not diracSiteName['OK']:
self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ces[0], diracSiteName['Message']))
continue
self.log.debug("Checking site %s (%s), aka %s" % (site, ces, diracSiteName['Value']))
if diracSiteName['Value'] in self.selectedSites:
continue
self.log.info("Dropping site %s, aka %s" % (site, diracSiteName))
ceBdiiDict.pop(site)
return
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = sorted(changeSet)
body = '\n'.join(["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
if body:
self.log.info('The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS", result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" % len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get('GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get('GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __updateSEs(self):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates(vo, bdiiInfo=seBdiiDict)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
for i in resultList:
if not cfg.isSection("Resources/Sites/%s" % i):
gLogger.error("Subsection /Resources/Sites/%s does not exists" % i)
continue
hasRun = True
if SORTBYNAME:
dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(ascending=not REVERSE)
else:
dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE)
if dirty:
isDirty = True
if not hasRun:
gLogger.notice("Failed to find suitable subsections with site names to sort")
DIRAC.exit(0)
if not isDirty:
gLogger.notice("Nothing to do, site names are already sorted")
DIRAC.exit(0)
timestamp = toString(dateTime())
stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp)
cs.setOptionComment("/Resources/Sites", stamp)
result = cs.commit()
if not result["OK"]:
gLogger.error("Failed to commit changes to CS", result["Message"])
DIRAC.exit(2)
gLogger.notice("Site names are sorted and committed to CS")
DIRAC.exit(0)
def checkUnusedSEs():
global vo, dry
result = getGridSRMs( vo, unUsed = True )
if not result['OK']:
gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] )
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set( result['Value'] )
else:
csVOs = set( [vo] )
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get( 'GlueServiceVersion', '' )
if not ( version and version.startswith( '2' ) ):
gLogger.debug( 'Skipping SRM service with version %s' % version )
continue
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) )
gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site )
continue
diracSites = result['Value']
yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
if not yn or yn.lower().startswith( 'y' ):
if len( diracSites ) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate( diracSites ):
prompt += '\n[%d] %s' % ( i, s )
prompt += '\nEnter your choice number: '
inp = raw_input( prompt )
try:
ind = int( inp )
except:
gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split( '.' )
recName = '%s-disk' % siteName
inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )
seSection = cfgPath( '/Resources/StorageElements', diracSEName )
changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )
bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
seVOs = csVOs.intersection( bdiiVOs )
changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
accessSection = cfgPath( seSection, 'AccessProtocol.1' )
changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
result = pfnparse( endPoint )
if not result['OK']:
gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
continue
host = result['Value']['Host']
port = result['Value']['Port']
changeSet.add( ( accessSection, 'Host', host ) )
changeSet.add( ( accessSection, 'Port', port ) )
changeSet.add( ( accessSection, 'Access', 'remote' ) )
voPathSection = cfgPath( accessSection, 'VOPath' )
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename( path )
if voFromPath != diracVO:
gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
changeSet.add( ( voPathSection, diracVO, path ) )
path = os.path.dirname( path )
else:
# Try to guess the Path
domain = '.'.join( host.split( '.' )[-2:] )
path = '/dpm/%s/home' % domain
changeSet.add( ( accessSection, 'Path', path ) )
changeSet.add( ( accessSection, 'SpaceToken', '' ) )
changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) )
gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
changeList = list( changeSet )
changeList.sort()
for entry in changeList:
gLogger.notice( entry )
yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )
if not yn or yn.lower().startswith( 'y' ):
changeSetFull = changeSetFull.union( changeSet )
if dry:
if changeSetFull:
gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
changeList = list( changeSetFull )
changeList.sort()
for section, option, value in changeList:
csAPI.setOption( cfgPath( section, option ), value )
yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
if not yn or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
class DDSimTarMaker( object ):
""" create a tarball of the DDSim release """
def __init__( self ):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.detmodels = {}
self.lcgeo_env="lcgeo_DIR"
self.ddhep_env="DD4HEP"
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.version = ''
self.platform = 'x86_64-slc5-gcc43-opt'
self.comment = ""
self.name = "ddsim"
self.csapi = CSAPI()
self.tarBallName = None
self.md5sum = None
def copyDetectorModels( self, basePath, folder, targetFolder ):
"""copy the compact folders to the targetFolder """
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import copyFolder
for root,dirs,_files in os.walk( os.path.join(basePath, folder) ):
for direct in dirs:
if root.endswith("compact"):
## the main xml file must have the same name as the folder, ILD and CLIC follow this convention already
xmlPath = os.path.join( root, direct, direct+".xml")
if os.path.exists( xmlPath ):
self.detmodels[direct] = "detectors/"+direct+"/"+direct+".xml"
copyFolder( os.path.join(root, direct), targetFolder )
def createTarBall( self, folder ):
"""create a tar ball from the folder
tar zcf $TARBALLNAME $LIBFOLDER/*
"""
##Create the Tarball
if os.path.exists(self.tarBallName):
os.remove(self.tarBallName)
gLogger.notice("Creating Tarball...")
myappTar = tarfile.open(self.tarBallName, "w:gz")
myappTar.add(folder, self.tarBallName[:-4])
myappTar.close()
self.md5sum = md5.md5(open( self.tarBallName, 'r' ).read()).hexdigest()
gLogger.notice("...Done")
return S_OK( "Created Tarball")
def parseArgs( self ):
""" parse the command line arguments"""
if len(sys.argv) != 3:
raise RuntimeError( "Wrong number of arguments in call: '%s'" % " ".join(sys.argv) )
self.name = sys.argv[1]
self.version = sys.argv[2]
self.tarBallName = "%s%s.tgz" % (self.name, self.version)
def checkEnvironment( self ):
""" check if dd4hep and lcgeo are in the environment """
for var in [ self.ddhep_env, self.lcgeo_env , 'ROOTSYS' ]:
if var not in os.environ:
raise RuntimeError( "%s is not set" % var )
return os.environ[self.ddhep_env], os.environ[self.lcgeo_env], os.environ['ROOTSYS']
def createCSEntry( self ):
"""add the entries for this version into the Configuration System
.. code::
<version>
{
TarBall = ddsim<version>.tgz
AdditionalEnvVar
{
ROOTSYS = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/root/5.34.30
G4INSTALL = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01
G4DATA = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01/share/Geant4-10.1.0/data
}
Overwrite = True
}
Operations/DDSimDetectorModels/<Version>
{
CLIC_o2_v03 = detectors/CLIC_o2_v03/CLIC_o2_v03.xml
...
}
"""
from ILCDIRAC.Core.Utilities.CheckAndGetProdProxy import checkOrGetGroupProxy
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import insertCSSection
#FIXME: Get root and geant4 location from environment, make sure it is cvmfs
csParameter = { "TarBall": self.tarBallName,
"AdditionalEnvVar": {
"ROOTSYS" : os.path.realpath( os.environ.get("ROOTSYS") ),
"G4INSTALL" : os.path.realpath( os.environ.get("G4INSTALL") ),
},
"Md5Sum": self.md5sum,
}
g4datavariables = [ "G4RADIOACTIVEDATA",
"G4NEUTRONHPDATA",
"G4LEDATA",
"G4LEVELGAMMADATA",
"G4RADIOACTIVEDATA",
"G4NEUTRONXSDATA",
"G4PIIDATA",
"G4REALSURFACEDATA",
"G4SAIDXSDATA",
"G4ABLADATA",
"G4ENSDFSTATEDATA",
]
for g4data in g4datavariables:
if os.environ.get(g4data) :
csParameter["AdditionalEnvVar"][g4data] = os.path.realpath( os.environ.get(g4data) )
pars = dict( platform=self.platform,
name="ddsim",
version=self.version
)
csPath = os.path.join( self.softSec , "%(platform)s/%(name)s/%(version)s/" % pars )
pprint(csParameter)
result = insertCSSection( self.csapi, csPath, csParameter )
csPathModels = "Operations/Defaults/DDSimDetectorModels"
csModels = { self.version : self.detmodels }
pprint(csModels)
result = insertCSSection( self.csapi, csPathModels, csModels )
if self.csapi is not None:
resProxy = checkOrGetGroupProxy( "diracAdmin" )
if not resProxy['OK']:
gLogger.error( "Failed to get AdminProxy", resProxy['Message'] )
raise RuntimeError( "Failed to get diracAdminProxy" )
self.csapi.commit()
if not result['OK']:
gLogger.error( "Failed to create CS Section", result['Message'] )
raise RuntimeError( "Failed to create CS Section" )
def createDDSimTarBall( self ):
""" do everything to create the DDSim tarball"""
from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import copyFolder, getLibraryPath, getFiles, getDependentLibraries, copyLibraries, getPythonStuff, killRPath, resolveLinks, removeSystemLibraries
self.parseArgs()
ddBase, lcgeoBase, _rootsys = self.checkEnvironment()
realTargetFolder = os.path.join( os.getcwd(), self.name+self.version )
targetFolder = os.path.join( os.getcwd(), "temp", self.name+self.version )
for folder in (targetFolder, targetFolder+"/lib"):
try:
os.makedirs( folder )
except OSError:
pass
libraries = set()
rootmaps = set()
dd4hepLibPath = getLibraryPath( ddBase )
lcgeoPath = getLibraryPath( lcgeoBase )
## FIXME: Automatically pick up folders with /compact/ in them
self.copyDetectorModels( lcgeoBase, "CLIC" , targetFolder+"/detectors" )
self.copyDetectorModels( lcgeoBase, "ILD" , targetFolder+"/detectors" )
self.copyDetectorModels( lcgeoBase, "SiD" , targetFolder+"/detectors" )
copyFolder( ddBase+"/DDDetectors/compact", realTargetFolder.rstrip("/")+"/DDDetectors")
copyFolder( ddBase+"/include", realTargetFolder.rstrip("/")+"")
libraries.update( getFiles( dd4hepLibPath, ".so") )
libraries.update( getFiles( lcgeoPath, ".so" ) )
rootmaps.update( getFiles( dd4hepLibPath, ".rootmap") )
rootmaps.update( getFiles( lcgeoPath, ".rootmap" ) )
rootmaps.update( getFiles( dd4hepLibPath, ".pcm") )
rootmaps.update( getFiles( lcgeoPath, ".pcm" ) )
rootmaps.update( getFiles( dd4hepLibPath, ".components") )
rootmaps.update( getFiles( lcgeoPath, ".components" ) )
pprint( libraries )
pprint( rootmaps )
allLibs = set()
for lib in libraries:
allLibs.update( getDependentLibraries(lib) )
### remote root and geant4 libraries, we pick them up from
allLibs = set( [ lib for lib in allLibs if not ( "/geant4/" in lib.lower() or "/root/" in lib.lower()) ] )
print(allLibs)
copyLibraries( libraries, targetFolder+"/lib" )
copyLibraries( allLibs, targetFolder+"/lib" )
copyLibraries( rootmaps, targetFolder+"/lib" )
getPythonStuff( ddBase+"/python" , targetFolder+"/lib/")
getPythonStuff( lcgeoBase+"/lib/python", targetFolder+"/lib/" )
getPythonStuff( lcgeoBase+"/bin/ddsim", targetFolder+"/bin/" )
##Should get this from CVMFS
#getRootStuff( rootsys, targetFolder+"/ROOT" )
copyFolder( targetFolder+"/", realTargetFolder.rstrip("/") )
killRPath( realTargetFolder )
resolveLinks( realTargetFolder+"/lib" )
removeSystemLibraries( realTargetFolder+"/lib" )
#removeSystemLibraries( realTargetFolder+"/ROOT/lib" )
print(self.detmodels)
self.createTarBall( realTargetFolder )
self.createCSEntry()
def main():
Script.registerSwitch(
"C", "country",
"Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)",
sortBy)
Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument([
"Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)"
],
mandatory=False)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
result = getProxyInfo()
if not result["OK"]:
gLogger.error("Failed to get proxy information", result["Message"])
DIRACExit(2)
proxy = result["Value"]
if proxy["secondsLeft"] < 1:
gLogger.error("Your proxy has expired, please create new one")
DIRACExit(2)
group = proxy["group"]
if "CSAdministrator" not in getPropertiesForGroup(group):
gLogger.error(
"You must be CSAdministrator user to execute this script")
gLogger.notice(
"Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'"
)
DIRACExit(2)
cs = CSAPI()
result = cs.getCurrentCFG()
if not result["OK"]:
gLogger.error("Failed to get copy of CS", result["Message"])
DIRACExit(2)
cfg = result["Value"]
if not cfg.isSection("Resources"):
gLogger.error("Section '/Resources' is absent in CS")
DIRACExit(2)
if not cfg.isSection("Resources/Sites"):
gLogger.error("Subsection '/Resources/Sites' is absent in CS")
DIRACExit(2)
if args and len(args) > 0:
resultList = args[:]
else:
resultList = cfg["Resources"]["Sites"].listSections()
hasRun = False
isDirty = False
for i in resultList:
if not cfg.isSection("Resources/Sites/%s" % i):
gLogger.error("Subsection /Resources/Sites/%s does not exists" % i)
continue
hasRun = True
if SORTBYNAME:
dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(
ascending=not REVERSE)
else:
dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country,
reverse=REVERSE)
if dirty:
isDirty = True
if not hasRun:
gLogger.notice(
"Failed to find suitable subsections with site names to sort")
DIRACExit(0)
if not isDirty:
gLogger.notice("Nothing to do, site names are already sorted")
DIRACExit(0)
timestamp = toString(dateTime())
stamp = "Site names are sorted by %s script at %s" % (Script.scriptName,
timestamp)
cs.setOptionComment("/Resources/Sites", stamp)
result = cs.commit()
if not result["OK"]:
gLogger.error("Failed to commit changes to CS", result["Message"])
DIRACExit(2)
gLogger.notice("Site names are sorted and committed to CS")
DIRACExit(0)
class MonitorAgents(AgentModule):
"""MonitorAgents class."""
def __init__(self, *args, **kwargs):
"""Initialize the agent, clients, default values."""
AgentModule.__init__(self, *args, **kwargs)
self.name = 'MonitorAgents'
self.setup = "Production"
self.enabled = False
self.restartAgents = False
self.restartExecutors = False
self.restartServices = False
self.controlComponents = False
self.commitURLs = False
self.diracLocation = "/opt/dirac/pro"
self.sysAdminClient = SystemAdministratorClient(socket.gethostname())
self.jobMonClient = JobMonitoringClient()
self.nClient = NotificationClient()
self.csAPI = None
self.agents = dict()
self.executors = dict()
self.services = dict()
self.errors = list()
self.accounting = defaultdict(dict)
self.addressTo = ["*****@*****.**"]
self.addressFrom = "*****@*****.**"
self.emailSubject = "MonitorAgents on %s" % socket.gethostname()
def logError(self, errStr, varMsg=''):
"""Append errors to a list, which is sent in email notification."""
self.log.error(errStr, varMsg)
self.errors.append(errStr + " " + varMsg)
def beginExecution(self):
"""Reload the configurations before every cycle."""
self.setup = self.am_getOption("Setup", self.setup)
self.enabled = self.am_getOption("EnableFlag", self.enabled)
self.restartAgents = self.am_getOption("RestartAgents", self.restartAgents)
self.restartExecutors = self.am_getOption("RestartExecutors", self.restartExecutors)
self.restartServices = self.am_getOption("RestartServices", self.restartServices)
self.diracLocation = os.environ.get("DIRAC", self.diracLocation)
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
self.controlComponents = self.am_getOption('ControlComponents', self.controlComponents)
self.commitURLs = self.am_getOption('CommitURLs', self.commitURLs)
self.csAPI = CSAPI()
res = self.getRunningInstances(instanceType='Agents')
if not res["OK"]:
return S_ERROR("Failure to get running agents")
self.agents = res["Value"]
res = self.getRunningInstances(instanceType='Executors')
if not res["OK"]:
return S_ERROR("Failure to get running executors")
self.executors = res["Value"]
res = self.getRunningInstances(instanceType='Services')
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
self.accounting.clear()
return S_OK()
def sendNotification(self):
"""Send email notification about changes done in the last cycle."""
if not(self.errors or self.accounting):
return S_OK()
emailBody = ""
rows = []
for instanceName, val in self.accounting.iteritems():
rows.append([[instanceName],
[val.get('Treatment', 'No Treatment')],
[str(val.get('LogAge', 'Not Relevant'))]])
if rows:
columns = ["Instance", "Treatment", "Log File Age (Minutes)"]
emailBody += printTable(columns, rows, printOut=False, numbering=False, columnSeparator=' | ')
if self.errors:
emailBody += "\n\nErrors:"
emailBody += "\n".join(self.errors)
self.log.notice("Sending Email:\n" + emailBody)
for address in self.addressTo:
res = self.nClient.sendMail(address, self.emailSubject, emailBody, self.addressFrom, localAttempt=False)
if not res['OK']:
self.log.error("Failure to send Email notification to ", address)
continue
self.errors = []
self.accounting.clear()
return S_OK()
def getRunningInstances(self, instanceType='Agents', runitStatus='Run'):
"""Return a dict of running agents, executors or services.
Key is agent's name, value contains dict with PollingTime, PID, Port, Module, RunitStatus, LogFileLocation
:param str instanceType: 'Agents', 'Executors', 'Services'
:param str runitStatus: Return only those instances with given RunitStatus or 'All'
:returns: Dictionary of running instances
"""
res = self.sysAdminClient.getOverallStatus()
if not res["OK"]:
self.logError("Failure to get %s from system administrator client" % instanceType, res["Message"])
return res
val = res['Value'][instanceType]
runningAgents = defaultdict(dict)
for system, agents in val.iteritems():
for agentName, agentInfo in agents.iteritems():
if agentInfo['Setup'] and agentInfo['Installed']:
if runitStatus != 'All' and agentInfo['RunitStatus'] != runitStatus:
continue
confPath = cfgPath('/Systems/' + system + '/' + self.setup + '/%s/' % instanceType + agentName)
for option, default in (('PollingTime', HOUR), ('Port', None)):
optPath = os.path.join(confPath, option)
runningAgents[agentName][option] = gConfig.getValue(optPath, default)
runningAgents[agentName]["LogFileLocation"] = \
os.path.join(self.diracLocation, 'runit', system, agentName, 'log', 'current')
runningAgents[agentName]["PID"] = agentInfo["PID"]
runningAgents[agentName]['Module'] = agentInfo['Module']
runningAgents[agentName]['RunitStatus'] = agentInfo['RunitStatus']
runningAgents[agentName]['System'] = system
return S_OK(runningAgents)
def on_terminate(self, agentName, process):
"""Execute callback when a process terminates gracefully."""
self.log.info("%s's process with ID: %s has been terminated successfully" % (agentName, process.pid))
def execute(self):
"""Execute checks for agents, executors, services."""
for instanceType in ('executor', 'agent', 'service'):
for name, options in getattr(self, instanceType + 's').iteritems():
# call checkAgent, checkExecutor, checkService
res = getattr(self, 'check' + instanceType.capitalize())(name, options)
if not res['OK']:
self.logError("Failure when checking %s" % instanceType, "%s, %s" % (name, res['Message']))
res = self.componentControl()
if not res['OK']:
if "Stopped does not exist" not in res['Message'] and \
"Running does not exist" not in res['Message']:
self.logError("Failure to control components", res['Message'])
if not self.errors:
res = self.checkURLs()
if not res['OK']:
self.logError("Failure to check URLs", res['Message'])
else:
self.logError('Something was wrong before, not checking URLs this time')
self.sendNotification()
if self.errors:
return S_ERROR("Error during this cycle, check log")
return S_OK()
@staticmethod
def getLastAccessTime(logFileLocation):
"""Return the age of log file."""
lastAccessTime = 0
try:
lastAccessTime = os.path.getmtime(logFileLocation)
lastAccessTime = datetime.fromtimestamp(lastAccessTime)
except OSError as e:
return S_ERROR('Failed to access logfile %s: %r' % (logFileLocation, e))
now = datetime.now()
age = now - lastAccessTime
return S_OK(age)
def restartInstance(self, pid, instanceName, enabled):
"""Kill a process which is then restarted automatically."""
if not (self.enabled and enabled):
self.log.info("Restarting is disabled, please restart %s manually" % instanceName)
self.accounting[instanceName]["Treatment"] = "Please restart it manually"
return S_OK(NO_RESTART)
try:
agentProc = psutil.Process(int(pid))
processesToTerminate = agentProc.children(recursive=True)
processesToTerminate.append(agentProc)
for proc in processesToTerminate:
proc.terminate()
_gone, alive = psutil.wait_procs(processesToTerminate, timeout=5,
callback=partial(self.on_terminate, instanceName))
for proc in alive:
self.log.info("Forcefully killing process %s" % proc.pid)
proc.kill()
return S_OK()
except psutil.Error as err:
self.logError("Exception occurred in terminating processes", "%s" % err)
return S_ERROR()
def checkService(self, serviceName, options):
"""Ping the service, restart if the ping does not respond."""
url = self._getURL(serviceName, options)
self.log.info("Pinging service", url)
pingRes = Client().ping(url=url)
if not pingRes['OK']:
self.log.info('Failure pinging service: %s: %s' % (url, pingRes['Message']))
res = self.restartInstance(int(options['PID']), serviceName, self.restartServices)
if not res["OK"]:
return res
elif res['OK'] and res['Value'] != NO_RESTART:
self.accounting[serviceName]["Treatment"] = "Successfully Restarted"
self.log.info("Agent %s has been successfully restarted" % serviceName)
self.log.info("Service responded OK")
return S_OK()
def checkAgent(self, agentName, options):
"""Check the age of agent's log file, if it is too old then restart the agent."""
pollingTime, currentLogLocation, pid = options['PollingTime'], options['LogFileLocation'], options['PID']
self.log.info("Checking Agent: %s" % agentName)
self.log.info("Polling Time: %s" % pollingTime)
self.log.info("Current Log File location: %s" % currentLogLocation)
res = self.getLastAccessTime(currentLogLocation)
if not res["OK"]:
return res
age = res["Value"]
self.log.info("Current log file for %s is %d minutes old" % (agentName, (age.seconds / MINUTES)))
maxLogAge = max(pollingTime + HOUR, 2 * HOUR)
if age.seconds < maxLogAge:
return S_OK()
self.log.info("Current log file is too old for Agent %s" % agentName)
self.accounting[agentName]["LogAge"] = age.seconds / MINUTES
res = self.restartInstance(int(pid), agentName, self.restartAgents)
if not res["OK"]:
return res
elif res['OK'] and res['Value'] != NO_RESTART:
self.accounting[agentName]["Treatment"] = "Successfully Restarted"
self.log.info("Agent %s has been successfully restarted" % agentName)
return S_OK()
def checkExecutor(self, executor, options):
"""Check the age of executor log file, if too old check for jobs in checking status, then restart the executors."""
currentLogLocation = options['LogFileLocation']
pid = options['PID']
self.log.info("Checking executor: %s" % executor)
self.log.info("Current Log File location: %s" % currentLogLocation)
res = self.getLastAccessTime(currentLogLocation)
if not res["OK"]:
return res
age = res["Value"]
self.log.info("Current log file for %s is %d minutes old" % (executor, (age.seconds / MINUTES)))
if age.seconds < 2 * HOUR:
return S_OK()
self.log.info("Current log file is too old for Executor %s" % executor)
self.accounting[executor]["LogAge"] = age.seconds / MINUTES
res = self.checkForCheckingJobs(executor)
if not res['OK']:
return res
if res['OK'] and res['Value'] == NO_CHECKING_JOBS:
self.accounting.pop(executor, None)
return S_OK(NO_RESTART)
res = self.restartInstance(int(pid), executor, self.restartExecutors)
if not res["OK"]:
return res
elif res['OK'] and res['Value'] != NO_RESTART:
self.accounting[executor]["Treatment"] = "Successfully Restarted"
self.log.info("Executor %s has been successfully restarted" % executor)
return S_OK()
def checkForCheckingJobs(self, executorName):
"""Check if there are checking jobs with the **executorName** as current MinorStatus."""
attrDict = {'Status': 'Checking', 'MinorStatus': executorName}
# returns list of jobs IDs
resJobs = self.jobMonClient.getJobs(attrDict)
if not resJobs['OK']:
self.logError("Could not get jobs for this executor", "%s: %s" % (executorName, resJobs['Message']))
return resJobs
if resJobs['Value']:
self.log.info("Found %d jobs in 'Checking' status for %s" % (len(resJobs['Value']), executorName))
return S_OK(CHECKING_JOBS)
self.log.info("Found no jobs in 'Checking' status for %s" % executorName)
return S_OK(NO_CHECKING_JOBS)
def componentControl(self):
"""Monitor and control component status as defined in the CS.
Check for running and stopped components and ensure they have the proper status as defined in the CS
Registry/Hosts/_HOST_/[Running|Stopped] sections
:returns: :func:`~DIRAC:DIRAC.Core.Utilities.ReturnValues.S_OK`,
:func:`~DIRAC:DIRAC.Core.Utilities.ReturnValues.S_ERROR`
"""
# get the current status of the components
resCurrent = self._getCurrentComponentStatus()
if not resCurrent['OK']:
return resCurrent
currentStatus = resCurrent['Value']
resDefault = self._getDefaultComponentStatus()
if not resDefault['OK']:
return resDefault
defaultStatus = resDefault['Value']
# ensure instances are in the right state
shouldBe = {}
shouldBe['Run'] = defaultStatus['Run'].intersection(currentStatus['Down'])
shouldBe['Down'] = defaultStatus['Down'].intersection(currentStatus['Run'])
shouldBe['Unknown'] = defaultStatus['All'].symmetric_difference(currentStatus['All'])
self._ensureComponentRunning(shouldBe['Run'])
self._ensureComponentDown(shouldBe['Down'])
for instance in shouldBe['Unknown']:
self.logError("Unknown instance", "%r, either uninstall or add to config" % instance)
return S_OK()
def _getCurrentComponentStatus(self):
"""Get current status for components."""
resOverall = self.sysAdminClient.getOverallStatus()
if not resOverall['OK']:
return resOverall
currentStatus = {'Down': set(), 'Run': set(), 'All': set()}
informationDict = resOverall['Value']
for systemsDict in informationDict.values():
for system, instancesDict in systemsDict.items():
for instanceName, instanceInfoDict in instancesDict.items():
identifier = '%s__%s' % (system, instanceName)
runitStatus = instanceInfoDict.get('RunitStatus')
if runitStatus in ('Run', 'Down'):
currentStatus[runitStatus].add(identifier)
currentStatus['All'] = currentStatus['Run'] | currentStatus['Down']
return S_OK(currentStatus)
def _getDefaultComponentStatus(self):
"""Get the configured status of the components."""
host = socket.gethostname()
defaultStatus = {'Down': set(), 'Run': set(), 'All': set()}
resRunning = gConfig.getOptionsDict(os.path.join('/Registry/Hosts/', host, 'Running'))
resStopped = gConfig.getOptionsDict(os.path.join('/Registry/Hosts/', host, 'Stopped'))
if not resRunning['OK']:
return resRunning
if not resStopped['OK']:
return resStopped
defaultStatus['Run'] = set(resRunning['Value'].keys())
defaultStatus['Down'] = set(resStopped['Value'].keys())
defaultStatus['All'] = defaultStatus['Run'] | defaultStatus['Down']
if defaultStatus['Run'].intersection(defaultStatus['Down']):
self.logError("Overlap in configuration", str(defaultStatus['Run'].intersection(defaultStatus['Down'])))
return S_ERROR("Bad host configuration")
return S_OK(defaultStatus)
def _ensureComponentRunning(self, shouldBeRunning):
"""Ensure the correct components are running."""
for instance in shouldBeRunning:
self.log.info("Starting instance %s" % instance)
system, name = instance.split('__')
if self.controlComponents:
res = self.sysAdminClient.startComponent(system, name)
if not res['OK']:
self.logError("Failed to start component:", "%s: %s" % (instance, res['Message']))
else:
self.accounting[instance]["Treatment"] = "Instance was down, started instance"
else:
self.accounting[instance]["Treatment"] = "Instance is down, should be started"
def _ensureComponentDown(self, shouldBeDown):
"""Ensure the correct components are not running."""
for instance in shouldBeDown:
self.log.info("Stopping instance %s" % instance)
system, name = instance.split('__')
if self.controlComponents:
res = self.sysAdminClient.stopComponent(system, name)
if not res['OK']:
self.logError("Failed to stop component:", "%s: %s" % (instance, res['Message']))
else:
self.accounting[instance]["Treatment"] = "Instance was running, stopped instance"
else:
self.accounting[instance]["Treatment"] = "Instance is running, should be stopped"
def checkURLs(self):
"""Ensure that the running services have their URL in the Config."""
self.log.info("Checking URLs")
# get services again, in case they were started/stop in controlComponents
gConfig.forceRefresh(fromMaster=True)
res = self.getRunningInstances(instanceType='Services', runitStatus='All')
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
for service, options in self.services.iteritems():
self.log.debug("Checking URL for %s with options %s" % (service, options))
# ignore SystemAdministrator, does not have URLs
if 'SystemAdministrator' in service:
continue
self._checkServiceURL(service, options)
if self.csAPI.csModified and self.commitURLs:
self.log.info("Commiting changes to the CS")
result = self.csAPI.commit()
if not result['OK']:
self.logError('Commit to CS failed', result['Message'])
return S_ERROR("Failed to commit to CS")
return S_OK()
def _checkServiceURL(self, serviceName, options):
"""Ensure service URL is properly configured in the CS."""
url = self._getURL(serviceName, options)
system = options['System']
module = options['Module']
self.log.info("Checking URLs for %s/%s" % (system, module))
urlsConfigPath = os.path.join('/Systems', system, self.setup, 'URLs', module)
urls = gConfig.getValue(urlsConfigPath, [])
self.log.debug("Found configured URLs for %s: %s" % (module, urls))
self.log.debug("This URL is %s" % url)
runitStatus = options['RunitStatus']
wouldHave = 'Would have ' if not self.commitURLs else ''
if runitStatus == 'Run' and url not in urls:
urls.append(url)
message = "%sAdded URL %s to URLs for %s/%s" % (wouldHave, url, system, module)
self.log.info(message)
self.accounting[serviceName + "/URL"]["Treatment"] = message
self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
if runitStatus == 'Down' and url in urls:
urls.remove(url)
message = "%sRemoved URL %s from URLs for %s/%s" % (wouldHave, url, system, module)
self.log.info(message)
self.accounting[serviceName + "/URL"]["Treatment"] = message
self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
@staticmethod
def _getURL(serviceName, options):
"""Return URL for the service."""
system = options['System']
port = options['Port']
host = socket.gethostname()
url = 'dips://%s:%s/%s/%s' % (host, port, system, serviceName)
return url
class CE2CSAgent(AgentModule):
""" !!!Out-dated!!! Moved to Bdii2CSAgent
"""
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
csAPI = None
def initialize(self):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', [])
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'TestManager')
self.voName = self.am_getOption('VirtualOrganization', [])
if not self.voName:
vo = getVO()
if vo:
self.voName = [vo]
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
self.log.info("Start Execution")
result = getProxyInfo()
if not result['OK']:
return result
infoDict = result['Value']
self.log.info(formatProxyInfoAsString(infoDict))
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data",
result['Message'])
self.__lookForCE()
self.__infoFromCE()
self.log.info("End Execution")
return S_OK()
def __checkAlternativeBDIISite(self, fun, *args):
if self.alternativeBDIIs:
self.log.warn("Trying to use alternative BDII sites")
for site in self.alternativeBDIIs:
self.log.info("Trying to contact alternative BDII", site)
if len(args) == 1:
result = fun(args[0], host=site)
elif len(args) == 2:
result = fun(args[0], vo=args[1], host=site)
if not result['OK']:
self.log.error("Problem contacting alternative BDII",
result['Message'])
elif result['OK']:
return result
self.log.warn("Also checking alternative BDII sites failed")
return result
def __lookForCE(self):
knownCEs = self.am_getOption('BannedCEs', [])
result = gConfig.getSections('/Resources/Sites')
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections('/Resources/Sites/%s' % grid)
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(grid, site))['Value']
ces = List.fromChar(opt.get('CE', ''))
knownCEs += ces
response = ''
for vo in self.voName:
self.log.info("Check for available CEs for VO", vo)
response = ldapCEState('', vo)
if not response['OK']:
self.log.error("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(ldapCEState, '', vo)
return response
newCEs = {}
for queue in response['Value']:
try:
queueName = queue['GlueCEUniqueID']
except:
continue
ceName = queueName.split(":")[0]
if not ceName in knownCEs:
newCEs[ceName] = None
self.log.debug("New CE", ceName)
body = ""
possibleNewSites = []
for ce in newCEs.iterkeys():
response = ldapCluster(ce)
if not response['OK']:
self.log.warn("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(ldapCluster, ce)
continue
clusters = {}
clusters = response['Value']
if len(clusters) != 1:
self.log.warn("Error in cluster length",
" CE %s Length %d" % (ce, len(clusters)))
if len(clusters) == 0:
continue
cluster = clusters[0]
fkey = cluster.get('GlueForeignKey', [])
if type(fkey) == type(''):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count('GlueSiteUniqueID'):
nameBDII = entry.split('=')[1]
break
if not nameBDII:
continue
ceString = "CE: %s, GOCDB Name: %s" % (ce, nameBDII)
self.log.info(ceString)
response = ldapCE(ce)
if not response['OK']:
self.log.warn("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
ceInfos = response['Value']
if len(ceInfos):
ceInfo = ceInfos[0]
systemName = ceInfo.get('GlueHostOperatingSystemName',
'Unknown')
systemVersion = ceInfo.get(
'GlueHostOperatingSystemVersion', 'Unknown')
systemRelease = ceInfo.get(
'GlueHostOperatingSystemRelease', 'Unknown')
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osString = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % (
systemName, systemVersion, systemRelease)
self.log.info(osString)
response = ldapCEState(ce, vo)
if not response['OK']:
self.log.warn("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(
ldapCEState, ce, vo)
continue
newCEString = "\n\n%s\n%s" % (ceString, osString)
usefull = False
ceStates = response['Value']
for ceState in ceStates:
queueName = ceState.get('GlueCEUniqueID', 'UnknownName')
queueStatus = ceState.get('GlueCEStateStatus',
'UnknownStatus')
queueString = "%s %s" % (queueName, queueStatus)
self.log.info(queueString)
newCEString += "\n%s" % queueString
if queueStatus.count('Production'):
usefull = True
if usefull:
body += newCEString
possibleNewSites.append(
'dirac-admin-add-site DIRACSiteName %s %s' %
(nameBDII, ce))
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % (body, possibleNewSite)
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return S_OK()
def __infoFromCE(self):
sitesSection = cfgPath('Resources', 'Sites')
result = gConfig.getSections(sitesSection)
if not result['OK']:
return
grids = result['Value']
changed = False
body = ""
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result['OK']:
return
sites = result['Value']
for site in sites:
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)['Value']
name = opt.get('Name', '')
if name:
coor = opt.get('Coordinates', 'Unknown')
mail = opt.get('Mail', 'Unknown')
result = ldapSite(name)
if not result['OK']:
self.log.warn("BDII site %s: %s" %
(name, result['Message']))
result = self.__checkAlternativeBDIISite(
ldapSite, name)
if result['OK']:
bdiiSites = result['Value']
if len(bdiiSites) == 0:
self.log.warn(name, "Error in BDII: leng = 0")
else:
if not len(bdiiSites) == 1:
self.log.warn(
name, "Warning in BDII: leng = %d" %
len(bdiiSites))
bdiiSite = bdiiSites[0]
try:
longitude = bdiiSite['GlueSiteLongitude']
latitude = bdiiSite['GlueSiteLatitude']
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in BDII coordinates")
newcoor = "Unknown"
try:
newmail = bdiiSite[
'GlueSiteSysAdminContact'].split(
":")[-1].strip()
except:
self.log.warn("Error in BDII mail")
newmail = "Unknown"
self.log.debug("%s %s %s" %
(name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name),
"%s -> %s" % (coor, newcoor))
if coor == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Coordinates'),
newcoor)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name),
"%s -> %s" % (mail, newmail))
if mail == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Mail'), newmail)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Mail'), newmail)
changed = True
ceList = List.fromChar(opt.get('CE', ''))
if not ceList:
self.log.warn(site, 'Empty site list')
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in ceList:
ceSection = cfgPath(siteSection, 'CEs', ce)
result = gConfig.getOptionsDict(ceSection)
if not result['OK']:
self.log.debug("Section CE", result['Message'])
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
ceType = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get('wnTmpDir', 'Unknown')
arch = ceopt.get('architecture', 'Unknown')
os = ceopt.get('OS', 'Unknown')
si00 = ceopt.get('SI00', 'Unknown')
pilot = ceopt.get('Pilot', 'Unknown')
ceType = ceopt.get('CEType', 'Unknown')
result = ldapCE(ce)
if not result['OK']:
self.log.warn('Error in BDII for %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiiCE = result['Value'][0]
except:
self.log.warn('Error in BDII for %s' % ce, result)
bdiiCE = None
if bdiiCE:
try:
newWNTmpDir = bdiiCE['GlueSubClusterWNTmpDir']
except:
newWNTmpDir = 'Unknown'
if wnTmpDir != newWNTmpDir and newWNTmpDir != 'Unknown':
section = cfgPath(ceSection, 'wnTmpDir')
self.log.info(section, " -> ".join(
(wnTmpDir, newWNTmpDir)))
if wnTmpDir == 'Unknown':
self.csAPI.setOption(section, newWNTmpDir)
else:
self.csAPI.modifyValue(section, newWNTmpDir)
changed = True
try:
newArch = bdiiCE[
'GlueHostArchitecturePlatformType']
except:
newArch = 'Unknown'
if arch != newArch and newArch != 'Unknown':
section = cfgPath(ceSection, 'architecture')
self.log.info(section, " -> ".join(
(arch, newArch)))
if arch == 'Unknown':
self.csAPI.setOption(section, newArch)
else:
self.csAPI.modifyValue(section, newArch)
changed = True
try:
newOS = '_'.join(
(bdiiCE['GlueHostOperatingSystemName'],
bdiiCE['GlueHostOperatingSystemVersion'],
bdiiCE['GlueHostOperatingSystemRelease']))
except:
newOS = 'Unknown'
if os != newOS and newOS != 'Unknown':
section = cfgPath(ceSection, 'OS')
self.log.info(section, " -> ".join((os, newOS)))
if os == 'Unknown':
self.csAPI.setOption(section, newOS)
else:
self.csAPI.modifyValue(section, newOS)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (
os, newOS, ce, site)
try:
newSI00 = bdiiCE['GlueHostBenchmarkSI00']
except:
newSI00 = 'Unknown'
if si00 != newSI00 and newSI00 != 'Unknown':
section = cfgPath(ceSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newSI00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
try:
rte = bdiiCE[
'GlueHostApplicationSoftwareRunTimeEnvironment']
for vo in self.voName:
if vo.lower() == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newPilot = 'True'
else:
newPilot = 'False'
else:
newPilot = 'Unknown'
except:
newPilot = 'Unknown'
if pilot != newPilot and newPilot != 'Unknown':
section = cfgPath(ceSection, 'Pilot')
self.log.info(section, " -> ".join(
(pilot, newPilot)))
if pilot == 'Unknown':
self.csAPI.setOption(section, newPilot)
else:
self.csAPI.modifyValue(section, newPilot)
changed = True
newVO = ''
for vo in self.voName:
result = ldapCEState(ce, vo) #getBDIICEVOView
if not result['OK']:
self.log.warn('Error in BDII for queue %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(
ldapCEState, ce, vo)
continue
try:
queues = result['Value']
except:
self.log.warn('Error in BDII for queue %s' % ce,
result['Massage'])
continue
newCEType = 'Unknown'
for queue in queues:
try:
queueType = queue['GlueCEImplementationName']
except:
queueType = 'Unknown'
if newCEType == 'Unknown':
newCEType = queueType
else:
if queueType != newCEType:
self.log.warn(
'Error in BDII for CE %s ' % ce,
'different CE types %s %s' %
(newCEType, queueType))
if newCEType == 'ARC-CE':
newCEType = 'ARC'
if ceType != newCEType and newCEType != 'Unknown':
section = cfgPath(ceSection, 'CEType')
self.log.info(section, " -> ".join(
(ceType, newCEType)))
if ceType == 'Unknown':
self.csAPI.setOption(section, newCEType)
else:
self.csAPI.modifyValue(section, newCEType)
changed = True
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split(
'/')[-1]
except:
self.log.warn('Error in queueName ', queue)
continue
try:
newMaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newMaxCPUTime = None
newSI00 = None
try:
caps = queue['GlueCECapability']
if type(caps) == type(''):
caps = [caps]
for cap in caps:
if cap.count('CPUScalingReferenceSI00'):
newSI00 = cap.split('=')[-1]
except:
newSI00 = None
queueSection = cfgPath(ceSection, 'Queues',
queueName)
result = gConfig.getOptionsDict(queueSection)
if not result['OK']:
self.log.warn("Section Queues",
result['Message'])
maxCPUTime = 'Unknown'
si00 = 'Unknown'
allowedVOs = ['']
else:
queueOpt = result['Value']
maxCPUTime = queueOpt.get(
'maxCPUTime', 'Unknown')
si00 = queueOpt.get('SI00', 'Unknown')
if newVO == '': # Remember previous iteration, if none - read from conf
allowedVOs = queueOpt.get('VO',
'').split(",")
else: # Else use newVO, as it can contain changes, which aren't in conf yet
allowedVOs = newVO.split(",")
if newMaxCPUTime and (maxCPUTime != newMaxCPUTime):
section = cfgPath(queueSection, 'maxCPUTime')
self.log.info(
section, " -> ".join(
(maxCPUTime, newMaxCPUTime)))
if maxCPUTime == 'Unknown':
self.csAPI.setOption(
section, newMaxCPUTime)
else:
self.csAPI.modifyValue(
section, newMaxCPUTime)
changed = True
if newSI00 and (si00 != newSI00):
section = cfgPath(queueSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newSI00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
modifyVO = True # Flag saying if we need VO option to change
newVO = ''
if allowedVOs != ['']:
for allowedVO in allowedVOs:
allowedVO = allowedVO.strip(
) # Get rid of spaces
newVO += allowedVO
if allowedVO == vo: # Current VO has been already in list
newVO = ''
modifyVO = False # Don't change anything
break # Skip next 'if', proceed to next VO
newVO += ', '
if modifyVO:
section = cfgPath(queueSection, 'VO')
newVO += vo
self.log.info(
section, " -> ".join(
('%s' % allowedVOs, newVO)))
if allowedVOs == ['']:
self.csAPI.setOption(section, newVO)
else:
self.csAPI.modifyValue(section, newVO)
changed = True
if changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return self.csAPI.commit()
else:
self.log.info("No changes found")
return S_OK()
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
Script.showHelp(exitCode=1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split('.')
except ValueError:
gLogger.error(
"The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch"
)
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result['OK'] and result['Value']:
if len(result['Value']) > 1:
gLogger.notice(
'%s GOC site name is associated with several DIRAC sites:' %
gridSiteName)
for i, dSite in enumerate(result['Value']):
gLogger.notice('%d: %s' % (i, dSite))
inp = six.moves.input('Enter your choice number: ')
try:
inp = int(inp)
except ValueError:
gLogger.error('You should enter an integer number')
DIRACExit(-1)
if 0 <= inp < len(result['Value']):
diracCSSite = result['Value'][inp]
else:
gLogger.error('Number out of range: %d' % inp)
DIRACExit(-1)
else:
diracCSSite = result['Value'][0]
if diracCSSite == diracSiteName:
gLogger.notice('Site with GOC name %s is already defined as %s' %
(gridSiteName, diracSiteName))
newSite = False
else:
gLogger.error(
'ERROR: Site with GOC name %s is already defined as %s' %
(gridSiteName, diracCSSite))
DIRACExit(-1)
else:
gLogger.error("ERROR getting DIRAC site name of %s" % gridSiteName,
result.get('Message'))
csAPI = CSAPI()
if newSite:
gLogger.notice("Site to CS: %s" % diracSiteName)
res = csAPI.addSite(diracSiteName, {"Name": gridSiteName})
if not res['OK']:
gLogger.error("Failed adding site to CS", res['Message'])
DIRACExit(1)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Failure committing to CS", res['Message'])
DIRACExit(3)
for ce in ces:
gLogger.notice("Adding CE %s" % ce)
res = csAPI.addCEtoSite(diracSiteName, ce)
if not res['OK']:
gLogger.error("Failed adding CE %s to CS" % ce, res['Message'])
DIRACExit(2)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Failure committing to CS", res['Message'])
DIRACExit(3)
class Bdii2CSAgent( AgentModule ):
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize( self ):
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
self.processCEs = self.am_getOption( 'ProcessCEs', True )
self.processSEs = self.am_getOption( 'ProcessSEs', False )
self.voName = self.am_getOption( 'VirtualOrganization', [] )
if not self.voName:
self.voName = self.am_getOption( 'VO', [] )
if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption( vo, "VOMSName" )
if vomsVO:
self.voName.append( vomsVO )
if self.voName:
self.log.info( "Agent will manage VO(s) %s" % self.voName )
else:
self.log.fatal( "VirtualOrganization option not defined for agent" )
return S_ERROR()
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute( self ):
""" General agent execution method
"""
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs( self ):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption( 'BannedCEs', [] )
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set( result['Value'] )
knownCEs = knownCEs.union( set( bannedCEs ) )
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs )
if not result['OK']:
self.log.error( 'Failed to get unused CEs', result['Message'] )
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set( siteDict[site].keys() )
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
ceString = "CE: %s, GOCDB Site Name: %s" % ( ce, site )
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % ( systemTuple )
newCEString = "\n%s\n%s\n" % ( ceString, osString )
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' )
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' )
queueString += " %s %s %s\n" % ( queue, queueStatus, ceType )
if queueString:
ceString = newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def __getBdiiCEInfo( self, vo ):
if vo in self.voBdiiCEDict:
return S_OK( self.voBdiiCEDict[vo] )
self.log.info( "Check for available CEs for VO", vo )
result = getBdiiCEInfo( vo )
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs :
result = getBdiiCEInfo( vo, host = bdii )
if result['OK']:
break
if not result['OK']:
if message:
self.log.error( "Error during BDII request", message )
else:
self.log.error( "Error during BDII request", result['Message'] )
else:
self.voBdiiCEDict[vo] = result['Value']
return result
def __getBdiiSEInfo( self, vo ):
if vo in self.voBdiiSEDict:
return S_OK( self.voBdiiSEDict[vo] )
self.log.info( "Check for available SEs for VO", vo )
result = getBdiiSEInfo( vo )
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs :
result = getBdiiSEInfo( vo, host = bdii )
if result['OK']:
break
if not result['OK']:
if message:
self.log.error( "Error during BDII request", message )
else:
self.log.error( "Error during BDII request", result['Message'] )
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs( self ):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates( vo, bdiiInfo = ceBdiiDict, log = self.log )
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union( result['Value'] )
# We have collected all the changes, consolidate VO settings
result = self.__updateCS( bdiiChangeSet )
return result
def __updateCS( self, bdiiChangeSet ):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option , _value, new_value = entry
if option == "VO":
queueVODict.setdefault( section, set() )
queueVODict[section] = queueVODict[section].union( set( new_value.split( ',' ) ) )
else:
changeSet.add( entry )
for section, VOs in queueVODict.items():
changeSet.add( ( section, 'VO', '', ','.join( VOs ) ) )
if changeSet:
changeList = list( changeSet )
changeList.sort()
body = '\n'.join( [ "%s/%s %s -> %s" % entry for entry in changeList ] )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if body:
self.log.info( 'The following configuration changes were detected:' )
self.log.info( body )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption( cfgPath( section, option ), new_value )
else:
self.csAPI.modifyValue( cfgPath( section, option ), new_value )
result = self.csAPI.commit()
if not result['OK']:
self.log.error( "Error while committing to CS", result['Message'] )
else:
self.log.info( "Successfully committed %d changes to CS" % len( changeList ) )
return result
else:
self.log.info( "No changes found" )
return S_OK()
def __lookForNewSEs( self ):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption( 'BannedSEs', [] )
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set( result['Value'] )
knownSEs = knownSEs.union( set( bannedSEs ) )
for vo in self.voName:
result = self.__getBdiiSEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs( vo, bdiiInfo = bdiiInfo, srmBlackList = knownSEs )
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set( siteDict[site].keys() )
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % ( se, site )
backend = siteDict[site][se]['SE'].get( 'GlueSEImplementationName', 'Unknown' )
size = siteDict[site][se]['SE'].get( 'GlueSESizeTotal', 'Unknown' )
body += ' Backend %s, Size %s' % ( backend, size )
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def __updateSEs( self ):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo( vo )
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates( vo, bdiiInfo = seBdiiDict )
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union( result['Value'] )
# We have collected all the changes, consolidate VO settings
result = self.__updateCS( bdiiChangeSet )
return result
class GOCDB2CSAgent(AgentModule):
""" Class to retrieve information about service endpoints
from GOCDB and update configuration stored by CS
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
super(GOCDB2CSAgent, self).__init__(*args, **kwargs)
self.GOCDBClient = None
self.csAPI = None
self.dryRun = False
def initialize(self):
""" Run at the agent initialization (normally every 500 cycles)
"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
"""
Execute GOCDB queries according to the function map
and user request (options in configuration).
"""
# __functionMap is at the end of the class definition
for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
optionValue = self.am_getOption(option, True)
if optionValue:
result = functionCall(self)
if not result['OK']:
self.log.error("%s() failed with message: %s" %
(functionCall.__name__, result['Message']))
else:
self.log.info("Successfully executed %s" %
functionCall.__name__)
return S_OK()
def updatePerfSONARConfiguration(self):
"""
Get current status of perfSONAR endpoints from GOCDB
and update CS configuration accordingly.
"""
log = self.log.getSubLogger('updatePerfSONAREndpoints')
log.debug('Begin function ...')
# get endpoints
result = self.__getPerfSONAREndpoints()
if not result['OK']:
log.error("__getPerfSONAREndpoints() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.')
endpointList = result['Value']
# add DIRAC site name
result = self.__addDIRACSiteName(endpointList)
if not result['OK']:
log.error("__addDIRACSiteName() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to extend the list with DIRAC site names.')
extendedEndpointList = result['Value']
# prepare dictionary with new configuration
result = self.__preparePerfSONARConfiguration(extendedEndpointList)
if not result['OK']:
log.error(
"__preparePerfSONARConfiguration() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to prepare a new perfSONAR configuration.')
finalConfiguration = result['Value']
# update configuration according to the final status of endpoints
self.__updateConfiguration(finalConfiguration)
log.debug("Configuration updated succesfully")
log.debug('End function.')
return S_OK()
def __getPerfSONAREndpoints(self):
"""
Retrieve perfSONAR endpoint information directly from GOCDB.
:return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB.
"""
log = self.log.getSubLogger('__getPerfSONAREndpoints')
log.debug('Begin function ...')
# get perfSONAR endpoints (latency and bandwidth) form GOCDB
endpointList = []
for endpointType in ['Latency', 'Bandwidth']:
result = self.GOCDBClient.getServiceEndpointInfo(
'service_type', 'net.perfSONAR.%s' % endpointType)
if not result['OK']:
log.error("getServiceEndpointInfo() failed with message: %s" %
result['Message'])
return S_ERROR('Could not fetch %s endpoints from GOCDB' %
endpointType.lower())
log.debug('Number of %s endpoints: %s' %
(endpointType.lower(), len(result['Value'])))
endpointList.extend(result['Value'])
log.debug('Number of perfSONAR endpoints: %s' % len(endpointList))
log.debug('End function.')
return S_OK(endpointList)
def __preparePerfSONARConfiguration(self, endpointList):
"""
Prepare a dictionary with a new CS configuration of perfSONAR endpoints.
:return: Dictionary where keys are configuration paths (options and sections)
and values are values of corresponding options
or None in case of a path pointing to a section.
"""
log = self.log.getSubLogger('__preparePerfSONARConfiguration')
log.debug('Begin function ...')
# static elements of a path
rootPath = '/Resources/Sites'
extPath = 'Network'
baseOptionName = 'Enabled'
options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'}
# enable GOCDB endpoints in configuration
newConfiguration = {}
for endpoint in endpointList:
if endpoint['DIRACSITENAME'] is None:
continue
split = endpoint['DIRACSITENAME'].split('.')
path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'],
extPath, endpoint['HOSTNAME'])
for name, defaultValue in options.iteritems():
newConfiguration[cfgPath(path, name)] = defaultValue
# get current configuration
currentConfiguration = {}
for option in options.iterkeys():
result = gConfig.getConfigurationTree(rootPath, extPath + '/',
'/' + option)
if not result['OK']:
log.error("getConfigurationTree() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
currentConfiguration.update(result['Value'])
# disable endpoints that disappeared in GOCDB
removedElements = set(currentConfiguration) - set(newConfiguration)
newElements = set(newConfiguration) - set(currentConfiguration)
addedEndpoints = len(newElements) / len(options)
disabledEndpoints = 0
for path in removedElements:
if baseOptionName in path:
newConfiguration[path] = 'False'
if currentConfiguration[path] != 'False':
disabledEndpoints = disabledEndpoints + 1
# inform what will be changed
if addedEndpoints > 0:
self.log.info(
"%s new perfSONAR endpoints will be added to the configuration"
% addedEndpoints)
if disabledEndpoints > 0:
self.log.info(
"%s old perfSONAR endpoints will be disable in the configuration"
% disabledEndpoints)
if addedEndpoints == 0 and disabledEndpoints == 0:
self.log.info("perfSONAR configuration is up-to-date")
log.debug('End function.')
return S_OK(newConfiguration)
def __addDIRACSiteName(self, inputList):
"""
Extend given list of GOCDB endpoints with DIRAC site name, i.e.
add an entry "DIRACSITENAME" in dictionaries that describe endpoints.
If given site name could not be found "DIRACSITENAME" is set to 'None'.
:return: List of perfSONAR endpoints (dictionaries).
"""
log = self.log.getSubLogger('__addDIRACSiteName')
log.debug('Begin function ...')
# get site name dictionary
result = getDIRACGOCDictionary()
if not result['OK']:
log.error("getDIRACGOCDictionary() failed with message: %s" %
result['Message'])
return S_ERROR('Could not get site name dictionary')
# reverse the dictionary (assume 1 to 1 relation)
DIRACGOCDict = result['Value']
GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys()))
# add DIRAC site names
outputList = []
for entry in inputList:
try:
entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']]
except KeyError:
self.log.warn("No dictionary entry for %s. " %
entry['SITENAME'])
entry['DIRACSITENAME'] = None
outputList.append(entry)
log.debug('End function.')
return S_OK(outputList)
def __updateConfiguration(self, setElements=None, delElements=None):
"""
Update configuration stored by CS.
"""
if setElements is None:
setElements = {}
if delElements is None:
delElements = []
log = self.log.getSubLogger('__updateConfiguration')
log.debug('Begin function ...')
# assure existence and proper value of a section or an option
for path, value in setElements.iteritems():
if value is None:
section = path
else:
split = path.rsplit('/', 1)
section = split[0]
try:
result = self.csAPI.createSection(section)
if not result['OK']:
log.error("createSection() failed with message: %s" %
result['Message'])
except Exception as e:
log.error("Exception in createSection(): %s" %
repr(e).replace(',)', ')'))
if value is not None:
try:
result = self.csAPI.setOption(path, value)
if not result['OK']:
log.error("setOption() failed with message: %s" %
result['Message'])
except Exception as e:
log.error("Exception in setOption(): %s" %
repr(e).replace(',)', ')'))
# delete elements in the configuration
for path in delElements:
result = self.csAPI.delOption(path)
if not result['OK']:
log.warn("delOption() failed with message: %s" %
result['Message'])
result = self.csAPI.delSection(path)
if not result['OK']:
log.warn("delSection() failed with message: %s" %
result['Message'])
if self.dryRun:
log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
# update configuration stored by CS
result = self.csAPI.commit()
if not result['OK']:
log.error("commit() failed with message: %s" %
result['Message'])
return S_ERROR("Could not commit changes to CS.")
else:
log.info("Committed changes to CS")
log.debug('End function.')
return S_OK()
# define mapping between an agent option in the configuration and a function call
__functionMap = {
'UpdatePerfSONARS': updatePerfSONARConfiguration,
}
# }
# }
# }
# }
for sct in [
'Systems/Bookkeeping', 'Systems/Bookkeeping/Production',
'Systems/Bookkeeping/Production/Databases',
'Systems/Bookkeeping/Production/Databases/BookkeepingDB'
]:
res = csAPI.createSection(sct)
if not res['OK']:
print(res['Message'])
exit(1)
csAPI.setOption(
'Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingTNS',
'FILL_ME')
csAPI.setOption(
'Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingUser',
'FILL_ME')
csAPI.setOption(
'Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingPassword',
'FILL_ME')
csAPI.setOption(
'Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingServer',
'FILL_ME')
# Commit
csAPI.commit()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result['OK']:
gLogger.error('Failed to look up SRMs in BDII', result['Message'])
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set(result['Value'])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get('GlueServiceVersion', '')
if not (version and version.startswith('2')):
gLogger.debug('Skipping SRM service with version %s' % version)
continue
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice('Unused se %s is detected at unused site %s' % (gridSE, site))
gLogger.notice('Consider adding site %s to the DIRAC CS' % site)
continue
diracSites = result['Value']
yn = raw_input(
'\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' %
(gridSE, str(diracSites)))
if not yn or yn.lower().startswith('y'):
if len(diracSites) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate(diracSites):
prompt += '\n[%d] %s' % (i, s)
prompt += '\nEnter your choice number: '
inp = raw_input(prompt)
try:
ind = int(inp)
except BaseException:
gLogger.notice('Can not interpret your choice: %s, try again later' % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split('.')
recName = '%s-disk' % siteName
inp = raw_input('Give a DIRAC name to the grid SE %s, default %s : ' % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice('Adding new SE %s at site %s' % (diracSEName, diracSite))
seSection = cfgPath('/Resources/StorageElements', diracSEName)
changeSet.add((seSection, 'BackendType', seDict.get('GlueSEImplementationName', 'Unknown')))
changeSet.add((seSection, 'Description', seDict.get('GlueSEName', 'Unknown')))
bdiiVOs = set([re.sub('^VO:', '', rule) for rule in srmDict.get('GlueServiceAccessControlBaseRule', [])])
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, 'VO', ','.join(seVOs)))
accessSection = cfgPath(seSection, 'AccessProtocol.1')
changeSet.add((accessSection, 'Protocol', 'srm'))
changeSet.add((accessSection, 'PluginName', 'SRM2'))
endPoint = srmDict.get('GlueServiceEndpoint', '')
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, 'Host', host))
changeSet.add((accessSection, 'Port', port))
changeSet.add((accessSection, 'Access', 'remote'))
voPathSection = cfgPath(accessSection, 'VOPath')
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice('\n!!! Warning: non-conventional VO path: %s\n' % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = '.'.join(host.split('.')[-2:])
path = '/dpm/%s/home' % domain
changeSet.add((accessSection, 'Path', path))
changeSet.add((accessSection, 'SpaceToken', ''))
changeSet.add((accessSection, 'WSUrl', '/srm/managerv2?SFN='))
gLogger.notice('SE %s will be added with the following parameters' % diracSEName)
changeList = sorted(changeSet)
for entry in changeList:
gLogger.notice(entry)
yn = raw_input('Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName)
if not yn or yn.lower().startswith('y'):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice('Skipping commit of the new SE data in a dry run')
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object', result['Message'])
DIRACExit(-1)
changeList = sorted(changeSetFull)
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input('New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ')
if not yn or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = self.am_getModuleParam('fullName')
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.host = 'lcg-bdii.cern.ch:2170'
self.glue2URLs = []
self.glue2Only = True
self.csAPI = None
# What to get
self.processCEs = True
self.selectedSites = []
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data", result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
for vo in self.voName:
# get the known CEs for a given VO, so we can know the unknowns, or no longer supported,
# for a VO
res = getQueues(community=vo)
if not res['OK']:
return res
knownCEs = set()
for _site, ces in res['Value'].items():
knownCEs.update(ces)
knownCEs.update(bannedCEs)
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
continue # next VO
siteDict = result['Value']
unknownCEs = set(result['UnknownCEs']) - set(bannedCEs)
body = ''
for site in siteDict:
newCEs = set(siteDict[site]) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if siteDict:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
if unknownCEs:
body += '\n\n'
body += 'There is no (longer) information about the following CEs for the %s VO.\n' % vo
body += '\n'.join(sorted(unknownCEs))
body += '\n\n'
if body:
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom,
localAttempt=False, avoidSpam=True)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo, host=self.host, glue2=self.glue2Only)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii", mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii, glue2=self.glue2Only)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii, resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
for glue2URL in self.glue2URLs:
if self.glue2Only:
break
resultGlue2 = getBdiiCEInfo(vo, host=glue2URL, glue2=True)
if resultGlue2['OK']:
totalResult['Value'].update(resultGlue2['Value'])
else:
self.log.error("Failed getting GLUE2 information for", "%s, %s: %s" %
(glue2URL, vo, resultGlue2['Message']))
message = (message + "\n" + resultGlue2['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult['Value'] and message: # Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
self.__purgeSites(totalResult['Value'])
return totalResult
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __purgeSites(self, ceBdiiDict):
"""Remove all sites that are not in self.selectedSites.
Modifies the ceBdiiDict!
"""
if not self.selectedSites:
return
for site in list(ceBdiiDict):
ces = list(ceBdiiDict[site]['CEs'])
if not ces:
self.log.error("No CE information for site:", site)
continue
siteInCS = 'Not_In_CS'
for ce in ces:
res = getCESiteMapping(ce)
if not res['OK']:
self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ce, res['Message']))
continue
# if the ce is not in the CS the returned value will be empty
if ce in res['Value']:
siteInCS = res['Value'][ce]
break
self.log.debug("Checking site %s (%s), aka %s" % (site, ces, siteInCS))
if siteInCS in self.selectedSites:
continue
self.log.info("Dropping site %s, aka %s" % (site, siteInCS))
ceBdiiDict.pop(site)
return
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items(): # can be an iterator
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = sorted(changeSet)
body = '\n'.join(["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False,
avoidSpam=True)
if body:
self.log.info('The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS", result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" % len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result["OK"]:
gLogger.error("Failed to look up SRMs in BDII", result["Message"])
siteSRMDict = result["Value"]
# Evaluate VOs
result = getVOs()
if result["OK"]:
csVOs = set(result["Value"])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]["SE"]
srmDict = siteSRMDict[site][gridSE]["SRM"]
# Check the SRM version
version = srmDict.get("GlueServiceVersion", "")
if not (version and version.startswith("2")):
gLogger.debug("Skipping SRM service with version %s" % version)
continue
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("Unused se %s is detected at unused site %s" % (gridSE, site))
gLogger.notice("Consider adding site %s to the DIRAC CS" % site)
continue
diracSites = result["Value"]
yn = raw_input(
"\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites))
)
if not yn or yn.lower().startswith("y"):
if len(diracSites) > 1:
prompt = "Which DIRAC site the new SE should be attached to ?"
for i, s in enumerate(diracSites):
prompt += "\n[%d] %s" % (i, s)
prompt += "\nEnter your choice number: "
inp = raw_input(prompt)
try:
ind = int(inp)
except:
gLogger.notice("Can not interpret your choice: %s, try again later" % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split(".")
recName = "%s-disk" % siteName
inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite))
seSection = cfgPath("/Resources/StorageElements", diracSEName)
changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown")))
changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown")))
bdiiVOs = set(
[re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])]
)
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, "VO", ",".join(seVOs)))
accessSection = cfgPath(seSection, "AccessProtocol.1")
changeSet.add((accessSection, "Protocol", "srm"))
changeSet.add((accessSection, "PluginName", "SRM2"))
endPoint = srmDict.get("GlueServiceEndpoint", "")
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, "Host", host))
changeSet.add((accessSection, "Port", port))
changeSet.add((accessSection, "Access", "remote"))
voPathSection = cfgPath(accessSection, "VOPath")
if "VOPath" in seDict:
path = seDict["VOPath"]
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = ".".join(host.split(".")[-2:])
path = "/dpm/%s/home" % domain
changeSet.add((accessSection, "Path", path))
changeSet.add((accessSection, "SpaceToken", ""))
changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN="))
gLogger.notice("SE %s will be added with the following parameters" % diracSEName)
changeList = list(changeSet)
changeList.sort()
for entry in changeList:
gLogger.notice(entry)
yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName)
if not yn or yn.lower().startswith("y"):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice("Skipping commit of the new SE data in a dry run")
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result["OK"]:
gLogger.error("Failed to initialize CSAPI object", result["Message"])
DIRACExit(-1)
changeList = list(changeSetFull)
changeList.sort()
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ")
if not yn or yn.lower().startswith("y"):
result = csAPI.commit()
if not result["OK"]:
gLogger.error("Error while commit to CS", result["Message"])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
