class VOMS2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__(*args, **kwargs)
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
def initialize(self):
""" Initialize the default parameters
"""
voNames = self.am_getOption('VO', [])
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs(voNames)
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice("VOs: %s" % self.__voDict.keys())
self.csapi = CSAPI()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.autoAddUsers = self.am_getOption('AutoAddUsers',
self.autoAddUsers)
self.autoModifyUsers = self.am_getOption('AutoModifyUsers',
self.autoModifyUsers)
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption('mailFrom', "DIRAC system"))
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info("There are changes to the CS ready to be committed")
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed")
else:
self.log.info("No changes to the CS recorded at this cycle")
return S_OK()
@executeWithUserProxy
def __syncCSWithVOMS(self, vo):
self.__adminMsgs = {'Errors': [], 'Info': []}
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
vomsSrv = VOMSService(vo)
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error('Could not retrieve VOMS VO name', "for %s" % vo)
return result
vomsVOName = result['Value'].lstrip('/')
self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName))
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
vomsUserDict = result['Value']
message = "There are %s registered users in VOMS VO %s" % (
len(vomsUserDict), vomsVOName)
self.__adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
diracUsers = getUsersInVO(vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % vo)
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
diracUserDict = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
existingDNs.append(dn)
if dn not in vomsUserDict:
obsoletedDNs.append(dn)
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOusers = list(set(allDiracUsers) - set(diracUsers))
result = self.csapi.describeUsers(nonVOusers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
for dn in vomsUserDict:
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
diracName = ''
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# We have a real new user
if not diracName:
nickName = ''
result = vomsSrv.attGetUserNickname(
dn, vomsUserDict[dn]['CA'])
if result['OK']:
nickName = result['Value']
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName(dn,
vomsUserDict[dn]['mail'])
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
group = vomsDIRACMapping.get(fullRole)
if group:
groupsWithRole.extend(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
self.__adminMsgs['Info'].append(
"Adding new user %s: %s" %
(newDiracName, str(userDict)))
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
continue
# We have an already existing user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
nonVOGroups = nonVOUserDict.get(diracName,
{}).get('Groups', [])
existingGroups = diracUserDict.get(diracName,
{}).get('Groups', [])
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
group = vomsDIRACMapping.get(fullRole)
if group:
groupsWithRole.extend(group)
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
role = diracVOMSMapping[group]
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = keepGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.voChanged = True
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dn = diracUserDict[user]['DN']
if not dn in vomsUserDict and not user in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if not group in noVOMSGroups:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
self.__adminMsgs['Info'].append(
'The following users to be checked for deletion: %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
return S_OK()
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = self.am_getModuleParam('fullName')
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.host = 'lcg-bdii.cern.ch:2170'
self.glue2URLs = []
self.glue2Only = True
self.csAPI = None
# What to get
self.processCEs = True
self.selectedSites = []
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data", result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
for vo in self.voName:
# get the known CEs for a given VO, so we can know the unknowns, or no longer supported,
# for a VO
res = getQueues(community=vo)
if not res['OK']:
return res
knownCEs = set()
for _site, ces in res['Value'].items():
knownCEs.update(ces)
knownCEs.update(bannedCEs)
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
continue # next VO
siteDict = result['Value']
unknownCEs = set(result['UnknownCEs']) - set(bannedCEs)
body = ''
for site in siteDict:
newCEs = set(siteDict[site]) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if siteDict:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
if unknownCEs:
body += '\n\n'
body += 'There is no (longer) information about the following CEs for the %s VO.\n' % vo
body += '\n'.join(sorted(unknownCEs))
body += '\n\n'
if body:
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom,
localAttempt=False, avoidSpam=True)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo, host=self.host, glue2=self.glue2Only)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii", mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii, glue2=self.glue2Only)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii, resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
for glue2URL in self.glue2URLs:
if self.glue2Only:
break
resultGlue2 = getBdiiCEInfo(vo, host=glue2URL, glue2=True)
if resultGlue2['OK']:
totalResult['Value'].update(resultGlue2['Value'])
else:
self.log.error("Failed getting GLUE2 information for", "%s, %s: %s" %
(glue2URL, vo, resultGlue2['Message']))
message = (message + "\n" + resultGlue2['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult['Value'] and message: # Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
self.__purgeSites(totalResult['Value'])
return totalResult
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __purgeSites(self, ceBdiiDict):
"""Remove all sites that are not in self.selectedSites.
Modifies the ceBdiiDict!
"""
if not self.selectedSites:
return
for site in list(ceBdiiDict):
ces = list(ceBdiiDict[site]['CEs'])
if not ces:
self.log.error("No CE information for site:", site)
continue
siteInCS = 'Not_In_CS'
for ce in ces:
res = getCESiteMapping(ce)
if not res['OK']:
self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ce, res['Message']))
continue
# if the ce is not in the CS the returned value will be empty
if ce in res['Value']:
siteInCS = res['Value'][ce]
break
self.log.debug("Checking site %s (%s), aka %s" % (site, ces, siteInCS))
if siteInCS in self.selectedSites:
continue
self.log.info("Dropping site %s, aka %s" % (site, siteInCS))
ceBdiiDict.pop(site)
return
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items(): # can be an iterator
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = sorted(changeSet)
body = '\n'.join(["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False,
avoidSpam=True)
if body:
self.log.info('The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS", result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" % len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = "Bdii2CSAgent"
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = None
# What to get
self.processCEs = True
self.processSEs = False
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs',
self.alternativeBDIIs)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data",
result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set(result['Value'])
knownCEs = knownCEs.union(set(bannedCEs))
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get(
'GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get(
'GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (
queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii",
mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii,
resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult[
'Value'] and message: ## Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiSEInfo(self, vo):
if vo in self.voBdiiSEDict:
return S_OK(self.voBdiiSEDict[vo])
self.log.info("Check for available SEs for VO", vo)
result = getBdiiSEInfo(vo)
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs:
result = getBdiiSEInfo(vo, host=bdii)
if result['OK']:
break
if not result['OK']:
if message:
self.log.error("Error during BDII request", message)
else:
self.log.error("Error during BDII request", result['Message'])
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(
set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = list(changeSet)
changeList.sort()
body = '\n'.join(
["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if body:
self.log.info(
'The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS",
result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" %
len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get(
'GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get(
'GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def __updateSEs(self):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates(vo, bdiiInfo=seBdiiDict)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
class VOMS2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__(*args, **kwargs)
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
self.autoDeleteUsers = False
self.detailedReport = True
self.makeFCEntry = False
def initialize(self):
""" Initialize the default parameters
"""
voNames = self.am_getOption('VO', [])
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs(voNames)
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice("VOs: %s" % self.__voDict.keys())
self.csapi = CSAPI()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.autoAddUsers = self.am_getOption('AutoAddUsers',
self.autoAddUsers)
self.autoModifyUsers = self.am_getOption('AutoModifyUsers',
self.autoModifyUsers)
self.autoDeleteUsers = self.am_getOption('AutoDeleteUsers',
self.autoDeleteUsers)
self.detailedReport = self.am_getOption('DetailedReport',
self.detailedReport)
self.makeFCEntry = self.am_getOption('MakeHomeDirectory',
self.makeFCEntry)
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
resultDict = result['Value']
newUsers = resultDict.get("NewUsers", [])
modUsers = resultDict.get("ModifiedUsers", [])
delUsers = resultDict.get("DeletedUsers", [])
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info(
"There are changes to the CS for vo %s ready to be committed"
% vo)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error(
"Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed for VO %s" % vo)
else:
self.log.info(
"No changes to the CS for VO %s recorded at this cycle" %
vo)
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info("Creating home directories for users %s" %
str(newUsers))
result = self.__addHomeDirectory(vo,
newUsers,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to create user home directories:',
'VO %s: %s' % (vo, result["Message"]))
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs['Info'].append(
"Created home directory for user %s" % user)
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
if self.detailedReport:
result = self.getVOUserReport(vo)
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption(
'MailFrom',
self.am_getOption('mailFrom', "DIRAC system")))
return S_OK()
def getVOUserData(self, vo, refreshFlag=False):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def getVOUserReport(self, vo):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData(vo, refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append(
(group, str(groupDict[group]), diracVOMSMapping.get(group,
'')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields,
records,
sortField='Group',
printOut=False,
numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
@executeWithUserProxy
def __syncCSWithVOMS(self, vo):
self.__adminMsgs = {'Errors': [], 'Info': []}
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(vo)
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error('Could not retrieve VOMS VO name', "for %s" % vo)
return result
vomsVOName = result['Value'].lstrip('/')
self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName))
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (
len(vomsUserDict), vomsVOName)
self.__adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append(dn)
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict.keys()))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname(dn, vomsUserDict[dn]['CA'])
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName(dn,
vomsUserDict[dn]['mail'],
vo)
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.__adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict[diracName]['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(
set(existingGroups) - set(diracVOMSMapping.keys()))
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(
diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(
diracUserDict.get(diracName, {}).get(
key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(
addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(
removedGroups)
else:
oldValue = str(
diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (
key, oldValue, str(userDict[key]))
if modMsg:
self.__adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" %
(diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(
vomsUserDict.keys())) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' %
str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.__adminMsgs['Info'].append(
'The following users are deleted from CS:\n %s\n' %
str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs['Errors'].append(
'Error in deleting users from CS:\n %s' %
str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.__adminMsgs['Info'].append(
'The following users to be checked for deletion:\n %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
return S_OK(resultDict)
@executeWithUserProxy
def __addHomeDirectory(self, vo, newUsers):
fc = FileCatalog(vo=vo)
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser(user)
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % (vo, user[0], user)
result = fc.createDirectory(dirName)
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner({dirName: user}, recursive=True)
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup({dirName: defaultVOGroup},
recursive=True)
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK({"Successful": successful, "Failed": failed})
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = "Bdii2CSAgent"
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.host = 'lcg-bdii.cern.ch:2170'
self.glue2URLs = []
self.glue2Only = False
self.csAPI = None
# What to get
self.processCEs = True
self.processSEs = False
self.selectedSites = []
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data", result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set(result['Value'])
knownCEs = knownCEs.union(set(bannedCEs))
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom,
localAttempt=False, avoidSpam=True)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo, host=self.host, glue2=self.glue2Only)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii", mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii, glue2=self.glue2Only)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii, resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
for glue2URL in self.glue2URLs:
if self.glue2Only:
break
resultGlue2 = getBdiiCEInfo(vo, host=glue2URL, glue2=True)
if resultGlue2['OK']:
totalResult['Value'].update(resultGlue2['Value'])
else:
self.log.error("Failed getting GLUE2 information for", "%s, %s: %s" %
(glue2URL, vo, resultGlue2['Message']))
message = (message + "\n" + resultGlue2['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult['Value'] and message: # Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiSEInfo(self, vo):
if vo in self.voBdiiSEDict:
return S_OK(self.voBdiiSEDict[vo])
self.log.info("Check for available SEs for VO", vo)
result = getBdiiSEInfo(vo)
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs:
result = getBdiiSEInfo(vo, host=bdii)
if result['OK']:
break
if not result['OK']:
if message:
self.log.error("Error during BDII request", message)
else:
self.log.error("Error during BDII request", result['Message'])
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
self.__purgeSites(ceBdiiDict)
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __purgeSites(self, ceBdiiDict):
"""Remove all sites that are not in self.selectedSites.
Modifies the ceBdiiDict!
"""
if not self.selectedSites:
return
for site in list(ceBdiiDict):
ces = list(ceBdiiDict[site]['CEs'])
if not ces:
self.log.error("No CE information for site:", site)
continue
diracSiteName = getSiteForCE(ces[0])
if not diracSiteName['OK']:
self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ces[0], diracSiteName['Message']))
continue
self.log.debug("Checking site %s (%s), aka %s" % (site, ces, diracSiteName['Value']))
if diracSiteName['Value'] in self.selectedSites:
continue
self.log.info("Dropping site %s, aka %s" % (site, diracSiteName))
ceBdiiDict.pop(site)
return
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = sorted(changeSet)
body = '\n'.join(["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
if body:
self.log.info('The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS", result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" % len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get('GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get('GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __updateSEs(self):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates(vo, bdiiInfo=seBdiiDict)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
class GOCDB2CSAgent(AgentModule):
""" Class to retrieve information about service endpoints
from GOCDB and update configuration stored by CS
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
super(GOCDB2CSAgent, self).__init__(*args, **kwargs)
self.GOCDBClient = None
self.csAPI = None
self.dryRun = False
def initialize(self):
""" Run at the agent initialization (normally every 500 cycles)
"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
"""
Execute GOCDB queries according to the function map
and user request (options in configuration).
"""
# __functionMap is at the end of the class definition
for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
optionValue = self.am_getOption(option, True)
if optionValue:
result = functionCall(self)
if not result['OK']:
self.log.error("%s() failed with message: %s" %
(functionCall.__name__, result['Message']))
else:
self.log.info("Successfully executed %s" %
functionCall.__name__)
return S_OK()
def updatePerfSONARConfiguration(self):
"""
Get current status of perfSONAR endpoints from GOCDB
and update CS configuration accordingly.
"""
log = self.log.getSubLogger('updatePerfSONAREndpoints')
log.debug('Begin function ...')
# get endpoints
result = self.__getPerfSONAREndpoints()
if not result['OK']:
log.error("__getPerfSONAREndpoints() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.')
endpointList = result['Value']
# add DIRAC site name
result = self.__addDIRACSiteName(endpointList)
if not result['OK']:
log.error("__addDIRACSiteName() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to extend the list with DIRAC site names.')
extendedEndpointList = result['Value']
# prepare dictionary with new configuration
result = self.__preparePerfSONARConfiguration(extendedEndpointList)
if not result['OK']:
log.error(
"__preparePerfSONARConfiguration() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to prepare a new perfSONAR configuration.')
finalConfiguration = result['Value']
# update configuration according to the final status of endpoints
self.__updateConfiguration(finalConfiguration)
log.debug("Configuration updated succesfully")
log.debug('End function.')
return S_OK()
def __getPerfSONAREndpoints(self):
"""
Retrieve perfSONAR endpoint information directly from GOCDB.
:return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB.
"""
log = self.log.getSubLogger('__getPerfSONAREndpoints')
log.debug('Begin function ...')
# get perfSONAR endpoints (latency and bandwidth) form GOCDB
endpointList = []
for endpointType in ['Latency', 'Bandwidth']:
result = self.GOCDBClient.getServiceEndpointInfo(
'service_type', 'net.perfSONAR.%s' % endpointType)
if not result['OK']:
log.error("getServiceEndpointInfo() failed with message: %s" %
result['Message'])
return S_ERROR('Could not fetch %s endpoints from GOCDB' %
endpointType.lower())
log.debug('Number of %s endpoints: %s' %
(endpointType.lower(), len(result['Value'])))
endpointList.extend(result['Value'])
log.debug('Number of perfSONAR endpoints: %s' % len(endpointList))
log.debug('End function.')
return S_OK(endpointList)
def __preparePerfSONARConfiguration(self, endpointList):
"""
Prepare a dictionary with a new CS configuration of perfSONAR endpoints.
:return: Dictionary where keys are configuration paths (options and sections)
and values are values of corresponding options
or None in case of a path pointing to a section.
"""
log = self.log.getSubLogger('__preparePerfSONARConfiguration')
log.debug('Begin function ...')
# static elements of a path
rootPath = '/Resources/Sites'
extPath = 'Network'
baseOptionName = 'Enabled'
options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'}
# enable GOCDB endpoints in configuration
newConfiguration = {}
for endpoint in endpointList:
if endpoint['DIRACSITENAME'] is None:
continue
split = endpoint['DIRACSITENAME'].split('.')
path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'],
extPath, endpoint['HOSTNAME'])
for name, defaultValue in options.iteritems():
newConfiguration[cfgPath(path, name)] = defaultValue
# get current configuration
currentConfiguration = {}
for option in options.iterkeys():
result = gConfig.getConfigurationTree(rootPath, extPath + '/',
'/' + option)
if not result['OK']:
log.error("getConfigurationTree() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
currentConfiguration.update(result['Value'])
# disable endpoints that disappeared in GOCDB
removedElements = set(currentConfiguration) - set(newConfiguration)
newElements = set(newConfiguration) - set(currentConfiguration)
addedEndpoints = len(newElements) / len(options)
disabledEndpoints = 0
for path in removedElements:
if baseOptionName in path:
newConfiguration[path] = 'False'
if currentConfiguration[path] != 'False':
disabledEndpoints = disabledEndpoints + 1
# inform what will be changed
if addedEndpoints > 0:
self.log.info(
"%s new perfSONAR endpoints will be added to the configuration"
% addedEndpoints)
if disabledEndpoints > 0:
self.log.info(
"%s old perfSONAR endpoints will be disable in the configuration"
% disabledEndpoints)
if addedEndpoints == 0 and disabledEndpoints == 0:
self.log.info("perfSONAR configuration is up-to-date")
log.debug('End function.')
return S_OK(newConfiguration)
def __addDIRACSiteName(self, inputList):
"""
Extend given list of GOCDB endpoints with DIRAC site name, i.e.
add an entry "DIRACSITENAME" in dictionaries that describe endpoints.
If given site name could not be found "DIRACSITENAME" is set to 'None'.
:return: List of perfSONAR endpoints (dictionaries).
"""
log = self.log.getSubLogger('__addDIRACSiteName')
log.debug('Begin function ...')
# get site name dictionary
result = getDIRACGOCDictionary()
if not result['OK']:
log.error("getDIRACGOCDictionary() failed with message: %s" %
result['Message'])
return S_ERROR('Could not get site name dictionary')
# reverse the dictionary (assume 1 to 1 relation)
DIRACGOCDict = result['Value']
GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys()))
# add DIRAC site names
outputList = []
for entry in inputList:
try:
entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']]
except KeyError:
self.log.warn("No dictionary entry for %s. " %
entry['SITENAME'])
entry['DIRACSITENAME'] = None
outputList.append(entry)
log.debug('End function.')
return S_OK(outputList)
def __updateConfiguration(self, setElements=None, delElements=None):
"""
Update configuration stored by CS.
"""
if setElements is None:
setElements = {}
if delElements is None:
delElements = []
log = self.log.getSubLogger('__updateConfiguration')
log.debug('Begin function ...')
# assure existence and proper value of a section or an option
for path, value in setElements.iteritems():
if value is None:
section = path
else:
split = path.rsplit('/', 1)
section = split[0]
try:
result = self.csAPI.createSection(section)
if not result['OK']:
log.error("createSection() failed with message: %s" %
result['Message'])
except Exception as e:
log.error("Exception in createSection(): %s" %
repr(e).replace(',)', ')'))
if value is not None:
try:
result = self.csAPI.setOption(path, value)
if not result['OK']:
log.error("setOption() failed with message: %s" %
result['Message'])
except Exception as e:
log.error("Exception in setOption(): %s" %
repr(e).replace(',)', ')'))
# delete elements in the configuration
for path in delElements:
result = self.csAPI.delOption(path)
if not result['OK']:
log.warn("delOption() failed with message: %s" %
result['Message'])
result = self.csAPI.delSection(path)
if not result['OK']:
log.warn("delSection() failed with message: %s" %
result['Message'])
if self.dryRun:
log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
# update configuration stored by CS
result = self.csAPI.commit()
if not result['OK']:
log.error("commit() failed with message: %s" %
result['Message'])
return S_ERROR("Could not commit changes to CS.")
else:
log.info("Committed changes to CS")
log.debug('End function.')
return S_OK()
# define mapping between an agent option in the configuration and a function call
__functionMap = {
'UpdatePerfSONARS': updatePerfSONARConfiguration,
}
class VOMS2CSAgent( AgentModule ):
def __init__( self, *args, **kwargs ):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__( *args, **kwargs )
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
self.autoDeleteUsers = False
self.detailedReport = True
self.makeFCEntry = False
def initialize( self ):
""" Initialize the default parameters
"""
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.csapi = CSAPI()
self.dryRun = self.am_getOption( 'DryRun', self.dryRun )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers )
self.autoDeleteUsers = self.am_getOption( 'AutoDeleteUsers', self.autoDeleteUsers )
self.detailedReport = self.am_getOption( 'DetailedReport', self.detailedReport )
self.makeFCEntry = self.am_getOption( 'MakeHomeDirectory', self.makeFCEntry )
return S_OK()
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
resultDict = result['Value']
newUsers = resultDict.get( "NewUsers", [] )
modUsers = resultDict.get( "ModifiedUsers", [] )
delUsers = resultDict.get( "DeletedUsers", [] )
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo )
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.notice( "Configuration committed for VO %s" % vo )
else:
self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo )
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info( "Creating home directories for users %s" % str( newUsers ) )
result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) )
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user )
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
if self.detailedReport:
result = self.getVOUserReport( vo )
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'MailFrom', self.am_getOption( 'mailFrom', "DIRAC system" ) ) )
return S_OK()
def getVOUserData( self, vo, refreshFlag = False ):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
def getVOUserReport( self, vo ):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData( vo, refreshFlag = True )
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict( int )
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar( userDict[user]['DN'] )
if len( dnList ) > 1:
multiDNUsers[user] = dnList
if userDict[user].get( 'Status', 'Active' ) == 'Suspended':
suspendedUsers.append( user )
for group in diracVOMSMapping:
records.append( ( group, str( groupDict[group] ), diracVOMSMapping.get( group, '' ) ) )
fields = [ 'Group', 'Number of users', 'VOMS Role' ]
output = printTable( fields, records, sortField = 'Group', printOut = False, numbering = False )
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len( suspendedUsers )
output += ' %s' % ','.join( suspendedUsers )
return S_OK( output )
@executeWithUserProxy
def __syncCSWithVOMS( self, vo ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
resultDict = defaultdict( list )
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService( vo )
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
return result
vomsVOName = result[ 'Value' ].lstrip( '/' )
self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
return result
vomsUserDict = result[ 'Value' ]
message = "There are %s user entries in VOMS for VO %s" % ( len( vomsUserDict ), vomsVOName )
self.__adminMsgs[ 'Info' ].append( message )
self.log.info( message )
# Get DIRAC users
result = self.getVOUserData( vo )
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) )
self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) )
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar( dn )
existingDNs.extend( dnList )
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append( dn )
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append( dn )
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list( set( allDiracUsers ) - set( diracUserDict.keys() ) )
if nonVOUsers:
result = self.csapi.describeUsers( nonVOUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] )
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName( dn, vomsUserDict[dn]['mail'], vo )
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % ( trialName, ind )
ind += 1
# We now have everything to add the new user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) )
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % ( key, str( userDict[key] ) )
self.__adminMsgs[ 'Info' ].append( message )
self.voChanged = True
if self.autoAddUsers:
self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True )
if not result['OK']:
self.log.warn( 'Failed adding new user %s' % newDiracName )
resultDict['NewUsers'].append( newDiracName )
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
if newDNForExistingUser:
userDict['DN'] = ','.join( [ dn, diracUserDict[diracName]['DN'] ] )
modified = True
existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] )
nonVOGroups = list( set( existingGroups ) - set( diracVOMSMapping.keys() ) )
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get( group, '' )
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append( group )
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append( group )
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append( group )
userDict['Groups'] = list( set( keepGroups ) )
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get( 'Groups', [] )
userDict['Groups'] = list( set( keepGroups + otherGroups ) )
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set( userDict[key] ) - set( diracUserDict.get( diracName, {} ).get( key, [] ) )
removedGroups = set( diracUserDict.get( diracName, {} ).get( key, [] ) ) - set( userDict[key] )
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join( addedGroups )
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups )
else:
oldValue = str( diracUserDict.get( diracName, {} ).get( key, '' ) )
if str( userDict[key] ) != oldValue:
modMsg += " %s: %s -> %s\n" % ( key, oldValue, str( userDict[key] ) )
if modMsg:
self.__adminMsgs[ 'Info' ].append( message + modMsg )
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser( diracName, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: %s" % ( diracName, str( userDict ) ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
if not dnSet.intersection( set( vomsUserDict.keys() ) ) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add( user )
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose( "Modified user %s: dropped DN %s" % ( user, dn ) )
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set( [dn] )
if modDNSet:
userDict['DN'] = ','.join( modDNSet )
result = self.csapi.modifyUser( user, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.__adminMsgs[ 'Info' ].append( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
else:
oldUsers.add( user )
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info( 'The following users will be deleted: %s' % str( oldUsers ) )
result = self.csapi.deleteUsers( oldUsers )
if result['OK']:
self.__adminMsgs[ 'Info' ].append( 'The following users are deleted from CS:\n %s\n' % str( oldUsers ) )
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs[ 'Errors' ].append( 'Error in deleting users from CS:\n %s' % str( oldUsers ) )
self.log.error( 'Error while user deletion from CS', result )
else:
self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion:\n %s' % str( oldUsers ) )
self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
return S_OK( resultDict )
@executeWithUserProxy
def __addHomeDirectory( self, vo, newUsers ):
fc = FileCatalog( vo = vo )
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser( user )
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % ( vo, user[0], user )
result = fc.createDirectory( dirName )
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner( { dirName: user }, recursive = True )
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup( { dirName: defaultVOGroup }, recursive = True )
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK( { "Successful": successful, "Failed": failed } )
class VOMS2CSAgent( AgentModule ):
def __init__( self, *args, **kwargs ):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__( *args, **kwargs )
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
def initialize( self ):
""" Initialize the default parameters
"""
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.csapi = CSAPI()
self.dryRun = self.am_getOption( 'DryRun', self.dryRun )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers )
return S_OK()
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'mailFrom', "DIRAC system" ) )
# We have accumulated all the changes, commit them now
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.info( "Configuration committed" )
return S_OK()
@executeWithUserProxy
def __syncCSWithVOMS( self, vo ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
vomsSrv = VOMSService( vo )
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
return result
vomsVOName = result[ 'Value' ].lstrip( '/' )
self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
return result
vomsUserDict = result[ 'Value' ]
message = "There are %s registered users in VOMS VO %s" % ( len( vomsUserDict ), vomsVOName )
self.__adminMsgs[ 'Info' ].append( message )
self.log.info( message )
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
diracUserDict = result['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) )
self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) )
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
existingDNs.append( dn )
if dn not in vomsUserDict:
obsoletedDNs.append( dn )
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append( dn )
allDiracUsers = getAllUsers()
nonVOusers = list( set( allDiracUsers ) - set(diracUsers) )
result = self.csapi.describeUsers( nonVOusers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
for dn in vomsUserDict:
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
diracName = ''
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# We have a real new user
if not diracName:
nickName = ''
result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] )
if result['OK']:
nickName = result['Value']
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName( dn, vomsUserDict[dn]['mail'] )
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % ( trialName, ind )
ind += 1
# We now have everything to add the new user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
group = vomsDIRACMapping.get( fullRole )
if group:
groupsWithRole.append( group )
userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) )
self.__adminMsgs[ 'Info' ].append( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
self.voChanged = True
if self.autoAddUsers:
self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True )
if not result['OK']:
self.log.warn( 'Failed adding new user %s' % newDiracName )
continue
# We have an already existing user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
nonVOGroups = nonVOUserDict.get( diracName, {} ).get( 'Groups', [] )
existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] )
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
group = vomsDIRACMapping.get( fullRole )
if group:
groupsWithRole.append( group )
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
role = diracVOMSMapping[group]
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append( group )
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append( group )
userDict['Groups'] = keepGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser( diracName, userDict )
if result['OK'] and result['Value']:
self.voChanged = True
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dn = diracUserDict[user]['DN']
if not dn in vomsUserDict and not user in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if not group in noVOMSGroups:
oldUsers.add( user )
if oldUsers:
self.voChanged = True
self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
return S_OK()
class GOCDB2CSAgent (AgentModule):
""" Class to retrieve information about service endpoints
from GOCDB and update configuration stored by CS
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
super(GOCDB2CSAgent, self).__init__(*args, **kwargs)
self.GOCDBClient = None
self.csAPI = None
self.dryRun = False
def initialize(self):
""" Run at the agent initialization (normally every 500 cycles)
"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
"""
Execute GOCDB queries according to the function map
and user request (options in configuration).
"""
# __functionMap is at the end of the class definition
for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
optionValue = self.am_getOption(option, True)
if optionValue:
result = functionCall(self)
if not result['OK']:
self.log.error("%s() failed with message: %s" % (functionCall.__name__, result['Message']))
else:
self.log.info("Successfully executed %s" % functionCall.__name__)
return S_OK()
def updatePerfSONARConfiguration(self):
"""
Get current status of perfSONAR endpoints from GOCDB
and update CS configuration accordingly.
"""
log = self.log.getSubLogger('updatePerfSONAREndpoints')
log.debug('Begin function ...')
# get endpoints
result = self.__getPerfSONAREndpoints()
if not result['OK']:
log.error("__getPerfSONAREndpoints() failed with message: %s" % result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.')
endpointList = result['Value']
# add DIRAC site name
result = self.__addDIRACSiteName(endpointList)
if not result['OK']:
log.error("__addDIRACSiteName() failed with message: %s" % result['Message'])
return S_ERROR('Unable to extend the list with DIRAC site names.')
extendedEndpointList = result['Value']
# prepare dictionary with new configuration
result = self.__preparePerfSONARConfiguration(extendedEndpointList)
if not result['OK']:
log.error("__preparePerfSONARConfiguration() failed with message: %s" % result['Message'])
return S_ERROR('Unable to prepare a new perfSONAR configuration.')
finalConfiguration = result['Value']
# update configuration according to the final status of endpoints
self.__updateConfiguration(finalConfiguration)
log.debug("Configuration updated succesfully")
log.debug('End function.')
return S_OK()
def __getPerfSONAREndpoints(self):
"""
Retrieve perfSONAR endpoint information directly from GOCDB.
:return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB.
"""
log = self.log.getSubLogger('__getPerfSONAREndpoints')
log.debug('Begin function ...')
# get perfSONAR endpoints (latency and bandwidth) form GOCDB
endpointList = []
for endpointType in ['Latency', 'Bandwidth']:
result = self.GOCDBClient.getServiceEndpointInfo('service_type', 'net.perfSONAR.%s' % endpointType)
if not result['OK']:
log.error("getServiceEndpointInfo() failed with message: %s" % result['Message'])
return S_ERROR('Could not fetch %s endpoints from GOCDB' % endpointType.lower())
log.debug('Number of %s endpoints: %s' % (endpointType.lower(), len(result['Value'])))
endpointList.extend(result['Value'])
log.debug('Number of perfSONAR endpoints: %s' % len(endpointList))
log.debug('End function.')
return S_OK(endpointList)
def __preparePerfSONARConfiguration(self, endpointList):
"""
Prepare a dictionary with a new CS configuration of perfSONAR endpoints.
:return: Dictionary where keys are configuration paths (options and sections)
and values are values of corresponding options
or None in case of a path pointing to a section.
"""
log = self.log.getSubLogger('__preparePerfSONARConfiguration')
log.debug('Begin function ...')
# static elements of a path
rootPath = '/Resources/Sites'
extPath = 'Network'
baseOptionName = 'Enabled'
options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'}
# enable GOCDB endpoints in configuration
newConfiguration = {}
for endpoint in endpointList:
if endpoint['DIRACSITENAME'] is None:
continue
split = endpoint['DIRACSITENAME'].split('.')
path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME'])
for name, defaultValue in options.iteritems():
newConfiguration[cfgPath(path, name)] = defaultValue
# get current configuration
currentConfiguration = {}
for option in options.iterkeys():
result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option)
if not result['OK']:
log.error("getConfigurationTree() failed with message: %s" % result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
currentConfiguration.update(result['Value'])
# disable endpoints that disappeared in GOCDB
removedElements = set(currentConfiguration) - set(newConfiguration)
newElements = set(newConfiguration) - set(currentConfiguration)
addedEndpoints = len(newElements) / len(options)
disabledEndpoints = 0
for path in removedElements:
if baseOptionName in path:
newConfiguration[path] = 'False'
if currentConfiguration[path] != 'False':
disabledEndpoints = disabledEndpoints + 1
# inform what will be changed
if addedEndpoints > 0:
self.log.info("%s new perfSONAR endpoints will be added to the configuration" % addedEndpoints)
if disabledEndpoints > 0:
self.log.info("%s old perfSONAR endpoints will be disable in the configuration" % disabledEndpoints)
if addedEndpoints == 0 and disabledEndpoints == 0:
self.log.info("perfSONAR configuration is up-to-date")
log.debug('End function.')
return S_OK(newConfiguration)
def __addDIRACSiteName(self, inputList):
"""
Extend given list of GOCDB endpoints with DIRAC site name, i.e.
add an entry "DIRACSITENAME" in dictionaries that describe endpoints.
If given site name could not be found "DIRACSITENAME" is set to 'None'.
:return: List of perfSONAR endpoints (dictionaries).
"""
log = self.log.getSubLogger('__addDIRACSiteName')
log.debug('Begin function ...')
# get site name dictionary
result = getDIRACGOCDictionary()
if not result['OK']:
log.error("getDIRACGOCDictionary() failed with message: %s" % result['Message'])
return S_ERROR('Could not get site name dictionary')
# reverse the dictionary (assume 1 to 1 relation)
DIRACGOCDict = result['Value']
GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys()))
# add DIRAC site names
outputList = []
for entry in inputList:
try:
entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']]
except KeyError:
self.log.warn("No dictionary entry for %s. " % entry['SITENAME'])
entry['DIRACSITENAME'] = None
outputList.append(entry)
log.debug('End function.')
return S_OK(outputList)
def __updateConfiguration(self, setElements=None, delElements=None):
"""
Update configuration stored by CS.
"""
if setElements is None:
setElements = {}
if delElements is None:
delElements = []
log = self.log.getSubLogger('__updateConfiguration')
log.debug('Begin function ...')
# assure existence and proper value of a section or an option
for path, value in setElements.iteritems():
if value is None:
section = path
else:
split = path.rsplit('/', 1)
section = split[0]
try:
result = self.csAPI.createSection(section)
if not result['OK']:
log.error("createSection() failed with message: %s" % result['Message'])
except Exception as e:
log.error("Exception in createSection(): %s" % repr(e).replace(',)', ')'))
if value is not None:
try:
result = self.csAPI.setOption(path, value)
if not result['OK']:
log.error("setOption() failed with message: %s" % result['Message'])
except Exception as e:
log.error("Exception in setOption(): %s" % repr(e).replace(',)', ')'))
# delete elements in the configuration
for path in delElements:
result = self.csAPI.delOption(path)
if not result['OK']:
log.warn("delOption() failed with message: %s" % result['Message'])
result = self.csAPI.delSection(path)
if not result['OK']:
log.warn("delSection() failed with message: %s" % result['Message'])
if self.dryRun:
log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
# update configuration stored by CS
result = self.csAPI.commit()
if not result['OK']:
log.error("commit() failed with message: %s" % result['Message'])
return S_ERROR("Could not commit changes to CS.")
else:
log.info("Committed changes to CS")
log.debug('End function.')
return S_OK()
# define mapping between an agent option in the configuration and a function call
__functionMap = {'UpdatePerfSONARS': updatePerfSONARConfiguration,
}