def notifyAboutNewSoftware(self):
"""Send an email to the mailing list if a new software version was defined"""
#Only send email when something was actually added
if not self.modifiedCS:
return
subject = '%s %s added to DIRAC CS' % (self.appName, self.appVersion)
msg = 'New application %s %s declared into Configuration service\n %s' % (
self.appName, self.appVersion, self.comment)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUserOption
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
notifyClient = NotificationClient()
gLogger.notice('Sending mail for software installation to %s' %
(self.mailadress))
res = getProxyInfo()
if not res['OK']:
sender = '*****@*****.**'
else:
if 'username' in res['Value']:
sender = getUserOption(res['Value']['username'], 'Email')
else:
sender = '*****@*****.**'
gLogger.info('*' * 80) # surround email with stars
res = notifyClient.sendMail(self.mailadress,
subject,
msg,
sender,
localAttempt=False)
gLogger.info('*' * 80)
if not res['OK']:
gLogger.error('The mail could not be sent: %s' % res['Message'])
def notifyAboutNewSoftware(self):
"""Send an email to the mailing list if a new software version was defined"""
#Only send email when something was actually added
if not self.modifiedCS:
return
subject = '%s %s added to DIRAC CS' % (self.appName, self.appVersion)
msg = 'New application %s %s declared into Configuration service\n %s' % (self.appName,
self.appVersion,
self.comment)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUserOption
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
notifyClient = NotificationClient()
gLogger.notice('Sending mail for software installation to %s' % (self.mailadress))
res = getProxyInfo()
if not res['OK']:
sender = '*****@*****.**'
else:
if 'username' in res['Value']:
sender = getUserOption(res['Value']['username'],'Email')
else:
sender = '*****@*****.**'
gLogger.info('*'*80)# surround email with stars
res = notifyClient.sendMail(self.mailadress, subject, msg, sender, localAttempt = False)
gLogger.info('*'*80)
if not res[ 'OK' ]:
gLogger.error('The mail could not be sent: %s' % res['Message'])
def initialize(self):
self.systemLoggingDB = SystemLoggingDB()
self.notification = NotificationClient()
userList = self.am_getOption("Reviewer", [])
self.log.debug("Users to be notified:", ', '.join(userList))
mailList = []
for user in userList:
mail = getUserOption(user, 'Email', '')
if not mail:
self.log.warn("Could not get user's mail", user)
else:
mailList.append(mail)
if not mailList:
mailList = Operations().getValue('EMail/Logging', [])
if not len(mailList):
errString = "There are no valid users in the mailing list"
varString = "[" + ','.join(userList) + "]"
self.log.warn(errString, varString)
self.log.info("List of mails to be notified", ','.join(mailList))
self._mailAddress = mailList
self._subject = 'New error messages were entered in the SystemLoggingDB'
return S_OK()
def initialize( self ):
self.systemLoggingDB = SystemLoggingDB()
self.notification = NotificationClient()
userList = self.am_getOption( "Reviewer", [] )
self.log.debug( "Users to be notified:", ', '.join( userList ) )
mailList = []
for user in userList:
mail = getUserOption( user, 'Email', '' )
if not mail:
self.log.warn( "Could not get user's mail", user )
else:
mailList.append( mail )
if not mailList:
mailList = Operations().getValue( 'EMail/Logging', [] )
if not len( mailList ):
errString = "There are no valid users in the mailing list"
varString = "[" + ','.join( userList ) + "]"
self.log.warn( errString, varString )
self.log.info( "List of mails to be notified", ','.join( mailList ) )
self._mailAddress = mailList
self._subject = 'New error messages were entered in the SystemLoggingDB'
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption('mailFrom', "DIRAC system"))
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info("There are changes to the CS ready to be committed")
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed")
else:
self.log.info("No changes to the CS recorded at this cycle")
return S_OK()
def _getMemberMails(group):
""" get members mails
"""
members = getUsersInGroup(group)
if members:
emails = []
for user in members:
email = getUserOption(user, 'Email')
if email:
emails.append(email)
return emails
def _sendReport(self, report):
"""sends a given report to the production manager
"""
if not self.email:
self.email = getUserOption(self.operations.getValue("Shifter/ProductionManager/User"), 'Email')
body = '\n'.join(report['body'])
res = self.notifyClient.sendMail(
self.email,
report['subject'],
body,
self.email,
localAttempt=False,
avoidSpam=True)
if not res['OK']:
self.log.error("sendMail failed", res['Message'])
else:
self.log.info('Mail summary sent to production manager')
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'mailFrom', "DIRAC system" ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info( "There are changes to the CS ready to be committed" )
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.notice( "Configuration committed" )
else:
self.log.info( "No changes to the CS recorded at this cycle" )
return S_OK()
def initialize(self):
self.systemLoggingDB = SystemLoggingDB()
self.agentName = self.am_getModuleParam('fullName')
self.notification = NotificationClient()
mailList = self.am_getOption("MailList", [])
userList = self.am_getOption("Reviewer", [])
self.log.debug("Users to be notified:", ', '.join(userList))
for user in userList:
mail = getUserOption(user, 'Email', '')
if not mail:
self.log.warn("Could not get user's mail", user)
else:
mailList.append(mail)
if not mailList:
mailList = Operations().getValue('EMail/Logging', [])
if not mailList:
errString = "There are no valid users in the list of email where to send the report"
errString += "\nPlease specify some in Operations/<default>/EMail/Logging"
varString = "[" + ','.join(userList) + "]"
self.log.error(errString, varString)
return S_ERROR(errString + varString)
self.log.info("List of mails to be notified", ','.join(mailList))
self._mailAddress = mailList
self._threshold = int(self.am_getOption('Threshold', 10))
self.__days = self.am_getOption('QueryPeriod', 7)
self._period = int(self.__days) * day
self._limit = int(self.am_getOption('NumberOfErrors', 10))
string = "The %i most common errors in the SystemLoggingDB" % self._limit
self._subject = string + " for the last %s days" % self.__days
return S_OK()
def initialize(self):
self.systemLoggingDB = SystemLoggingDB()
self.agentName = self.am_getModuleParam('fullName')
self.notification = NotificationClient()
mailList = self.am_getOption("MailList", [])
userList = self.am_getOption("Reviewer", [])
self.log.debug("Users to be notified:", ', '.join(userList))
for user in userList:
mail = getUserOption(user, 'Email', '')
if not mail:
self.log.warn("Could not get user's mail", user)
else:
mailList.append(mail)
if not mailList:
mailList = Operations().getValue('EMail/Logging', [])
if not mailList:
errString = "There are no valid users in the list of email where to send the report"
errString += "\nPlease specify some in Operations/<default>/EMail/Logging"
varString = "[" + ','.join(userList) + "]"
self.log.error(errString, varString)
return S_ERROR(errString + varString)
self.log.info("List of mails to be notified", ','.join(mailList))
self._mailAddress = mailList
self._threshold = int(self.am_getOption('Threshold', 10))
self.__days = self.am_getOption('QueryPeriod', 7)
self._period = int(self.__days) * day
self._limit = int(self.am_getOption('NumberOfErrors', 10))
string = "The %i most common errors in the SystemLoggingDB" % self._limit
self._subject = string + " for the last %s days" % self.__days
return S_OK()
def initialize(self):
self.systemLoggingDB = SystemLoggingDB()
self.agentName = self.am_getModuleParam("fullName")
self.notification = NotificationClient()
mailList = self.am_getOption("MailList", [])
userList = self.am_getOption("Reviewer", [])
self.log.debug("Users to be notified:", ", ".join(userList))
for user in userList:
mail = getUserOption(user, "Email", "")
if not mail:
self.log.warn("Could not get user's mail", user)
else:
mailList.append(mail)
if not mailList:
mailList = Operations().getValue("EMail/Logging", [])
if not len(mailList):
errString = "There are no valid users in the list"
varString = "[" + ",".join(userList) + "]"
self.log.error(errString, varString)
return S_ERROR(errString + varString)
self.log.info("List of mails to be notified", ",".join(mailList))
self._mailAddress = mailList
self._threshold = int(self.am_getOption("Threshold", 10))
self.__days = self.am_getOption("QueryPeriod", 7)
self._period = int(self.__days) * day
self._limit = int(self.am_getOption("NumberOfErrors", 10))
string = "The %i most common errors in the SystemLoggingDB" % self._limit
self._subject = string + " for the last %s days" % self.__days
return S_OK()
def execute(self):
for vo in self.voList:
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
"Performing VOMS sync", "for VO %s with credentials %s@%s" %
(vo, voAdminUser, voAdminGroup))
autoAddUsers = getVOOption(vo, "AutoAddUsers", self.autoAddUsers)
autoModifyUsers = getVOOption(vo, "AutoModifyUsers",
self.autoModifyUsers)
autoDeleteUsers = getVOOption(vo, "AutoDeleteUsers",
self.autoDeleteUsers)
autoLiftSuspendedStatus = getVOOption(vo,
"AutoLiftSuspendedStatus",
self.autoLiftSuspendedStatus)
syncPluginName = getVOOption(vo, "SyncPluginName",
self.syncPluginName)
vomsSync = VOMS2CSSynchronizer(
vo,
autoAddUsers=autoAddUsers,
autoModifyUsers=autoModifyUsers,
autoDeleteUsers=autoDeleteUsers,
autoLiftSuspendedStatus=autoLiftSuspendedStatus,
syncPluginName=syncPluginName,
)
result = self.__syncCSWithVOMS( # pylint: disable=unexpected-keyword-arg
vomsSync,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup,
)
if not result["OK"]:
self.log.error("Failed to perform VOMS to CS synchronization:",
"VO %s: %s" % (vo, result["Message"]))
continue
resultDict = result["Value"]
newUsers = resultDict.get("NewUsers", [])
modUsers = resultDict.get("ModifiedUsers", [])
delUsers = resultDict.get("DeletedUsers", [])
susUsers = resultDict.get("SuspendedUsers", [])
csapi = resultDict.get("CSAPI")
adminMessages = resultDict.get("AdminMessages", {
"Errors": [],
"Info": []
})
voChanged = resultDict.get("VOChanged", False)
self.log.info(
"Run user results",
": new %d, modified %d, deleted %d, new/suspended %d" %
(len(newUsers), len(modUsers), len(delUsers), len(susUsers)),
)
if csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info(
"There are changes to the CS ready to be committed",
"for VO %s" % vo)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
csapi.showDiff()
else:
result = csapi.commitChanges()
if not result["OK"]:
self.log.error(
"Could not commit configuration changes",
result["Message"])
return result
self.log.notice("Configuration committed",
"for VO %s" % vo)
else:
self.log.info("No changes to the CS recorded at this cycle",
"for VO %s" % vo)
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info("Creating home directories for users",
str(newUsers))
result = self.__addHomeDirectory( # pylint: disable=unexpected-keyword-arg
vo,
newUsers,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup,
)
if not result["OK"]:
self.log.error("Failed to create user home directories:",
"VO %s: %s" % (vo, result["Message"]))
else:
for user in result["Value"]["Failed"]:
self.log.error(
"Failed to create home directory",
"user: %s, operation: %s" %
(user, result["Value"]["Failed"][user]),
)
adminMessages["Errors"].append(
"Failed to create home directory for user %s: operation %s"
% (user, result["Value"]["Failed"][user]))
for user in result["Value"]["Successful"]:
adminMessages["Info"].append(
"Created home directory for user %s" % user)
if voChanged or self.detailedReport:
mailMsg = ""
if adminMessages["Errors"]:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
adminMessages["Errors"])
if adminMessages["Info"]:
mailMsg += "\nRun result:\n %s" % "\n ".join(
adminMessages["Info"])
if self.detailedReport:
result = vomsSync.getVOUserReport()
if result["OK"]:
mailMsg += "\n\n"
mailMsg += result["Value"]
else:
mailMsg += "Failed to produce a detailed user report"
mailMsg += result["Message"]
if self.dryRun:
self.log.info("Dry Run: mail won't be sent")
self.log.info(mailMsg)
else:
NotificationClient().sendMail(
self.am_getOption("MailTo", voAdminMail),
"VOMS2CSAgent run log", mailMsg, self.mailFrom)
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
resultDict = result['Value']
newUsers = resultDict.get("NewUsers", [])
modUsers = resultDict.get("ModifiedUsers", [])
delUsers = resultDict.get("DeletedUsers", [])
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info(
"There are changes to the CS for vo %s ready to be committed"
% vo)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error(
"Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed for VO %s" % vo)
else:
self.log.info(
"No changes to the CS for VO %s recorded at this cycle" %
vo)
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info("Creating home directories for users %s" %
str(newUsers))
result = self.__addHomeDirectory(vo,
newUsers,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to create user home directories:',
'VO %s: %s' % (vo, result["Message"]))
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs['Info'].append(
"Created home directory for user %s" % user)
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
if self.detailedReport:
result = self.getVOUserReport(vo)
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption(
'MailFrom',
self.am_getOption('mailFrom', "DIRAC system")))
return S_OK()
comment)
if not result['OK']:
gLogger.error("Error setting comment in CS")
#Commit the changes if nothing has failed and the CS has been modified
if modifiedCS:
result = diracAdmin.csCommitChanges(False)
if not result[ 'OK' ]:
gLogger.error('Commit failed with message = %s' % (result[ 'Message' ]))
dexit(255)
else:
gLogger.info('Successfully committed changes to CS')
notifyClient = NotificationClient()
gLogger.info('Sending mail for software installation %s' % (mailadress))
res = getProxyInfo()
if not res['OK']:
sender = '*****@*****.**'
else:
#sender = res['Value']['']
if 'username' in res['Value']:
sender = getUserOption(res['Value']['username'],'Email')
else:
sender = '*****@*****.**'
res = notifyClient.sendMail(mailadress, subject, msg, sender, localAttempt = False)
if not res[ 'OK' ]:
gLogger.error('The mail could not be sent')
else:
gLogger.info('No modifications to CS required')
gLogger.notice("All done!")
dexit(0)
def syncCSWithVOMS(self):
"""Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result["OK"]:
return result
vomsDIRACMapping = result["Value"]["VOMSDIRAC"]
diracVOMSMapping = result["Value"]["DIRACVOMS"]
noVOMSGroups = result["Value"]["NoVOMS"]
noSyncVOMSGroups = result["Value"]["NoSyncVOMS"]
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result["OK"]:
self.log.error("Could not retrieve user information from VOMS", result["Message"])
return result
self.vomsUserDict = result["Value"]
message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName)
self.adminMsgs["Info"].append(message)
self.log.info("VOMS user entries", message)
self.log.debug(self.vomsUserDict)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result["OK"]:
return result
diracUserDict = result["Value"]
self.adminMsgs["Info"].append(
"There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo)
)
self.log.info(
"Users already registered", ": there are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo)
)
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]["DN"]
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result["OK"]:
self.log.error("Could not retrieve CS User description")
return result
nonVOUserDict = result["Value"]
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ""
diracName = ""
if dn in existingDNs:
for user in diracUserDict:
if dn in fromChar(diracUserDict[user]["DN"]):
diracName = user
break
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn in fromChar(nonVOUserDict[user]["DN"]):
diracName = user
diracUserDict[diracName] = nonVOUserDict[user]
break
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get("nickname")
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]["CA"], "Email": self.vomsUserDict[dn]["mail"]}
groupsWithRole = []
for role in self.vomsUserDict[dn]["Roles"]:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict["Groups"] = list(set(groupsWithRole + [defaultVOGroup]))
# Run the sync plugins for extra info and/or validations
if self.syncPlugin:
try:
self.syncPlugin.verifyAndUpdateUserInfo(newDiracName, userDict)
except ValueError as e:
self.log.error("Error validating new user", "nickname %s\n error %s" % (newDiracName, e))
self.adminMsgs["Errors"].append(
"Error validating new user %s: %s\n %s" % (newDiracName, userDict, e)
)
continue
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs["Info"].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict)))
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
if not result["OK"]:
self.log.warn("Failed adding new user %s" % newDiracName)
resultDict["NewUsers"].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
suspendedInVOMS = self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]
suspendedVOList = getUserOption(diracName, "Suspended", [])
knownEmail = getUserOption(diracName, "Email", None)
userDict = {
"DN": diracUserDict[diracName]["DN"],
"CA": diracUserDict[diracName]["CA"],
"Email": self.vomsUserDict[dn].get("mail", self.vomsUserDict[dn].get("emailAddress")) or knownEmail,
}
# Set Suspended status for the user for this particular VO
if suspendedInVOMS and self.vo not in suspendedVOList:
suspendedVOList.append(self.vo)
userDict["Suspended"] = ",".join(suspendedVOList)
modified = True
# Remove the lifted Suspended status
if not suspendedInVOMS and self.vo in suspendedVOList and self.autoLiftSuspendedStatus:
newList = []
for vo in suspendedVOList:
if vo != self.vo:
newList.append(vo)
if not newList:
newList = ["None"]
userDict["Suspended"] = ",".join(newList)
modified = True
if newDNForExistingUser:
userDict["DN"] = ",".join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))["DN"]])
userDict["CA"] = ",".join(
[self.vomsUserDict[dn]["CA"], diracUserDict.get(diracName, newAddedUserDict.get(diracName))["CA"]]
)
modified = True
existingGroups = diracUserDict.get(diracName, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]["Roles"]:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
if quarantineVOGroup and quarantineVOGroup in existingGroups:
keepGroups = [quarantineVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, "")
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict["Groups"] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get("Groups", [])
userDict["Groups"] = list(set(keepGroups + otherGroups))
modified = True
if not existingGroups and diracName in allDiracUsers:
groups = getGroupsForUser(diracName)
if groups["OK"]:
self.log.info("Found groups for user %s %s" % (diracName, groups["Value"]))
userDict["Groups"] = list(set(groups["Value"] + keepGroups))
addedGroups = list(set(userDict["Groups"]) - set(groups["Value"]))
modified = True
message = "\n Modified user %s:\n" % diracName
message += " Added to group(s) %s\n" % ",".join(addedGroups)
self.adminMsgs["Info"].append(message)
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ""
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ",".join(addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ",".join(removedGroups)
elif key == "Suspended":
if userDict["Suspended"] == "None":
modMsg += " Suspended status removed\n"
else:
modMsg += " User Suspended in VOs: %s\n" % userDict["Suspended"]
else:
oldValue = str(diracUserDict.get(diracName, {}).get(key, ""))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs["Info"].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: %s" % (diracName, str(userDict)))
self.voChanged = True
resultDict["ModifiedUsers"].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]["DN"]))
if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict:
existingGroups = diracUserDict.get(user, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
removedGroups = list(set(existingGroups) - set(nonVOGroups))
if removedGroups:
self.log.info("Checking user for deletion", "%s: %s" % (user, existingGroups))
self.log.info("User has groups in other VOs", "%s: %s" % (user, nonVOGroups))
userDict = diracUserDict[user]
userDict["Groups"] = nonVOGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser(user, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: %s" % (user, str(userDict)))
self.voChanged = True
message = "\n Modified user %s:\n" % user
modMsg = " Removed from group(s) %s\n" % ",".join(removedGroups)
self.adminMsgs["Info"].append(message + modMsg)
resultDict["ModifiedUsers"].append(user)
continue
if not any(group in noVOMSGroups for group in existingGroups):
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]["DN"]))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
existingGroups = diracUserDict.get(user, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
if nonVOGroups:
self.log.verbose("User has groups in other VOs", "%s: %s" % (user, nonVOGroups))
continue
self.log.verbose("Modified user %s: dropped DN %s" % (user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict["DN"] = ",".join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: dropped DN %s" % (user, dn))
self.adminMsgs["Info"].append("Modified user %s: dropped DN %s" % (user, dn))
self.voChanged = True
resultDict["ModifiedUsers"].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info("The following users will be deleted: %s" % str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result["OK"]:
self.adminMsgs["Info"].append("The following users are deleted from CS:\n %s\n" % str(oldUsers))
resultDict["DeletedUsers"] = oldUsers
else:
self.adminMsgs["Errors"].append("Error in deleting users from CS:\n %s" % str(oldUsers))
self.log.error("Error while user deletion from CS", result)
else:
self.adminMsgs["Info"].append(
"The following users to be checked for deletion:\n\t%s" % "\n\t".join(sorted(oldUsers))
)
self.log.info("The following users to be checked for deletion:", "\n\t".join(sorted(oldUsers)))
resultDict["CSAPI"] = self.csapi
resultDict["AdminMessages"] = self.adminMsgs
resultDict["VOChanged"] = self.voChanged
return S_OK(resultDict)
def informPeople(rec, oldstate, state, author, inform):
""" inform utility
"""
if not state or state == 'New':
return # was no state change or resurrect
reqId = rec['RequestID']
csS = PathFinder.getServiceSection(
'ProductionManagement/ProductionRequest')
if not csS:
gLogger.error('No ProductionRequest section in configuration')
return
fromAddress = gConfig.getValue('%s/fromAddress' % csS, '')
if not fromAddress:
gLogger.error('No fromAddress is defined in CS path %s/fromAddress' %
csS)
return
sendNotifications = gConfig.getValue('%s/sendNotifications' % csS, 'Yes')
if sendNotifications != 'Yes':
gLogger.info('No notifications will be send')
return
footer = "\n\nNOTE: it is an automated notification."
footer += " Don't reply please.\n"
footer += "DIRAC Web portal: https://lhcb-portal-dirac.cern.ch/DIRAC/s:%s/g:" % \
PathFinder.getDIRACSetup()
ppath = "/?view=tabs&theme=Grey&url_state=1|*LHCbDIRAC.ProductionRequestManager.classes.ProductionRequestManager:,\n\n"
ppath += 'The request details:\n'
ppath += ' Type: %s' % str(rec['RequestType'])
ppath += ' Name: %s\n' % str(rec['RequestName'])
ppath += ' Conditions: %s\n' % str(rec['SimCondition'])
ppath += ' Processing pass: %s\n' % str(rec['ProPath'])
gLogger.info(".... %s ...." % ppath)
authorMail = getUserOption(author, 'Email')
if authorMail:
if not state in ['BK Check', 'Submitted']:
if state == 'BK OK':
subj = 'DIRAC: please resign your Production Request %s' % reqId
body = '\n'.join([
'Customized Simulation Conditions in your request was registered.',
'Since Bookkeeping expert could make changes in your request,',
'you are asked to confirm it.'
])
else:
subj = "DIRAC: the state of Production Request %s is changed to '%s'; %s;%s" % (
reqId, state, rec.get('RequestWG',
''), rec.get('RequestName', ''))
body = '\n'.join([
'The state of your request is changed.',
'This mail is for information only.'
])
notification = NotificationClient()
res = notification.sendMail(authorMail, subj,
body + footer + 'lhcb_user' + ppath,
fromAddress, True)
if not res['OK']:
gLogger.error("_inform_people: can't send email: %s" %
res['Message'])
if inform:
subj = "DIRAC: the state of %s Production Request %s is changed to '%s'; %s;%s" % (
rec['RequestType'], reqId, state, rec.get(
'RequestWG', ''), rec.get('RequestName', ''))
body = '\n'.join([
'You have received this mail because you are'
'in the subscription list for this request'
])
for x in inform.replace(" ", ",").split(","):
if x:
if x.find("@") > 0:
eMail = x
else:
eMail = getUserOption(x, 'Email')
if eMail:
notification = NotificationClient()
res = notification.sendMail(
eMail, subj, body + footer + 'lhcb_user' + ppath,
fromAddress, True)
if not res['OK']:
gLogger.error("_inform_people: can't send email: %s" %
res['Message'])
if state == 'Accepted':
subj = "DIRAC: the Production Request %s is accepted; %s;%s" % (
reqId, rec.get('RequestWG', ''), rec.get('RequestName', ''))
body = '\n'.join([
"The Production Request is signed and ready to process",
"You are informed as member of %s group"
])
groups = ['lhcb_prmgr']
for group in groups:
for man in _getMemberMails(group):
notification = NotificationClient()
res = notification.sendMail(
man, subj, body % group + footer + group + ppath,
fromAddress, True)
if not res['OK']:
gLogger.error("_inform_people: can't send email: %s" %
res['Message'])
elif state == 'PPG OK' and oldstate == 'Accepted':
subj = "DIRAC: returned Production Request %s; %s;%s" % (
reqId, rec.get('RequestWG', ''), rec.get('RequestName', ''))
body = '\n'.join([
"Production Request is returned by Production Manager.",
"As member of %s group, your are asked to correct and sign",
"or to reject it.", "",
"In case some other member of the group has already",
"done that, please ignore this mail."
])
groups = ['lhcb_tech']
for group in groups:
for man in _getMemberMails(group):
notification = NotificationClient()
res = notification.sendMail(
man, subj, body % group + footer + group + ppath,
fromAddress, True)
if not res['OK']:
gLogger.error("_inform_people: can't send email: %s" %
res['Message'])
elif state == 'BK Check':
groups = ['lhcb_bk']
_aggregate(reqId, rec.get('RequestType', ''), rec.get('RequestWG', ''),
rec.get('RequestName', ''), rec['SimCondition'],
rec['ProPath'], groups, rec.get('reqInform', inform))
elif state == 'Submitted':
groups = ['lhcb_ppg', 'lhcb_tech']
_aggregate(reqId, rec.get('RequestType', ''), rec.get('RequestWG', ''),
rec.get('RequestName', ''), rec['SimCondition'],
rec['ProPath'], groups, rec.get('reqInform', inform))
else:
return
def executeForVO(self, vo):
"""
Execute one SE and user synchronisation cycle for a VO.
:param str vo: Virtual organisation name.
:return: S_OK or S_ERROR
:rtype: dict
"""
valid_protocols = ["srm", "gsiftp", "davs", "https", "root"]
default_email = None
try:
try:
client = Client(account="root", auth_type="userpass")
except Exception as err:
self.log.info(
"Login to Rucio as root with password failed. Will try host cert/key",
str(err))
certKeyTuple = Locations.getHostCertificateAndKeyLocation()
if not certKeyTuple:
self.log.error("Hostcert/key location not set")
return S_ERROR("Hostcert/key location not set")
hostcert, hostkey = certKeyTuple
self.log.info("Logging in with a host cert/key pair:")
self.log.debug("account: ",
self.clientConfig[vo]["privilegedAccount"])
self.log.debug("rucio host: ",
self.clientConfig[vo]["rucioHost"])
self.log.debug("auth host: ",
self.clientConfig[vo]["authHost"])
self.log.debug("CA cert path: ", self.caCertPath)
self.log.debug("Cert location: ", hostcert)
self.log.debug("Key location: ", hostkey)
self.log.debug("VO: ", vo)
client = Client(
account=self.clientConfig[vo]["privilegedAccount"],
rucio_host=self.clientConfig[vo]["rucioHost"],
auth_host=self.clientConfig[vo]["authHost"],
ca_cert=self.caCertPath,
auth_type="x509",
creds={
"client_cert": hostcert,
"client_key": hostkey
},
timeout=600,
user_agent="rucio-clients",
vo=vo,
)
self.log.info("Rucio client instantiated for VO:", vo)
# Get the storage elements from Dirac Configuration and create them in Rucio
newRSE = False
self.log.info("Synchronizing SEs for VO ", vo)
result = getStorageElements(vo)
if result["OK"]:
rses = [rse["rse"] for rse in client.list_rses()]
for se in result["Value"]:
if se not in rses:
# The SE doesn't exist. Will create it
newRSE = True
self.log.info(
"Rucio Storage Element does not exist and will be created:",
se)
try:
client.add_rse(rse=se,
deterministic=True,
volatile=False)
except Exception as err:
self.log.error(
"Cannot create RSE",
"[RSE: %s, Error: %s]" % (se, str(err)))
continue
# Add RSE attributes for the new RSE
ret = gConfig.getOptionsDict(
"Resources/FTSEndpoints/FTS3")
ftsList = ""
if ret["OK"]:
ftsList = ",".join(ret["Value"].values())
dictRSEAttributes = {
"naming_convention": "BelleII",
"ANY": True,
"fts": ftsList
}
for key in dictRSEAttributes:
self.log.info(
"Setting RSE attributes",
"[RSE: %s, Attr. name: %s, Value: %s]" %
(se, key, dictRSEAttributes[key]),
)
client.add_rse_attribute(
se, key, value=dictRSEAttributes[key])
client.set_local_account_limit("root", se,
100000000000000000)
# Create the protocols
try:
protocols = client.get_protocols(se)
except RSEProtocolNotSupported as err:
self.log.info("Cannot get protocols for",
"[RSE %s, Error: %s]" % (se, str(err)))
protocols = []
existing_protocols = []
for prot in protocols:
existing_protocols.append(
(str(prot["scheme"]), str(prot["hostname"]),
str(prot["port"]), str(prot["prefix"])))
protocols_to_create = []
for params in result["Value"][se]:
prot = (
str(params["scheme"]),
str(params["hostname"]),
str(params["port"]),
str(params["prefix"]),
)
protocols_to_create.append(prot)
if prot not in existing_protocols and prot[
0] in valid_protocols:
# The protocol defined in Dirac does not exist in Rucio. Will be created
self.log.info(
"Will create new protocol:",
"%s://%s:%s%s on %s" %
(params["scheme"], params["hostname"],
params["port"], params["prefix"], se),
)
try:
client.add_protocol(rse=se, params=params)
except Duplicate as err:
self.log.info(
"Protocol already exists on",
"[RSE: %s, schema:%s]" %
(se, params["scheme"]))
except Exception as err:
self.log.error(
"Cannot create protocol on RSE",
"[RSE: %s, Error: %s]" % (se, str(err)))
else:
update = False
for protocol in protocols:
if prot == (
str(protocol["scheme"]),
str(protocol["hostname"]),
str(protocol["port"]),
str(protocol["prefix"]),
):
# Check if the protocol defined in Dirac has the same priority as the one defined in Rucio
for domain in ["lan", "wan"]:
for activity in [
"read", "write", "delete"
]:
if (params["domains"][domain]
[activity] !=
protocol["domains"][domain]
[activity]):
update = True
break
if (params["domains"]["wan"]
["third_party_copy"] !=
protocol["domains"]["wan"]
["third_party_copy"]):
update = True
if update:
data = {
"prefix":
params["prefix"],
"read_lan":
params["domains"]["lan"]["read"],
"read_wan":
params["domains"]["wan"]["read"],
"write_lan":
params["domains"]["lan"]["write"],
"write_wan":
params["domains"]["wan"]["write"],
"delete_lan":
params["domains"]["lan"]["delete"],
"delete_wan":
params["domains"]["wan"]["delete"],
"third_party_copy":
params["domains"]["wan"]["write"],
}
self.log.info(
"Will update protocol:",
"%s://%s:%s%s on %s" % (
params["scheme"],
params["hostname"],
params["port"],
params["prefix"],
se,
),
)
client.update_protocols(
rse=se,
scheme=params["scheme"],
data=data,
hostname=params["hostname"],
port=params["port"],
)
for prot in existing_protocols:
if prot not in protocols_to_create:
self.log.info(
"Will delete protocol:", "%s://%s:%s%s on %s" %
(prot[0], prot[1], prot[2], prot[3], se))
client.delete_protocols(se,
scheme=prot[0],
hostname=prot[1],
port=prot[2])
else:
self.log.error("Cannot get SEs:", result["Message"])
# If new RSE added, add distances
rses = [rse["rse"] for rse in client.list_rses()]
if newRSE:
self.log.info("Adding distances")
for src_rse, dest_rse in permutations(rses, r=2):
try:
client.add_distance(src_rse, dest_rse, {
"ranking": 1,
"distance": 10
})
except Exception as err:
self.log.error(
"Cannot add distance for",
"Source RSE: %s, Dest RSE: %s, Error:%s" %
(src_rse, dest_rse, str(err)),
)
# Collect the shares from Dirac Configuration and create them in Rucio
self.log.info("Synchronizing shares")
result = Operations().getOptionsDict("Production/SEshares")
if result["OK"]:
rseDict = result["Value"]
for rse in rses:
try:
self.log.info("Setting productionSEshare for",
"[RSE: %s : Share: %s", rse,
rseDict.get(rse, 0))
client.add_rse_attribute(rse, "productionSEshare",
rseDict.get(rse, 0))
except Exception as err:
self.log.error(
"Cannot create productionSEshare for RSE:", rse)
else:
self.log.error("Cannot get SEs", result["Message"])
result = Operations().getSections("Shares")
if result["OK"]:
for dataLevel in result["Value"]:
result = Operations().getOptionsDict("Shares/%s" %
dataLevel)
if not result["OK"]:
self.log.error("Cannot get SEs:" % result["Message"])
continue
rseDict = result["Value"]
for rse in rses:
try:
self.log.info(
"Setting", "%sShare for %s : %s" %
(dataLevel, rse, rseDict.get(rse, 0)))
client.add_rse_attribute(rse,
"%sShare" % dataLevel,
rseDict.get(rse, 0))
except Exception as err:
self.log.error("Cannot create share:",
"%sShare for %s", dataLevel, rse)
else:
self.log.error("Cannot get shares:", result["Message"])
# Create the RSE attribute PrimaryDataSE and OccupancyLFN
result = gConfig.getValue(
"Resources/StorageElementGroups/PrimarySEs")
result = getStorageElements(vo)
if result["OK"]:
allSEs = result["Value"]
primarySEs = resolveSEGroup("PrimarySEs", allSEs)
self.log.info("Will set primarySEs flag to:", str(primarySEs))
for rse in rses:
if rse in allSEs:
storage = StorageElement(rse)
if not storage.valid:
self.log.warn(
"Storage element is not valid. Skipped RSE:",
rse)
continue
occupancyLFN = storage.options.get("OccupancyLFN")
try:
client.add_rse_attribute(rse, "OccupancyLFN",
occupancyLFN)
except Exception as err:
self.log.error(
"Cannot create RSE attribute OccupancyLFN for",
"[RSE: %s, Error: %s]" % (rse, str(err)))
if rse in primarySEs:
try:
client.add_rse_attribute(rse, "PrimaryDataSE",
True)
except Exception as err:
self.log.error(
"Cannot create RSE attribute PrimaryDataSE for",
"[RSE: %s, Error: %s]" % (rse, str(err)),
)
else:
try:
client.delete_rse_attribute(rse, "PrimaryDataSE")
except RSEAttributeNotFound:
pass
except Exception as err:
self.log.error(
"Cannot remove RSE attribute PrimaryDataSE for",
"[RSE: %s, Error: %s]" % (rse, str(err)),
)
self.log.info("RSEs synchronized for VO: ", vo)
# Collect the user accounts from Dirac Configuration and create user accounts in Rucio
self.log.info("Synchronizing accounts for VO", vo)
listAccounts = [
str(acc["account"]) for acc in client.list_accounts()
]
listScopes = [str(scope) for scope in client.list_scopes()]
dnMapping = {}
diracUsers = getUsersInVO(vo)
self.log.debug(" Will consider following Dirac users for",
"[VO: %s, Dirac users: %s]" % (vo, diracUsers))
for account in diracUsers:
dn = getUserOption(account, "DN")
email = getUserOption(account, "Email")
dnMapping[dn] = email
if account not in listAccounts:
self.log.info("Will create account with associated DN ",
"[account: %s, DN: %s]" % (account, dn))
try:
client.add_account(account, "USER", email)
listAccounts.append(account)
except Exception as err:
self.log.error(
"Cannot create account",
"[account: %s, Error: %s]" % (account, str(err)))
try:
client.add_identity(account=account,
identity=dn,
authtype="X509",
email=email,
default=True)
except Exception as err:
self.log.error(
"Cannot add identity for account",
"[Identity: dn=%s, account:%s, Error: %s]" %
(dn, account, str(err)),
)
self.log.error(
" Account/identity skipped (it will not be created in Rucio)",
"[%s/%s]" % (account, dn))
continue
for rse in rses:
client.set_local_account_limit(account, rse,
1000000000000000)
else:
try:
client.add_identity(account=account,
identity=dn,
authtype="X509",
email=email,
default=True)
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account",
"[DN: %s, account: %s, Error: %s]" %
(dn, account, str(err)),
)
scope = "user." + account
if scope not in listScopes:
try:
self.log.info("Will create a scope",
"[Scope: %s]" % scope)
client.add_scope(account, scope)
self.log.info("Scope successfully added",
"[Scope: %s]" % scope)
except Exception as err:
self.log.error(
"Cannot create a scope",
"[Scope: %s, Error: %s]" % (scope, str(err)))
# Collect the group accounts from Dirac Configuration and create service accounts in Rucio
result = getGroupsForVO(vo)
if result["OK"]:
groups = result["Value"]
self.log.debug(" Will consider following Dirac groups for",
"[%s VO: %s]" % (vo, groups))
else:
groups = []
self.log.debug("No Dirac groups for", "%s VO " % vo)
self.log.debug("No Rucio service accounts will be created")
for group in groups:
if group not in listAccounts:
self.log.info(
"Will create SERVICE account for Dirac group:",
str(group))
try:
client.add_account(group, "SERVICE", None)
listAccounts.append(group)
except Exception as err:
self.log.error(
"Cannot create SERVICE account for",
"[group: %s, Error: %s]" % (group, str(err)))
for rse in rses:
client.set_local_account_limit(account, rse,
1000000000000000)
for dn in getDNsInGroup(group):
try:
client.add_identity(account=group,
identity=dn,
authtype="X509",
email=dnMapping.get(
dn, default_email))
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account",
"[identity %s, account %s, Error: %s]" %
(dn, group, str(err)),
)
self.log.error(format_exc())
# Collect the group accounts from Dirac Configuration and create service accounts in Rucio
result = getHosts()
if not result["OK"]:
self.log.error("Cannot get host accounts:",
"%s" % result["Message"])
else:
hosts = result["Value"]
for host in hosts:
dn = getHostOption(host, "DN")
email = dnMapping.get(dn, default_email)
try:
client.add_identity(account="dirac_srv",
identity=dn,
authtype="X509",
email=email)
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account dirac_srv:",
"[DN: %s, Error: %s]" % (dn, str(err)))
self.log.error(format_exc())
return S_OK()
except Exception as exc:
self.log.exception("Synchronisation for VO failed. VO skipped ",
"VO=%s" % vo,
lException=exc)
return S_ERROR(str(format_exc()))
res = csAPI.setOption( "%s/%s/Status" % ( catalogCFGBase, site ), "InActive" )
if not res['OK']:
gLogger.error( "Failed to update %s catalog status to InActive" % site )
else:
gLogger.debug( "Successfully updated %s catalog status to InActive" % site )
banned.append( site )
if not banned:
gLogger.error( "Failed to ban any catalog mirrors" )
DIRAC.exit( -1 )
res = csAPI.commitChanges()
if not res['OK']:
gLogger.error( "Failed to commit changes to CS", res['Message'] )
DIRAC.exit( -1 )
subject = '%d catalog instance(s) banned for use' % len( banned )
addressPath = 'EMail/Production'
address = Operations().getValue( addressPath, '' )
body = 'The catalog mirrors at the following sites were banned'
for site in banned:
body = "%s\n%s" % ( body, site )
if not address:
gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body )
DIRAC.exit( 0 )
NotificationClient().sendMail( address, subject, body, getUserOption( userName, 'Email', '' ) )
DIRAC.exit( 0 )
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
resultDict = result['Value']
newUsers = resultDict.get( "NewUsers", [] )
modUsers = resultDict.get( "ModifiedUsers", [] )
delUsers = resultDict.get( "DeletedUsers", [] )
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo )
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.notice( "Configuration committed for VO %s" % vo )
else:
self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo )
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info( "Creating home directories for users %s" % str( newUsers ) )
result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) )
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user )
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
if self.detailedReport:
result = self.getVOUserReport( vo )
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'MailFrom', self.am_getOption( 'mailFrom', "DIRAC system" ) ) )
return S_OK()
if not res['OK']:
gLogger.error( "Failed to update %s catalog status to Active" % site )
else:
gLogger.debug( "Successfully updated %s catalog status to Active" % site )
allowed.append( site )
if not allowed:
gLogger.error( "Failed to allow any catalog mirrors" )
DIRAC.exit( -1 )
res = csAPI.commitChanges()
if not res['OK']:
gLogger.error( "Failed to commit changes to CS", res['Message'] )
DIRAC.exit( -1 )
subject = '%d catalog instance(s) allowed for use' % len( allowed )
addressPath = 'EMail/Production'
address = Operations().getValue( addressPath, '' )
body = 'The catalog mirrors at the following sites were allowed'
for site in allowed:
body = "%s\n%s" % ( body, site )
if not address:
gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body )
DIRAC.exit( 0 )
NotificationClient().sendMail( address, subject, body, getUserOption( userName, 'Email', '' ) )
DIRAC.exit( 0 )
gLogger.debug("Successfully updated %s catalog status to Active" %
site)
allowed.append(site)
if not allowed:
gLogger.error("Failed to allow any catalog mirrors")
DIRAC.exit(-1)
res = csAPI.commitChanges()
if not res['OK']:
gLogger.error("Failed to commit changes to CS", res['Message'])
DIRAC.exit(-1)
subject = '%d catalog instance(s) allowed for use' % len(allowed)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = 'The catalog mirrors at the following sites were allowed'
for site in allowed:
body = "%s\n%s" % (body, site)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" % addressPath,
body)
DIRAC.exit(0)
NotificationClient().sendMail(address, subject, body,
getUserOption(userName, 'Email', ''))
DIRAC.exit(0)
