class VOMS2CSSynchronizer(object):
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False):
self.log = gLogger.getSubLogger("VOMS2CSSynchronizer")
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {'Errors': [], 'Info': []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
def syncCSWithVOMS(self):
""" Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS', result['Message'])
return result
self.vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName)
self.adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.adminMsgs['Info'].append("There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo))
self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get('nickname')
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[dn]['certSuspended']:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict)))
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' % newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(removedGroups)
else:
oldValue = str(diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" % (diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose("Modified user %s: dropped DN %s" % (user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict['DN'] = ','.join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: dropped DN %s" % (user, dn))
self.adminMsgs['Info'].append("Modified user %s: dropped DN %s" % (user, dn))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' % str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.adminMsgs['Info'].append('The following users are deleted from CS:\n %s\n' % str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.adminMsgs['Errors'].append('Error in deleting users from CS:\n %s' % str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.adminMsgs['Info'].append('The following users to be checked for deletion:\n %s' % str(oldUsers))
self.log.info('The following users to be checked for deletion: %s' % str(oldUsers))
resultDict['CSAPI'] = self.csapi
resultDict['AdminMessages'] = self.adminMsgs
return S_OK(resultDict)
def getVOUserData(self, refreshFlag=False):
""" Get a report for users of a given VO
:param bool refreshFlag: flag to indicate that the configuration must be refreshed
before looking up user data
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(self.vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % self.vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def getVOUserReport(self):
""" Get a report string with the current status of the DIRAC Registry for the
Virtual Organization
:return: S_OK with the report string as Value
"""
result = self.getVOUserData(refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, '')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields, records, sortField='Group', printOut=False, numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
def getUserName(self, dn):
""" Utility to construct user name
:param str dn: user DN
:return str: user name
"""
name = self.vomsUserDict[dn].get('name')
surname = self.vomsUserDict[dn].get('surname')
if name and surname:
surnameName = _getUserNameFromSurname(name, surname)
return surnameName
mail = self.vomsUserDict[dn]['mail']
dnName = _getUserNameFromDN(dn, self.vo)
mailName = _getUserNameFromMail(mail)
# If robot, take the dn based name
if dnName.startswith('robot'):
return dnName
# Is mailName reasonable ?
if len(mailName) > 5 and mailName.isalpha():
return mailName
# dnName too long
if len(dnName) >= 12:
return dnName[:11]
# May be the mail name is still more reasonable
if len(dnName) < len(mailName) and mailName.isalpha():
return mailName
return dnName
def __syncCSWithVOMS( self ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
#Get DIRAC VOMS Mapping
self.log.info( "Getting DIRAC VOMS mapping" )
mappingSection = '/Registry/VOMS/Mapping'
ret = gConfig.getOptionsDict( mappingSection )
if not ret['OK']:
self.log.fatal( 'No VOMS to DIRAC Group Mapping Available' )
return ret
vomsMapping = ret['Value']
self.log.info( "There are %s registered voms mappings in DIRAC" % len( vomsMapping ) )
#Get VOMS VO name
self.log.info( "Getting VOMS VO name" )
result = self.vomsSrv.admGetVOName()
if not ret['OK']:
self.log.fatal( 'Could not retrieve VOMS VO name' )
voNameInVOMS = result[ 'Value' ]
self.log.info( "VOMS VO Name is %s" % voNameInVOMS )
#Get VOMS roles
self.log.info( "Getting the list of registered roles in VOMS" )
result = self.vomsSrv.admListRoles()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered roles in VOMS' )
rolesInVOMS = result[ 'Value' ]
self.log.info( "There are %s registered roles in VOMS" % len( rolesInVOMS ) )
print rolesInVOMS
rolesInVOMS.append( '' )
#Map VOMS roles
vomsRoles = {}
for role in rolesInVOMS:
if role:
role = "%s/%s" % ( voNameInVOMS, role )
else:
role = voNameInVOMS
groupsForRole = []
for group in vomsMapping:
if vomsMapping[ group ] == role:
groupsForRole.append( group )
if groupsForRole:
vomsRoles[ role ] = { 'Groups' : groupsForRole, 'Users' : [] }
self.log.info( "DIRAC valid VOMS roles are:\n\t", "\n\t ".join( vomsRoles.keys() ) )
#Get DIRAC users
self.log.info( "Getting the list of registered users in DIRAC" )
csapi = CSAPI()
ret = csapi.listUsers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve current list of Users' )
return ret
currentUsers = ret['Value']
ret = csapi.describeUsers( currentUsers )
if not ret['OK']:
self.log.fatal( 'Could not retrieve current User description' )
return ret
currentUsers = ret['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC" % len( currentUsers ) )
self.log.info( "There are %s registered users in DIRAC" % len( currentUsers ) )
#Get VOMS user entries
self.log.info( "Getting the list of registered user entries in VOMS" )
result = self.vomsSrv.admListMembers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered user entries in VOMS' )
usersInVOMS = result[ 'Value' ]
self.__adminMsgs[ 'Info' ].append( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
self.log.info( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
#Consolidate users by nickname
usersData = {}
newUserNames = []
knownUserNames = []
obsoleteUserNames = []
self.log.info( "Retrieving usernames..." )
usersInVOMS.sort()
for iUPos in range( len( usersInVOMS ) ):
userName = ''
user = usersInVOMS[ iUPos ]
for oldUser in currentUsers:
if user[ 'DN' ].strip() in List.fromChar( currentUsers[oldUser][ 'DN' ] ):
userName = oldUser
if not userName:
result = self.vomsSrv.attGetUserNickname( user[ 'DN' ], user[ 'CA' ] )
if result[ 'OK' ]:
userName = result[ 'Value' ]
else:
self.__adminMsgs[ 'Errors' ].append( "Could not retrieve nickname for DN %s" % user[ 'DN' ] )
self.log.error( "Could not get nickname for DN", user[ 'DN' ] )
userName = user[ 'mail' ][:user[ 'mail' ].find( '@' )]
if not userName:
self.log.error( "Empty nickname for DN", user[ 'DN' ] )
self.__adminMsgs[ 'Errors' ].append( "Empty nickname for DN %s" % user[ 'DN' ] )
continue
self.log.info( " (%02d%%) Found username %s : %s " % ( ( iUPos * 100 / len( usersInVOMS ) ), userName, user[ 'DN' ] ) )
if userName not in usersData:
usersData[ userName ] = { 'DN': [], 'CA': [], 'Email': [], 'Groups' : ['user'] }
for key in ( 'DN', 'CA', 'mail' ):
value = user[ key ]
if value:
if key == "mail":
List.appendUnique( usersData[ userName ][ 'Email' ], value )
else:
usersData[ userName ][ key ].append( value.strip() )
if userName not in currentUsers:
List.appendUnique( newUserNames, userName )
else:
List.appendUnique( knownUserNames, userName )
self.log.info( "Finished retrieving usernames" )
if newUserNames:
self.log.info( "There are %s new users" % len( newUserNames ) )
else:
self.log.info( "There are no new users" )
#Get the list of users for each group
result = csapi.listGroups()
if not result[ 'OK' ]:
self.log.error( "Could not get the list of groups in DIRAC", result[ 'Message' ] )
return result
staticGroups = result[ 'Value' ]
vomsGroups = []
self.log.info( "Mapping users in VOMS to groups" )
for vomsRole in vomsRoles:
self.log.info( " Getting users for role %s" % vomsRole )
groupsForRole = vomsRoles[ vomsRole ][ 'Groups' ]
vomsMap = vomsRole.split( "Role=" )
for g in groupsForRole:
if g in staticGroups:
staticGroups.pop( staticGroups.index( g ) )
else:
vomsGroups.append( g )
if len( vomsMap ) == 1:
# no Role
users = usersInVOMS
else:
vomsGroup = "Role=".join( vomsMap[:-1] )
if vomsGroup[-1] == "/":
vomsGroup = vomsGroup[:-1]
vomsRole = "Role=%s" % vomsMap[-1]
result = self.vomsSrv.admListUsersWithRole( vomsGroup, vomsRole )
if not result[ 'OK' ]:
errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[ group ] )
self.__adminMsgs[ 'Errors' ].append( errorMsg )
self.log.error( errorMsg, result[ 'Message' ] )
return result
users = result['Value']
numUsersInGroup = 0
for vomsUser in users:
for userName in usersData:
if vomsUser[ 'DN' ] in usersData[ userName ][ 'DN' ]:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].extend( groupsForRole )
infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join( groupsForRole ), vomsRole )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
self.log.info( "Checking static groups" )
staticUsers = []
for group in staticGroups:
self.log.info( " Checking static group %s" % group )
numUsersInGroup = 0
result = csapi.listUsers( group )
if not result[ 'OK' ]:
self.log.error( "Could not get the list of users in DIRAC group %s" % group , result[ 'Message' ] )
return result
for userName in result[ 'Value' ]:
if userName in usersData:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].append( group )
else:
if group not in vomsGroups and userName not in staticUsers:
staticUsers.append( userName )
infoMsg = "There are %s users in group %s" % ( numUsersInGroup, group )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
if staticUsers:
infoMsg = "There are %s static users: %s" % ( len( staticUsers ) , ', '.join( staticUsers ) )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( "%s" % infoMsg )
for user in currentUsers:
if user not in usersData and user not in staticUsers:
self.log.info( 'User %s is no longer valid' % user )
obsoleteUserNames.append( user )
#Do the CS Sync
self.log.info( "Updating CS..." )
ret = csapi.downloadCSData()
if not ret['OK']:
self.log.fatal( 'Can not update from CS', ret['Message'] )
return ret
usersWithMoreThanOneDN = {}
for user in usersData:
csUserData = dict( usersData[ user ] )
if len( csUserData[ 'DN' ] ) > 1:
usersWithMoreThanOneDN[ user ] = csUserData[ 'DN' ]
result = csapi.describeUsers( [ user ] )
if result[ 'OK' ]:
if result[ 'Value' ]:
prevUser = result[ 'Value' ][ user ]
prevDNs = List.fromChar( prevUser[ 'DN' ] )
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
if DN not in prevDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has new DN %s" % ( user, DN ) )
for DN in prevDNs:
if DN not in newDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has lost a DN %s" % ( user, DN ) )
else:
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
self.__adminMsgs[ 'Info' ].append( "New user %s has new DN %s" % ( user, DN ) )
for k in ( 'DN', 'CA', 'Email' ):
csUserData[ k ] = ", ".join( csUserData[ k ] )
result = csapi.modifyUser( user, csUserData, createIfNonExistant = True )
if not result[ 'OK' ]:
self.__adminMsgs[ 'Error' ].append( "Cannot modify user %s: %s" % ( user, result[ 'Message' ] ) )
self.log.error( "Cannot modify user", user )
if usersWithMoreThanOneDN:
self.__adminMsgs[ 'Info' ].append( "\nUsers with more than one DN:" )
for uwmtod in sorted( usersWithMoreThanOneDN ):
self.__adminMsgs[ 'Info' ].append( " %s" % uwmtod )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersWithMoreThanOneDN[uwmtod]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
if obsoleteUserNames:
self.__adminMsgs[ 'Info' ].append( "\nObsolete users:" )
address = self.am_getOption( 'MailTo', '*****@*****.**' )
fromAddress = self.am_getOption( 'mailFrom', '*****@*****.**' )
subject = 'Obsolete LFC Users found'
body = 'Delete entries into LFC: \n'
for obsoleteUser in obsoleteUserNames:
self.log.info( subject, ", ".join( obsoleteUserNames ) )
body += 'for ' + obsoleteUser + '\n'
self.__adminMsgs[ 'Info' ].append( " %s" % obsoleteUser )
self.log.info( "Deleting %s users" % len( obsoleteUserNames ) )
NotificationClient().sendMail( address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress )
csapi.deleteUsers( obsoleteUserNames )
if newUserNames:
self.__adminMsgs[ 'Info' ].append( "\nNew users:" )
for newUser in newUserNames:
self.__adminMsgs[ 'Info' ].append( " %s" % newUser )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersData[newUser][ 'DN' ]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
self.__adminMsgs[ 'Info' ].append( " + EMail: %s" % usersData[newUser][ 'Email' ] )
result = csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.info( "Configuration committed" )
#LFC Check
if self.am_getOption( "LFCCheckEnabled", True ):
result = self.checkLFCRegisteredUsers( usersData )
if not result[ 'OK' ]:
return result
return S_OK()
class DiracAdmin( API ):
""" Administrative functionalities
"""
#############################################################################
def __init__( self ):
"""Internal initialization of the DIRAC Admin API.
"""
super( DiracAdmin, self ).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue( self.section + '/ScratchDir', '/tmp' )
self.currentDir = os.getcwd()
#############################################################################
def uploadProxy( self, group ):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('lhcb_pilot')
{'OK': True, 'Value': 0L}
:param group: DIRAC Group
:type job: string
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy( diracGroup = group )
#############################################################################
def setProxyPersistency( self, userDN, userGroup, persistent = True ):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency( userDN, userGroup, persistent )
#############################################################################
def checkProxyUploaded( self, userDN, userGroup, requiredTime ):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy( userDN, userGroup, requiredTime )
#############################################################################
def getSiteMask( self, printOutput = False ):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> print diracAdmin.getSiteMask()
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getSiteMask()
if result['OK']:
sites = result['Value']
if printOutput:
sites.sort()
for site in sites:
print site
return result
#############################################################################
def getBannedSites( self, gridType = [], printOutput = False ):
"""Retrieve current list of banned sites.
Example usage:
>>> print diracAdmin.getBannedSites()
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
bannedSites = []
totalList = []
result = wmsAdmin.getSiteMask()
if not result['OK']:
self.log.warn( result['Message'] )
return result
sites = result['Value']
if not gridType:
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return result
gridType = result['Value']
for grid in gridType:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return result
totalList += result['Value']
for site in totalList:
if not site in sites:
bannedSites.append( site )
bannedSites.sort()
if printOutput:
print '\n'.join( bannedSites )
return S_OK( bannedSites )
#############################################################################
def getSiteSection( self, site, printOutput = False ):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> print diracAdmin.getSiteSection('LCG.CERN.ch')
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split( '.' )[0]
if not gConfig.getSections( '/Resources/Sites/%s' % ( gridType ) )['OK']:
return S_ERROR( '/Resources/Sites/%s is not a valid site section' % ( gridType ) )
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( gridType, site ) )
if printOutput and result['OK']:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def addSiteInMask( self, site, comment, printOutput = False ):
"""Adds the site to the site mask.
Example usage:
>>> print diracAdmin.addSiteInMask()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
mask = self.getSiteMask()
if not mask['OK']:
return mask
siteMask = mask['Value']
if site in siteMask:
return S_ERROR( 'Site %s already in mask of allowed sites' % site )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.allowSite( site, comment )
if not result['OK']:
return result
if printOutput:
print 'Allowing %s in site mask' % site
return result
#############################################################################
def getSiteMaskLogging( self, site = None, printOutput = False ):
"""Retrieves site mask logging information.
Example usage:
>>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getSiteMaskLogging( site )
if not result['OK']:
return result
if site:
if not result['Value'].has_key( site ):
return S_ERROR( 'Site mask information not available for %s' % ( site ) )
if printOutput:
if site:
print '\nSite Mask Logging Info for %s\n' % site
else:
print '\nAll Site Mask Logging Info\n'
siteDict = result['Value']
for site, tupleList in siteDict.iteritems():
if not site:
print '\n===> %s\n' % site
for tup in tupleList:
print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \
'( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"'
print ' '
return result
#############################################################################
def banSiteFromMask( self, site, comment, printOutput = False ):
"""Removes the site from the site mask.
Example usage:
>>> print diracAdmin.banSiteFromMask()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
mask = self.getSiteMask()
if not mask['OK']:
return mask
siteMask = mask['Value']
if not site in siteMask:
return S_ERROR( 'Site %s is already banned' % site )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.banSite( site, comment )
if not result['OK']:
return result
if printOutput:
print 'Removing %s from site mask' % site
return result
#############################################################################
@classmethod
def __checkSiteIsValid( self, site ):
"""Internal function to check that a site name is valid.
"""
sites = getSiteCEMapping()
if not sites['OK']:
return S_ERROR( 'Could not get site CE mapping' )
siteList = sites['Value'].keys()
if not site in siteList:
return S_ERROR( 'Specified site %s is not in list of defined sites' % site )
return S_OK( '%s is valid' % site )
#############################################################################
def clearMask( self ):
"""Removes all sites from the site mask. Should be used with care.
Example usage:
>>> print diracAdmin.clearMask()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.clearMask()
return result
#############################################################################
def getServicePorts( self, setup = '', printOutput = False ):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue( '/DIRAC/Setup', '' )
setupList = gConfig.getSections( '/DIRAC/Setups', [] )
if not setupList['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups sections' )
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) )
serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup )
if not serviceSetups['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup )
serviceSetups = serviceSetups['Value'] # dict
systemList = gConfig.getSections( '/Systems' )
if not systemList['OK']:
return S_ERROR( 'Could not get Systems sections' )
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key( system ):
path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] )
servicesList = gConfig.getSections( path )
if not servicesList['OK']:
self.log.warn( 'Could not get sections in %s' % path )
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) )
for service in servicesList:
spath = '%s/%s/Port' % ( path, service )
servicePort = gConfig.getValue( spath, 0 )
if servicePort:
self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) )
result['%s/%s' % ( system, service )] = servicePort
else:
self.log.warn( 'No port found for %s' % spath )
else:
self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) )
if printOutput:
print self.pPrint.pformat( result )
return S_OK( result )
#############################################################################
def getProxy( self, userDN, userGroup, validity = 43200, limited = False ):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy( userDN, userGroup, limited = limited,
requiredTimeLeft = validity )
#############################################################################
def getVOMSProxy( self, userDN, userGroup, vomsAttr = False, validity = 43200, limited = False ):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy( userDN, userGroup, limited = limited,
requiredVOMSAttribute = vomsAttr,
requiredTimeLeft = validity )
#############################################################################
def getPilotProxy( self, userDN, userGroup, validity = 43200 ):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft = validity )
#############################################################################
def resetJob( self, jobID ):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> print dirac.reset(12345)
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance( jobID, basestring ):
try:
jobID = int( jobID )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobID' )
elif isinstance( jobID, list ):
try:
jobID = [int( job ) for job in jobID]
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobIDs' )
jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = False )
result = jobManager.resetJob( jobID )
return result
#############################################################################
def getJobPilotOutput( self, jobID, directory = '' ):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists( directory ):
return self._errorReport( 'Directory %s does not exist' % directory )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getJobPilotOutput( jobID )
if not result['OK']:
return result
outputPath = '%s/pilot_%s' % ( directory, jobID )
if os.path.exists( outputPath ):
self.log.info( 'Remove %s and retry to continue' % outputPath )
return S_ERROR( 'Remove %s and retry to continue' % outputPath )
if not os.path.exists( outputPath ):
self.log.verbose( 'Creating directory %s' % outputPath )
os.mkdir( outputPath )
outputs = result['Value']
if outputs.has_key( 'StdOut' ):
stdout = '%s/std.out' % ( outputPath )
with open( stdout, 'w' ) as fopen:
fopen.write( outputs['StdOut'] )
self.log.verbose( 'Standard output written to %s' % ( stdout ) )
else:
self.log.warn( 'No standard output returned' )
if outputs.has_key( 'StdError' ):
stderr = '%s/std.err' % ( outputPath )
with open( stderr, 'w' ) as fopen:
fopen.write( outputs['StdError'] )
self.log.verbose( 'Standard error written to %s' % ( stderr ) )
else:
self.log.warn( 'No standard error returned' )
self.log.always( 'Outputs retrieved in %s' % outputPath )
return result
#############################################################################
def getPilotOutput( self, gridReference, directory = '' ):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
if not directory:
directory = self.currentDir
if not os.path.exists( directory ):
return self._errorReport( 'Directory %s does not exist' % directory )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotOutput( gridReference )
if not result['OK']:
return result
gridReferenceSmall = gridReference.split( '/' )[-1]
if not gridReferenceSmall:
gridReferenceSmall = 'reference'
outputPath = '%s/pilot_%s' % ( directory, gridReferenceSmall )
if os.path.exists( outputPath ):
self.log.info( 'Remove %s and retry to continue' % outputPath )
return S_ERROR( 'Remove %s and retry to continue' % outputPath )
if not os.path.exists( outputPath ):
self.log.verbose( 'Creating directory %s' % outputPath )
os.mkdir( outputPath )
outputs = result['Value']
if outputs.has_key( 'StdOut' ):
stdout = '%s/std.out' % ( outputPath )
with open( stdout, 'w' ) as fopen:
fopen.write( outputs['StdOut'] )
self.log.info( 'Standard output written to %s' % ( stdout ) )
else:
self.log.warn( 'No standard output returned' )
if outputs.has_key( 'StdErr' ):
stderr = '%s/std.err' % ( outputPath )
with open( stderr, 'w' ) as fopen:
fopen.write( outputs['StdErr'] )
self.log.info( 'Standard error written to %s' % ( stderr ) )
else:
self.log.warn( 'No standard error returned' )
self.log.always( 'Outputs retrieved in %s' % outputPath )
return result
#############################################################################
def getPilotInfo( self, gridReference ):
"""Retrieve info relative to a pilot reference
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotInfo( gridReference )
return result
#############################################################################
def killPilot( self, gridReference ):
"""Kill the pilot specified
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.killPilot( gridReference )
return result
#############################################################################
def getPilotLoggingInfo( self, gridReference ):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> print dirac.getPilotLoggingInfo(12345)
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if type( gridReference ) not in types.StringTypes:
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
return wmsAdmin.getPilotLoggingInfo( gridReference )
#############################################################################
def getJobPilots( self, jobID ):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> print dirac.getJobPilots()
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance( jobID, basestring ):
try:
jobID = int( jobID )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or string for existing jobID' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilots( jobID )
if result['OK']:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def getPilotSummary( self, startDate = '', endDate = '' ):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> print dirac.getPilotSummary()
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotSummary( startDate, endDate )
if not result['OK']:
return result
ceDict = result['Value']
headers = 'CE'.ljust( 28 )
i = 0
for ce, summary in ceDict.iteritems():
states = summary.keys()
if len( states ) > i:
i = len( states )
for i in xrange( i ):
headers += 'Status'.ljust( 12 ) + 'Count'.ljust( 12 )
print headers
for ce, summary in ceDict.iteritems():
line = ce.ljust( 28 )
states = summary.keys()
states.sort()
for state in states:
count = str( summary[state] )
line += state.ljust( 12 ) + count.ljust( 12 )
print line
return result
#############################################################################
def selectRequests( self, jobID = None, requestID = None, requestName = None,
requestType = None, status = None, operation = None, ownerDN = None,
ownerGroup = None, requestStart = 0, limit = 100, printOutput = False ):
"""Select requests from the request management system. A few notes on the selection criteria:
- jobID is the WMS JobID for the request (if applicable)
- requestID is assigned during submission of the request
- requestName is the corresponding XML file name
- requestType e.g. 'transfer'
- status e.g. Done
- operation e.g. replicateAndRegister
- requestStart e.g. the first request to consider (start from 0 by default)
- limit e.g. selection limit (default 100)
>>> dirac.selectRequests(jobID='4894')
{'OK': True, 'Value': [[<Requests>]]}
"""
options = {'RequestID':requestID, 'RequestName':requestName, 'JobID':jobID, 'OwnerDN':ownerDN,
'OwnerGroup':ownerGroup, 'RequestType':requestType, 'Status':status, 'Operation':operation}
conditions = {}
for key, value in options.iteritems():
if value:
try:
conditions[key] = str( value )
except Exception as x:
return self._errorReport( str( x ), 'Expected string for %s field' % key )
try:
requestStart = int( requestStart )
limit = int( limit )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer for %s field' % limit )
self.log.verbose( 'Will select requests with the following conditions' )
self.log.verbose( self.pPrint.pformat( conditions ) )
requestClient = RPCClient( "RequestManagement/centralURL" )
result = requestClient.getRequestSummaryWeb( conditions, [], requestStart, limit )
if not result['OK']:
self.log.warn( result['Message'] )
return result
requestIDs = result['Value']
conds = []
for key, value in conditions.iteritems():
if value:
conds.append( '%s = %s' % ( key, value ) )
self.log.verbose( '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ),
', '.join( conds ), limit ) )
if printOutput:
requests = []
if len( requestIDs['Records'] ) > limit:
requestList = requestIDs['Records']
requests = requestList[:limit]
else:
requests = requestIDs['Records']
print '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ),
', '.join( conds ), limit )
print requestIDs['ParameterNames']
for request in requests:
print request
if not requestIDs:
return S_ERROR( 'No requests selected for conditions: %s' % conditions )
else:
return result
#############################################################################
def getRequestSummary( self, printOutput = False ):
"""
Get a summary of the requests in the request DB.
"""
requestClient = RPCClient( "RequestManagement/centralURL", timeout = 120 )
result = requestClient.getDBSummary()
if not result['OK']:
self.log.warn( result['Message'] )
return result
if printOutput:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def getExternalPackageVersions( self ):
"""
Simple function that attempts to obtain the external versions for
the local DIRAC installation (frequently needed for debugging purposes).
"""
gLogger.info( 'DIRAC version v%dr%d build %d' % ( DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel ) )
try:
import lcg_util
infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__
gLogger.info( infoStr )
infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % ( x )
gLogger.exception( errStr )
try:
import gfalthr as gfal
infoStr = "Using gfalthr from: \n%s" % gfal.__file__
gLogger.info( infoStr )
infoStr = "The version of gfalthr is %s" % gfal.gfal_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % ( x )
gLogger.warn( errStr )
try:
import gfal
infoStr = "Using gfal from: %s" % gfal.__file__
gLogger.info( infoStr )
infoStr = "The version of gfal is %s" % gfal.gfal_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % ( x )
gLogger.exception( errStr )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
gLogger.info( 'Default list of protocols are: %s' % ( ', '.join( defaultProtocols ) ) )
return S_OK()
#############################################################################
def getSiteProtocols( self, site, printOutput = False ):
"""
Allows to check the defined protocols for each site SE.
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site )
siteSEs = gConfig.getValue( siteSection, [] )
if not siteSEs:
return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
self.log.verbose( 'Default list of protocols are' ', '.join( defaultProtocols ) )
seInfo = {}
siteSEs.sort()
for se in siteSEs:
sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) )
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section )
seProtocols = gConfig.getValue( path, [] )
if not seProtocols:
seProtocols = defaultProtocols
seInfo[se] = seProtocols
if printOutput:
print '\nSummary of protocols for StorageElements at site %s' % site
print '\nStorageElement'.ljust( 30 ) + 'ProtocolsList'.ljust( 30 ) + '\n'
for se, protocols in seInfo.iteritems():
print se.ljust( 30 ) + ', '.join( protocols ).ljust( 30 )
return S_OK( seInfo )
#############################################################################
def setSiteProtocols( self, site, protocolsList, printOutput = False ):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site )
siteSEs = gConfig.getValue( siteSection, [] )
if not siteSEs:
return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
self.log.verbose( 'Default list of protocols are', ', '.join( defaultProtocols ) )
for protocol in protocolsList:
if not protocol in defaultProtocols:
return S_ERROR( 'Requested to set protocol %s in list but %s is not '
'in default list of protocols:\n%s' % ( protocol, protocol, ', '.join( defaultProtocols ) ) )
modifiedCS = False
result = promptUser( 'Do you want to add the following default protocols:'
' %s for SE(s):\n%s' % ( ', '.join( protocolsList ), ', '.join( siteSEs ) ) )
if not result['OK']:
return result
if result['Value'].lower() != 'y':
self.log.always( 'No protocols will be added' )
return S_OK()
for se in siteSEs:
sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) )
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section )
self.log.verbose( 'Setting %s to %s' % ( path, ', '.join( protocolsList ) ) )
result = self.csSetOption( path, ', '.join( protocolsList ) )
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges( False )
if not result[ 'OK' ]:
return S_ERROR( 'CS Commit failed with message = %s' % ( result[ 'Message' ] ) )
else:
if printOutput:
print 'Successfully committed changes to CS'
else:
if printOutput:
print 'No modifications to CS required'
return S_OK()
#############################################################################
def csSetOption( self, optionPath, optionValue ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption( optionPath, optionValue )
#############################################################################
def csSetOptionComment( self, optionPath, comment ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment( optionPath, comment )
#############################################################################
def csModifyValue( self, optionPath, newValue ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue( optionPath, newValue )
#############################################################################
def csRegisterUser( self, username, properties ):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser( username, properties )
#############################################################################
def csDeleteUser( self, user ):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers( user )
#############################################################################
def csModifyUser( self, username, properties, createIfNonExistant = False ):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser( username, properties, createIfNonExistant )
#############################################################################
def csListUsers( self, group = False ):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers( group )
#############################################################################
def csDescribeUsers( self, mask = False ):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers( mask )
#############################################################################
def csModifyGroup( self, groupname, properties, createIfNonExistant = False ):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup( groupname, properties, createIfNonExistant )
#############################################################################
def csListHosts( self ):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts( self, mask = False ):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts( mask )
#############################################################################
def csModifyHost( self, hostname, properties, createIfNonExistant = False ):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost( hostname, properties, createIfNonExistant )
#############################################################################
def csListGroups( self ):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups( self, mask = False ):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups( mask )
#############################################################################
def csSyncUsersWithCFG( self, usersCFG ):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG( usersCFG )
#############################################################################
def csCommitChanges( self, sortUsers = True ):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges( sortUsers = False )
#############################################################################
def sendMail( self, address, subject, body, fromAddress = None, localAttempt = True, html = False ):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail( address, subject, body, fromAddress, localAttempt, html )
#############################################################################
def sendSMS( self, userName, body, fromAddress = None ):
"""
Send mail to specified address with body.
"""
if len( body ) > 160:
return S_ERROR( 'Exceeded maximum SMS length of 160 characters' )
notification = NotificationClient()
return notification.sendSMS( userName, body, fromAddress )
#############################################################################
def getBDIISite( self, site, host = None ):
"""
Get information about site from BDII at host
"""
return ldapSite( site, host = host )
#############################################################################
def getBDIICluster( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapCluster( ce, host = host )
#############################################################################
def getBDIICE( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapCE( ce, host = host )
#############################################################################
def getBDIIService( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapService( ce, host = host )
#############################################################################
def getBDIICEState( self, ce, useVO = voName, host = None ):
"""
Get information about ce state from BDII at host
"""
return ldapCEState( ce, useVO, host = host )
#############################################################################
def getBDIICEVOView( self, ce, useVO = voName, host = None ):
"""
Get information about ce voview from BDII at host
"""
return ldapCEVOView( ce, useVO, host = host )
#############################################################################
def getBDIISE( self, site, useVO = voName, host = None ):
"""
Get information about SA from BDII at host
"""
return ldapSE( site, useVO, host = host )
class VOMS2CSAgent( AgentModule ):
def __init__( self, *args, **kwargs ):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__( *args, **kwargs )
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
self.autoDeleteUsers = False
self.detailedReport = True
self.makeFCEntry = False
def initialize( self ):
""" Initialize the default parameters
"""
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.csapi = CSAPI()
self.dryRun = self.am_getOption( 'DryRun', self.dryRun )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers )
self.autoDeleteUsers = self.am_getOption( 'AutoDeleteUsers', self.autoDeleteUsers )
self.detailedReport = self.am_getOption( 'DetailedReport', self.detailedReport )
self.makeFCEntry = self.am_getOption( 'MakeHomeDirectory', self.makeFCEntry )
return S_OK()
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
resultDict = result['Value']
newUsers = resultDict.get( "NewUsers", [] )
modUsers = resultDict.get( "ModifiedUsers", [] )
delUsers = resultDict.get( "DeletedUsers", [] )
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo )
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.notice( "Configuration committed for VO %s" % vo )
else:
self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo )
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info( "Creating home directories for users %s" % str( newUsers ) )
result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) )
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user )
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
if self.detailedReport:
result = self.getVOUserReport( vo )
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'MailFrom', self.am_getOption( 'mailFrom', "DIRAC system" ) ) )
return S_OK()
def getVOUserData( self, vo, refreshFlag = False ):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
def getVOUserReport( self, vo ):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData( vo, refreshFlag = True )
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict( int )
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar( userDict[user]['DN'] )
if len( dnList ) > 1:
multiDNUsers[user] = dnList
if userDict[user].get( 'Status', 'Active' ) == 'Suspended':
suspendedUsers.append( user )
for group in diracVOMSMapping:
records.append( ( group, str( groupDict[group] ), diracVOMSMapping.get( group, '' ) ) )
fields = [ 'Group', 'Number of users', 'VOMS Role' ]
output = printTable( fields, records, sortField = 'Group', printOut = False, numbering = False )
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len( suspendedUsers )
output += ' %s' % ','.join( suspendedUsers )
return S_OK( output )
@executeWithUserProxy
def __syncCSWithVOMS( self, vo ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
resultDict = defaultdict( list )
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService( vo )
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
return result
vomsVOName = result[ 'Value' ].lstrip( '/' )
self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
return result
vomsUserDict = result[ 'Value' ]
message = "There are %s user entries in VOMS for VO %s" % ( len( vomsUserDict ), vomsVOName )
self.__adminMsgs[ 'Info' ].append( message )
self.log.info( message )
# Get DIRAC users
result = self.getVOUserData( vo )
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) )
self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) )
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar( dn )
existingDNs.extend( dnList )
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append( dn )
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append( dn )
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list( set( allDiracUsers ) - set( diracUserDict.keys() ) )
if nonVOUsers:
result = self.csapi.describeUsers( nonVOUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] )
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName( dn, vomsUserDict[dn]['mail'], vo )
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % ( trialName, ind )
ind += 1
# We now have everything to add the new user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) )
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % ( key, str( userDict[key] ) )
self.__adminMsgs[ 'Info' ].append( message )
self.voChanged = True
if self.autoAddUsers:
self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True )
if not result['OK']:
self.log.warn( 'Failed adding new user %s' % newDiracName )
resultDict['NewUsers'].append( newDiracName )
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
if newDNForExistingUser:
userDict['DN'] = ','.join( [ dn, diracUserDict[diracName]['DN'] ] )
modified = True
existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] )
nonVOGroups = list( set( existingGroups ) - set( diracVOMSMapping.keys() ) )
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get( group, '' )
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append( group )
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append( group )
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append( group )
userDict['Groups'] = list( set( keepGroups ) )
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get( 'Groups', [] )
userDict['Groups'] = list( set( keepGroups + otherGroups ) )
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set( userDict[key] ) - set( diracUserDict.get( diracName, {} ).get( key, [] ) )
removedGroups = set( diracUserDict.get( diracName, {} ).get( key, [] ) ) - set( userDict[key] )
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join( addedGroups )
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups )
else:
oldValue = str( diracUserDict.get( diracName, {} ).get( key, '' ) )
if str( userDict[key] ) != oldValue:
modMsg += " %s: %s -> %s\n" % ( key, oldValue, str( userDict[key] ) )
if modMsg:
self.__adminMsgs[ 'Info' ].append( message + modMsg )
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser( diracName, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: %s" % ( diracName, str( userDict ) ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
if not dnSet.intersection( set( vomsUserDict.keys() ) ) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add( user )
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose( "Modified user %s: dropped DN %s" % ( user, dn ) )
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set( [dn] )
if modDNSet:
userDict['DN'] = ','.join( modDNSet )
result = self.csapi.modifyUser( user, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.__adminMsgs[ 'Info' ].append( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
else:
oldUsers.add( user )
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info( 'The following users will be deleted: %s' % str( oldUsers ) )
result = self.csapi.deleteUsers( oldUsers )
if result['OK']:
self.__adminMsgs[ 'Info' ].append( 'The following users are deleted from CS:\n %s\n' % str( oldUsers ) )
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs[ 'Errors' ].append( 'Error in deleting users from CS:\n %s' % str( oldUsers ) )
self.log.error( 'Error while user deletion from CS', result )
else:
self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion:\n %s' % str( oldUsers ) )
self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
return S_OK( resultDict )
@executeWithUserProxy
def __addHomeDirectory( self, vo, newUsers ):
fc = FileCatalog( vo = vo )
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser( user )
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % ( vo, user[0], user )
result = fc.createDirectory( dirName )
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner( { dirName: user }, recursive = True )
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup( { dirName: defaultVOGroup }, recursive = True )
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK( { "Successful": successful, "Failed": failed } )
class DiracAdmin(API):
"""Administrative functionalities"""
#############################################################################
def __init__(self):
"""Internal initialization of the DIRAC Admin API."""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + "/LogLevel", "DEBUG") == "DEBUG":
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + "/ScratchDir",
"/tmp")
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
#############################################################################
def uploadProxy(self):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('dteam_pilot')
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy()
#############################################################################
def setProxyPersistency(self, userDN, userGroup, persistent=True):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ))
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency(userDN, userGroup, persistent)
#############################################################################
def checkProxyUploaded(self, userDN, userGroup, requiredTime):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ))
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy(userDN, userGroup, requiredTime)
#############################################################################
def getSiteMask(self, printOutput=False, status="Active"):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteMask())
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
result = self.sitestatus.getSites(siteState=status)
if result["OK"]:
sites = result["Value"]
if printOutput:
sites.sort()
for site in sites:
gLogger.notice(site)
return result
#############################################################################
def getBannedSites(self, printOutput=False):
"""Retrieve current list of banned and probing sites.
Example usage:
>>> gLogger.notice(diracAdmin.getBannedSites())
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
bannedSites = self.sitestatus.getSites(siteState="Banned")
if not bannedSites["OK"]:
return bannedSites
probingSites = self.sitestatus.getSites(siteState="Probing")
if not probingSites["OK"]:
return probingSites
mergedList = sorted(bannedSites["Value"] + probingSites["Value"])
if printOutput:
gLogger.notice("\n".join(mergedList))
return S_OK(mergedList)
#############################################################################
def getSiteSection(self, site, printOutput=False):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteSection('LCG.CERN.ch'))
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split(".")[0]
if not gConfig.getSections("/Resources/Sites/%s" % (gridType))["OK"]:
return S_ERROR("/Resources/Sites/%s is not a valid site section" %
(gridType))
result = gConfig.getOptionsDict("/Resources/Sites/%s/%s" %
(gridType, site))
if printOutput and result["OK"]:
gLogger.notice(self.pPrint.pformat(result["Value"]))
return result
#############################################################################
def allowSite(self, site, comment, printOutput=False):
"""Adds the site to the site mask.
Example usage:
>>> gLogger.notice(diracAdmin.allowSite())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self._checkSiteIsValid(site)
if not result["OK"]:
return result
result = self.getSiteMask(status="Active")
if not result["OK"]:
return result
siteMask = result["Value"]
if site in siteMask:
if printOutput:
gLogger.notice("Site %s is already Active" % site)
return S_OK("Site %s is already Active" % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, "Active", comment)
else:
result = WMSAdministratorClient().allowSite(site, comment)
if not result["OK"]:
return result
if printOutput:
gLogger.notice("Site %s status is set to Active" % site)
return result
#############################################################################
def getSiteMaskLogging(self, site=None, printOutput=False):
"""Retrieves site mask logging information.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteMaskLogging('LCG.AUVER.fr'))
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self._checkSiteIsValid(site)
if not result["OK"]:
return result
if self.rssFlag:
result = ResourceStatusClient().selectStatusElement("Site",
"History",
name=site)
else:
result = WMSAdministratorClient().getSiteMaskLogging(site)
if not result["OK"]:
return result
if printOutput:
if site:
gLogger.notice("\nSite Mask Logging Info for %s\n" % site)
else:
gLogger.notice("\nAll Site Mask Logging Info\n")
sitesLogging = result["Value"]
if isinstance(sitesLogging, dict):
for siteName, tupleList in sitesLogging.items(
): # can be an iterator
if not siteName:
gLogger.notice("\n===> %s\n" % siteName)
for tup in tupleList:
stup = str(tup[0]).ljust(8) + str(tup[1]).ljust(20)
stup += "( " + str(tup[2]).ljust(len(str(
tup[2]))) + ' ) "' + str(tup[3]) + '"'
gLogger.notice(stup)
gLogger.notice(" ")
elif isinstance(sitesLogging, list):
sitesLoggingList = [(sl[1], sl[3], sl[4])
for sl in sitesLogging]
for siteLog in sitesLoggingList:
gLogger.notice(siteLog)
return S_OK()
#############################################################################
def banSite(self, site, comment, printOutput=False):
"""Removes the site from the site mask.
Example usage:
>>> gLogger.notice(diracAdmin.banSite())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self._checkSiteIsValid(site)
if not result["OK"]:
return result
mask = self.getSiteMask(status="Banned")
if not mask["OK"]:
return mask
siteMask = mask["Value"]
if site in siteMask:
if printOutput:
gLogger.notice("Site %s is already Banned" % site)
return S_OK("Site %s is already Banned" % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, "Banned", comment)
else:
result = WMSAdministratorClient().banSite(site, comment)
if not result["OK"]:
return result
if printOutput:
gLogger.notice("Site %s status is set to Banned" % site)
return result
#############################################################################
def getServicePorts(self, setup="", printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> gLogger.notice(diracAdmin.getServicePorts())
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "")
setupList = gConfig.getSections("/DIRAC/Setups", [])
if not setupList["OK"]:
return S_ERROR("Could not get /DIRAC/Setups sections")
setupList = setupList["Value"]
if setup not in setupList:
return S_ERROR("Setup %s is not in allowed list: %s" %
(setup, ", ".join(setupList)))
serviceSetups = gConfig.getOptionsDict("/DIRAC/Setups/%s" % setup)
if not serviceSetups["OK"]:
return S_ERROR("Could not get /DIRAC/Setups/%s options" % setup)
serviceSetups = serviceSetups["Value"] # dict
systemList = gConfig.getSections("/Systems")
if not systemList["OK"]:
return S_ERROR("Could not get Systems sections")
systemList = systemList["Value"]
result = {}
for system in systemList:
if system in serviceSetups:
path = "/Systems/%s/%s/Services" % (system,
serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList["OK"]:
self.log.warn("Could not get sections in %s" % path)
else:
servicesList = servicesList["Value"]
if not servicesList:
servicesList = []
self.log.verbose("System: %s ServicesList: %s" %
(system, ", ".join(servicesList)))
for service in servicesList:
spath = "%s/%s/Port" % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose("Found port for %s/%s = %s" %
(system, service, servicePort))
result["%s/%s" % (system, service)] = servicePort
else:
self.log.warn("No port found for %s" % spath)
else:
self.log.warn("%s is not defined in /DIRAC/Setups/%s" %
(system, setup))
if printOutput:
gLogger.notice(self.pPrint.pformat(result))
return S_OK(result)
#############################################################################
def getProxy(self, userDN, userGroup, validity=43200, limited=False):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy(userDN,
userGroup,
limited=limited,
requiredTimeLeft=validity)
#############################################################################
def getVOMSProxy(self,
userDN,
userGroup,
vomsAttr=False,
validity=43200,
limited=False):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getVOMSProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy(userDN,
userGroup,
limited=limited,
requiredVOMSAttribute=vomsAttr,
requiredTimeLeft=validity)
#############################################################################
def getPilotProxy(self, userDN, userGroup, validity=43200):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getVOMSProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup(
userDN, userGroup, requiredTimeLeft=validity)
#############################################################################
def resetJob(self, jobID):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> gLogger.notice(dirac.reset(12345))
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance(jobID, six.string_types):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x),
"Expected integer or convertible integer for existing jobID"
)
elif isinstance(jobID, list):
try:
jobID = [int(job) for job in jobID]
except Exception as x:
return self._errorReport(
str(x),
"Expected integer or convertible integer for existing jobIDs"
)
result = JobManagerClient(useCertificates=False).resetJob(jobID)
return result
#############################################################################
def getJobPilotOutput(self, jobID, directory=""):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> gLogger.notice(dirac.getJobPilotOutput(12345))
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport("Directory %s does not exist" % directory)
result = WMSAdministratorClient().getJobPilotOutput(jobID)
if not result["OK"]:
return result
outputPath = "%s/pilot_%s" % (directory, jobID)
if os.path.exists(outputPath):
self.log.info("Remove %s and retry to continue" % outputPath)
return S_ERROR("Remove %s and retry to continue" % outputPath)
if not os.path.exists(outputPath):
self.log.verbose("Creating directory %s" % outputPath)
os.mkdir(outputPath)
outputs = result["Value"]
if "StdOut" in outputs:
stdout = "%s/std.out" % (outputPath)
with open(stdout, "w") as fopen:
fopen.write(outputs["StdOut"])
self.log.verbose("Standard output written to %s" % (stdout))
else:
self.log.warn("No standard output returned")
if "StdError" in outputs:
stderr = "%s/std.err" % (outputPath)
with open(stderr, "w") as fopen:
fopen.write(outputs["StdError"])
self.log.verbose("Standard error written to %s" % (stderr))
else:
self.log.warn("No standard error returned")
self.log.always("Outputs retrieved in %s" % outputPath)
return result
#############################################################################
def getPilotOutput(self, gridReference, directory=""):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> gLogger.notice(dirac.getJobPilotOutput(12345))
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport("Expected string for pilot reference")
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport("Directory %s does not exist" % directory)
result = PilotManagerClient().getPilotOutput(gridReference)
if not result["OK"]:
return result
gridReferenceSmall = gridReference.split("/")[-1]
if not gridReferenceSmall:
gridReferenceSmall = "reference"
outputPath = "%s/pilot_%s" % (directory, gridReferenceSmall)
if os.path.exists(outputPath):
self.log.info("Remove %s and retry to continue" % outputPath)
return S_ERROR("Remove %s and retry to continue" % outputPath)
if not os.path.exists(outputPath):
self.log.verbose("Creating directory %s" % outputPath)
os.mkdir(outputPath)
outputs = result["Value"]
if "StdOut" in outputs:
stdout = "%s/std.out" % (outputPath)
with open(stdout, "w") as fopen:
fopen.write(outputs["StdOut"])
self.log.info("Standard output written to %s" % (stdout))
else:
self.log.warn("No standard output returned")
if "StdErr" in outputs:
stderr = "%s/std.err" % (outputPath)
with open(stderr, "w") as fopen:
fopen.write(outputs["StdErr"])
self.log.info("Standard error written to %s" % (stderr))
else:
self.log.warn("No standard error returned")
self.log.always("Outputs retrieved in %s" % outputPath)
return result
#############################################################################
def getPilotInfo(self, gridReference):
"""Retrieve info relative to a pilot reference
>>> gLogger.notice(dirac.getPilotInfo(12345))
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport("Expected string for pilot reference")
result = PilotManagerClient().getPilotInfo(gridReference)
return result
#############################################################################
def killPilot(self, gridReference):
"""Kill the pilot specified
>>> gLogger.notice(dirac.getPilotInfo(12345))
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport("Expected string for pilot reference")
result = PilotManagerClient().killPilot(gridReference)
return result
#############################################################################
def getPilotLoggingInfo(self, gridReference):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> gLogger.notice(dirac.getPilotLoggingInfo(12345))
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport("Expected string for pilot reference")
return PilotManagerClient().getPilotLoggingInfo(gridReference)
#############################################################################
def getJobPilots(self, jobID):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> gLogger.notice(dirac.getJobPilots())
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance(jobID, six.string_types):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x), "Expected integer or string for existing jobID")
result = PilotManagerClient().getPilots(jobID)
if result["OK"]:
gLogger.notice(self.pPrint.pformat(result["Value"]))
return result
#############################################################################
def getPilotSummary(self, startDate="", endDate=""):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> gLogger.notice(dirac.getPilotSummary())
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
result = PilotManagerClient().getPilotSummary(startDate, endDate)
if not result["OK"]:
return result
ceDict = result["Value"]
headers = "CE".ljust(28)
i = 0
for ce, summary in ceDict.items():
states = list(summary)
if len(states) > i:
i = len(states)
for i in range(i):
headers += "Status".ljust(12) + "Count".ljust(12)
gLogger.notice(headers)
for ce, summary in ceDict.items():
line = ce.ljust(28)
states = sorted(summary)
for state in states:
count = str(summary[state])
line += state.ljust(12) + count.ljust(12)
gLogger.notice(line)
return result
#############################################################################
def setSiteProtocols(self, site, protocolsList, printOutput=False):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self._checkSiteIsValid(site)
if not result["OK"]:
return result
siteSection = "/Resources/Sites/%s/%s/SE" % (site.split(".")[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR("No SEs found for site %s in section %s" %
(site, siteSection))
defaultProtocols = gConfig.getValue(
"/Resources/StorageElements/DefaultProtocols", [])
self.log.verbose("Default list of protocols are",
", ".join(defaultProtocols))
for protocol in protocolsList:
if protocol not in defaultProtocols:
return S_ERROR(
"Requested to set protocol %s in list but %s is not "
"in default list of protocols:\n%s" %
(protocol, protocol, ", ".join(defaultProtocols)))
modifiedCS = False
result = promptUser(
"Do you want to add the following default protocols:"
" %s for SE(s):\n%s" %
(", ".join(protocolsList), ", ".join(siteSEs)))
if not result["OK"]:
return result
if result["Value"].lower() != "y":
self.log.always("No protocols will be added")
return S_OK()
for se in siteSEs:
sections = gConfig.getSections("/Resources/StorageElements/%s/" %
(se))
if not sections["OK"]:
return sections
for section in sections["Value"]:
if gConfig.getValue(
"/Resources/StorageElements/%s/%s/ProtocolName" %
(se, section), "") == "SRM2":
path = "/Resources/StorageElements/%s/%s/ProtocolsList" % (
se, section)
self.log.verbose("Setting %s to %s" %
(path, ", ".join(protocolsList)))
result = self.csSetOption(path, ", ".join(protocolsList))
if not result["OK"]:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges(False)
if not result["OK"]:
return S_ERROR("CS Commit failed with message = %s" %
(result["Message"]))
else:
if printOutput:
gLogger.notice("Successfully committed changes to CS")
else:
if printOutput:
gLogger.notice("No modifications to CS required")
return S_OK()
#############################################################################
def csSetOption(self, optionPath, optionValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption(optionPath, optionValue)
#############################################################################
def csSetOptionComment(self, optionPath, comment):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment(optionPath, comment)
#############################################################################
def csModifyValue(self, optionPath, newValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue(optionPath, newValue)
#############################################################################
def csRegisterUser(self, username, properties):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser(username, properties)
#############################################################################
def csDeleteUser(self, user):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers(user)
#############################################################################
def csModifyUser(self, username, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser(username, properties, createIfNonExistant)
#############################################################################
def csListUsers(self, group=False):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers(group)
#############################################################################
def csDescribeUsers(self, mask=False):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers(mask)
#############################################################################
def csModifyGroup(self, groupname, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup(groupname, properties,
createIfNonExistant)
#############################################################################
def csListHosts(self):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts(self, mask=False):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts(mask)
#############################################################################
def csModifyHost(self, hostname, properties, createIfNonExistant=False):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost(hostname, properties, createIfNonExistant)
#############################################################################
def csListGroups(self):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups(self, mask=False):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups(mask)
#############################################################################
def csSyncUsersWithCFG(self, usersCFG):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG(usersCFG)
#############################################################################
def csCommitChanges(self, sortUsers=True):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges(sortUsers=False)
#############################################################################
def sendMail(self,
address,
subject,
body,
fromAddress=None,
localAttempt=True,
html=False):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail(address, subject, body, fromAddress,
localAttempt, html)
#############################################################################
def sendSMS(self, userName, body, fromAddress=None):
"""
Send mail to specified address with body.
"""
if len(body) > 160:
return S_ERROR("Exceeded maximum SMS length of 160 characters")
notification = NotificationClient()
return notification.sendSMS(userName, body, fromAddress)
class VOMS2CSSynchronizer(object):
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False):
self.log = gLogger.getSubLogger("VOMS2CSSynchronizer")
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {'Errors': [], 'Info': []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
self.voChanged = False
def syncCSWithVOMS(self):
""" Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS', result['Message'])
return result
self.vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName)
self.adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.adminMsgs['Info'].append("There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo))
self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get('nickname')
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[dn]['certSuspended']:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict)))
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' % newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(removedGroups)
else:
oldValue = str(diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" % (diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose("Modified user %s: dropped DN %s" % (user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict['DN'] = ','.join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: dropped DN %s" % (user, dn))
self.adminMsgs['Info'].append("Modified user %s: dropped DN %s" % (user, dn))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' % str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.adminMsgs['Info'].append('The following users are deleted from CS:\n %s\n' % str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.adminMsgs['Errors'].append('Error in deleting users from CS:\n %s' % str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.adminMsgs['Info'].append('The following users to be checked for deletion:\n\t%s' %
"\n\t".join(sorted(oldUsers)))
self.log.info('The following users to be checked for deletion: %s' % str(oldUsers))
resultDict['CSAPI'] = self.csapi
resultDict['AdminMessages'] = self.adminMsgs
resultDict['VOChanged'] = self.voChanged
return S_OK(resultDict)
def getVOUserData(self, refreshFlag=False):
""" Get a report for users of a given VO
:param bool refreshFlag: flag to indicate that the configuration must be refreshed
before looking up user data
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(self.vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % self.vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def getVOUserReport(self):
""" Get a report string with the current status of the DIRAC Registry for the
Virtual Organization
:return: S_OK with the report string as Value
"""
result = self.getVOUserData(refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, '')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields, records, sortField='Group', printOut=False, numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
def getUserName(self, dn):
""" Utility to construct user name
:param str dn: user DN
:return str: user name
"""
name = self.vomsUserDict[dn].get('name')
surname = self.vomsUserDict[dn].get('surname')
if name and surname:
surnameName = _getUserNameFromSurname(name, surname)
return surnameName
mail = self.vomsUserDict[dn]['mail']
dnName = _getUserNameFromDN(dn, self.vo)
mailName = _getUserNameFromMail(mail)
# If robot, take the dn based name
if dnName.startswith('robot'):
return dnName
# Is mailName reasonable ?
if len(mailName) > 5 and mailName.isalpha():
return mailName
# dnName too long
if len(dnName) >= 12:
return dnName[:11]
# May be the mail name is still more reasonable
if len(dnName) < len(mailName) and mailName.isalpha():
return mailName
return dnName