def __setCSElementStatus( self, elementName, elementType, statusType, status ): """ Sets on the CS the Elements status """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing if elementType in ('ComputingElement', 'FTS'): return S_OK() # If we are here it is because elementType is either 'StorageElement' or 'Catalog' statuses = self.rssConfig.getConfigStatusType( elementType ) if statusType not in statuses: gLogger.error( "%s is not a valid statusType" % statusType ) return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) ) if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" #FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs) # but needs to be VO-aware statusType = 'Status' csAPI = CSAPI() csAPI.setOption( "%s/%s/%s/%s" % ( cs_path, elementName, elementType, statusType ), status ) res = csAPI.commitChanges() if not res[ 'OK' ]: gLogger.warn( 'CS: %s' % res[ 'Message' ] ) return res
def __setCSElementStatus(self, elementName, elementType, statusType, status): """ Sets on the CS the Elements status """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing if elementType in ('ComputingElement', 'FTS'): return S_OK() # If we are here it is because elementType is either 'StorageElement' or 'Catalog' statuses = self.rssConfig.getConfigStatusType(elementType) if statusType not in statuses: gLogger.error("%s is not a valid statusType" % statusType) return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses)) if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" # FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs) # but needs to be VO-aware statusType = 'Status' csAPI = CSAPI() csAPI.setOption("%s/%s/%s/%s" % (cs_path, elementName, elementType, statusType), status) res = csAPI.commitChanges() if not res['OK']: gLogger.warn('CS: %s' % res['Message']) return res
def test_configurationAutoUpdate(value1, value2): """ Test if service refresh his configuration. It sent a random value to the CS and check if Service can return it. """ csapi = CSAPI() # SETTING FIRST VALUE csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value1) csapi.commitChanges() # Wait for automatic refresh (+1 to be sure that request is done) time.sleep(gConfigurationData.getPropagationTime() + 1) RPCClient("Framework/User").getTestValue() assert RPCClient("Framework/User").getTestValue()["Value"] == value1 # SETTING SECOND VALUE csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value2) csapi.commitChanges() time.sleep(gConfigurationData.getPropagationTime() + 1) assert RPCClient("Framework/User").getTestValue()["Value"] == value2
def __setCSStorageElementStatus( self, elementName, statusType, status ): """ Sets on the CS the StorageElements status """ statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) if not statusType in statuses: gLogger.error( "%s is not a valid statusType" % statusType ) return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) ) csAPI = CSAPI() cs_path = "/Resources/StorageElements" csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status ) res = csAPI.commitChanges() if not res[ 'OK' ]: gLogger.warn( 'CS: %s' % res[ 'Message' ] ) return res
def __syncCSWithVOMS( self ): self.__adminMsgs = { 'Errors' : [], 'Info' : [] } #Get DIRAC VOMS Mapping self.log.info( "Getting DIRAC VOMS mapping" ) mappingSection = '/Registry/VOMS/Mapping' ret = gConfig.getOptionsDict( mappingSection ) if not ret['OK']: self.log.fatal( 'No VOMS to DIRAC Group Mapping Available' ) return ret vomsMapping = ret['Value'] self.log.info( "There are %s registered voms mappings in DIRAC" % len( vomsMapping ) ) #Get VOMS VO name self.log.info( "Getting VOMS VO name" ) result = self.vomsSrv.admGetVOName() if not ret['OK']: self.log.fatal( 'Could not retrieve VOMS VO name' ) voNameInVOMS = result[ 'Value' ] self.log.info( "VOMS VO Name is %s" % voNameInVOMS ) #Get VOMS roles self.log.info( "Getting the list of registered roles in VOMS" ) result = self.vomsSrv.admListRoles() if not ret['OK']: self.log.fatal( 'Could not retrieve registered roles in VOMS' ) rolesInVOMS = result[ 'Value' ] self.log.info( "There are %s registered roles in VOMS" % len( rolesInVOMS ) ) print rolesInVOMS rolesInVOMS.append( '' ) #Map VOMS roles vomsRoles = {} for role in rolesInVOMS: if role: role = "%s/%s" % ( voNameInVOMS, role ) else: role = voNameInVOMS groupsForRole = [] for group in vomsMapping: if vomsMapping[ group ] == role: groupsForRole.append( group ) if groupsForRole: vomsRoles[ role ] = { 'Groups' : groupsForRole, 'Users' : [] } self.log.info( "DIRAC valid VOMS roles are:\n\t", "\n\t ".join( vomsRoles.keys() ) ) #Get DIRAC users self.log.info( "Getting the list of registered users in DIRAC" ) csapi = CSAPI() ret = csapi.listUsers() if not ret['OK']: self.log.fatal( 'Could not retrieve current list of Users' ) return ret currentUsers = ret['Value'] ret = csapi.describeUsers( currentUsers ) if not ret['OK']: self.log.fatal( 'Could not retrieve current User description' ) return ret currentUsers = ret['Value'] self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC" % len( currentUsers ) ) self.log.info( "There are %s registered users in DIRAC" % len( currentUsers ) ) #Get VOMS user entries self.log.info( "Getting the list of registered user entries in VOMS" ) result = self.vomsSrv.admListMembers() if not ret['OK']: self.log.fatal( 'Could not retrieve registered user entries in VOMS' ) usersInVOMS = result[ 'Value' ] self.__adminMsgs[ 'Info' ].append( "There are %s registered user entries in VOMS" % len( usersInVOMS ) ) self.log.info( "There are %s registered user entries in VOMS" % len( usersInVOMS ) ) #Consolidate users by nickname usersData = {} newUserNames = [] knownUserNames = [] obsoleteUserNames = [] self.log.info( "Retrieving usernames..." ) usersInVOMS.sort() for iUPos in range( len( usersInVOMS ) ): userName = '' user = usersInVOMS[ iUPos ] for oldUser in currentUsers: if user[ 'DN' ].strip() in List.fromChar( currentUsers[oldUser][ 'DN' ] ): userName = oldUser if not userName: result = self.vomsSrv.attGetUserNickname( user[ 'DN' ], user[ 'CA' ] ) if result[ 'OK' ]: userName = result[ 'Value' ] else: self.__adminMsgs[ 'Errors' ].append( "Could not retrieve nickname for DN %s" % user[ 'DN' ] ) self.log.error( "Could not get nickname for DN", user[ 'DN' ] ) userName = user[ 'mail' ][:user[ 'mail' ].find( '@' )] if not userName: self.log.error( "Empty nickname for DN", user[ 'DN' ] ) self.__adminMsgs[ 'Errors' ].append( "Empty nickname for DN %s" % user[ 'DN' ] ) continue self.log.info( " (%02d%%) Found username %s : %s " % ( ( iUPos * 100 / len( usersInVOMS ) ), userName, user[ 'DN' ] ) ) if userName not in usersData: usersData[ userName ] = { 'DN': [], 'CA': [], 'Email': [], 'Groups' : ['user'] } for key in ( 'DN', 'CA', 'mail' ): value = user[ key ] if value: if key == "mail": List.appendUnique( usersData[ userName ][ 'Email' ], value ) else: usersData[ userName ][ key ].append( value.strip() ) if userName not in currentUsers: List.appendUnique( newUserNames, userName ) else: List.appendUnique( knownUserNames, userName ) self.log.info( "Finished retrieving usernames" ) if newUserNames: self.log.info( "There are %s new users" % len( newUserNames ) ) else: self.log.info( "There are no new users" ) #Get the list of users for each group result = csapi.listGroups() if not result[ 'OK' ]: self.log.error( "Could not get the list of groups in DIRAC", result[ 'Message' ] ) return result staticGroups = result[ 'Value' ] vomsGroups = [] self.log.info( "Mapping users in VOMS to groups" ) for vomsRole in vomsRoles: self.log.info( " Getting users for role %s" % vomsRole ) groupsForRole = vomsRoles[ vomsRole ][ 'Groups' ] vomsMap = vomsRole.split( "Role=" ) for g in groupsForRole: if g in staticGroups: staticGroups.pop( staticGroups.index( g ) ) else: vomsGroups.append( g ) if len( vomsMap ) == 1: # no Role users = usersInVOMS else: vomsGroup = "Role=".join( vomsMap[:-1] ) if vomsGroup[-1] == "/": vomsGroup = vomsGroup[:-1] vomsRole = "Role=%s" % vomsMap[-1] result = self.vomsSrv.admListUsersWithRole( vomsGroup, vomsRole ) if not result[ 'OK' ]: errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[ group ] ) self.__adminMsgs[ 'Errors' ].append( errorMsg ) self.log.error( errorMsg, result[ 'Message' ] ) return result users = result['Value'] numUsersInGroup = 0 for vomsUser in users: for userName in usersData: if vomsUser[ 'DN' ] in usersData[ userName ][ 'DN' ]: numUsersInGroup += 1 usersData[ userName ][ 'Groups' ].extend( groupsForRole ) infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join( groupsForRole ), vomsRole ) self.__adminMsgs[ 'Info' ].append( infoMsg ) self.log.info( " %s" % infoMsg ) self.log.info( "Checking static groups" ) staticUsers = [] for group in staticGroups: self.log.info( " Checking static group %s" % group ) numUsersInGroup = 0 result = csapi.listUsers( group ) if not result[ 'OK' ]: self.log.error( "Could not get the list of users in DIRAC group %s" % group , result[ 'Message' ] ) return result for userName in result[ 'Value' ]: if userName in usersData: numUsersInGroup += 1 usersData[ userName ][ 'Groups' ].append( group ) else: if group not in vomsGroups and userName not in staticUsers: staticUsers.append( userName ) infoMsg = "There are %s users in group %s" % ( numUsersInGroup, group ) self.__adminMsgs[ 'Info' ].append( infoMsg ) self.log.info( " %s" % infoMsg ) if staticUsers: infoMsg = "There are %s static users: %s" % ( len( staticUsers ) , ', '.join( staticUsers ) ) self.__adminMsgs[ 'Info' ].append( infoMsg ) self.log.info( "%s" % infoMsg ) for user in currentUsers: if user not in usersData and user not in staticUsers: self.log.info( 'User %s is no longer valid' % user ) obsoleteUserNames.append( user ) #Do the CS Sync self.log.info( "Updating CS..." ) ret = csapi.downloadCSData() if not ret['OK']: self.log.fatal( 'Can not update from CS', ret['Message'] ) return ret usersWithMoreThanOneDN = {} for user in usersData: csUserData = dict( usersData[ user ] ) if len( csUserData[ 'DN' ] ) > 1: usersWithMoreThanOneDN[ user ] = csUserData[ 'DN' ] result = csapi.describeUsers( [ user ] ) if result[ 'OK' ]: if result[ 'Value' ]: prevUser = result[ 'Value' ][ user ] prevDNs = List.fromChar( prevUser[ 'DN' ] ) newDNs = csUserData[ 'DN' ] for DN in newDNs: if DN not in prevDNs: self.__adminMsgs[ 'Info' ].append( "User %s has new DN %s" % ( user, DN ) ) for DN in prevDNs: if DN not in newDNs: self.__adminMsgs[ 'Info' ].append( "User %s has lost a DN %s" % ( user, DN ) ) else: newDNs = csUserData[ 'DN' ] for DN in newDNs: self.__adminMsgs[ 'Info' ].append( "New user %s has new DN %s" % ( user, DN ) ) for k in ( 'DN', 'CA', 'Email' ): csUserData[ k ] = ", ".join( csUserData[ k ] ) result = csapi.modifyUser( user, csUserData, createIfNonExistant = True ) if not result[ 'OK' ]: self.__adminMsgs[ 'Error' ].append( "Cannot modify user %s: %s" % ( user, result[ 'Message' ] ) ) self.log.error( "Cannot modify user", user ) if usersWithMoreThanOneDN: self.__adminMsgs[ 'Info' ].append( "\nUsers with more than one DN:" ) for uwmtod in sorted( usersWithMoreThanOneDN ): self.__adminMsgs[ 'Info' ].append( " %s" % uwmtod ) self.__adminMsgs[ 'Info' ].append( " + DN list:" ) for DN in usersWithMoreThanOneDN[uwmtod]: self.__adminMsgs[ 'Info' ].append( " - %s" % DN ) if obsoleteUserNames: self.__adminMsgs[ 'Info' ].append( "\nObsolete users:" ) address = self.am_getOption( 'MailTo', '*****@*****.**' ) fromAddress = self.am_getOption( 'mailFrom', '*****@*****.**' ) subject = 'Obsolete LFC Users found' body = 'Delete entries into LFC: \n' for obsoleteUser in obsoleteUserNames: self.log.info( subject, ", ".join( obsoleteUserNames ) ) body += 'for ' + obsoleteUser + '\n' self.__adminMsgs[ 'Info' ].append( " %s" % obsoleteUser ) self.log.info( "Deleting %s users" % len( obsoleteUserNames ) ) NotificationClient().sendMail( address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress ) csapi.deleteUsers( obsoleteUserNames ) if newUserNames: self.__adminMsgs[ 'Info' ].append( "\nNew users:" ) for newUser in newUserNames: self.__adminMsgs[ 'Info' ].append( " %s" % newUser ) self.__adminMsgs[ 'Info' ].append( " + DN list:" ) for DN in usersData[newUser][ 'DN' ]: self.__adminMsgs[ 'Info' ].append( " - %s" % DN ) self.__adminMsgs[ 'Info' ].append( " + EMail: %s" % usersData[newUser][ 'Email' ] ) result = csapi.commitChanges() if not result[ 'OK' ]: self.log.error( "Could not commit configuration changes", result[ 'Message' ] ) return result self.log.info( "Configuration committed" ) #LFC Check if self.am_getOption( "LFCCheckEnabled", True ): result = self.checkLFCRegisteredUsers( usersData ) if not result[ 'OK' ]: return result return S_OK()
class VOMS2CSAgent(AgentModule): def __init__(self, *args, **kwargs): """ Defines default parameters """ super(VOMS2CSAgent, self).__init__(*args, **kwargs) self.__voDict = {} self.__adminMsgs = {} self.csapi = None self.voChanged = False self.dryRun = False self.autoAddUsers = False self.autoModifyUsers = False self.autoDeleteUsers = False self.detailedReport = True self.makeFCEntry = False def initialize(self): """ Initialize the default parameters """ voNames = self.am_getOption('VO', []) if not voNames[0].lower() == "none": if voNames[0].lower() == "any": voNames = [] result = getVOMSVOs(voNames) if not result['OK']: return result self.__voDict = result['Value'] self.log.notice("VOs: %s" % self.__voDict.keys()) self.csapi = CSAPI() self.dryRun = self.am_getOption('DryRun', self.dryRun) self.autoAddUsers = self.am_getOption('AutoAddUsers', self.autoAddUsers) self.autoModifyUsers = self.am_getOption('AutoModifyUsers', self.autoModifyUsers) self.autoDeleteUsers = self.am_getOption('AutoDeleteUsers', self.autoDeleteUsers) self.detailedReport = self.am_getOption('DetailedReport', self.detailedReport) self.makeFCEntry = self.am_getOption('MakeHomeDirectory', self.makeFCEntry) return S_OK() def execute(self): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption(vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption(voAdminUser, "Email") voAdminGroup = getVOOption(vo, "VOAdminGroup", getVOOption(vo, "DefaultGroup")) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % (vo, voAdminUser, voAdminGroup)) result = self.__syncCSWithVOMS(vo, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error('Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % (vo, result["Message"])) continue resultDict = result['Value'] newUsers = resultDict.get("NewUsers", []) modUsers = resultDict.get("ModifiedUsers", []) delUsers = resultDict.get("DeletedUsers", []) self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \ ( len( newUsers ), len( modUsers ), len( delUsers ) ) ) if self.csapi.csModified: # We have accumulated all the changes, commit them now self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo) if self.dryRun: self.log.info("Dry Run: CS won't be updated") self.csapi.showDiff() else: result = self.csapi.commitChanges() if not result['OK']: self.log.error( "Could not commit configuration changes", result['Message']) return result self.log.notice("Configuration committed for VO %s" % vo) else: self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo) # Add user home directory in the file catalog if self.makeFCEntry and newUsers: self.log.info("Creating home directories for users %s" % str(newUsers)) result = self.__addHomeDirectory(vo, newUsers, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error('Failed to create user home directories:', 'VO %s: %s' % (vo, result["Message"])) else: for user in result['Value']['Failed']: self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \ ( user, result['Value']['Failed'][user] ) ) self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \ ( user, result['Value']['Failed'][user] ) ) for user in result['Value']['Successful']: self.__adminMsgs['Info'].append( "Created home directory for user %s" % user) if self.voChanged or self.detailedReport: mailMsg = "" if self.__adminMsgs['Errors']: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs['Errors']) if self.__adminMsgs['Info']: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs['Info']) if self.detailedReport: result = self.getVOUserReport(vo) if result['OK']: mailMsg += '\n\n' mailMsg += result['Value'] else: mailMsg += 'Failed to produce a detailed user report' mailMsg += result['Message'] NotificationClient().sendMail( self.am_getOption('MailTo', voAdminMail), "VOMS2CSAgent run log", mailMsg, self.am_getOption( 'MailFrom', self.am_getOption('mailFrom', "DIRAC system"))) return S_OK() def getVOUserData(self, vo, refreshFlag=False): """ Get a report for users of a given VO :param str vo: VO name :return: S_OK/S_ERROR, Value = user description dictionary """ if refreshFlag: gConfig.forceRefresh() # Get DIRAC users diracUsers = getUsersInVO(vo) if not diracUsers: return S_ERROR("No VO users found for %s" % vo) if refreshFlag: result = self.csapi.downloadCSData() if not result['OK']: return result result = self.csapi.describeUsers(diracUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result def getVOUserReport(self, vo): """ :param str vo: VO name :return: report string """ result = self.getVOUserData(vo, refreshFlag=True) if not result['OK']: return result userDict = result['Value'] # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(vo) if not result['OK']: return result diracVOMSMapping = result['Value']['DIRACVOMS'] records = [] groupDict = defaultdict(int) multiDNUsers = {} suspendedUsers = [] for user in userDict: for group in userDict[user]['Groups']: groupDict[group] += 1 dnList = fromChar(userDict[user]['DN']) if len(dnList) > 1: multiDNUsers[user] = dnList if userDict[user].get('Status', 'Active') == 'Suspended': suspendedUsers.append(user) for group in diracVOMSMapping: records.append( (group, str(groupDict[group]), diracVOMSMapping.get(group, ''))) fields = ['Group', 'Number of users', 'VOMS Role'] output = printTable(fields, records, sortField='Group', printOut=False, numbering=False) if multiDNUsers: output += '\nUsers with multiple DNs:\n' for user in multiDNUsers: output += ' %s:\n' % user for dn in multiDNUsers[user]: output += ' %s\n' % dn if suspendedUsers: output += '\n%d suspended users:\n' % len(suspendedUsers) output += ' %s' % ','.join(suspendedUsers) return S_OK(output) @executeWithUserProxy def __syncCSWithVOMS(self, vo): self.__adminMsgs = {'Errors': [], 'Info': []} resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService(vo) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error('Could not retrieve VOMS VO name', "for %s" % vo) return result vomsVOName = result['Value'].lstrip('/') self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName)) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result vomsUserDict = result['Value'] message = "There are %s user entries in VOMS for VO %s" % ( len(vomsUserDict), vomsVOName) self.__adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users result = self.getVOUserData(vo) if not result['OK']: return result diracUserDict = result['Value'] self.__adminMsgs['Info'].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in vomsUserDict: obsoletedDNs.append(dn) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict.keys())) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) newAddedUserDict = {} for dn in vomsUserDict: nickName = '' newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN result = vomsSrv.attGetUserNickname(dn, vomsUserDict[dn]['CA']) if result['OK']: nickName = result['Value'] if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = getUserName(dn, vomsUserDict[dn]['mail'], vo) # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) groupList = vomsDIRACMapping.get(fullRole, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict['Groups'] = list( set(groupsWithRole + [defaultVOGroup])) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.__adminMsgs['Info'].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) resultDict['NewUsers'].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } if newDNForExistingUser: userDict['DN'] = ','.join([dn, diracUserDict[diracName]['DN']]) modified = True existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) nonVOGroups = list( set(existingGroups) - set(diracVOMSMapping.keys())) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) groupList = vomsDIRACMapping.get(fullRole, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, '') # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict['Groups'] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get('Groups', []) userDict['Groups'] = list(set(keepGroups + otherGroups)) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set( diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set( diracUserDict.get(diracName, {}).get( key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join( addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups) else: oldValue = str( diracUserDict.get(diracName, {}).get(key, '')) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % ( key, oldValue, str(userDict[key])) if modMsg: self.__adminMsgs['Info'].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) if not dnSet.intersection(set( vomsUserDict.keys())) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info('The following users will be deleted: %s' % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result['OK']: self.__adminMsgs['Info'].append( 'The following users are deleted from CS:\n %s\n' % str(oldUsers)) resultDict['DeletedUsers'] = oldUsers else: self.__adminMsgs['Errors'].append( 'Error in deleting users from CS:\n %s' % str(oldUsers)) self.log.error('Error while user deletion from CS', result) else: self.__adminMsgs['Info'].append( 'The following users to be checked for deletion:\n %s' % str(oldUsers)) self.log.info( 'The following users to be checked for deletion: %s' % str(oldUsers)) return S_OK(resultDict) @executeWithUserProxy def __addHomeDirectory(self, vo, newUsers): fc = FileCatalog(vo=vo) defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) failed = {} successful = {} for user in newUsers: result = fc.addUser(user) if not result['OK']: failed[user] = "addUser" continue dirName = '/%s/user/%s/%s' % (vo, user[0], user) result = fc.createDirectory(dirName) if not result['OK']: failed[user] = "createDirectory" continue result = fc.changePathOwner({dirName: user}, recursive=True) if not result['OK']: failed[user] = "changePathOwner" continue result = fc.changePathGroup({dirName: defaultVOGroup}, recursive=True) if not result['OK']: failed[user] = "changePathGroup" continue successful[user] = True return S_OK({"Successful": successful, "Failed": failed})
class DiracAdmin(API): """ Administrative functionalities """ ############################################################################# def __init__(self): """Internal initialization of the DIRAC Admin API. """ super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue(self.section + '/ScratchDir', '/tmp') self.currentDir = os.getcwd() self.rssFlag = ResourceStatus().rssFlag self.sitestatus = SiteStatus() ############################################################################# def uploadProxy(self, group): """Upload a proxy to the DIRAC WMS. This method Example usage: >>> print diracAdmin.uploadProxy('lhcb_pilot') {'OK': True, 'Value': 0L} :param group: DIRAC Group :type job: string :return: S_OK,S_ERROR :param permanent: Indefinitely update proxy :type permanent: boolean """ return gProxyManager.uploadProxy(diracGroup=group) ############################################################################# def setProxyPersistency(self, userDN, userGroup, persistent=True): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param persistent: Persistent flag :type persistent: boolean :return: S_OK,S_ERROR """ return gProxyManager.setPersistency(userDN, userGroup, persistent) ############################################################################# def checkProxyUploaded(self, userDN, userGroup, requiredTime): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True, 'Value' : True/False } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param requiredTime: Required life time of the uploaded proxy :type requiredTime: boolean :return: S_OK,S_ERROR """ return gProxyManager.userHasProxy(userDN, userGroup, requiredTime) ############################################################################# def getSiteMask(self, printOutput=False, status='Active'): """Retrieve current site mask from WMS Administrator service. Example usage: >>> print diracAdmin.getSiteMask() {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR """ result = self.sitestatus.getSites(siteState=status) if result['OK']: sites = result['Value'] if printOutput: sites.sort() for site in sites: print site return result ############################################################################# def getBannedSites(self, gridType=[], printOutput=False): """Retrieve current list of banned and probing sites. Example usage: >>> print diracAdmin.getBannedSites() {'OK': True, 'Value': []} :return: S_OK,S_ERROR """ bannedSites = self.sitestatus.getSites(siteState='Banned') if not bannedSites['OK']: return bannedSites probingSites = self.sitestatus.getSites(siteState='Probing') if not probingSites['OK']: return probingSites mergedList = bannedSites['Value'] + probingSites['Value'] mergedList.sort() if printOutput: print '\n'.join(mergedList) return S_OK(mergedList) ############################################################################# def getSiteSection(self, site, printOutput=False): """Simple utility to get the list of CEs for DIRAC site name. Example usage: >>> print diracAdmin.getSiteSection('LCG.CERN.ch') {'OK': True, 'Value':} :return: S_OK,S_ERROR """ gridType = site.split('.')[0] if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']: return S_ERROR('/Resources/Sites/%s is not a valid site section' % (gridType)) result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' % (gridType, site)) if printOutput and result['OK']: print self.pPrint.pformat(result['Value']) return result ############################################################################# def allowSite(self, site, comment, printOutput=False): """Adds the site to the site mask. Example usage: >>> print diracAdmin.allowSite() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result result = self.getSiteMask(status='Active') if not result['OK']: return result siteMask = result['Value'] if site in siteMask: if printOutput: print 'Site %s is already Active' % site return S_OK('Site %s is already Active' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Active', comment) else: wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.allowSite(site, comment) if not result['OK']: return result if printOutput: print 'Site %s status is set to Active' % site return result ############################################################################# def getSiteMaskLogging(self, site=None, printOutput=False): """Retrieves site mask logging information. Example usage: >>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr') {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getSiteMaskLogging(site) if not result['OK']: return result if site: if not result['Value'].has_key(site): return S_ERROR('Site mask information not available for %s' % (site)) if printOutput: if site: print '\nSite Mask Logging Info for %s\n' % site else: print '\nAll Site Mask Logging Info\n' siteDict = result['Value'] for site, tupleList in siteDict.iteritems(): if not site: print '\n===> %s\n' % site for tup in tupleList: print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \ '( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"' print ' ' return result ############################################################################# def banSite(self, site, comment, printOutput=False): """Removes the site from the site mask. Example usage: >>> print diracAdmin.banSite() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result mask = self.getSiteMask(status='Banned') if not mask['OK']: return mask siteMask = mask['Value'] if site in siteMask: if printOutput: print 'Site %s is already Banned' % site return S_OK('Site %s is already Banned' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Banned', comment) else: wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.banSite(site, comment) if not result['OK']: return result if printOutput: print 'Site %s status is set to Banned' % site return result ############################################################################# def __checkSiteIsValid(self, site): """Internal function to check that a site name is valid. """ sites = getSiteCEMapping() if not sites['OK']: return S_ERROR('Could not get site CE mapping') siteList = sites['Value'].keys() if not site in siteList: return S_ERROR( 'Specified site %s is not in list of defined sites' % site) return S_OK('%s is valid' % site) ############################################################################# def clearMask(self): """Removes all sites from the site mask. Should be used with care. Example usage: >>> print diracAdmin.clearMask() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.clearMask() return result ############################################################################# def getServicePorts(self, setup='', printOutput=False): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> print diracAdmin.getServicePorts() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if not setup in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict systemList = gConfig.getSections('/Systems') if not systemList['OK']: return S_ERROR('Could not get Systems sections') systemList = systemList['Value'] result = {} for system in systemList: if serviceSetups.has_key(system): path = '/Systems/%s/%s/Services' % (system, serviceSetups[system]) servicesList = gConfig.getSections(path) if not servicesList['OK']: self.log.warn('Could not get sections in %s' % path) else: servicesList = servicesList['Value'] if not servicesList: servicesList = [] self.log.verbose('System: %s ServicesList: %s' % (system, ', '.join(servicesList))) for service in servicesList: spath = '%s/%s/Port' % (path, service) servicePort = gConfig.getValue(spath, 0) if servicePort: self.log.verbose('Found port for %s/%s = %s' % (system, service, servicePort)) result['%s/%s' % (system, service)] = servicePort else: self.log.warn('No port found for %s' % spath) else: self.log.warn('%s is not defined in /DIRAC/Setups/%s' % (system, setup)) if printOutput: print self.pPrint.pformat(result) return S_OK(result) ############################################################################# def getProxy(self, userDN, userGroup, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadProxy(userDN, userGroup, limited=limited, requiredTimeLeft=validity) ############################################################################# def getVOMSProxy(self, userDN, userGroup, vomsAttr=False, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and VOMS extensions and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=limited, requiredVOMSAttribute=vomsAttr, requiredTimeLeft=validity) ############################################################################# def getPilotProxy(self, userDN, userGroup, validity=43200): """Retrieves a pilot proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft=validity) ############################################################################# def resetJob(self, jobID): """Reset a job or list of jobs in the WMS. This operation resets the reschedule counter for a job or list of jobs and allows them to run as new. Example:: >>> print dirac.reset(12345) {'OK': True, 'Value': [12345]} :param job: JobID :type job: integer or list of integers :return: S_OK,S_ERROR """ if isinstance(jobID, basestring): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobID' ) elif isinstance(jobID, list): try: jobID = [int(job) for job in jobID] except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobIDs' ) jobManager = RPCClient('WorkloadManagement/JobManager', useCertificates=False) result = jobManager.resetJob(jobID) return result ############################################################################# def getJobPilotOutput(self, jobID, directory=''): """Retrieve the pilot output for an existing job in the WMS. The output will be retrieved in a local directory unless otherwise specified. >>> print dirac.getJobPilotOutput(12345) {'OK': True, StdOut:'',StdError:''} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getJobPilotOutput(jobID) if not result['OK']: return result outputPath = '%s/pilot_%s' % (directory, jobID) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if outputs.has_key('StdOut'): stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.verbose('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if outputs.has_key('StdError'): stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdError']) self.log.verbose('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotOutput(self, gridReference, directory=''): """Retrieve the pilot output (std.out and std.err) for an existing job in the WMS. >>> print dirac.getJobPilotOutput(12345) {'OK': True, 'Value': {}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not isinstance(gridReference, basestring): return self._errorReport('Expected string for pilot reference') if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilotOutput(gridReference) if not result['OK']: return result gridReferenceSmall = gridReference.split('/')[-1] if not gridReferenceSmall: gridReferenceSmall = 'reference' outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if outputs.has_key('StdOut'): stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.info('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if outputs.has_key('StdErr'): stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdErr']) self.log.info('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotInfo(self, gridReference): """Retrieve info relative to a pilot reference >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance(gridReference, basestring): return self._errorReport('Expected string for pilot reference') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilotInfo(gridReference) return result ############################################################################# def killPilot(self, gridReference): """Kill the pilot specified >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :return: S_OK,S_ERROR """ if not isinstance(gridReference, basestring): return self._errorReport('Expected string for pilot reference') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.killPilot(gridReference) return result ############################################################################# def getPilotLoggingInfo(self, gridReference): """Retrieve the pilot logging info for an existing job in the WMS. >>> print dirac.getPilotLoggingInfo(12345) {'OK': True, 'Value': {"The output of the command"}} :param gridReference: Gridp pilot job reference Id :type gridReference: string :return: S_OK,S_ERROR """ if type(gridReference) not in types.StringTypes: return self._errorReport('Expected string for pilot reference') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') return wmsAdmin.getPilotLoggingInfo(gridReference) ############################################################################# def getJobPilots(self, jobID): """Extract the list of submitted pilots and their status for a given jobID from the WMS. Useful information is printed to the screen. >>> print dirac.getJobPilots() {'OK': True, 'Value': {PilotID:{StatusDict}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if isinstance(jobID, basestring): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or string for existing jobID') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilots(jobID) if result['OK']: print self.pPrint.pformat(result['Value']) return result ############################################################################# def getPilotSummary(self, startDate='', endDate=''): """Retrieve the pilot output for an existing job in the WMS. Summary is printed at INFO level, full dictionary of results also returned. >>> print dirac.getPilotSummary() {'OK': True, 'Value': {CE:{Status:Count}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilotSummary(startDate, endDate) if not result['OK']: return result ceDict = result['Value'] headers = 'CE'.ljust(28) i = 0 for ce, summary in ceDict.iteritems(): states = summary.keys() if len(states) > i: i = len(states) for i in xrange(i): headers += 'Status'.ljust(12) + 'Count'.ljust(12) print headers for ce, summary in ceDict.iteritems(): line = ce.ljust(28) states = summary.keys() states.sort() for state in states: count = str(summary[state]) line += state.ljust(12) + count.ljust(12) print line return result ############################################################################# def selectRequests(self, jobID=None, requestID=None, requestName=None, requestType=None, status=None, operation=None, ownerDN=None, ownerGroup=None, requestStart=0, limit=100, printOutput=False): """Select requests from the request management system. A few notes on the selection criteria: - jobID is the WMS JobID for the request (if applicable) - requestID is assigned during submission of the request - requestName is the corresponding XML file name - requestType e.g. 'transfer' - status e.g. Done - operation e.g. replicateAndRegister - requestStart e.g. the first request to consider (start from 0 by default) - limit e.g. selection limit (default 100) >>> dirac.selectRequests(jobID='4894') {'OK': True, 'Value': [[<Requests>]]} """ options = { 'RequestID': requestID, 'RequestName': requestName, 'JobID': jobID, 'OwnerDN': ownerDN, 'OwnerGroup': ownerGroup, 'RequestType': requestType, 'Status': status, 'Operation': operation } conditions = {} for key, value in options.iteritems(): if value: try: conditions[key] = str(value) except Exception as x: return self._errorReport( str(x), 'Expected string for %s field' % key) try: requestStart = int(requestStart) limit = int(limit) except Exception as x: return self._errorReport(str(x), 'Expected integer for %s field' % limit) self.log.verbose('Will select requests with the following conditions') self.log.verbose(self.pPrint.pformat(conditions)) requestClient = RPCClient("RequestManagement/centralURL") result = requestClient.getRequestSummaryWeb(conditions, [], requestStart, limit) if not result['OK']: self.log.warn(result['Message']) return result requestIDs = result['Value'] conds = [] for key, value in conditions.iteritems(): if value: conds.append('%s = %s' % (key, value)) self.log.verbose( '%s request(s) selected with conditions %s and limit %s' % (len(requestIDs['Records']), ', '.join(conds), limit)) if printOutput: requests = [] if len(requestIDs['Records']) > limit: requestList = requestIDs['Records'] requests = requestList[:limit] else: requests = requestIDs['Records'] print '%s request(s) selected with conditions %s and limit %s' % ( len(requestIDs['Records']), ', '.join(conds), limit) print requestIDs['ParameterNames'] for request in requests: print request if not requestIDs: return S_ERROR('No requests selected for conditions: %s' % conditions) else: return result ############################################################################# def getRequestSummary(self, printOutput=False): """ Get a summary of the requests in the request DB. """ requestClient = RPCClient("RequestManagement/centralURL", timeout=120) result = requestClient.getDBSummary() if not result['OK']: self.log.warn(result['Message']) return result if printOutput: print self.pPrint.pformat(result['Value']) return result ############################################################################# def getExternalPackageVersions(self): """ Simple function that attempts to obtain the external versions for the local DIRAC installation (frequently needed for debugging purposes). """ gLogger.info( 'DIRAC version v%dr%d build %d' % (DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel)) try: import lcg_util infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__ gLogger.info(infoStr) infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version( ) gLogger.info(infoStr) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % ( x) gLogger.exception(errStr) try: import gfalthr as gfal infoStr = "Using gfalthr from: \n%s" % gfal.__file__ gLogger.info(infoStr) infoStr = "The version of gfalthr is %s" % gfal.gfal_version() gLogger.info(infoStr) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % ( x) gLogger.warn(errStr) try: import gfal infoStr = "Using gfal from: %s" % gfal.__file__ gLogger.info(infoStr) infoStr = "The version of gfal is %s" % gfal.gfal_version() gLogger.info(infoStr) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % ( x) gLogger.exception(errStr) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) gLogger.info('Default list of protocols are: %s' % (', '.join(defaultProtocols))) return S_OK() ############################################################################# def getSiteProtocols(self, site, printOutput=False): """ Allows to check the defined protocols for each site SE. """ result = self.__checkSiteIsValid(site) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site) siteSEs = gConfig.getValue(siteSection, []) if not siteSEs: return S_ERROR('No SEs found for site %s in section %s' % (site, siteSection)) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) self.log.verbose('Default list of protocols are' ', '.join(defaultProtocols)) seInfo = {} siteSEs.sort() for se in siteSEs: sections = gConfig.getSections('/Resources/StorageElements/%s/' % (se)) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % (se, section), '') == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section) seProtocols = gConfig.getValue(path, []) if not seProtocols: seProtocols = defaultProtocols seInfo[se] = seProtocols if printOutput: print '\nSummary of protocols for StorageElements at site %s' % site print '\nStorageElement'.ljust(30) + 'ProtocolsList'.ljust( 30) + '\n' for se, protocols in seInfo.iteritems(): print se.ljust(30) + ', '.join(protocols).ljust(30) return S_OK(seInfo) ############################################################################# def setSiteProtocols(self, site, protocolsList, printOutput=False): """ Allows to set the defined protocols for each SE for a given site. """ result = self.__checkSiteIsValid(site) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site) siteSEs = gConfig.getValue(siteSection, []) if not siteSEs: return S_ERROR('No SEs found for site %s in section %s' % (site, siteSection)) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) self.log.verbose('Default list of protocols are', ', '.join(defaultProtocols)) for protocol in protocolsList: if not protocol in defaultProtocols: return S_ERROR( 'Requested to set protocol %s in list but %s is not ' 'in default list of protocols:\n%s' % (protocol, protocol, ', '.join(defaultProtocols))) modifiedCS = False result = promptUser( 'Do you want to add the following default protocols:' ' %s for SE(s):\n%s' % (', '.join(protocolsList), ', '.join(siteSEs))) if not result['OK']: return result if result['Value'].lower() != 'y': self.log.always('No protocols will be added') return S_OK() for se in siteSEs: sections = gConfig.getSections('/Resources/StorageElements/%s/' % (se)) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % (se, section), '') == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section) self.log.verbose('Setting %s to %s' % (path, ', '.join(protocolsList))) result = self.csSetOption(path, ', '.join(protocolsList)) if not result['OK']: return result modifiedCS = True if modifiedCS: result = self.csCommitChanges(False) if not result['OK']: return S_ERROR('CS Commit failed with message = %s' % (result['Message'])) else: if printOutput: print 'Successfully committed changes to CS' else: if printOutput: print 'No modifications to CS required' return S_OK() ############################################################################# def csSetOption(self, optionPath, optionValue): """ Function to modify an existing value in the CS. """ return self.csAPI.setOption(optionPath, optionValue) ############################################################################# def csSetOptionComment(self, optionPath, comment): """ Function to modify an existing value in the CS. """ return self.csAPI.setOptionComment(optionPath, comment) ############################################################################# def csModifyValue(self, optionPath, newValue): """ Function to modify an existing value in the CS. """ return self.csAPI.modifyValue(optionPath, newValue) ############################################################################# def csRegisterUser(self, username, properties): """ Registers a user in the CS. - username: Username of the user (easy;) - properties: Dict containing: - DN - groups : list/tuple of groups the user belongs to - <others> : More properties of the user, like mail """ return self.csAPI.addUser(username, properties) ############################################################################# def csDeleteUser(self, user): """ Deletes a user from the CS. Can take a list of users """ return self.csAPI.deleteUsers(user) ############################################################################# def csModifyUser(self, username, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addUser and applies the changes """ return self.csAPI.modifyUser(username, properties, createIfNonExistant) ############################################################################# def csListUsers(self, group=False): """ Lists the users in the CS. If no group is specified return all users. """ return self.csAPI.listUsers(group) ############################################################################# def csDescribeUsers(self, mask=False): """ List users and their properties in the CS. If a mask is given, only users in the mask will be returned """ return self.csAPI.describeUsers(mask) ############################################################################# def csModifyGroup(self, groupname, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addGroup and applies the changes """ return self.csAPI.modifyGroup(groupname, properties, createIfNonExistant) ############################################################################# def csListHosts(self): """ Lists the hosts in the CS """ return self.csAPI.listHosts() ############################################################################# def csDescribeHosts(self, mask=False): """ Gets extended info for the hosts in the CS """ return self.csAPI.describeHosts(mask) ############################################################################# def csModifyHost(self, hostname, properties, createIfNonExistant=False): """ Modify a host in the CS. Takes the same params as in addHost and applies the changes """ return self.csAPI.modifyHost(hostname, properties, createIfNonExistant) ############################################################################# def csListGroups(self): """ Lists groups in the CS """ return self.csAPI.listGroups() ############################################################################# def csDescribeGroups(self, mask=False): """ List groups and their properties in the CS. If a mask is given, only groups in the mask will be returned """ return self.csAPI.describeGroups(mask) ############################################################################# def csSyncUsersWithCFG(self, usersCFG): """ Synchronize users in cfg with its contents """ return self.csAPI.syncUsersWithCFG(usersCFG) ############################################################################# def csCommitChanges(self, sortUsers=True): """ Commit the changes in the CS """ return self.csAPI.commitChanges(sortUsers=False) ############################################################################# def sendMail(self, address, subject, body, fromAddress=None, localAttempt=True, html=False): """ Send mail to specified address with body. """ notification = NotificationClient() return notification.sendMail(address, subject, body, fromAddress, localAttempt, html) ############################################################################# def sendSMS(self, userName, body, fromAddress=None): """ Send mail to specified address with body. """ if len(body) > 160: return S_ERROR('Exceeded maximum SMS length of 160 characters') notification = NotificationClient() return notification.sendSMS(userName, body, fromAddress) ############################################################################# def getBDIISite(self, site, host=None): """ Get information about site from BDII at host """ return ldapSite(site, host=host) ############################################################################# def getBDIICluster(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCluster(ce, host=host) ############################################################################# def getBDIICE(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCE(ce, host=host) ############################################################################# def getBDIIService(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapService(ce, host=host) ############################################################################# def getBDIICEState(self, ce, useVO=voName, host=None): """ Get information about ce state from BDII at host """ return ldapCEState(ce, useVO, host=host) ############################################################################# def getBDIICEVOView(self, ce, useVO=voName, host=None): """ Get information about ce voview from BDII at host """ return ldapCEVOView(ce, useVO, host=host) ############################################################################# def getBDIISE(self, site, useVO=voName, host=None): """ Get information about SA from BDII at host """ return ldapSE(site, useVO, host=host)
def __syncCSWithVOMS(self): self.__adminMsgs = {'Errors': [], 'Info': []} #Get DIRAC VOMS Mapping self.log.info("Getting DIRAC VOMS mapping") mappingSection = '/Registry/VOMS/Mapping' ret = gConfig.getOptionsDict(mappingSection) if not ret['OK']: self.log.fatal('No VOMS to DIRAC Group Mapping Available') return ret vomsMapping = ret['Value'] self.log.info("There are %s registered voms mappings in DIRAC" % len(vomsMapping)) #Get VOMS VO name self.log.info("Getting VOMS VO name") result = self.vomsSrv.admGetVOName() if not ret['OK']: self.log.fatal('Could not retrieve VOMS VO name') voNameInVOMS = result['Value'] self.log.info("VOMS VO Name is %s" % voNameInVOMS) #Get VOMS roles self.log.info("Getting the list of registered roles in VOMS") result = self.vomsSrv.admListRoles() if not ret['OK']: self.log.fatal('Could not retrieve registered roles in VOMS') rolesInVOMS = result['Value'] self.log.info("There are %s registered roles in VOMS" % len(rolesInVOMS)) print rolesInVOMS rolesInVOMS.append('') #Map VOMS roles vomsRoles = {} for role in rolesInVOMS: if role: role = "%s/%s" % (voNameInVOMS, role) else: role = voNameInVOMS groupsForRole = [] for group in vomsMapping: if vomsMapping[group] == role: groupsForRole.append(group) if groupsForRole: vomsRoles[role] = {'Groups': groupsForRole, 'Users': []} self.log.info("DIRAC valid VOMS roles are:\n\t", "\n\t ".join(vomsRoles.keys())) #Get DIRAC users self.log.info("Getting the list of registered users in DIRAC") csapi = CSAPI() ret = csapi.listUsers() if not ret['OK']: self.log.fatal('Could not retrieve current list of Users') return ret currentUsers = ret['Value'] ret = csapi.describeUsers(currentUsers) if not ret['OK']: self.log.fatal('Could not retrieve current User description') return ret currentUsers = ret['Value'] self.__adminMsgs['Info'].append( "There are %s registered users in DIRAC" % len(currentUsers)) self.log.info("There are %s registered users in DIRAC" % len(currentUsers)) #Get VOMS user entries self.log.info("Getting the list of registered user entries in VOMS") result = self.vomsSrv.admListMembers() if not ret['OK']: self.log.fatal( 'Could not retrieve registered user entries in VOMS') usersInVOMS = result['Value'] self.__adminMsgs['Info'].append( "There are %s registered user entries in VOMS" % len(usersInVOMS)) self.log.info("There are %s registered user entries in VOMS" % len(usersInVOMS)) #Consolidate users by nickname usersData = {} newUserNames = [] knownUserNames = [] obsoleteUserNames = [] self.log.info("Retrieving usernames...") usersInVOMS.sort() for iUPos in range(len(usersInVOMS)): userName = '' user = usersInVOMS[iUPos] for oldUser in currentUsers: if user['DN'].strip() in List.fromChar( currentUsers[oldUser]['DN']): userName = oldUser if not userName: result = self.vomsSrv.attGetUserNickname( user['DN'], user['CA']) if result['OK']: userName = result['Value'] else: self.__adminMsgs['Errors'].append( "Could not retrieve nickname for DN %s" % user['DN']) self.log.error("Could not get nickname for DN", user['DN']) userName = user['mail'][:user['mail'].find('@')] if not userName: self.log.error("Empty nickname for DN", user['DN']) self.__adminMsgs['Errors'].append("Empty nickname for DN %s" % user['DN']) continue self.log.info( " (%02d%%) Found username %s : %s " % ((iUPos * 100 / len(usersInVOMS)), userName, user['DN'])) if userName not in usersData: usersData[userName] = { 'DN': [], 'CA': [], 'Email': [], 'Groups': ['user'] } for key in ('DN', 'CA', 'mail'): value = user[key] if value: if key == "mail": List.appendUnique(usersData[userName]['Email'], value) else: usersData[userName][key].append(value.strip()) if userName not in currentUsers: List.appendUnique(newUserNames, userName) else: List.appendUnique(knownUserNames, userName) self.log.info("Finished retrieving usernames") if newUserNames: self.log.info("There are %s new users" % len(newUserNames)) else: self.log.info("There are no new users") #Get the list of users for each group result = csapi.listGroups() if not result['OK']: self.log.error("Could not get the list of groups in DIRAC", result['Message']) return result staticGroups = result['Value'] vomsGroups = [] self.log.info("Mapping users in VOMS to groups") for vomsRole in vomsRoles: self.log.info(" Getting users for role %s" % vomsRole) groupsForRole = vomsRoles[vomsRole]['Groups'] vomsMap = vomsRole.split("Role=") for g in groupsForRole: if g in staticGroups: staticGroups.pop(staticGroups.index(g)) else: vomsGroups.append(g) if len(vomsMap) == 1: # no Role users = usersInVOMS else: vomsGroup = "Role=".join(vomsMap[:-1]) if vomsGroup[-1] == "/": vomsGroup = vomsGroup[:-1] vomsRole = "Role=%s" % vomsMap[-1] result = self.vomsSrv.admListUsersWithRole(vomsGroup, vomsRole) if not result['OK']: errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[group]) self.__adminMsgs['Errors'].append(errorMsg) self.log.error(errorMsg, result['Message']) return result users = result['Value'] numUsersInGroup = 0 for vomsUser in users: for userName in usersData: if vomsUser['DN'] in usersData[userName]['DN']: numUsersInGroup += 1 usersData[userName]['Groups'].extend(groupsForRole) infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join(groupsForRole), vomsRole) self.__adminMsgs['Info'].append(infoMsg) self.log.info(" %s" % infoMsg) self.log.info("Checking static groups") staticUsers = [] for group in staticGroups: self.log.info(" Checking static group %s" % group) numUsersInGroup = 0 result = csapi.listUsers(group) if not result['OK']: self.log.error( "Could not get the list of users in DIRAC group %s" % group, result['Message']) return result for userName in result['Value']: if userName in usersData: numUsersInGroup += 1 usersData[userName]['Groups'].append(group) else: if group not in vomsGroups and userName not in staticUsers: staticUsers.append(userName) infoMsg = "There are %s users in group %s" % (numUsersInGroup, group) self.__adminMsgs['Info'].append(infoMsg) self.log.info(" %s" % infoMsg) if staticUsers: infoMsg = "There are %s static users: %s" % ( len(staticUsers), ', '.join(staticUsers)) self.__adminMsgs['Info'].append(infoMsg) self.log.info("%s" % infoMsg) for user in currentUsers: if user not in usersData and user not in staticUsers: self.log.info('User %s is no longer valid' % user) obsoleteUserNames.append(user) #Do the CS Sync self.log.info("Updating CS...") ret = csapi.downloadCSData() if not ret['OK']: self.log.fatal('Can not update from CS', ret['Message']) return ret usersWithMoreThanOneDN = {} for user in usersData: csUserData = dict(usersData[user]) if len(csUserData['DN']) > 1: usersWithMoreThanOneDN[user] = csUserData['DN'] result = csapi.describeUsers([user]) if result['OK']: if result['Value']: prevUser = result['Value'][user] prevDNs = List.fromChar(prevUser['DN']) newDNs = csUserData['DN'] for DN in newDNs: if DN not in prevDNs: self.__adminMsgs['Info'].append( "User %s has new DN %s" % (user, DN)) for DN in prevDNs: if DN not in newDNs: self.__adminMsgs['Info'].append( "User %s has lost a DN %s" % (user, DN)) else: newDNs = csUserData['DN'] for DN in newDNs: self.__adminMsgs['Info'].append( "New user %s has new DN %s" % (user, DN)) for k in ('DN', 'CA', 'Email'): csUserData[k] = ", ".join(csUserData[k]) result = csapi.modifyUser(user, csUserData, createIfNonExistant=True) if not result['OK']: self.__adminMsgs['Error'].append("Cannot modify user %s: %s" % (user, result['Message'])) self.log.error("Cannot modify user", user) if usersWithMoreThanOneDN: self.__adminMsgs['Info'].append("\nUsers with more than one DN:") for uwmtod in sorted(usersWithMoreThanOneDN): self.__adminMsgs['Info'].append(" %s" % uwmtod) self.__adminMsgs['Info'].append(" + DN list:") for DN in usersWithMoreThanOneDN[uwmtod]: self.__adminMsgs['Info'].append(" - %s" % DN) if obsoleteUserNames: self.__adminMsgs['Info'].append("\nObsolete users:") address = self.am_getOption('MailTo', '*****@*****.**') fromAddress = self.am_getOption('mailFrom', '*****@*****.**') subject = 'Obsolete LFC Users found' body = 'Delete entries into LFC: \n' for obsoleteUser in obsoleteUserNames: self.log.info(subject, ", ".join(obsoleteUserNames)) body += 'for ' + obsoleteUser + '\n' self.__adminMsgs['Info'].append(" %s" % obsoleteUser) self.log.info("Deleting %s users" % len(obsoleteUserNames)) NotificationClient().sendMail(address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress) csapi.deleteUsers(obsoleteUserNames) if newUserNames: self.__adminMsgs['Info'].append("\nNew users:") for newUser in newUserNames: self.__adminMsgs['Info'].append(" %s" % newUser) self.__adminMsgs['Info'].append(" + DN list:") for DN in usersData[newUser]['DN']: self.__adminMsgs['Info'].append(" - %s" % DN) self.__adminMsgs['Info'].append(" + EMail: %s" % usersData[newUser]['Email']) result = csapi.commitChanges() if not result['OK']: self.log.error("Could not commit configuration changes", result['Message']) return result self.log.info("Configuration committed") #LFC Check if self.am_getOption("LFCCheckEnabled", True): result = self.checkLFCRegisteredUsers(usersData) if not result['OK']: return result return S_OK()
class CE2CSAgent( AgentModule ): def __init__( self, agentName, baseAgentName, properties ): AgentModule.__init__( self, agentName, baseAgentName, properties ) self.addressTo = '' self.addressFrom = '' self.voName = '' self.csAPI = CSAPI() self.subject = "CE2CSAgent" self.alternativeBDIIs = [] def initialize( self ): # TODO: Have no default and if no mail is found then use the diracAdmin group # and resolve all associated mail addresses. self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" # This sets the Default Proxy to used as that defined under # /Operations/Shifter/SAMManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'SAMManager' ) self.voName = self.am_getOption( 'VirtualOrganization', self.voName ) if not self.voName: self.voName = getVO() if not self.voName: self.log.fatal( "VO option not defined for agent" ) return S_ERROR() return S_OK() def execute( self ): self.log.info( "Start Execution" ) result = getProxyInfo() if not result[ 'OK' ]: return result infoDict = result[ 'Value' ] self.log.info( formatProxyInfoAsString( infoDict ) ) self.__lookForCE() self.__infoFromCE() self.log.info( "End Execution" ) return S_OK() def __checkAlternativeBDIISite( self, fun, *args ): if self.alternativeBDIIs: self.log.warn( "Trying to use alternative bdii sites" ) for site in self.alternativeBDIIs : self.log.info( "Trying to contact alternative bdii ", site ) if len( args ) == 1 : result = fun( args[0], host = site ) elif len( args ) == 2 : result = fun( args[0], vo = args[1], host = site ) if not result['OK'] : self.log.error ( "Problem contacting alternative bddii", result['Message'] ) elif result['OK'] : return result self.log.warn( "Also checking alternative BDII sites failed" ) return result def __lookForCE( self ): knownces = self.am_getOption( 'BannedCEs', [] ) result = gConfig.getSections( '/Resources/Sites' ) if not result['OK']: return grids = result['Value'] for grid in grids: result = gConfig.getSections( '/Resources/Sites/%s' % grid ) if not result['OK']: return sites = result['Value'] for site in sites: opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value'] ces = List.fromChar( opt.get( 'CE', '' ) ) knownces += ces response = ldapCEState( '', vo = self.voName ) if not response['OK']: self.log.error( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName ) return response newces = {} for queue in response['Value']: try: queuename = queue['GlueCEUniqueID'] except: continue cename = queuename.split( ":" )[0] if not cename in knownces: newces[cename] = None self.log.debug( "newce", cename ) body = "" possibleNewSites = [] for ce in newces.iterkeys(): response = ldapCluster( ce ) if not response['OK']: self.log.warn( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCluster, ce ) continue clusters = response['Value'] if len( clusters ) != 1: self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) ) if len( clusters ) == 0: continue cluster = clusters[0] fkey = cluster.get( 'GlueForeignKey', [] ) if type( fkey ) == type( '' ): fkey = [fkey] nameBDII = None for entry in fkey: if entry.count( 'GlueSiteUniqueID' ): nameBDII = entry.split( '=' )[1] break if not nameBDII: continue cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII ) self.log.info( cestring ) response = ldapCE( ce ) if not response['OK']: self.log.warn( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCE, ce ) continue ceinfos = response['Value'] if len( ceinfos ): ceinfo = ceinfos[0] systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' ) systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' ) systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' ) else: systemName = "Unknown" systemVersion = "Unknown" systemRelease = "Unknown" osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease ) self.log.info( osstring ) response = ldapCEState( ce, vo = self.voName ) if not response['OK']: self.log.warn( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName ) continue newcestring = "\n\n%s\n%s" % ( cestring, osstring ) usefull = False cestates = response['Value'] for cestate in cestates: queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' ) queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' ) queuestring = "%s %s" % ( queuename, queuestatus ) self.log.info( queuestring ) newcestring += "\n%s" % queuestring if queuestatus.count( 'Production' ): usefull = True if usefull: body += newcestring possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) ) if body: body = "We are glade to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body body += "\n\nTo suppress information about CE add its name to BannedCEs list." for possibleNewSite in possibleNewSites: body = "%s\n%s" % ( body, possibleNewSite ) self.log.info( body ) if self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) return S_OK() def __infoFromCE( self ): sitesSection = cfgPath( 'Resources', 'Sites' ) result = gConfig.getSections( sitesSection ) if not result['OK']: return grids = result['Value'] for grid in grids: gridSection = cfgPath( sitesSection, grid ) result = gConfig.getSections( gridSection ) if not result['OK']: return sites = result['Value'] changed = False body = "" for site in sites: # if site[-2:]!='ru': # continue siteSection = cfgPath( gridSection, site ) opt = gConfig.getOptionsDict( siteSection )['Value'] name = opt.get( 'Name', '' ) if name: coor = opt.get( 'Coordinates', 'Unknown' ) mail = opt.get( 'Mail', 'Unknown' ) result = ldapSite( name ) if not result['OK']: self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) ) result = self.__checkAlternativeBDIISite( ldapSite, name ) if result['OK']: bdiisites = result['Value'] if len( bdiisites ) == 0: self.log.warn( name, "Error in bdii: leng = 0" ) else: if not len( bdiisites ) == 1: self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) ) bdiisite = bdiisites[0] try: longitude = bdiisite['GlueSiteLongitude'] latitude = bdiisite['GlueSiteLatitude'] newcoor = "%s:%s" % ( longitude, latitude ) except: self.log.warn( "Error in bdii coor" ) newcoor = "Unknown" try: newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip() except: self.log.warn( "Error in bdii mail" ) newmail = "Unknown" self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) ) if newcoor != coor: self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) ) if coor == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor ) changed = True if newmail != mail: self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) ) if mail == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newmail ) changed = True celist = List.fromChar( opt.get( 'CE', '' ) ) if not celist: self.log.warn( site, 'Empty site list' ) continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath( siteSection, 'CEs', ce ) result = gConfig.getOptionsDict( ceSection ) if not result['OK']: self.log.debug( "Section CE", result['Message'] ) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' cetype = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' ) arch = ceopt.get( 'architecture', 'Unknown' ) os = ceopt.get( 'OS', 'Unknown' ) si00 = ceopt.get( 'SI00', 'Unknown' ) pilot = ceopt.get( 'Pilot', 'Unknown' ) cetype = ceopt.get( 'CEType', 'Unknown' ) result = ldapCE( ce ) if not result['OK']: self.log.warn( 'Error in bdii for %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCE, ce ) continue try: bdiice = result['Value'][0] except: self.log.warn( 'Error in bdii for %s' % ce, result ) bdiice = None if bdiice: try: newwnTmpDir = bdiice['GlueSubClusterWNTmpDir'] except: newwnTmpDir = 'Unknown' if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown': section = cfgPath( ceSection, 'wnTmpDir' ) self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) ) if wnTmpDir == 'Unknown': self.csAPI.setOption( section, newwnTmpDir ) else: self.csAPI.modifyValue( section, newwnTmpDir ) changed = True try: newarch = bdiice['GlueHostArchitecturePlatformType'] except: newarch = 'Unknown' if arch != newarch and newarch != 'Unknown': section = cfgPath( ceSection, 'architecture' ) self.log.info( section, " -> ".join( ( arch, newarch ) ) ) if arch == 'Unknown': self.csAPI.setOption( section, newarch ) else: self.csAPI.modifyValue( section, newarch ) changed = True try: newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'], bdiice['GlueHostOperatingSystemVersion'], bdiice['GlueHostOperatingSystemRelease'] ) ) except: newos = 'Unknown' if os != newos and newos != 'Unknown': section = cfgPath( ceSection, 'OS' ) self.log.info( section, " -> ".join( ( os, newos ) ) ) if os == 'Unknown': self.csAPI.setOption( section, newos ) else: self.csAPI.modifyValue( section, newos ) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site ) try: newsi00 = bdiice['GlueHostBenchmarkSI00'] except: newsi00 = 'Unknown' if si00 != newsi00 and newsi00 != 'Unknown': section = cfgPath( ceSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True try: rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment'] if self.voName == 'lhcb': if 'VO-lhcb-pilot' in rte: newpilot = 'True' else: newpilot = 'False' else: newpilot = 'Unknown' except: newpilot = 'Unknown' if pilot != newpilot and newpilot != 'Unknown': section = cfgPath( ceSection, 'Pilot' ) self.log.info( section, " -> ".join( ( pilot, newpilot ) ) ) if pilot == 'Unknown': self.csAPI.setOption( section, newpilot ) else: self.csAPI.modifyValue( section, newpilot ) changed = True result = ldapService( ce ) if not result['OK'] : result = self.__checkAlternativeBDIISite( ldapService, ce ) if result['OK']: services = result['Value'] newcetype = 'LCG' for service in services: if service['GlueServiceType'].count( 'CREAM' ): newcetype = "CREAM" else: newcetype = 'Unknown' if cetype != newcetype and newcetype != 'Unknown': section = cfgPath( ceSection, 'CEType' ) self.log.info( section, " -> ".join( ( cetype, newcetype ) ) ) if cetype == 'Unknown': self.csAPI.setOption( section, newcetype ) else: self.csAPI.modifyValue( section, newcetype ) changed = True result = ldapCEState( ce, vo = self.voName ) #getBDIICEVOView if not result['OK']: self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName ) continue try: queues = result['Value'] except: self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] ) continue for queue in queues: try: queueName = queue['GlueCEUniqueID'].split( '/' )[-1] except: self.log.warn( 'error in queuename ', queue ) continue try: newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newmaxCPUTime = None newsi00 = None try: caps = queue['GlueCECapability'] if type( caps ) == type( '' ): caps = [caps] for cap in caps: if cap.count( 'CPUScalingReferenceSI00' ): newsi00 = cap.split( '=' )[-1] except: newsi00 = None queueSection = cfgPath( ceSection, 'Queues', queueName ) result = gConfig.getOptionsDict( queueSection ) if not result['OK']: self.log.warn( "Section Queues", result['Message'] ) maxCPUTime = 'Unknown' si00 = 'Unknown' else: queueopt = result['Value'] maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' ) si00 = queueopt.get( 'SI00', 'Unknown' ) if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ): section = cfgPath( queueSection, 'maxCPUTime' ) self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) ) if maxCPUTime == 'Unknown': self.csAPI.setOption( section, newmaxCPUTime ) else: self.csAPI.modifyValue( section, newmaxCPUTime ) changed = True if newsi00 and ( si00 != newsi00 ): section = cfgPath( queueSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True if changed: self.log.info( body ) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) return self.csAPI.commitChanges( sortUsers = False ) else: self.log.info( "No changes found" ) return S_OK()
class DiracAdmin(API): """ Administrative functionalities """ ############################################################################# def __init__(self): """Internal initialization of the DIRAC Admin API. """ super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue(self.section + '/ScratchDir', '/tmp') self.currentDir = os.getcwd() self.rssFlag = ResourceStatus().rssFlag self.sitestatus = SiteStatus() self._siteSet = set(getSites().get('Value', [])) ############################################################################# def uploadProxy(self): """Upload a proxy to the DIRAC WMS. This method Example usage: >>> print diracAdmin.uploadProxy('dteam_pilot') {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR :param permanent: Indefinitely update proxy :type permanent: boolean """ return gProxyManager.uploadProxy() ############################################################################# def setProxyPersistency(self, userDN, userGroup, persistent=True): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )) {'OK': True } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param persistent: Persistent flag :type persistent: boolean :return: S_OK,S_ERROR """ return gProxyManager.setPersistency(userDN, userGroup, persistent) ############################################################################# def checkProxyUploaded(self, userDN, userGroup, requiredTime): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )) {'OK': True, 'Value' : True/False } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param requiredTime: Required life time of the uploaded proxy :type requiredTime: boolean :return: S_OK,S_ERROR """ return gProxyManager.userHasProxy(userDN, userGroup, requiredTime) ############################################################################# def getSiteMask(self, printOutput=False, status='Active'): """Retrieve current site mask from WMS Administrator service. Example usage: >>> gLogger.notice(diracAdmin.getSiteMask()) {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR """ result = self.sitestatus.getSites(siteState=status) if result['OK']: sites = result['Value'] if printOutput: sites.sort() for site in sites: gLogger.notice(site) return result ############################################################################# def getBannedSites(self, printOutput=False): """Retrieve current list of banned and probing sites. Example usage: >>> gLogger.notice(diracAdmin.getBannedSites()) {'OK': True, 'Value': []} :return: S_OK,S_ERROR """ bannedSites = self.sitestatus.getSites(siteState='Banned') if not bannedSites['OK']: return bannedSites probingSites = self.sitestatus.getSites(siteState='Probing') if not probingSites['OK']: return probingSites mergedList = sorted(bannedSites['Value'] + probingSites['Value']) if printOutput: gLogger.notice('\n'.join(mergedList)) return S_OK(mergedList) ############################################################################# def getSiteSection(self, site, printOutput=False): """Simple utility to get the list of CEs for DIRAC site name. Example usage: >>> gLogger.notice(diracAdmin.getSiteSection('LCG.CERN.ch')) {'OK': True, 'Value':} :return: S_OK,S_ERROR """ gridType = site.split('.')[0] if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']: return S_ERROR('/Resources/Sites/%s is not a valid site section' % (gridType)) result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' % (gridType, site)) if printOutput and result['OK']: gLogger.notice(self.pPrint.pformat(result['Value'])) return result ############################################################################# def allowSite(self, site, comment, printOutput=False): """Adds the site to the site mask. Example usage: >>> gLogger.notice(diracAdmin.allowSite()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result result = self.getSiteMask(status='Active') if not result['OK']: return result siteMask = result['Value'] if site in siteMask: if printOutput: gLogger.notice('Site %s is already Active' % site) return S_OK('Site %s is already Active' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Active', comment) else: result = WMSAdministratorClient().allowSite(site, comment) if not result['OK']: return result if printOutput: gLogger.notice('Site %s status is set to Active' % site) return result ############################################################################# def getSiteMaskLogging(self, site=None, printOutput=False): """Retrieves site mask logging information. Example usage: >>> gLogger.notice(diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result if self.rssFlag: result = ResourceStatusClient().selectStatusElement('Site', 'History', name=site) else: result = WMSAdministratorClient().getSiteMaskLogging(site) if not result['OK']: return result if printOutput: if site: gLogger.notice('\nSite Mask Logging Info for %s\n' % site) else: gLogger.notice('\nAll Site Mask Logging Info\n') sitesLogging = result['Value'] if isinstance(sitesLogging, dict): for siteName, tupleList in sitesLogging.items( ): # can be an iterator if not siteName: gLogger.notice('\n===> %s\n' % siteName) for tup in tupleList: stup = str(tup[0]).ljust(8) + str(tup[1]).ljust(20) stup += '( ' + str(tup[2]).ljust(len(str( tup[2]))) + ' ) "' + str(tup[3]) + '"' gLogger.notice(stup) gLogger.notice(' ') elif isinstance(sitesLogging, list): sitesLoggingList = [(sl[1], sl[3], sl[4]) for sl in sitesLogging] for siteLog in sitesLoggingList: gLogger.notice(siteLog) return S_OK() ############################################################################# def banSite(self, site, comment, printOutput=False): """Removes the site from the site mask. Example usage: >>> gLogger.notice(diracAdmin.banSite()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result mask = self.getSiteMask(status='Banned') if not mask['OK']: return mask siteMask = mask['Value'] if site in siteMask: if printOutput: gLogger.notice('Site %s is already Banned' % site) return S_OK('Site %s is already Banned' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Banned', comment) else: result = WMSAdministratorClient().banSite(site, comment) if not result['OK']: return result if printOutput: gLogger.notice('Site %s status is set to Banned' % site) return result ############################################################################# def __checkSiteIsValid(self, site): """Internal function to check that a site name is valid. """ if isinstance(site, (list, set, dict)): site = set(site) - self._siteSet if not site: return S_OK() elif site in self._siteSet: return S_OK() return S_ERROR('Specified site %s is not in list of defined sites' % str(site)) ############################################################################# def getServicePorts(self, setup='', printOutput=False): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> gLogger.notice(diracAdmin.getServicePorts()) {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if setup not in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict systemList = gConfig.getSections('/Systems') if not systemList['OK']: return S_ERROR('Could not get Systems sections') systemList = systemList['Value'] result = {} for system in systemList: if system in serviceSetups: path = '/Systems/%s/%s/Services' % (system, serviceSetups[system]) servicesList = gConfig.getSections(path) if not servicesList['OK']: self.log.warn('Could not get sections in %s' % path) else: servicesList = servicesList['Value'] if not servicesList: servicesList = [] self.log.verbose('System: %s ServicesList: %s' % (system, ', '.join(servicesList))) for service in servicesList: spath = '%s/%s/Port' % (path, service) servicePort = gConfig.getValue(spath, 0) if servicePort: self.log.verbose('Found port for %s/%s = %s' % (system, service, servicePort)) result['%s/%s' % (system, service)] = servicePort else: self.log.warn('No port found for %s' % spath) else: self.log.warn('%s is not defined in /DIRAC/Setups/%s' % (system, setup)) if printOutput: gLogger.notice(self.pPrint.pformat(result)) return S_OK(result) ############################################################################# def getProxy(self, userDN, userGroup, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> gLogger.notice(diracAdmin.getProxy()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadProxy(userDN, userGroup, limited=limited, requiredTimeLeft=validity) ############################################################################# def getVOMSProxy(self, userDN, userGroup, vomsAttr=False, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and VOMS extensions and stores this in a file in the local directory by default. Example usage: >>> gLogger.notice(diracAdmin.getVOMSProxy()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=limited, requiredVOMSAttribute=vomsAttr, requiredTimeLeft=validity) ############################################################################# def getPilotProxy(self, userDN, userGroup, validity=43200): """Retrieves a pilot proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> gLogger.notice(diracAdmin.getVOMSProxy()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft=validity) ############################################################################# def resetJob(self, jobID): """Reset a job or list of jobs in the WMS. This operation resets the reschedule counter for a job or list of jobs and allows them to run as new. Example:: >>> gLogger.notice(dirac.reset(12345)) {'OK': True, 'Value': [12345]} :param job: JobID :type job: integer or list of integers :return: S_OK,S_ERROR """ if isinstance(jobID, six.string_types): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobID' ) elif isinstance(jobID, list): try: jobID = [int(job) for job in jobID] except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobIDs' ) result = JobManagerClient(useCertificates=False).resetJob(jobID) return result ############################################################################# def getJobPilotOutput(self, jobID, directory=''): """Retrieve the pilot output for an existing job in the WMS. The output will be retrieved in a local directory unless otherwise specified. >>> gLogger.notice(dirac.getJobPilotOutput(12345)) {'OK': True, StdOut:'',StdError:''} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) result = WMSAdministratorClient().getJobPilotOutput(jobID) if not result['OK']: return result outputPath = '%s/pilot_%s' % (directory, jobID) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if 'StdOut' in outputs: stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.verbose('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if 'StdError' in outputs: stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdError']) self.log.verbose('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotOutput(self, gridReference, directory=''): """Retrieve the pilot output (std.out and std.err) for an existing job in the WMS. >>> gLogger.notice(dirac.getJobPilotOutput(12345)) {'OK': True, 'Value': {}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) result = PilotManagerClient().getPilotOutput(gridReference) if not result['OK']: return result gridReferenceSmall = gridReference.split('/')[-1] if not gridReferenceSmall: gridReferenceSmall = 'reference' outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if 'StdOut' in outputs: stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.info('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if 'StdErr' in outputs: stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdErr']) self.log.info('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotInfo(self, gridReference): """Retrieve info relative to a pilot reference >>> gLogger.notice(dirac.getPilotInfo(12345)) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') result = PilotManagerClient().getPilotInfo(gridReference) return result ############################################################################# def killPilot(self, gridReference): """Kill the pilot specified >>> gLogger.notice(dirac.getPilotInfo(12345)) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') result = PilotManagerClient().killPilot(gridReference) return result ############################################################################# def getPilotLoggingInfo(self, gridReference): """Retrieve the pilot logging info for an existing job in the WMS. >>> gLogger.notice(dirac.getPilotLoggingInfo(12345)) {'OK': True, 'Value': {"The output of the command"}} :param gridReference: Gridp pilot job reference Id :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') return PilotManagerClient().getPilotLoggingInfo(gridReference) ############################################################################# def getJobPilots(self, jobID): """Extract the list of submitted pilots and their status for a given jobID from the WMS. Useful information is printed to the screen. >>> gLogger.notice(dirac.getJobPilots()) {'OK': True, 'Value': {PilotID:{StatusDict}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if isinstance(jobID, six.string_types): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or string for existing jobID') result = PilotManagerClient().getPilots(jobID) if result['OK']: gLogger.notice(self.pPrint.pformat(result['Value'])) return result ############################################################################# def getPilotSummary(self, startDate='', endDate=''): """Retrieve the pilot output for an existing job in the WMS. Summary is printed at INFO level, full dictionary of results also returned. >>> gLogger.notice(dirac.getPilotSummary()) {'OK': True, 'Value': {CE:{Status:Count}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ result = PilotManagerClient().getPilotSummary(startDate, endDate) if not result['OK']: return result ceDict = result['Value'] headers = 'CE'.ljust(28) i = 0 for ce, summary in ceDict.iteritems(): states = summary.keys() if len(states) > i: i = len(states) for i in xrange(i): headers += 'Status'.ljust(12) + 'Count'.ljust(12) gLogger.notice(headers) for ce, summary in ceDict.iteritems(): line = ce.ljust(28) states = sorted(summary) for state in states: count = str(summary[state]) line += state.ljust(12) + count.ljust(12) gLogger.notice(line) return result ############################################################################# def setSiteProtocols(self, site, protocolsList, printOutput=False): """ Allows to set the defined protocols for each SE for a given site. """ result = self.__checkSiteIsValid(site) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site) siteSEs = gConfig.getValue(siteSection, []) if not siteSEs: return S_ERROR('No SEs found for site %s in section %s' % (site, siteSection)) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) self.log.verbose('Default list of protocols are', ', '.join(defaultProtocols)) for protocol in protocolsList: if protocol not in defaultProtocols: return S_ERROR( 'Requested to set protocol %s in list but %s is not ' 'in default list of protocols:\n%s' % (protocol, protocol, ', '.join(defaultProtocols))) modifiedCS = False result = promptUser( 'Do you want to add the following default protocols:' ' %s for SE(s):\n%s' % (', '.join(protocolsList), ', '.join(siteSEs))) if not result['OK']: return result if result['Value'].lower() != 'y': self.log.always('No protocols will be added') return S_OK() for se in siteSEs: sections = gConfig.getSections('/Resources/StorageElements/%s/' % (se)) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % (se, section), '') == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section) self.log.verbose('Setting %s to %s' % (path, ', '.join(protocolsList))) result = self.csSetOption(path, ', '.join(protocolsList)) if not result['OK']: return result modifiedCS = True if modifiedCS: result = self.csCommitChanges(False) if not result['OK']: return S_ERROR('CS Commit failed with message = %s' % (result['Message'])) else: if printOutput: gLogger.notice('Successfully committed changes to CS') else: if printOutput: gLogger.notice('No modifications to CS required') return S_OK() ############################################################################# def csSetOption(self, optionPath, optionValue): """ Function to modify an existing value in the CS. """ return self.csAPI.setOption(optionPath, optionValue) ############################################################################# def csSetOptionComment(self, optionPath, comment): """ Function to modify an existing value in the CS. """ return self.csAPI.setOptionComment(optionPath, comment) ############################################################################# def csModifyValue(self, optionPath, newValue): """ Function to modify an existing value in the CS. """ return self.csAPI.modifyValue(optionPath, newValue) ############################################################################# def csRegisterUser(self, username, properties): """ Registers a user in the CS. - username: Username of the user (easy;) - properties: Dict containing: - DN - groups : list/tuple of groups the user belongs to - <others> : More properties of the user, like mail """ return self.csAPI.addUser(username, properties) ############################################################################# def csDeleteUser(self, user): """ Deletes a user from the CS. Can take a list of users """ return self.csAPI.deleteUsers(user) ############################################################################# def csModifyUser(self, username, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addUser and applies the changes """ return self.csAPI.modifyUser(username, properties, createIfNonExistant) ############################################################################# def csListUsers(self, group=False): """ Lists the users in the CS. If no group is specified return all users. """ return self.csAPI.listUsers(group) ############################################################################# def csDescribeUsers(self, mask=False): """ List users and their properties in the CS. If a mask is given, only users in the mask will be returned """ return self.csAPI.describeUsers(mask) ############################################################################# def csModifyGroup(self, groupname, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addGroup and applies the changes """ return self.csAPI.modifyGroup(groupname, properties, createIfNonExistant) ############################################################################# def csListHosts(self): """ Lists the hosts in the CS """ return self.csAPI.listHosts() ############################################################################# def csDescribeHosts(self, mask=False): """ Gets extended info for the hosts in the CS """ return self.csAPI.describeHosts(mask) ############################################################################# def csModifyHost(self, hostname, properties, createIfNonExistant=False): """ Modify a host in the CS. Takes the same params as in addHost and applies the changes """ return self.csAPI.modifyHost(hostname, properties, createIfNonExistant) ############################################################################# def csListGroups(self): """ Lists groups in the CS """ return self.csAPI.listGroups() ############################################################################# def csDescribeGroups(self, mask=False): """ List groups and their properties in the CS. If a mask is given, only groups in the mask will be returned """ return self.csAPI.describeGroups(mask) ############################################################################# def csSyncUsersWithCFG(self, usersCFG): """ Synchronize users in cfg with its contents """ return self.csAPI.syncUsersWithCFG(usersCFG) ############################################################################# def csCommitChanges(self, sortUsers=True): """ Commit the changes in the CS """ return self.csAPI.commitChanges(sortUsers=False) ############################################################################# def sendMail(self, address, subject, body, fromAddress=None, localAttempt=True, html=False): """ Send mail to specified address with body. """ notification = NotificationClient() return notification.sendMail(address, subject, body, fromAddress, localAttempt, html) ############################################################################# def sendSMS(self, userName, body, fromAddress=None): """ Send mail to specified address with body. """ if len(body) > 160: return S_ERROR('Exceeded maximum SMS length of 160 characters') notification = NotificationClient() return notification.sendSMS(userName, body, fromAddress) ############################################################################# def getBDIISite(self, site, host=None): """ Get information about site from BDII at host """ return ldapSite(site, host=host) ############################################################################# def getBDIICluster(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCluster(ce, host=host) ############################################################################# def getBDIICE(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCE(ce, host=host) ############################################################################# def getBDIIService(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapService(ce, host=host) ############################################################################# def getBDIICEState(self, ce, useVO=voName, host=None): """ Get information about ce state from BDII at host """ return ldapCEState(ce, useVO, host=host) ############################################################################# def getBDIICEVOView(self, ce, useVO=voName, host=None): """ Get information about ce voview from BDII at host """ return ldapCEVOView(ce, useVO, host=host)
change = False if newSite: gLogger.notice("Adding new site to CS: %s" % diracSiteName) csAPI.setOption("%s/Name" % cfgBase, gridSiteName) gLogger.notice("Adding CEs: %s" % ",".join(ces)) csAPI.setOption("%s/CE" % cfgBase, ",".join(ces)) change = True else: cesCS = set(gConfig.getValue("%s/CE" % cfgBase, [])) ces = set(ces) newCEs = ces - cesCS if newCEs: cesCS = cesCS.union(ces) gLogger.notice("Adding CEs %s" % ",".join(newCEs)) cesCS = cesCS.union(ces) csAPI.modifyValue("%s/CE" % cfgBase, ",".join(cesCS)) change = True if change: res = csAPI.commitChanges() if not res["OK"]: gLogger.error("Failed to commit changes to CS", res["Message"]) DIRACExit(-1) else: if newSite: gLogger.notice( "Successfully added site %s to the CS with name %s and CEs: %s" % (diracSiteName, gridSiteName, ",".join(ces)) ) else: gLogger.notice("Successfully added new CEs to site %s: %s" % (diracSiteName, ",".join(newCEs)))
class VOMS2CSAgent( AgentModule ): def __init__( self, *args, **kwargs ): """ Defines default parameters """ super(VOMS2CSAgent, self).__init__( *args, **kwargs ) self.__voDict = {} self.__adminMsgs = {} self.csapi = None self.voChanged = False self.dryRun = False self.autoAddUsers = False self.autoModifyUsers = False self.autoDeleteUsers = False self.detailedReport = True self.makeFCEntry = False def initialize( self ): """ Initialize the default parameters """ voNames = self.am_getOption( 'VO', [] ) if not voNames[0].lower() == "none": if voNames[0].lower() == "any": voNames = [] result = getVOMSVOs( voNames ) if not result['OK']: return result self.__voDict = result['Value'] self.log.notice( "VOs: %s" % self.__voDict.keys() ) self.csapi = CSAPI() self.dryRun = self.am_getOption( 'DryRun', self.dryRun ) self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers ) self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers ) self.autoDeleteUsers = self.am_getOption( 'AutoDeleteUsers', self.autoDeleteUsers ) self.detailedReport = self.am_getOption( 'DetailedReport', self.detailedReport ) self.makeFCEntry = self.am_getOption( 'MakeHomeDirectory', self.makeFCEntry ) return S_OK() def execute( self ): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption( vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption( voAdminUser, "Email") voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) ) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) ) result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) ) continue resultDict = result['Value'] newUsers = resultDict.get( "NewUsers", [] ) modUsers = resultDict.get( "ModifiedUsers", [] ) delUsers = resultDict.get( "DeletedUsers", [] ) self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \ ( len( newUsers ), len( modUsers ), len( delUsers ) ) ) if self.csapi.csModified: # We have accumulated all the changes, commit them now self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo ) if self.dryRun: self.log.info( "Dry Run: CS won't be updated" ) self.csapi.showDiff() else: result = self.csapi.commitChanges() if not result[ 'OK' ]: self.log.error( "Could not commit configuration changes", result[ 'Message' ] ) return result self.log.notice( "Configuration committed for VO %s" % vo ) else: self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo ) # Add user home directory in the file catalog if self.makeFCEntry and newUsers: self.log.info( "Creating home directories for users %s" % str( newUsers ) ) result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) ) else: for user in result['Value']['Failed']: self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \ ( user, result['Value']['Failed'][user] ) ) self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \ ( user, result['Value']['Failed'][user] ) ) for user in result['Value']['Successful']: self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user ) if self.voChanged or self.detailedReport: mailMsg = "" if self.__adminMsgs[ 'Errors' ]: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] ) if self.__adminMsgs[ 'Info' ]: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] ) if self.detailedReport: result = self.getVOUserReport( vo ) if result['OK']: mailMsg += '\n\n' mailMsg += result['Value'] else: mailMsg += 'Failed to produce a detailed user report' mailMsg += result['Message'] NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ), "VOMS2CSAgent run log", mailMsg, self.am_getOption( 'MailFrom', self.am_getOption( 'mailFrom', "DIRAC system" ) ) ) return S_OK() def getVOUserData( self, vo, refreshFlag = False ): """ Get a report for users of a given VO :param str vo: VO name :return: S_OK/S_ERROR, Value = user description dictionary """ if refreshFlag: gConfig.forceRefresh() # Get DIRAC users diracUsers = getUsersInVO( vo ) if not diracUsers: return S_ERROR( "No VO users found for %s" % vo ) if refreshFlag: result = self.csapi.downloadCSData() if not result['OK']: return result result = self.csapi.describeUsers( diracUsers ) if not result['OK']: self.log.error( 'Could not retrieve CS User description' ) return result def getVOUserReport( self, vo ): """ :param str vo: VO name :return: report string """ result = self.getVOUserData( vo, refreshFlag = True ) if not result['OK']: return result userDict = result['Value'] # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping( vo ) if not result['OK']: return result diracVOMSMapping = result['Value']['DIRACVOMS'] records = [] groupDict = defaultdict( int ) multiDNUsers = {} suspendedUsers = [] for user in userDict: for group in userDict[user]['Groups']: groupDict[group] += 1 dnList = fromChar( userDict[user]['DN'] ) if len( dnList ) > 1: multiDNUsers[user] = dnList if userDict[user].get( 'Status', 'Active' ) == 'Suspended': suspendedUsers.append( user ) for group in diracVOMSMapping: records.append( ( group, str( groupDict[group] ), diracVOMSMapping.get( group, '' ) ) ) fields = [ 'Group', 'Number of users', 'VOMS Role' ] output = printTable( fields, records, sortField = 'Group', printOut = False, numbering = False ) if multiDNUsers: output += '\nUsers with multiple DNs:\n' for user in multiDNUsers: output += ' %s:\n' % user for dn in multiDNUsers[user]: output += ' %s\n' % dn if suspendedUsers: output += '\n%d suspended users:\n' % len( suspendedUsers ) output += ' %s' % ','.join( suspendedUsers ) return S_OK( output ) @executeWithUserProxy def __syncCSWithVOMS( self, vo ): self.__adminMsgs = { 'Errors' : [], 'Info' : [] } resultDict = defaultdict( list ) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping( vo ) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService( vo ) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo ) return result vomsVOName = result[ 'Value' ].lstrip( '/' ) self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) ) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error( 'Could not retrieve user information from VOMS', result['Message'] ) return result vomsUserDict = result[ 'Value' ] message = "There are %s user entries in VOMS for VO %s" % ( len( vomsUserDict ), vomsVOName ) self.__adminMsgs[ 'Info' ].append( message ) self.log.info( message ) # Get DIRAC users result = self.getVOUserData( vo ) if not result['OK']: return result diracUserDict = result['Value'] self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) ) self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) ) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar( dn ) existingDNs.extend( dnList ) for dn in dnList: if dn not in vomsUserDict: obsoletedDNs.append( dn ) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append( dn ) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list( set( allDiracUsers ) - set( diracUserDict.keys() ) ) if nonVOUsers: result = self.csapi.describeUsers( nonVOUsers ) if not result['OK']: self.log.error( 'Could not retrieve CS User description' ) return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo ) newAddedUserDict = {} for dn in vomsUserDict: nickName = '' newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] ) if result['OK']: nickName = result['Value'] if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = getUserName( dn, vomsUserDict[dn]['mail'], vo ) # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % ( trialName, ind ) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % ( vomsVOName, role ) groupList = vomsDIRACMapping.get( fullRole, [] ) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append( group ) userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) ) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % ( key, str( userDict[key] ) ) self.__adminMsgs[ 'Info' ].append( message ) self.voChanged = True if self.autoAddUsers: self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) ) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True ) if not result['OK']: self.log.warn( 'Failed adding new user %s' % newDiracName ) resultDict['NewUsers'].append( newDiracName ) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } if newDNForExistingUser: userDict['DN'] = ','.join( [ dn, diracUserDict[diracName]['DN'] ] ) modified = True existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] ) nonVOGroups = list( set( existingGroups ) - set( diracVOMSMapping.keys() ) ) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % ( vomsVOName, role ) groupList = vomsDIRACMapping.get( fullRole, [] ) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append( group ) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get( group, '' ) # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append( group ) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append( group ) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append( group ) userDict['Groups'] = list( set( keepGroups ) ) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get( 'Groups', [] ) userDict['Groups'] = list( set( keepGroups + otherGroups ) ) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set( userDict[key] ) - set( diracUserDict.get( diracName, {} ).get( key, [] ) ) removedGroups = set( diracUserDict.get( diracName, {} ).get( key, [] ) ) - set( userDict[key] ) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join( addedGroups ) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups ) else: oldValue = str( diracUserDict.get( diracName, {} ).get( key, '' ) ) if str( userDict[key] ) != oldValue: modMsg += " %s: %s -> %s\n" % ( key, oldValue, str( userDict[key] ) ) if modMsg: self.__adminMsgs[ 'Info' ].append( message + modMsg ) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser( diracName, userDict ) if result['OK'] and result['Value']: self.log.info( "Modified user %s: %s" % ( diracName, str( userDict ) ) ) self.voChanged = True resultDict['ModifiedUsers'].append( diracName ) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set( fromChar( diracUserDict[user]['DN'] ) ) if not dnSet.intersection( set( vomsUserDict.keys() ) ) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add( user ) # Check for obsoleted DNs for user in diracUserDict: dnSet = set( fromChar( diracUserDict[user]['DN'] ) ) for dn in dnSet: if dn in obsoletedDNs and user not in oldUsers: self.log.verbose( "Modified user %s: dropped DN %s" % ( user, dn ) ) if self.autoModifyUsers: userDict = diracUserDict[user] modDNSet = dnSet - set( [dn] ) if modDNSet: userDict['DN'] = ','.join( modDNSet ) result = self.csapi.modifyUser( user, userDict ) if result['OK'] and result['Value']: self.log.info( "Modified user %s: dropped DN %s" % ( user, dn ) ) self.__adminMsgs[ 'Info' ].append( "Modified user %s: dropped DN %s" % ( user, dn ) ) self.voChanged = True resultDict['ModifiedUsers'].append( diracName ) else: oldUsers.add( user ) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info( 'The following users will be deleted: %s' % str( oldUsers ) ) result = self.csapi.deleteUsers( oldUsers ) if result['OK']: self.__adminMsgs[ 'Info' ].append( 'The following users are deleted from CS:\n %s\n' % str( oldUsers ) ) resultDict['DeletedUsers'] = oldUsers else: self.__adminMsgs[ 'Errors' ].append( 'Error in deleting users from CS:\n %s' % str( oldUsers ) ) self.log.error( 'Error while user deletion from CS', result ) else: self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion:\n %s' % str( oldUsers ) ) self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) ) return S_OK( resultDict ) @executeWithUserProxy def __addHomeDirectory( self, vo, newUsers ): fc = FileCatalog( vo = vo ) defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo ) failed = {} successful = {} for user in newUsers: result = fc.addUser( user ) if not result['OK']: failed[user] = "addUser" continue dirName = '/%s/user/%s/%s' % ( vo, user[0], user ) result = fc.createDirectory( dirName ) if not result['OK']: failed[user] = "createDirectory" continue result = fc.changePathOwner( { dirName: user }, recursive = True ) if not result['OK']: failed[user] = "changePathOwner" continue result = fc.changePathGroup( { dirName: defaultVOGroup }, recursive = True ) if not result['OK']: failed[user] = "changePathGroup" continue successful[user] = True return S_OK( { "Successful": successful, "Failed": failed } )
if not res['OK']: gLogger.error( "The provided site (%s) does not have an associated catalog." % site) continue res = csAPI.setOption("%s/%s/Status" % (storageCFGBase, site), "InActive") if not res['OK']: gLogger.error("Failed to update %s catalog status to InActive" % site) else: gLogger.debug("Successfully updated %s catalog status to InActive" % site) banned.append(site) if not banned: gLogger.error("Failed to ban any catalog mirrors") DIRAC.exit(-1) res = csAPI.commitChanges() if not res['OK']: gLogger.error("Failed to commit changes to CS", res['Message']) DIRAC.exit(-1) subject = '%d catalog instance(s) banned for use' % len(banned) address = gConfig.getValue('/Operations/EMail/Production', '*****@*****.**') body = 'The catalog mirrors at the following sites were banned' for site in banned: body = "%s\n%s" % (body, site) NotificationClient().sendMail(address, subject, body, '*****@*****.**' % userName) DIRAC.exit(0)
siteName = result['Value'] gLogger.notice( 'Processing CondDB endpoint at site %s' % siteName ) csapi.copySection( '/Resources/CondDB/%s' % site, '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION,siteName) ) csapi.setOptionComment( '%s/Sites/%s/DBServer' % (RESOURCES_NEW_SECTION,siteName), 'Database server resource' ) csapi.setOptionComment( '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION,siteName), 'Conditions database' ) if __name__ == '__main__': if computingFlag: result = convertSites() if storageFlag: result = convertSEs() if catalogFlag: result = convertCatalogs() if transferFlag: result = convertTransfers() if dbFlag: result = convertDBServers() csapi.commitChanges() print csapi.getCurrentCFG()['Value'].serialize()
class VOMS2CSAgent( AgentModule ): def initialize( self ): self.__voDict = {} voNames = self.am_getOption( 'VO', [] ) if not voNames[0].lower() == "none": if voNames[0].lower() == "any": voNames = [] result = getVOMSVOs( voNames ) if not result['OK']: return result self.__voDict = result['Value'] self.__adminMsgs = {} self.csapi = CSAPI() self.voChanged = False self.log.notice( "VOs: %s" % self.__voDict.keys() ) self.autoAddUsers = self.am_getOption( 'AutoAddUsers', False ) self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', False ) self.autoSuspendUsers = self.am_getOption( 'AutoSuspendUsers', False ) return S_OK() def execute( self ): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption( vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption( voAdminUser, "Email") voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) ) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) ) result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) ) continue if self.voChanged: mailMsg = "" if self.__adminMsgs[ 'Errors' ]: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] ) if self.__adminMsgs[ 'Info' ]: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] ) NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ), "VOMS2CSAgent run log", mailMsg, self.am_getOption( 'mailFrom', "DIRAC system" ) ) # We have accumulated all the changes, commit them now result = self.csapi.commitChanges() if not result[ 'OK' ]: self.log.error( "Could not commit configuration changes", result[ 'Message' ] ) return result self.log.info( "Configuration committed" ) return S_OK() @executeWithUserProxy def __syncCSWithVOMS( self, vo ): self.__adminMsgs = { 'Errors' : [], 'Info' : [] } # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping( vo ) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] vomsSrv = VOMSService( vo ) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo ) return result vomsVOName = result[ 'Value' ].lstrip( '/' ) self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) ) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error( 'Could not retrieve user information from VOMS', result['Message'] ) return result vomsUserDict = result[ 'Value' ] message = "There are %s registered users in VOMS VO %s" % ( len( vomsUserDict ), vomsVOName ) self.__adminMsgs[ 'Info' ].append( message ) self.log.info( message ) # Get DIRAC users diracUsers = getUsersInVO( vo ) if not diracUsers: return S_ERROR( "No VO users found for %s" % vo ) result = self.csapi.describeUsers( diracUsers ) if not result['OK']: self.log.error( 'Could not retrieve CS User description' ) return result diracUserDict = result['Value'] self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) ) self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) ) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] existingDNs.append( dn ) if dn not in vomsUserDict: obsoletedDNs.append( dn ) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append( dn ) allDiracUsers = getAllUsers() nonVOusers = list( set( allDiracUsers ) - set(diracUsers) ) result = self.csapi.describeUsers( nonVOusers ) if not result['OK']: self.log.error( 'Could not retrieve CS User description' ) return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo ) for dn in vomsUserDict: if dn in newDNs: # Find if the DN is already registered in the DIRAC CS diracName = '' for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # We have a real new user if not diracName: nickName = '' result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] ) if result['OK']: nickName = result['Value'] if nickName: newDiracName = nickName else: newDiracName = getUserName( dn, vomsUserDict[dn]['mail'] ) ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % ( trialName, ind ) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % ( vomsVOName, role ) group = vomsDIRACMapping.get( fullRole ) if group: groupsWithRole.append( group ) userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) ) self.__adminMsgs[ 'Info' ].append( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) ) self.voChanged = True if self.autoAddUsers: self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) ) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True ) if not result['OK']: self.log.warn( 'Failed adding new user %s' % newDiracName ) continue # We have an already existing user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } nonVOGroups = nonVOUserDict.get( diracName, {} ).get( 'Groups', [] ) existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] ) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % ( vomsVOName, role ) group = vomsDIRACMapping.get( fullRole ) if group: groupsWithRole.append( group ) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: role = diracVOMSMapping[group] # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append( group ) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append( group ) userDict['Groups'] = keepGroups if self.autoModifyUsers: result = self.csapi.modifyUser( diracName, userDict ) if result['OK'] and result['Value']: self.voChanged = True # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dn = diracUserDict[user]['DN'] if not dn in vomsUserDict and not user in nonVOUserDict: for group in diracUserDict[user]['Groups']: if not group in noVOMSGroups: oldUsers.add( user ) if oldUsers: self.voChanged = True self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion: %s' % str( oldUsers ) ) self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) ) return S_OK()
class DiracAdmin( API ): """ Administrative functionalities """ ############################################################################# def __init__( self ): """Internal initialization of the DIRAC Admin API. """ super( DiracAdmin, self ).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue( self.section + '/ScratchDir', '/tmp' ) self.currentDir = os.getcwd() ############################################################################# def uploadProxy( self, group ): """Upload a proxy to the DIRAC WMS. This method Example usage: >>> print diracAdmin.uploadProxy('lhcb_pilot') {'OK': True, 'Value': 0L} :param group: DIRAC Group :type job: string :return: S_OK,S_ERROR :param permanent: Indefinitely update proxy :type permanent: boolean """ return gProxyManager.uploadProxy( diracGroup = group ) ############################################################################# def setProxyPersistency( self, userDN, userGroup, persistent = True ): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param persistent: Persistent flag :type persistent: boolean :return: S_OK,S_ERROR """ return gProxyManager.setPersistency( userDN, userGroup, persistent ) ############################################################################# def checkProxyUploaded( self, userDN, userGroup, requiredTime ): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True, 'Value' : True/False } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param requiredTime: Required life time of the uploaded proxy :type requiredTime: boolean :return: S_OK,S_ERROR """ return gProxyManager.userHasProxy( userDN, userGroup, requiredTime ) ############################################################################# def getSiteMask( self, printOutput = False ): """Retrieve current site mask from WMS Administrator service. Example usage: >>> print diracAdmin.getSiteMask() {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getSiteMask() if result['OK']: sites = result['Value'] if printOutput: sites.sort() for site in sites: print site return result ############################################################################# def getBannedSites( self, gridType = [], printOutput = False ): """Retrieve current list of banned sites. Example usage: >>> print diracAdmin.getBannedSites() {'OK': True, 'Value': []} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) bannedSites = [] totalList = [] result = wmsAdmin.getSiteMask() if not result['OK']: self.log.warn( result['Message'] ) return result sites = result['Value'] if not gridType: result = gConfig.getSections( '/Resources/Sites' ) if not result['OK']: return result gridType = result['Value'] for grid in gridType: result = gConfig.getSections( '/Resources/Sites/%s' % grid ) if not result['OK']: return result totalList += result['Value'] for site in totalList: if not site in sites: bannedSites.append( site ) bannedSites.sort() if printOutput: print '\n'.join( bannedSites ) return S_OK( bannedSites ) ############################################################################# def getSiteSection( self, site, printOutput = False ): """Simple utility to get the list of CEs for DIRAC site name. Example usage: >>> print diracAdmin.getSiteSection('LCG.CERN.ch') {'OK': True, 'Value':} :return: S_OK,S_ERROR """ gridType = site.split( '.' )[0] if not gConfig.getSections( '/Resources/Sites/%s' % ( gridType ) )['OK']: return S_ERROR( '/Resources/Sites/%s is not a valid site section' % ( gridType ) ) result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( gridType, site ) ) if printOutput and result['OK']: print self.pPrint.pformat( result['Value'] ) return result ############################################################################# def addSiteInMask( self, site, comment, printOutput = False ): """Adds the site to the site mask. Example usage: >>> print diracAdmin.addSiteInMask() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result mask = self.getSiteMask() if not mask['OK']: return mask siteMask = mask['Value'] if site in siteMask: return S_ERROR( 'Site %s already in mask of allowed sites' % site ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.allowSite( site, comment ) if not result['OK']: return result if printOutput: print 'Allowing %s in site mask' % site return result ############################################################################# def getSiteMaskLogging( self, site = None, printOutput = False ): """Retrieves site mask logging information. Example usage: >>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr') {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getSiteMaskLogging( site ) if not result['OK']: return result if site: if not result['Value'].has_key( site ): return S_ERROR( 'Site mask information not available for %s' % ( site ) ) if printOutput: if site: print '\nSite Mask Logging Info for %s\n' % site else: print '\nAll Site Mask Logging Info\n' siteDict = result['Value'] for site, tupleList in siteDict.iteritems(): if not site: print '\n===> %s\n' % site for tup in tupleList: print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \ '( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"' print ' ' return result ############################################################################# def banSiteFromMask( self, site, comment, printOutput = False ): """Removes the site from the site mask. Example usage: >>> print diracAdmin.banSiteFromMask() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result mask = self.getSiteMask() if not mask['OK']: return mask siteMask = mask['Value'] if not site in siteMask: return S_ERROR( 'Site %s is already banned' % site ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.banSite( site, comment ) if not result['OK']: return result if printOutput: print 'Removing %s from site mask' % site return result ############################################################################# @classmethod def __checkSiteIsValid( self, site ): """Internal function to check that a site name is valid. """ sites = getSiteCEMapping() if not sites['OK']: return S_ERROR( 'Could not get site CE mapping' ) siteList = sites['Value'].keys() if not site in siteList: return S_ERROR( 'Specified site %s is not in list of defined sites' % site ) return S_OK( '%s is valid' % site ) ############################################################################# def clearMask( self ): """Removes all sites from the site mask. Should be used with care. Example usage: >>> print diracAdmin.clearMask() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.clearMask() return result ############################################################################# def getServicePorts( self, setup = '', printOutput = False ): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> print diracAdmin.getServicePorts() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue( '/DIRAC/Setup', '' ) setupList = gConfig.getSections( '/DIRAC/Setups', [] ) if not setupList['OK']: return S_ERROR( 'Could not get /DIRAC/Setups sections' ) setupList = setupList['Value'] if not setup in setupList: return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) ) serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup ) if not serviceSetups['OK']: return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup ) serviceSetups = serviceSetups['Value'] # dict systemList = gConfig.getSections( '/Systems' ) if not systemList['OK']: return S_ERROR( 'Could not get Systems sections' ) systemList = systemList['Value'] result = {} for system in systemList: if serviceSetups.has_key( system ): path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] ) servicesList = gConfig.getSections( path ) if not servicesList['OK']: self.log.warn( 'Could not get sections in %s' % path ) else: servicesList = servicesList['Value'] if not servicesList: servicesList = [] self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) ) for service in servicesList: spath = '%s/%s/Port' % ( path, service ) servicePort = gConfig.getValue( spath, 0 ) if servicePort: self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) ) result['%s/%s' % ( system, service )] = servicePort else: self.log.warn( 'No port found for %s' % spath ) else: self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) ) if printOutput: print self.pPrint.pformat( result ) return S_OK( result ) ############################################################################# def getProxy( self, userDN, userGroup, validity = 43200, limited = False ): """Retrieves a proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadProxy( userDN, userGroup, limited = limited, requiredTimeLeft = validity ) ############################################################################# def getVOMSProxy( self, userDN, userGroup, vomsAttr = False, validity = 43200, limited = False ): """Retrieves a proxy with default 12hr validity and VOMS extensions and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadVOMSProxy( userDN, userGroup, limited = limited, requiredVOMSAttribute = vomsAttr, requiredTimeLeft = validity ) ############################################################################# def getPilotProxy( self, userDN, userGroup, validity = 43200 ): """Retrieves a pilot proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft = validity ) ############################################################################# def resetJob( self, jobID ): """Reset a job or list of jobs in the WMS. This operation resets the reschedule counter for a job or list of jobs and allows them to run as new. Example:: >>> print dirac.reset(12345) {'OK': True, 'Value': [12345]} :param job: JobID :type job: integer or list of integers :return: S_OK,S_ERROR """ if isinstance( jobID, basestring ): try: jobID = int( jobID ) except Exception as x: return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobID' ) elif isinstance( jobID, list ): try: jobID = [int( job ) for job in jobID] except Exception as x: return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobIDs' ) jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = False ) result = jobManager.resetJob( jobID ) return result ############################################################################# def getJobPilotOutput( self, jobID, directory = '' ): """Retrieve the pilot output for an existing job in the WMS. The output will be retrieved in a local directory unless otherwise specified. >>> print dirac.getJobPilotOutput(12345) {'OK': True, StdOut:'',StdError:''} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not directory: directory = self.currentDir if not os.path.exists( directory ): return self._errorReport( 'Directory %s does not exist' % directory ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getJobPilotOutput( jobID ) if not result['OK']: return result outputPath = '%s/pilot_%s' % ( directory, jobID ) if os.path.exists( outputPath ): self.log.info( 'Remove %s and retry to continue' % outputPath ) return S_ERROR( 'Remove %s and retry to continue' % outputPath ) if not os.path.exists( outputPath ): self.log.verbose( 'Creating directory %s' % outputPath ) os.mkdir( outputPath ) outputs = result['Value'] if outputs.has_key( 'StdOut' ): stdout = '%s/std.out' % ( outputPath ) with open( stdout, 'w' ) as fopen: fopen.write( outputs['StdOut'] ) self.log.verbose( 'Standard output written to %s' % ( stdout ) ) else: self.log.warn( 'No standard output returned' ) if outputs.has_key( 'StdError' ): stderr = '%s/std.err' % ( outputPath ) with open( stderr, 'w' ) as fopen: fopen.write( outputs['StdError'] ) self.log.verbose( 'Standard error written to %s' % ( stderr ) ) else: self.log.warn( 'No standard error returned' ) self.log.always( 'Outputs retrieved in %s' % outputPath ) return result ############################################################################# def getPilotOutput( self, gridReference, directory = '' ): """Retrieve the pilot output (std.out and std.err) for an existing job in the WMS. >>> print dirac.getJobPilotOutput(12345) {'OK': True, 'Value': {}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not isinstance( gridReference, basestring ): return self._errorReport( 'Expected string for pilot reference' ) if not directory: directory = self.currentDir if not os.path.exists( directory ): return self._errorReport( 'Directory %s does not exist' % directory ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilotOutput( gridReference ) if not result['OK']: return result gridReferenceSmall = gridReference.split( '/' )[-1] if not gridReferenceSmall: gridReferenceSmall = 'reference' outputPath = '%s/pilot_%s' % ( directory, gridReferenceSmall ) if os.path.exists( outputPath ): self.log.info( 'Remove %s and retry to continue' % outputPath ) return S_ERROR( 'Remove %s and retry to continue' % outputPath ) if not os.path.exists( outputPath ): self.log.verbose( 'Creating directory %s' % outputPath ) os.mkdir( outputPath ) outputs = result['Value'] if outputs.has_key( 'StdOut' ): stdout = '%s/std.out' % ( outputPath ) with open( stdout, 'w' ) as fopen: fopen.write( outputs['StdOut'] ) self.log.info( 'Standard output written to %s' % ( stdout ) ) else: self.log.warn( 'No standard output returned' ) if outputs.has_key( 'StdErr' ): stderr = '%s/std.err' % ( outputPath ) with open( stderr, 'w' ) as fopen: fopen.write( outputs['StdErr'] ) self.log.info( 'Standard error written to %s' % ( stderr ) ) else: self.log.warn( 'No standard error returned' ) self.log.always( 'Outputs retrieved in %s' % outputPath ) return result ############################################################################# def getPilotInfo( self, gridReference ): """Retrieve info relative to a pilot reference >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance( gridReference, basestring ): return self._errorReport( 'Expected string for pilot reference' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilotInfo( gridReference ) return result ############################################################################# def killPilot( self, gridReference ): """Kill the pilot specified >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :return: S_OK,S_ERROR """ if not isinstance( gridReference, basestring ): return self._errorReport( 'Expected string for pilot reference' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.killPilot( gridReference ) return result ############################################################################# def getPilotLoggingInfo( self, gridReference ): """Retrieve the pilot logging info for an existing job in the WMS. >>> print dirac.getPilotLoggingInfo(12345) {'OK': True, 'Value': {"The output of the command"}} :param gridReference: Gridp pilot job reference Id :type gridReference: string :return: S_OK,S_ERROR """ if type( gridReference ) not in types.StringTypes: return self._errorReport( 'Expected string for pilot reference' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) return wmsAdmin.getPilotLoggingInfo( gridReference ) ############################################################################# def getJobPilots( self, jobID ): """Extract the list of submitted pilots and their status for a given jobID from the WMS. Useful information is printed to the screen. >>> print dirac.getJobPilots() {'OK': True, 'Value': {PilotID:{StatusDict}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if isinstance( jobID, basestring ): try: jobID = int( jobID ) except Exception as x: return self._errorReport( str( x ), 'Expected integer or string for existing jobID' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilots( jobID ) if result['OK']: print self.pPrint.pformat( result['Value'] ) return result ############################################################################# def getPilotSummary( self, startDate = '', endDate = '' ): """Retrieve the pilot output for an existing job in the WMS. Summary is printed at INFO level, full dictionary of results also returned. >>> print dirac.getPilotSummary() {'OK': True, 'Value': {CE:{Status:Count}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilotSummary( startDate, endDate ) if not result['OK']: return result ceDict = result['Value'] headers = 'CE'.ljust( 28 ) i = 0 for ce, summary in ceDict.iteritems(): states = summary.keys() if len( states ) > i: i = len( states ) for i in xrange( i ): headers += 'Status'.ljust( 12 ) + 'Count'.ljust( 12 ) print headers for ce, summary in ceDict.iteritems(): line = ce.ljust( 28 ) states = summary.keys() states.sort() for state in states: count = str( summary[state] ) line += state.ljust( 12 ) + count.ljust( 12 ) print line return result ############################################################################# def selectRequests( self, jobID = None, requestID = None, requestName = None, requestType = None, status = None, operation = None, ownerDN = None, ownerGroup = None, requestStart = 0, limit = 100, printOutput = False ): """Select requests from the request management system. A few notes on the selection criteria: - jobID is the WMS JobID for the request (if applicable) - requestID is assigned during submission of the request - requestName is the corresponding XML file name - requestType e.g. 'transfer' - status e.g. Done - operation e.g. replicateAndRegister - requestStart e.g. the first request to consider (start from 0 by default) - limit e.g. selection limit (default 100) >>> dirac.selectRequests(jobID='4894') {'OK': True, 'Value': [[<Requests>]]} """ options = {'RequestID':requestID, 'RequestName':requestName, 'JobID':jobID, 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup, 'RequestType':requestType, 'Status':status, 'Operation':operation} conditions = {} for key, value in options.iteritems(): if value: try: conditions[key] = str( value ) except Exception as x: return self._errorReport( str( x ), 'Expected string for %s field' % key ) try: requestStart = int( requestStart ) limit = int( limit ) except Exception as x: return self._errorReport( str( x ), 'Expected integer for %s field' % limit ) self.log.verbose( 'Will select requests with the following conditions' ) self.log.verbose( self.pPrint.pformat( conditions ) ) requestClient = RPCClient( "RequestManagement/centralURL" ) result = requestClient.getRequestSummaryWeb( conditions, [], requestStart, limit ) if not result['OK']: self.log.warn( result['Message'] ) return result requestIDs = result['Value'] conds = [] for key, value in conditions.iteritems(): if value: conds.append( '%s = %s' % ( key, value ) ) self.log.verbose( '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ), ', '.join( conds ), limit ) ) if printOutput: requests = [] if len( requestIDs['Records'] ) > limit: requestList = requestIDs['Records'] requests = requestList[:limit] else: requests = requestIDs['Records'] print '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ), ', '.join( conds ), limit ) print requestIDs['ParameterNames'] for request in requests: print request if not requestIDs: return S_ERROR( 'No requests selected for conditions: %s' % conditions ) else: return result ############################################################################# def getRequestSummary( self, printOutput = False ): """ Get a summary of the requests in the request DB. """ requestClient = RPCClient( "RequestManagement/centralURL", timeout = 120 ) result = requestClient.getDBSummary() if not result['OK']: self.log.warn( result['Message'] ) return result if printOutput: print self.pPrint.pformat( result['Value'] ) return result ############################################################################# def getExternalPackageVersions( self ): """ Simple function that attempts to obtain the external versions for the local DIRAC installation (frequently needed for debugging purposes). """ gLogger.info( 'DIRAC version v%dr%d build %d' % ( DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel ) ) try: import lcg_util infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__ gLogger.info( infoStr ) infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version() gLogger.info( infoStr ) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % ( x ) gLogger.exception( errStr ) try: import gfalthr as gfal infoStr = "Using gfalthr from: \n%s" % gfal.__file__ gLogger.info( infoStr ) infoStr = "The version of gfalthr is %s" % gfal.gfal_version() gLogger.info( infoStr ) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % ( x ) gLogger.warn( errStr ) try: import gfal infoStr = "Using gfal from: %s" % gfal.__file__ gLogger.info( infoStr ) infoStr = "The version of gfal is %s" % gfal.gfal_version() gLogger.info( infoStr ) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % ( x ) gLogger.exception( errStr ) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] ) gLogger.info( 'Default list of protocols are: %s' % ( ', '.join( defaultProtocols ) ) ) return S_OK() ############################################################################# def getSiteProtocols( self, site, printOutput = False ): """ Allows to check the defined protocols for each site SE. """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site ) siteSEs = gConfig.getValue( siteSection, [] ) if not siteSEs: return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) ) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] ) self.log.verbose( 'Default list of protocols are' ', '.join( defaultProtocols ) ) seInfo = {} siteSEs.sort() for se in siteSEs: sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) ) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section ) seProtocols = gConfig.getValue( path, [] ) if not seProtocols: seProtocols = defaultProtocols seInfo[se] = seProtocols if printOutput: print '\nSummary of protocols for StorageElements at site %s' % site print '\nStorageElement'.ljust( 30 ) + 'ProtocolsList'.ljust( 30 ) + '\n' for se, protocols in seInfo.iteritems(): print se.ljust( 30 ) + ', '.join( protocols ).ljust( 30 ) return S_OK( seInfo ) ############################################################################# def setSiteProtocols( self, site, protocolsList, printOutput = False ): """ Allows to set the defined protocols for each SE for a given site. """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site ) siteSEs = gConfig.getValue( siteSection, [] ) if not siteSEs: return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) ) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] ) self.log.verbose( 'Default list of protocols are', ', '.join( defaultProtocols ) ) for protocol in protocolsList: if not protocol in defaultProtocols: return S_ERROR( 'Requested to set protocol %s in list but %s is not ' 'in default list of protocols:\n%s' % ( protocol, protocol, ', '.join( defaultProtocols ) ) ) modifiedCS = False result = promptUser( 'Do you want to add the following default protocols:' ' %s for SE(s):\n%s' % ( ', '.join( protocolsList ), ', '.join( siteSEs ) ) ) if not result['OK']: return result if result['Value'].lower() != 'y': self.log.always( 'No protocols will be added' ) return S_OK() for se in siteSEs: sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) ) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section ) self.log.verbose( 'Setting %s to %s' % ( path, ', '.join( protocolsList ) ) ) result = self.csSetOption( path, ', '.join( protocolsList ) ) if not result['OK']: return result modifiedCS = True if modifiedCS: result = self.csCommitChanges( False ) if not result[ 'OK' ]: return S_ERROR( 'CS Commit failed with message = %s' % ( result[ 'Message' ] ) ) else: if printOutput: print 'Successfully committed changes to CS' else: if printOutput: print 'No modifications to CS required' return S_OK() ############################################################################# def csSetOption( self, optionPath, optionValue ): """ Function to modify an existing value in the CS. """ return self.csAPI.setOption( optionPath, optionValue ) ############################################################################# def csSetOptionComment( self, optionPath, comment ): """ Function to modify an existing value in the CS. """ return self.csAPI.setOptionComment( optionPath, comment ) ############################################################################# def csModifyValue( self, optionPath, newValue ): """ Function to modify an existing value in the CS. """ return self.csAPI.modifyValue( optionPath, newValue ) ############################################################################# def csRegisterUser( self, username, properties ): """ Registers a user in the CS. - username: Username of the user (easy;) - properties: Dict containing: - DN - groups : list/tuple of groups the user belongs to - <others> : More properties of the user, like mail """ return self.csAPI.addUser( username, properties ) ############################################################################# def csDeleteUser( self, user ): """ Deletes a user from the CS. Can take a list of users """ return self.csAPI.deleteUsers( user ) ############################################################################# def csModifyUser( self, username, properties, createIfNonExistant = False ): """ Modify a user in the CS. Takes the same params as in addUser and applies the changes """ return self.csAPI.modifyUser( username, properties, createIfNonExistant ) ############################################################################# def csListUsers( self, group = False ): """ Lists the users in the CS. If no group is specified return all users. """ return self.csAPI.listUsers( group ) ############################################################################# def csDescribeUsers( self, mask = False ): """ List users and their properties in the CS. If a mask is given, only users in the mask will be returned """ return self.csAPI.describeUsers( mask ) ############################################################################# def csModifyGroup( self, groupname, properties, createIfNonExistant = False ): """ Modify a user in the CS. Takes the same params as in addGroup and applies the changes """ return self.csAPI.modifyGroup( groupname, properties, createIfNonExistant ) ############################################################################# def csListHosts( self ): """ Lists the hosts in the CS """ return self.csAPI.listHosts() ############################################################################# def csDescribeHosts( self, mask = False ): """ Gets extended info for the hosts in the CS """ return self.csAPI.describeHosts( mask ) ############################################################################# def csModifyHost( self, hostname, properties, createIfNonExistant = False ): """ Modify a host in the CS. Takes the same params as in addHost and applies the changes """ return self.csAPI.modifyHost( hostname, properties, createIfNonExistant ) ############################################################################# def csListGroups( self ): """ Lists groups in the CS """ return self.csAPI.listGroups() ############################################################################# def csDescribeGroups( self, mask = False ): """ List groups and their properties in the CS. If a mask is given, only groups in the mask will be returned """ return self.csAPI.describeGroups( mask ) ############################################################################# def csSyncUsersWithCFG( self, usersCFG ): """ Synchronize users in cfg with its contents """ return self.csAPI.syncUsersWithCFG( usersCFG ) ############################################################################# def csCommitChanges( self, sortUsers = True ): """ Commit the changes in the CS """ return self.csAPI.commitChanges( sortUsers = False ) ############################################################################# def sendMail( self, address, subject, body, fromAddress = None, localAttempt = True, html = False ): """ Send mail to specified address with body. """ notification = NotificationClient() return notification.sendMail( address, subject, body, fromAddress, localAttempt, html ) ############################################################################# def sendSMS( self, userName, body, fromAddress = None ): """ Send mail to specified address with body. """ if len( body ) > 160: return S_ERROR( 'Exceeded maximum SMS length of 160 characters' ) notification = NotificationClient() return notification.sendSMS( userName, body, fromAddress ) ############################################################################# def getBDIISite( self, site, host = None ): """ Get information about site from BDII at host """ return ldapSite( site, host = host ) ############################################################################# def getBDIICluster( self, ce, host = None ): """ Get information about ce from BDII at host """ return ldapCluster( ce, host = host ) ############################################################################# def getBDIICE( self, ce, host = None ): """ Get information about ce from BDII at host """ return ldapCE( ce, host = host ) ############################################################################# def getBDIIService( self, ce, host = None ): """ Get information about ce from BDII at host """ return ldapService( ce, host = host ) ############################################################################# def getBDIICEState( self, ce, useVO = voName, host = None ): """ Get information about ce state from BDII at host """ return ldapCEState( ce, useVO, host = host ) ############################################################################# def getBDIICEVOView( self, ce, useVO = voName, host = None ): """ Get information about ce voview from BDII at host """ return ldapCEVOView( ce, useVO, host = host ) ############################################################################# def getBDIISE( self, site, useVO = voName, host = None ): """ Get information about SA from BDII at host """ return ldapSE( site, useVO, host = host )
class VOMS2CSAgent(AgentModule): def __init__(self, *args, **kwargs): """ Defines default parameters """ super(VOMS2CSAgent, self).__init__(*args, **kwargs) self.__voDict = {} self.__adminMsgs = {} self.csapi = None self.voChanged = False self.dryRun = False self.autoAddUsers = False self.autoModifyUsers = False def initialize(self): """ Initialize the default parameters """ voNames = self.am_getOption('VO', []) if not voNames[0].lower() == "none": if voNames[0].lower() == "any": voNames = [] result = getVOMSVOs(voNames) if not result['OK']: return result self.__voDict = result['Value'] self.log.notice("VOs: %s" % self.__voDict.keys()) self.csapi = CSAPI() self.dryRun = self.am_getOption('DryRun', self.dryRun) self.autoAddUsers = self.am_getOption('AutoAddUsers', self.autoAddUsers) self.autoModifyUsers = self.am_getOption('AutoModifyUsers', self.autoModifyUsers) return S_OK() def execute(self): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption(vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption(voAdminUser, "Email") voAdminGroup = getVOOption(vo, "VOAdminGroup", getVOOption(vo, "DefaultGroup")) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % (vo, voAdminUser, voAdminGroup)) result = self.__syncCSWithVOMS(vo, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error('Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % (vo, result["Message"])) continue if self.voChanged: mailMsg = "" if self.__adminMsgs['Errors']: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs['Errors']) if self.__adminMsgs['Info']: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs['Info']) NotificationClient().sendMail( self.am_getOption('MailTo', voAdminMail), "VOMS2CSAgent run log", mailMsg, self.am_getOption('mailFrom', "DIRAC system")) if self.csapi.csModified: # We have accumulated all the changes, commit them now self.log.info("There are changes to the CS ready to be committed") if self.dryRun: self.log.info("Dry Run: CS won't be updated") self.csapi.showDiff() else: result = self.csapi.commitChanges() if not result['OK']: self.log.error("Could not commit configuration changes", result['Message']) return result self.log.notice("Configuration committed") else: self.log.info("No changes to the CS recorded at this cycle") return S_OK() @executeWithUserProxy def __syncCSWithVOMS(self, vo): self.__adminMsgs = {'Errors': [], 'Info': []} # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] vomsSrv = VOMSService(vo) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error('Could not retrieve VOMS VO name', "for %s" % vo) return result vomsVOName = result['Value'].lstrip('/') self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName)) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result vomsUserDict = result['Value'] message = "There are %s registered users in VOMS VO %s" % ( len(vomsUserDict), vomsVOName) self.__adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users diracUsers = getUsersInVO(vo) if not diracUsers: return S_ERROR("No VO users found for %s" % vo) result = self.csapi.describeUsers(diracUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result diracUserDict = result['Value'] self.__adminMsgs['Info'].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] existingDNs.append(dn) if dn not in vomsUserDict: obsoletedDNs.append(dn) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOusers = list(set(allDiracUsers) - set(diracUsers)) result = self.csapi.describeUsers(nonVOusers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) for dn in vomsUserDict: if dn in newDNs: # Find if the DN is already registered in the DIRAC CS diracName = '' for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # We have a real new user if not diracName: nickName = '' result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA']) if result['OK']: nickName = result['Value'] if nickName: newDiracName = nickName else: newDiracName = getUserName(dn, vomsUserDict[dn]['mail']) ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) group = vomsDIRACMapping.get(fullRole) if group: groupsWithRole.extend(group) userDict['Groups'] = list( set(groupsWithRole + [defaultVOGroup])) self.__adminMsgs['Info'].append( "Adding new user %s: %s" % (newDiracName, str(userDict))) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) continue # We have an already existing user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } nonVOGroups = nonVOUserDict.get(diracName, {}).get('Groups', []) existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) group = vomsDIRACMapping.get(fullRole) if group: groupsWithRole.extend(group) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: role = diracVOMSMapping[group] # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) userDict['Groups'] = keepGroups if self.autoModifyUsers: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.voChanged = True # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dn = diracUserDict[user]['DN'] if not dn in vomsUserDict and not user in nonVOUserDict: for group in diracUserDict[user]['Groups']: if not group in noVOMSGroups: oldUsers.add(user) if oldUsers: self.voChanged = True self.__adminMsgs['Info'].append( 'The following users to be checked for deletion: %s' % str(oldUsers)) self.log.info( 'The following users to be checked for deletion: %s' % str(oldUsers)) return S_OK()
for site in sites: result = getSiteName(site) siteName = result['Value'] gLogger.notice('Processing CondDB endpoint at site %s' % siteName) csapi.copySection( '/Resources/CondDB/%s' % site, '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION, siteName)) csapi.setOptionComment( '%s/Sites/%s/DBServer' % (RESOURCES_NEW_SECTION, siteName), 'Database server resource') csapi.setOptionComment( '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION, siteName), 'Conditions database') if __name__ == '__main__': if computingFlag: result = convertSites() if storageFlag: result = convertSEs() if catalogFlag: result = convertCatalogs() if transferFlag: result = convertTransfers() if dbFlag: result = convertDBServers() csapi.commitChanges() print csapi.getCurrentCFG()['Value'].serialize()