def execute(self): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption(vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption(voAdminUser, "Email") voAdminGroup = getVOOption(vo, "VOAdminGroup", getVOOption(vo, "DefaultGroup")) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % (vo, voAdminUser, voAdminGroup)) result = self.__syncCSWithVOMS(vo, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error('Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % (vo, result["Message"])) continue if self.voChanged: mailMsg = "" if self.__adminMsgs['Errors']: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs['Errors']) if self.__adminMsgs['Info']: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs['Info']) NotificationClient().sendMail( self.am_getOption('MailTo', voAdminMail), "VOMS2CSAgent run log", mailMsg, self.am_getOption('mailFrom', "DIRAC system")) if self.csapi.csModified: # We have accumulated all the changes, commit them now self.log.info("There are changes to the CS ready to be committed") if self.dryRun: self.log.info("Dry Run: CS won't be updated") self.csapi.showDiff() else: result = self.csapi.commitChanges() if not result['OK']: self.log.error("Could not commit configuration changes", result['Message']) return result self.log.notice("Configuration committed") else: self.log.info("No changes to the CS recorded at this cycle") return S_OK()
def main(): signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) global vo, dry, doCEs, ceBdiiDict processScriptSwitches() if not vo: gLogger.error('No VO specified') DIRACExit(-1) vo = getVOOption(vo, 'VOMSName', vo) if doCEs: yn = six.moves.input( 'Do you want to check/add new sites to CS ? [default yes] [yes|no]: ' ) yn = yn.strip() if yn == '' or yn.lower().startswith('y'): checkUnusedCEs() yn = six.moves.input( 'Do you want to update CE details in the CS ? [default yes] [yes|no]: ' ) yn = yn.strip() if yn == '' or yn.lower().startswith('y'): updateSites()
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False, autoLiftSuspendedStatus=False): """ VOMS2CSSynchronizer class constructor :param str vo: VO to be synced :param boolean autoModifyUsers: flag to automatically modify user data in CS :param autoAddUsers: flag to automatically add new users to CS :param autoDeleteUsers: flag to automatically delete users from CS if no more in VOMS :param autoLiftSuspendedStatus: flag to automatically remove Suspended status in CS :return: None """ self.log = gLogger.getSubLogger("VOMS2CSSynchronizer") self.csapi = CSAPI() self.vo = vo self.vomsVOName = getVOOption(vo, "VOMSName", "") if not self.vomsVOName: raise Exception("VOMS name not defined for VO %s" % vo) self.adminMsgs = {'Errors': [], 'Info': []} self.vomsUserDict = {} self.autoModifyUsers = autoModifyUsers self.autoAddUsers = autoAddUsers self.autoDeleteUsers = autoDeleteUsers self.autoLiftSuspendedStatus = autoLiftSuspendedStatus self.voChanged = False
def __addHomeDirectory(self, vo, newUsers): fc = FileCatalog(vo=vo) defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) failed = {} successful = {} for user in newUsers: result = fc.addUser(user) if not result['OK']: failed[user] = "addUser" continue dirName = '/%s/user/%s/%s' % (vo, user[0], user) result = fc.createDirectory(dirName) if not result['OK']: failed[user] = "createDirectory" continue result = fc.changePathOwner({dirName: user}, recursive=True) if not result['OK']: failed[user] = "changePathOwner" continue result = fc.changePathGroup({dirName: defaultVOGroup}, recursive=True) if not result['OK']: failed[user] = "changePathGroup" continue successful[user] = True return S_OK({"Successful": successful, "Failed": failed})
def __addHomeDirectory( self, vo, newUsers ): fc = FileCatalog( vo = vo ) defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo ) failed = {} successful = {} for user in newUsers: result = fc.addUser( user ) if not result['OK']: failed[user] = "addUser" continue dirName = '/%s/user/%s/%s' % ( vo, user[0], user ) result = fc.createDirectory( dirName ) if not result['OK']: failed[user] = "createDirectory" continue result = fc.changePathOwner( { dirName: user }, recursive = True ) if not result['OK']: failed[user] = "changePathOwner" continue result = fc.changePathGroup( { dirName: defaultVOGroup }, recursive = True ) if not result['OK']: failed[user] = "changePathGroup" continue successful[user] = True return S_OK( { "Successful": successful, "Failed": failed } )
def __init__(self, vo=None): """ c'tor :param str vo: name of the virtual organization (community) """ if vo is None: vo = getVO() if not vo: raise Exception('No VO name given') self.vo = vo self.vomsVO = getVOOption(vo, "VOMSName") if not self.vomsVO: raise Exception("Can not get VOMS name for VO %s" % vo) self.urls = [] result = gConfig.getSections('/Registry/VO/%s/VOMSServers' % self.vo) if result['OK']: for server in result['Value']: gLogger.verbose("Adding 'https://%s:8443/voms/%s/apiv2/users'" % (server, self.vomsVO)) self.urls.append('https://%s:8443/voms/%s/apiv2/users' % (server, self.vomsVO)) else: gLogger.error("Section '/Registry/VO/%s/VOMSServers' not found" % self.vo) self.userDict = None
def __init__(self, vo=None): """c'tor :param str vo: name of the virtual organization (community) """ if vo is None: vo = getVO() if not vo: raise Exception("No VO name given") self.vo = vo self.vomsVO = getVOOption(vo, "VOMSName") if not self.vomsVO: raise Exception("Can not get VOMS name for VO %s" % vo) self.urls = [] result = gConfig.getSections("/Registry/VO/%s/VOMSServers" % self.vo) if result["OK"]: for server in result["Value"]: gLogger.verbose("Adding 'https://%s:8443/voms/%s/apiv2/users'" % (server, self.vomsVO)) self.urls.append("https://%s:8443/voms/%s/apiv2/users" % (server, self.vomsVO)) else: gLogger.error("Section '/Registry/VO/%s/VOMSServers' not found" % self.vo) self.userDict = None
def execute( self ): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption( vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption( voAdminUser, "Email") voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) ) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) ) result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) ) continue if self.voChanged: mailMsg = "" if self.__adminMsgs[ 'Errors' ]: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] ) if self.__adminMsgs[ 'Info' ]: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] ) NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ), "VOMS2CSAgent run log", mailMsg, self.am_getOption( 'mailFrom', "DIRAC system" ) ) if self.csapi.csModified: # We have accumulated all the changes, commit them now self.log.info( "There are changes to the CS ready to be committed" ) if self.dryRun: self.log.info( "Dry Run: CS won't be updated" ) self.csapi.showDiff() else: result = self.csapi.commitChanges() if not result[ 'OK' ]: self.log.error( "Could not commit configuration changes", result[ 'Message' ] ) return result self.log.notice( "Configuration committed" ) else: self.log.info( "No changes to the CS recorded at this cycle" ) return S_OK()
def __init__(self, vo=None, adminUrl=False, attributesUrl=False, certificatesUrl=False): if vo is None: vo = getVO() if not vo: raise Exception('No VO name given') self.vo = vo self.vomsVO = getVOOption(vo, "VOMSName") if not self.vomsVO: raise Exception("Can not get VOMS name for VO %s" % vo) self.__soapClients = {} for key, url in (('Admin', adminUrl), ('Attributes', attributesUrl), ('Certificates', certificatesUrl)): urls = [] if not url: url = gConfig.getValue( "/Registry/VO/%s/VOMSServices/VOMS%s" % (self.vo, key), "") if not url: result = gConfig.getSections('/Registry/VO/%s/VOMSServers' % self.vo) if result['OK']: vomsServers = result['Value'] for server in vomsServers: urls.append('https://%s:8443/voms/%s/services/VOMS%s' % (server, self.vomsVO, key)) else: urls = [url] gotURL = False for url in urls: retries = 3 while retries: retries -= 1 try: client = getSOAPClient("%s?wsdl" % url) client.set_options(headers={"X-VOMS-CSRF-GUARD": "1"}) self.__soapClients[key] = client gotURL = True break except Exception: pass if gotURL: break if not gotURL: raise Exception( 'Could not connect to the %s service for VO %s' % (key, self.vo))
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False): self.log = gLogger.getSubLogger("VOMS2CSSynchronizer") self.csapi = CSAPI() self.vo = vo self.vomsVOName = getVOOption(vo, "VOMSName", "") if not self.vomsVOName: raise Exception("VOMS name not defined for VO %s" % vo) self.adminMsgs = {'Errors': [], 'Info': []} self.vomsUserDict = {} self.autoModifyUsers = autoModifyUsers self.autoAddUsers = autoAddUsers self.autoDeleteUsers = autoDeleteUsers
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False): self.log = gLogger.getSubLogger("VOMS2CSSynchronizer") self.csapi = CSAPI() self.vo = vo self.vomsVOName = getVOOption(vo, "VOMSName", "") if not self.vomsVOName: raise Exception("VOMS name not defined for VO %s" % vo) self.adminMsgs = {'Errors': [], 'Info': []} self.vomsUserDict = {} self.autoModifyUsers = autoModifyUsers self.autoAddUsers = autoAddUsers self.autoDeleteUsers = autoDeleteUsers
def initialize(self): """ Gets run paramaters from the configuration """ self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs) self.host = self.am_getOption('Host', self.host) self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs) self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(':')[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.processCEs = self.am_getOption('ProcessCEs', self.processCEs) self.processSEs = self.am_getOption('ProcessSEs', self.processSEs) self.selectedSites = self.am_getOption('SelectedSites', []) self.dryRun = self.am_getOption('DryRun', self.dryRun) self.voName = self.am_getOption('VirtualOrganization', self.voName) if not self.voName: self.voName = self.am_getOption('VO', []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def getGridVOs(): """ Get all the VOMS VO names served by this DIRAC service """ voNames = [] result = getVOs() if not result['OK']: return result else: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: voNames.append(vomsVO) return S_OK(voNames)
def getGridVOs(): """ Get all the VOMS VO names served by this DIRAC service """ voNames = [] result = getVOs() if not result['OK']: return result else: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: voNames.append(vomsVO) return S_OK(voNames)
def initialize(self): """Gets run paramaters from the configuration""" self.addressTo = self.am_getOption("MailTo", self.addressTo) self.addressFrom = self.am_getOption("MailFrom", self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs", self.alternativeBDIIs) self.host = self.am_getOption("Host", self.host) self.injectSingleCoreQueues = self.am_getOption( "InjectSingleCoreQueues", self.injectSingleCoreQueues) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(":")[-1].isdigit(): self.alternativeBDIIs[index] += ":2170" if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.processCEs = self.am_getOption("ProcessCEs", self.processCEs) self.selectedSites = self.am_getOption("SelectedSites", []) self.dryRun = self.am_getOption("DryRun", self.dryRun) self.voName = self.am_getOption("VirtualOrganization", self.voName) if not self.voName: self.voName = self.am_getOption("VO", []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == "all"): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result["OK"]: vos = result["Value"] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def initialize(self): """ Gets run paramaters from the configuration """ self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs) self.host = self.am_getOption('Host', self.host) self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs) self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(':')[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.processCEs = self.am_getOption('ProcessCEs', self.processCEs) self.processSEs = self.am_getOption('ProcessSEs', self.processSEs) self.selectedSites = self.am_getOption('SelectedSites', []) self.dryRun = self.am_getOption('DryRun', self.dryRun) self.voName = self.am_getOption('VirtualOrganization', self.voName) if not self.voName: self.voName = self.am_getOption('VO', []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def getGridVOs(): """Get all the VOMS VO names served by this DIRAC service :return: S_OK(list)/S_ERROR() """ voNames = [] result = getVOs() if not result["OK"]: return result else: vos = result["Value"] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: voNames.append(vomsVO) return S_OK(voNames)
def __init__( self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False, autoLiftSuspendedStatus=False, syncPluginName=None, ): """VOMS2CSSynchronizer class constructor :param str vo: VO to be synced :param boolean autoModifyUsers: flag to automatically modify user data in CS :param autoAddUsers: flag to automatically add new users to CS :param autoDeleteUsers: flag to automatically delete users from CS if no more in VOMS :param autoLiftSuspendedStatus: flag to automatically remove Suspended status in CS :param syncPluginName: name of the plugin to validate or extend users' info :return: None """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.csapi = CSAPI() self.vo = vo self.vomsVOName = getVOOption(vo, "VOMSName", "") if not self.vomsVOName: raise Exception("VOMS name not defined for VO %s" % vo) self.adminMsgs = {"Errors": [], "Info": []} self.vomsUserDict = {} self.autoModifyUsers = autoModifyUsers self.autoAddUsers = autoAddUsers self.autoDeleteUsers = autoDeleteUsers self.autoLiftSuspendedStatus = autoLiftSuspendedStatus self.voChanged = False self.syncPlugin = None if syncPluginName: objLoader = ObjectLoader() _class = objLoader.loadObject( "ConfigurationSystem.Client.SyncPlugins.%sSyncPlugin" % syncPluginName, "%sSyncPlugin" % syncPluginName ) if not _class["OK"]: raise Exception(_class["Message"]) self.syncPlugin = _class["Value"]()
def initialize(self): self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', []) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(':')[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.subject = "Bdii2CSAgent" self.processCEs = self.am_getOption('ProcessCEs', True) self.processSEs = self.am_getOption('ProcessSEs', False) self.voName = self.am_getOption('VirtualOrganization', []) if not self.voName: self.voName = self.am_getOption('VO', []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.voBdiiCEDict = {} self.voBdiiSEDict = {} self.csAPI = CSAPI() return self.csAPI.initialize()
def getVOMSVOs( voList = [] ): """ Get all VOs that have VOMS correspondence :return: dictonary of the VO -> VOMSName correspondence """ voDict = {} if not voList: result = gConfig.getSections( '/Registry/VO' ) if not result['OK']: return result voList = result['Value'] for vo in voList: vomsName = getVOOption( vo, 'VOMSName' ) if vomsName: voDict[vo] = vomsName return S_OK( voDict )
def initialize( self ): self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" self.processCEs = self.am_getOption( 'ProcessCEs', True ) self.processSEs = self.am_getOption( 'ProcessSEs', False ) self.voName = self.am_getOption( 'VirtualOrganization', [] ) if not self.voName: self.voName = self.am_getOption( 'VO', [] ) if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption( vo, "VOMSName" ) if vomsVO: self.voName.append( vomsVO ) if self.voName: self.log.info( "Agent will manage VO(s) %s" % self.voName ) else: self.log.fatal( "VirtualOrganization option not defined for agent" ) return S_ERROR() self.voBdiiCEDict = {} self.voBdiiSEDict = {} self.csAPI = CSAPI() return self.csAPI.initialize()
def __init__( self, vo = None, adminUrl = False, attributesUrl = False, certificatesUrl = False ): if vo is None: vo = getVO() if not vo: raise Exception( 'No VO name given' ) self.vo = vo self.vomsVO = getVOOption( vo, "VOMSName" ) if not self.vomsVO: raise Exception( "Can not get VOMS name for VO %s" % vo ) self.__soapClients = {} for key, url in ( ( 'Admin', adminUrl ), ( 'Attributes', attributesUrl ), ( 'Certificates', certificatesUrl ) ): urls = [] if not url: url = gConfig.getValue( "/Registry/VO/%s/VOMSServices/VOMS%s" % ( self.vo, key ), "" ) if not url: result = gConfig.getSections( '/Registry/VO/%s/VOMSServers' % self.vo ) if result['OK']: vomsServers = result['Value'] for server in vomsServers: urls.append( 'https://%s:8443/voms/%s/services/VOMS%s' % ( server, self.vomsVO, key ) ) else: urls = [url] gotURL = False for url in urls: retries = 3 while retries: retries -= 1 try: client = getSOAPClient( "%s?wsdl" % url ) client.set_options(headers={"X-VOMS-CSRF-GUARD":"1"}) self.__soapClients[ key ] = client gotURL = True break except Exception: pass if gotURL: break if not gotURL: raise Exception( 'Could not connect to the %s service for VO %s' % ( key, self.vo ) )
def getVOMSVOs(voList=None): """ Get all VOs that have VOMS correspondence :return: dictonary of the VO -> VOMSName correspondence """ if voList is None: voList = [] voDict = {} if not voList: result = gConfig.getSections('/Registry/VO') if not result['OK']: return result voList = result['Value'] for vo in voList: vomsName = getVOOption(vo, 'VOMSName') if vomsName: voDict[vo] = vomsName return S_OK(voDict)
def __init__(self, vo=None): if vo is None: vo = getVO() if not vo: raise Exception('No VO name given') self.vo = vo self.vomsVO = getVOOption(vo, "VOMSName") if not self.vomsVO: raise Exception("Can not get VOMS name for VO %s" % vo) self.urls = [] result = gConfig.getSections('/Registry/VO/%s/VOMSServers' % self.vo) if result['OK']: vomsServers = result['Value'] for server in vomsServers: self.urls.append('https://%s:8443/voms/%s/apiv2/users' % (server, self.vomsVO)) self.userDict = None
signal.signal(signal.SIGINT, handler) vo = '' dry = False doCEs = False doSEs = False ceBdiiDict = None processScriptSwitches() if not vo: gLogger.error('No VO specified') DIRACExit(-1) diracVO = vo vo = getVOOption(vo, 'VOMSName', vo) if doCEs: yn = raw_input('Do you want to check/add new sites to CS ? [default yes] [yes|no]: ') yn = yn.strip() if yn == '' or yn.lower().startswith('y'): checkUnusedCEs() yn = raw_input('Do you want to update CE details in the CS ? [default yes] [yes|no]: ') yn = yn.strip() if yn == '' or yn.lower().startswith('y'): updateSites() if doSEs: yn = raw_input('Do you want to check/add new storage elements to CS ? [default yes] [yes|no]: ') yn = yn.strip()
def getVOAdmin(voName): voAdminUser = getVOOption(voName, "VOAdmin") voAdminGroup = getVOOption(voName, "VOAdminGroup", getVOOption(voName, "DefaultGroup")) return voAdminUser, voAdminGroup
def __syncCSWithVOMS(self, vo): self.__adminMsgs = {'Errors': [], 'Info': []} # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] vomsSrv = VOMSService(vo) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error('Could not retrieve VOMS VO name', "for %s" % vo) return result vomsVOName = result['Value'].lstrip('/') self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName)) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result vomsUserDict = result['Value'] message = "There are %s registered users in VOMS VO %s" % ( len(vomsUserDict), vomsVOName) self.__adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users diracUsers = getUsersInVO(vo) if not diracUsers: return S_ERROR("No VO users found for %s" % vo) result = self.csapi.describeUsers(diracUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result diracUserDict = result['Value'] self.__adminMsgs['Info'].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] existingDNs.append(dn) if dn not in vomsUserDict: obsoletedDNs.append(dn) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOusers = list(set(allDiracUsers) - set(diracUsers)) result = self.csapi.describeUsers(nonVOusers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) for dn in vomsUserDict: if dn in newDNs: # Find if the DN is already registered in the DIRAC CS diracName = '' for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # We have a real new user if not diracName: nickName = '' result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA']) if result['OK']: nickName = result['Value'] if nickName: newDiracName = nickName else: newDiracName = getUserName(dn, vomsUserDict[dn]['mail']) ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) group = vomsDIRACMapping.get(fullRole) if group: groupsWithRole.extend(group) userDict['Groups'] = list( set(groupsWithRole + [defaultVOGroup])) self.__adminMsgs['Info'].append( "Adding new user %s: %s" % (newDiracName, str(userDict))) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) continue # We have an already existing user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } nonVOGroups = nonVOUserDict.get(diracName, {}).get('Groups', []) existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) group = vomsDIRACMapping.get(fullRole) if group: groupsWithRole.extend(group) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: role = diracVOMSMapping[group] # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) userDict['Groups'] = keepGroups if self.autoModifyUsers: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.voChanged = True # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dn = diracUserDict[user]['DN'] if not dn in vomsUserDict and not user in nonVOUserDict: for group in diracUserDict[user]['Groups']: if not group in noVOMSGroups: oldUsers.add(user) if oldUsers: self.voChanged = True self.__adminMsgs['Info'].append( 'The following users to be checked for deletion: %s' % str(oldUsers)) self.log.info( 'The following users to be checked for deletion: %s' % str(oldUsers)) return S_OK()
def execute(self): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption(vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption(voAdminUser, "Email") voAdminGroup = getVOOption(vo, "VOAdminGroup", getVOOption(vo, "DefaultGroup")) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % (vo, voAdminUser, voAdminGroup)) result = self.__syncCSWithVOMS(vo, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error('Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % (vo, result["Message"])) continue resultDict = result['Value'] newUsers = resultDict.get("NewUsers", []) modUsers = resultDict.get("ModifiedUsers", []) delUsers = resultDict.get("DeletedUsers", []) self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \ ( len( newUsers ), len( modUsers ), len( delUsers ) ) ) if self.csapi.csModified: # We have accumulated all the changes, commit them now self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo) if self.dryRun: self.log.info("Dry Run: CS won't be updated") self.csapi.showDiff() else: result = self.csapi.commitChanges() if not result['OK']: self.log.error( "Could not commit configuration changes", result['Message']) return result self.log.notice("Configuration committed for VO %s" % vo) else: self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo) # Add user home directory in the file catalog if self.makeFCEntry and newUsers: self.log.info("Creating home directories for users %s" % str(newUsers)) result = self.__addHomeDirectory(vo, newUsers, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error('Failed to create user home directories:', 'VO %s: %s' % (vo, result["Message"])) else: for user in result['Value']['Failed']: self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \ ( user, result['Value']['Failed'][user] ) ) self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \ ( user, result['Value']['Failed'][user] ) ) for user in result['Value']['Successful']: self.__adminMsgs['Info'].append( "Created home directory for user %s" % user) if self.voChanged or self.detailedReport: mailMsg = "" if self.__adminMsgs['Errors']: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs['Errors']) if self.__adminMsgs['Info']: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs['Info']) if self.detailedReport: result = self.getVOUserReport(vo) if result['OK']: mailMsg += '\n\n' mailMsg += result['Value'] else: mailMsg += 'Failed to produce a detailed user report' mailMsg += result['Message'] NotificationClient().sendMail( self.am_getOption('MailTo', voAdminMail), "VOMS2CSAgent run log", mailMsg, self.am_getOption( 'MailFrom', self.am_getOption('mailFrom', "DIRAC system"))) return S_OK()
def main(): Script.registerSwitch("V:", "vo=", "VO name", setVO) Script.registerSwitch("D", "dryRun", "Dry run", setDryRun) Script.parseCommandLine(ignoreErrors=True) @executeWithUserProxy def syncCSWithVOMS(vomsSync): return vomsSync.syncCSWithVOMS() voAdminUser = getVOOption(voName, "VOAdmin") voAdminGroup = getVOOption(voName, "VOAdminGroup", getVOOption(voName, "DefaultGroup")) vomsSync = VOMS2CSSynchronizer(voName) result = syncCSWithVOMS( # pylint: disable=unexpected-keyword-arg vomsSync, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup) if not result["OK"]: gLogger.error("Failed to synchronize user data") DIRACExit(-1) resultDict = result["Value"] newUsers = resultDict.get("NewUsers", []) modUsers = resultDict.get("ModifiedUsers", []) delUsers = resultDict.get("DeletedUsers", []) susUsers = resultDict.get("SuspendedUsers", []) gLogger.notice( "\nUser results: new %d, modified %d, deleted %d, new/suspended %d" % (len(newUsers), len(modUsers), len(delUsers), len(susUsers))) for msg in resultDict["AdminMessages"]["Info"]: gLogger.notice(msg) csapi = resultDict.get("CSAPI") if csapi and csapi.csModified: if dryRun: gLogger.notice( "There are changes to Registry ready to commit, skipped because of dry run" ) else: yn = input( "There are changes to Registry ready to commit, do you want to proceed ? [Y|n]:" ) if yn == "" or yn[0].lower() == "y": result = csapi.commitChanges() if not result["OK"]: gLogger.error("Could not commit configuration changes", result["Message"]) else: gLogger.notice("Registry changes committed for VO %s" % voName) else: gLogger.notice("Registry changes are not committed") else: gLogger.notice("No changes to Registry for VO %s" % voName) result = vomsSync.getVOUserReport() if not result["OK"]: gLogger.error("Failed to generate user data report") DIRACExit(-1) gLogger.notice("\n" + result["Value"])
def __syncCSWithVOMS(self, vo): self.__adminMsgs = {'Errors': [], 'Info': []} resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService(vo) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error('Could not retrieve VOMS VO name', "for %s" % vo) return result vomsVOName = result['Value'].lstrip('/') self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName)) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result vomsUserDict = result['Value'] message = "There are %s user entries in VOMS for VO %s" % ( len(vomsUserDict), vomsVOName) self.__adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users result = self.getVOUserData(vo) if not result['OK']: return result diracUserDict = result['Value'] self.__adminMsgs['Info'].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in vomsUserDict: obsoletedDNs.append(dn) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict.keys())) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) newAddedUserDict = {} for dn in vomsUserDict: nickName = '' newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN result = vomsSrv.attGetUserNickname(dn, vomsUserDict[dn]['CA']) if result['OK']: nickName = result['Value'] if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = getUserName(dn, vomsUserDict[dn]['mail'], vo) # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) groupList = vomsDIRACMapping.get(fullRole, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict['Groups'] = list( set(groupsWithRole + [defaultVOGroup])) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.__adminMsgs['Info'].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) resultDict['NewUsers'].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } if newDNForExistingUser: userDict['DN'] = ','.join([dn, diracUserDict[diracName]['DN']]) modified = True existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) nonVOGroups = list( set(existingGroups) - set(diracVOMSMapping.keys())) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) groupList = vomsDIRACMapping.get(fullRole, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, '') # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict['Groups'] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get('Groups', []) userDict['Groups'] = list(set(keepGroups + otherGroups)) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set( diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set( diracUserDict.get(diracName, {}).get( key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join( addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups) else: oldValue = str( diracUserDict.get(diracName, {}).get(key, '')) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % ( key, oldValue, str(userDict[key])) if modMsg: self.__adminMsgs['Info'].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) if not dnSet.intersection(set( vomsUserDict.keys())) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info('The following users will be deleted: %s' % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result['OK']: self.__adminMsgs['Info'].append( 'The following users are deleted from CS:\n %s\n' % str(oldUsers)) resultDict['DeletedUsers'] = oldUsers else: self.__adminMsgs['Errors'].append( 'Error in deleting users from CS:\n %s' % str(oldUsers)) self.log.error('Error while user deletion from CS', result) else: self.__adminMsgs['Info'].append( 'The following users to be checked for deletion:\n %s' % str(oldUsers)) self.log.info( 'The following users to be checked for deletion: %s' % str(oldUsers)) return S_OK(resultDict)
def execute( self ): self.__adminMsgs = {} self.csapi.downloadCSData() for vo in self.__voDict: self.voChanged = False voAdminUser = getVOOption( vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption( voAdminUser, "Email") voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) ) self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) ) result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) ) continue resultDict = result['Value'] newUsers = resultDict.get( "NewUsers", [] ) modUsers = resultDict.get( "ModifiedUsers", [] ) delUsers = resultDict.get( "DeletedUsers", [] ) self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \ ( len( newUsers ), len( modUsers ), len( delUsers ) ) ) if self.csapi.csModified: # We have accumulated all the changes, commit them now self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo ) if self.dryRun: self.log.info( "Dry Run: CS won't be updated" ) self.csapi.showDiff() else: result = self.csapi.commitChanges() if not result[ 'OK' ]: self.log.error( "Could not commit configuration changes", result[ 'Message' ] ) return result self.log.notice( "Configuration committed for VO %s" % vo ) else: self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo ) # Add user home directory in the file catalog if self.makeFCEntry and newUsers: self.log.info( "Creating home directories for users %s" % str( newUsers ) ) result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg if not result['OK']: self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) ) else: for user in result['Value']['Failed']: self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \ ( user, result['Value']['Failed'][user] ) ) self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \ ( user, result['Value']['Failed'][user] ) ) for user in result['Value']['Successful']: self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user ) if self.voChanged or self.detailedReport: mailMsg = "" if self.__adminMsgs[ 'Errors' ]: mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] ) if self.__adminMsgs[ 'Info' ]: mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] ) if self.detailedReport: result = self.getVOUserReport( vo ) if result['OK']: mailMsg += '\n\n' mailMsg += result['Value'] else: mailMsg += 'Failed to produce a detailed user report' mailMsg += result['Message'] NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ), "VOMS2CSAgent run log", mailMsg, self.am_getOption( 'MailFrom', self.am_getOption( 'mailFrom', "DIRAC system" ) ) ) return S_OK()
def __syncCSWithVOMS( self, vo ): self.__adminMsgs = { 'Errors' : [], 'Info' : [] } # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping( vo ) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] vomsSrv = VOMSService( vo ) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo ) return result vomsVOName = result[ 'Value' ].lstrip( '/' ) self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) ) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error( 'Could not retrieve user information from VOMS', result['Message'] ) return result vomsUserDict = result[ 'Value' ] message = "There are %s registered users in VOMS VO %s" % ( len( vomsUserDict ), vomsVOName ) self.__adminMsgs[ 'Info' ].append( message ) self.log.info( message ) # Get DIRAC users diracUsers = getUsersInVO( vo ) if not diracUsers: return S_ERROR( "No VO users found for %s" % vo ) result = self.csapi.describeUsers( diracUsers ) if not result['OK']: self.log.error( 'Could not retrieve CS User description' ) return result diracUserDict = result['Value'] self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) ) self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) ) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] existingDNs.append( dn ) if dn not in vomsUserDict: obsoletedDNs.append( dn ) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append( dn ) allDiracUsers = getAllUsers() nonVOusers = list( set( allDiracUsers ) - set(diracUsers) ) result = self.csapi.describeUsers( nonVOusers ) if not result['OK']: self.log.error( 'Could not retrieve CS User description' ) return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo ) for dn in vomsUserDict: if dn in newDNs: # Find if the DN is already registered in the DIRAC CS diracName = '' for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # We have a real new user if not diracName: nickName = '' result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] ) if result['OK']: nickName = result['Value'] if nickName: newDiracName = nickName else: newDiracName = getUserName( dn, vomsUserDict[dn]['mail'] ) ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % ( trialName, ind ) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % ( vomsVOName, role ) group = vomsDIRACMapping.get( fullRole ) if group: groupsWithRole.append( group ) userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) ) self.__adminMsgs[ 'Info' ].append( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) ) self.voChanged = True if self.autoAddUsers: self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) ) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True ) if not result['OK']: self.log.warn( 'Failed adding new user %s' % newDiracName ) continue # We have an already existing user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } nonVOGroups = nonVOUserDict.get( diracName, {} ).get( 'Groups', [] ) existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] ) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % ( vomsVOName, role ) group = vomsDIRACMapping.get( fullRole ) if group: groupsWithRole.append( group ) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: role = diracVOMSMapping[group] # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append( group ) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append( group ) userDict['Groups'] = keepGroups if self.autoModifyUsers: result = self.csapi.modifyUser( diracName, userDict ) if result['OK'] and result['Value']: self.voChanged = True # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dn = diracUserDict[user]['DN'] if not dn in vomsUserDict and not user in nonVOUserDict: for group in diracUserDict[user]['Groups']: if not group in noVOMSGroups: oldUsers.add( user ) if oldUsers: self.voChanged = True self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion: %s' % str( oldUsers ) ) self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) ) return S_OK()
def execute(self): for vo in self.voList: voAdminUser = getVOOption(vo, "VOAdmin") voAdminMail = None if voAdminUser: voAdminMail = getUserOption(voAdminUser, "Email") voAdminGroup = getVOOption(vo, "VOAdminGroup", getVOOption(vo, "DefaultGroup")) self.log.info( "Performing VOMS sync", "for VO %s with credentials %s@%s" % (vo, voAdminUser, voAdminGroup)) autoAddUsers = getVOOption(vo, "AutoAddUsers", self.autoAddUsers) autoModifyUsers = getVOOption(vo, "AutoModifyUsers", self.autoModifyUsers) autoDeleteUsers = getVOOption(vo, "AutoDeleteUsers", self.autoDeleteUsers) autoLiftSuspendedStatus = getVOOption(vo, "AutoLiftSuspendedStatus", self.autoLiftSuspendedStatus) syncPluginName = getVOOption(vo, "SyncPluginName", self.syncPluginName) vomsSync = VOMS2CSSynchronizer( vo, autoAddUsers=autoAddUsers, autoModifyUsers=autoModifyUsers, autoDeleteUsers=autoDeleteUsers, autoLiftSuspendedStatus=autoLiftSuspendedStatus, syncPluginName=syncPluginName, ) result = self.__syncCSWithVOMS( # pylint: disable=unexpected-keyword-arg vomsSync, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup, ) if not result["OK"]: self.log.error("Failed to perform VOMS to CS synchronization:", "VO %s: %s" % (vo, result["Message"])) continue resultDict = result["Value"] newUsers = resultDict.get("NewUsers", []) modUsers = resultDict.get("ModifiedUsers", []) delUsers = resultDict.get("DeletedUsers", []) susUsers = resultDict.get("SuspendedUsers", []) csapi = resultDict.get("CSAPI") adminMessages = resultDict.get("AdminMessages", { "Errors": [], "Info": [] }) voChanged = resultDict.get("VOChanged", False) self.log.info( "Run user results", ": new %d, modified %d, deleted %d, new/suspended %d" % (len(newUsers), len(modUsers), len(delUsers), len(susUsers)), ) if csapi.csModified: # We have accumulated all the changes, commit them now self.log.info( "There are changes to the CS ready to be committed", "for VO %s" % vo) if self.dryRun: self.log.info("Dry Run: CS won't be updated") csapi.showDiff() else: result = csapi.commitChanges() if not result["OK"]: self.log.error( "Could not commit configuration changes", result["Message"]) return result self.log.notice("Configuration committed", "for VO %s" % vo) else: self.log.info("No changes to the CS recorded at this cycle", "for VO %s" % vo) # Add user home directory in the file catalog if self.makeFCEntry and newUsers: self.log.info("Creating home directories for users", str(newUsers)) result = self.__addHomeDirectory( # pylint: disable=unexpected-keyword-arg vo, newUsers, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup, ) if not result["OK"]: self.log.error("Failed to create user home directories:", "VO %s: %s" % (vo, result["Message"])) else: for user in result["Value"]["Failed"]: self.log.error( "Failed to create home directory", "user: %s, operation: %s" % (user, result["Value"]["Failed"][user]), ) adminMessages["Errors"].append( "Failed to create home directory for user %s: operation %s" % (user, result["Value"]["Failed"][user])) for user in result["Value"]["Successful"]: adminMessages["Info"].append( "Created home directory for user %s" % user) if voChanged or self.detailedReport: mailMsg = "" if adminMessages["Errors"]: mailMsg += "\nErrors list:\n %s" % "\n ".join( adminMessages["Errors"]) if adminMessages["Info"]: mailMsg += "\nRun result:\n %s" % "\n ".join( adminMessages["Info"]) if self.detailedReport: result = vomsSync.getVOUserReport() if result["OK"]: mailMsg += "\n\n" mailMsg += result["Value"] else: mailMsg += "Failed to produce a detailed user report" mailMsg += result["Message"] if self.dryRun: self.log.info("Dry Run: mail won't be sent") self.log.info(mailMsg) else: NotificationClient().sendMail( self.am_getOption("MailTo", voAdminMail), "VOMS2CSAgent run log", mailMsg, self.mailFrom) return S_OK()
def syncCSWithVOMS(self): """ Performs the synchronization of the DIRAC registry with the VOMS data. The resulting CSAPI object containing modifications is returned as part of the output dictionary. Those changes can be applied by the caller depending on the mode (dry or a real run) :return: S_OK with a dictionary containing the results of the synchronization operation """ resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService(self.vo) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result self.vomsUserDict = result['Value'] message = "There are %s user entries in VOMS for VO %s" % (len( self.vomsUserDict), self.vomsVOName) self.adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users result = self.getVOUserData(self.vo) if not result['OK']: return result diracUserDict = result['Value'] self.adminMsgs['Info'].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in self.vomsUserDict: obsoletedDNs.append(dn) for dn in self.vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict)) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo) # If a user is (previously put by hand) in a "QuarantineGroup", # then the default group will be ignored. # So, this option is only considered for the case of existing users. quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup") newAddedUserDict = {} for dn in self.vomsUserDict: newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN nickName = self.vomsUserDict[dn].get('nickname') if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = self.getUserName(dn) # Do not consider users with Suspended status in VOMS if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[ dn]['certSuspended']: resultDict["SuspendedUsers"].append(newDiracName) continue # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict['Groups'] = list( set(groupsWithRole + [defaultVOGroup])) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.adminMsgs['Info'].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) resultDict['NewUsers'].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = { "DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail'] } if newDNForExistingUser: userDict['DN'] = ','.join([ dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN'] ]) modified = True existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole if not quarantineVOGroup or quarantineVOGroup not in existingGroups: keepGroups += [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, '') # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if "Role" not in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict['Groups'] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get('Groups', []) userDict['Groups'] = list(set(keepGroups + otherGroups)) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set( diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set( diracUserDict.get(diracName, {}).get( key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join( addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups) else: oldValue = str( diracUserDict.get(diracName, {}).get(key, '')) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % ( key, oldValue, str(userDict[key])) if modMsg: self.adminMsgs['Info'].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) if not dnSet.intersection(set( self.vomsUserDict)) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add(user) # Check for obsoleted DNs for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) for dn in dnSet: if dn in obsoletedDNs and user not in oldUsers: self.log.verbose("Modified user %s: dropped DN %s" % (user, dn)) if self.autoModifyUsers: userDict = diracUserDict[user] modDNSet = dnSet - set([dn]) if modDNSet: userDict['DN'] = ','.join(modDNSet) result = self.csapi.modifyUser(user, userDict) if result['OK'] and result['Value']: self.log.info( "Modified user %s: dropped DN %s" % (user, dn)) self.adminMsgs['Info'].append( "Modified user %s: dropped DN %s" % (user, dn)) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) else: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info('The following users will be deleted: %s' % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result['OK']: self.adminMsgs['Info'].append( 'The following users are deleted from CS:\n %s\n' % str(oldUsers)) resultDict['DeletedUsers'] = oldUsers else: self.adminMsgs['Errors'].append( 'Error in deleting users from CS:\n %s' % str(oldUsers)) self.log.error('Error while user deletion from CS', result) else: self.adminMsgs['Info'].append( 'The following users to be checked for deletion:\n %s' % str(oldUsers)) self.log.info( 'The following users to be checked for deletion: %s' % str(oldUsers)) resultDict['CSAPI'] = self.csapi resultDict['AdminMessages'] = self.adminMsgs return S_OK(resultDict)
def syncCSWithVOMS(self): """ Performs the synchronization of the DIRAC registry with the VOMS data. The resulting CSAPI object containing modifications is returned as part of the output dictionary. Those changes can be applied by the caller depending on the mode (dry or a real run) :return: S_OK with a dictionary containing the results of the synchronization operation """ resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService(self.vo) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result self.vomsUserDict = result['Value'] message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName) self.adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users result = self.getVOUserData(self.vo) if not result['OK']: return result diracUserDict = result['Value'] self.adminMsgs['Info'].append("There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in self.vomsUserDict: obsoletedDNs.append(dn) for dn in self.vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict)) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo) # If a user is (previously put by hand) in a "QuarantineGroup", # then the default group will be ignored. # So, this option is only considered for the case of existing users. quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup") newAddedUserDict = {} for dn in self.vomsUserDict: newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN nickName = self.vomsUserDict[dn].get('nickname') if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = self.getUserName(dn) # Do not consider users with Suspended status in VOMS if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[dn]['certSuspended']: resultDict["SuspendedUsers"].append(newDiracName) continue # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']} groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict['Groups'] = list(set(groupsWithRole + [defaultVOGroup])) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.adminMsgs['Info'].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) resultDict['NewUsers'].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']} if newDNForExistingUser: userDict['DN'] = ','.join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN']]) modified = True existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole if not quarantineVOGroup or quarantineVOGroup not in existingGroups: keepGroups += [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, '') # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if "Role" not in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict['Groups'] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get('Groups', []) userDict['Groups'] = list(set(keepGroups + otherGroups)) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join(addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join(removedGroups) else: oldValue = str(diracUserDict.get(diracName, {}).get(key, '')) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key])) if modMsg: self.adminMsgs['Info'].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add(user) # Check for obsoleted DNs for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) for dn in dnSet: if dn in obsoletedDNs and user not in oldUsers: self.log.verbose("Modified user %s: dropped DN %s" % (user, dn)) if self.autoModifyUsers: userDict = diracUserDict[user] modDNSet = dnSet - set([dn]) if modDNSet: userDict['DN'] = ','.join(modDNSet) result = self.csapi.modifyUser(user, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: dropped DN %s" % (user, dn)) self.adminMsgs['Info'].append("Modified user %s: dropped DN %s" % (user, dn)) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) else: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info('The following users will be deleted: %s' % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result['OK']: self.adminMsgs['Info'].append('The following users are deleted from CS:\n %s\n' % str(oldUsers)) resultDict['DeletedUsers'] = oldUsers else: self.adminMsgs['Errors'].append('Error in deleting users from CS:\n %s' % str(oldUsers)) self.log.error('Error while user deletion from CS', result) else: self.adminMsgs['Info'].append('The following users to be checked for deletion:\n %s' % str(oldUsers)) self.log.info('The following users to be checked for deletion: %s' % str(oldUsers)) resultDict['CSAPI'] = self.csapi resultDict['AdminMessages'] = self.adminMsgs return S_OK(resultDict)
def syncCSWithVOMS(self): """Performs the synchronization of the DIRAC registry with the VOMS data. The resulting CSAPI object containing modifications is returned as part of the output dictionary. Those changes can be applied by the caller depending on the mode (dry or a real run) :return: S_OK with a dictionary containing the results of the synchronization operation """ resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result["OK"]: return result vomsDIRACMapping = result["Value"]["VOMSDIRAC"] diracVOMSMapping = result["Value"]["DIRACVOMS"] noVOMSGroups = result["Value"]["NoVOMS"] noSyncVOMSGroups = result["Value"]["NoSyncVOMS"] vomsSrv = VOMSService(self.vo) # Get VOMS users result = vomsSrv.getUsers() if not result["OK"]: self.log.error("Could not retrieve user information from VOMS", result["Message"]) return result self.vomsUserDict = result["Value"] message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName) self.adminMsgs["Info"].append(message) self.log.info("VOMS user entries", message) self.log.debug(self.vomsUserDict) # Get DIRAC users result = self.getVOUserData(self.vo) if not result["OK"]: return result diracUserDict = result["Value"] self.adminMsgs["Info"].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo) ) self.log.info( "Users already registered", ": there are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo) ) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]["DN"] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in self.vomsUserDict: obsoletedDNs.append(dn) for dn in self.vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict)) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result["OK"]: self.log.error("Could not retrieve CS User description") return result nonVOUserDict = result["Value"] # Process users defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo) # If a user is (previously put by hand) in a "QuarantineGroup", # then the default group will be ignored. # So, this option is only considered for the case of existing users. quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup") newAddedUserDict = {} for dn in self.vomsUserDict: newDNForExistingUser = "" diracName = "" if dn in existingDNs: for user in diracUserDict: if dn in fromChar(diracUserDict[user]["DN"]): diracName = user break if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn in fromChar(nonVOUserDict[user]["DN"]): diracName = user diracUserDict[diracName] = nonVOUserDict[user] break # Check the nickName in the same VO to see if the user is already registered # with another DN nickName = self.vomsUserDict[dn].get("nickname") if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = self.getUserName(dn) # Do not consider users with Suspended status in VOMS if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]: resultDict["SuspendedUsers"].append(newDiracName) continue # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = {"DN": dn, "CA": self.vomsUserDict[dn]["CA"], "Email": self.vomsUserDict[dn]["mail"]} groupsWithRole = [] for role in self.vomsUserDict[dn]["Roles"]: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict["Groups"] = list(set(groupsWithRole + [defaultVOGroup])) # Run the sync plugins for extra info and/or validations if self.syncPlugin: try: self.syncPlugin.verifyAndUpdateUserInfo(newDiracName, userDict) except ValueError as e: self.log.error("Error validating new user", "nickname %s\n error %s" % (newDiracName, e)) self.adminMsgs["Errors"].append( "Error validating new user %s: %s\n %s" % (newDiracName, userDict, e) ) continue message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.adminMsgs["Info"].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True) if not result["OK"]: self.log.warn("Failed adding new user %s" % newDiracName) resultDict["NewUsers"].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False suspendedInVOMS = self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"] suspendedVOList = getUserOption(diracName, "Suspended", []) knownEmail = getUserOption(diracName, "Email", None) userDict = { "DN": diracUserDict[diracName]["DN"], "CA": diracUserDict[diracName]["CA"], "Email": self.vomsUserDict[dn].get("mail", self.vomsUserDict[dn].get("emailAddress")) or knownEmail, } # Set Suspended status for the user for this particular VO if suspendedInVOMS and self.vo not in suspendedVOList: suspendedVOList.append(self.vo) userDict["Suspended"] = ",".join(suspendedVOList) modified = True # Remove the lifted Suspended status if not suspendedInVOMS and self.vo in suspendedVOList and self.autoLiftSuspendedStatus: newList = [] for vo in suspendedVOList: if vo != self.vo: newList.append(vo) if not newList: newList = ["None"] userDict["Suspended"] = ",".join(newList) modified = True if newDNForExistingUser: userDict["DN"] = ",".join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))["DN"]]) userDict["CA"] = ",".join( [self.vomsUserDict[dn]["CA"], diracUserDict.get(diracName, newAddedUserDict.get(diracName))["CA"]] ) modified = True existingGroups = diracUserDict.get(diracName, {}).get("Groups", []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) groupsWithRole = [] for role in self.vomsUserDict[dn]["Roles"]: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole if not quarantineVOGroup or quarantineVOGroup not in existingGroups: keepGroups += [defaultVOGroup] if quarantineVOGroup and quarantineVOGroup in existingGroups: keepGroups = [quarantineVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, "") # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if "Role" not in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict["Groups"] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get("Groups", []) userDict["Groups"] = list(set(keepGroups + otherGroups)) modified = True if not existingGroups and diracName in allDiracUsers: groups = getGroupsForUser(diracName) if groups["OK"]: self.log.info("Found groups for user %s %s" % (diracName, groups["Value"])) userDict["Groups"] = list(set(groups["Value"] + keepGroups)) addedGroups = list(set(userDict["Groups"]) - set(groups["Value"])) modified = True message = "\n Modified user %s:\n" % diracName message += " Added to group(s) %s\n" % ",".join(addedGroups) self.adminMsgs["Info"].append(message) # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = "" for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ",".join(addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ",".join(removedGroups) elif key == "Suspended": if userDict["Suspended"] == "None": modMsg += " Suspended status removed\n" else: modMsg += " User Suspended in VOs: %s\n" % userDict["Suspended"] else: oldValue = str(diracUserDict.get(diracName, {}).get(key, "")) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key])) if modMsg: self.adminMsgs["Info"].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result["OK"] and result["Value"]: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict["ModifiedUsers"].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]["DN"])) if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict: existingGroups = diracUserDict.get(user, {}).get("Groups", []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) removedGroups = list(set(existingGroups) - set(nonVOGroups)) if removedGroups: self.log.info("Checking user for deletion", "%s: %s" % (user, existingGroups)) self.log.info("User has groups in other VOs", "%s: %s" % (user, nonVOGroups)) userDict = diracUserDict[user] userDict["Groups"] = nonVOGroups if self.autoModifyUsers: result = self.csapi.modifyUser(user, userDict) if result["OK"] and result["Value"]: self.log.info("Modified user %s: %s" % (user, str(userDict))) self.voChanged = True message = "\n Modified user %s:\n" % user modMsg = " Removed from group(s) %s\n" % ",".join(removedGroups) self.adminMsgs["Info"].append(message + modMsg) resultDict["ModifiedUsers"].append(user) continue if not any(group in noVOMSGroups for group in existingGroups): oldUsers.add(user) # Check for obsoleted DNs for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]["DN"])) for dn in dnSet: if dn in obsoletedDNs and user not in oldUsers: existingGroups = diracUserDict.get(user, {}).get("Groups", []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) if nonVOGroups: self.log.verbose("User has groups in other VOs", "%s: %s" % (user, nonVOGroups)) continue self.log.verbose("Modified user %s: dropped DN %s" % (user, dn)) if self.autoModifyUsers: userDict = diracUserDict[user] modDNSet = dnSet - set([dn]) if modDNSet: userDict["DN"] = ",".join(modDNSet) result = self.csapi.modifyUser(user, userDict) if result["OK"] and result["Value"]: self.log.info("Modified user %s: dropped DN %s" % (user, dn)) self.adminMsgs["Info"].append("Modified user %s: dropped DN %s" % (user, dn)) self.voChanged = True resultDict["ModifiedUsers"].append(diracName) else: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info("The following users will be deleted: %s" % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result["OK"]: self.adminMsgs["Info"].append("The following users are deleted from CS:\n %s\n" % str(oldUsers)) resultDict["DeletedUsers"] = oldUsers else: self.adminMsgs["Errors"].append("Error in deleting users from CS:\n %s" % str(oldUsers)) self.log.error("Error while user deletion from CS", result) else: self.adminMsgs["Info"].append( "The following users to be checked for deletion:\n\t%s" % "\n\t".join(sorted(oldUsers)) ) self.log.info("The following users to be checked for deletion:", "\n\t".join(sorted(oldUsers))) resultDict["CSAPI"] = self.csapi resultDict["AdminMessages"] = self.adminMsgs resultDict["VOChanged"] = self.voChanged return S_OK(resultDict)
signal.signal(signal.SIGINT, handler) vo = "" dry = False doCEs = False doSEs = False ceBdiiDict = None processScriptSwitches() if not vo: gLogger.error("No VO specified") DIRACExit(-1) diracVO = vo vo = getVOOption(vo, "VOMSName", vo) if doCEs: yn = raw_input("Do you want to check/add new sites to CS ? [default yes] [yes|no]: ") yn = yn.strip() if yn == "" or yn.lower().startswith("y"): checkUnusedCEs() yn = raw_input("Do you want to update CE details in the CS ? [default yes] [yes|no]: ") yn = yn.strip() if yn == "" or yn.lower().startswith("y"): updateSites() if doSEs: yn = raw_input("Do you want to check/add new storage elements to CS ? [default yes] [yes|no]: ") yn = yn.strip()
signal.signal( signal.SIGINT, handler ) vo = '' dry = False doCEs = False doSEs = False ceBdiiDict = None processScriptSwitches() if not vo: gLogger.error( 'No VO specified' ) DIRACExit( -1 ) diracVO = vo vo = getVOOption( vo, 'VOMSName', vo ) if doCEs: yn = raw_input( 'Do you want to check/add new sites to CS ? [default yes] [yes|no]: ' ) yn = yn.strip() if yn == '' or yn.lower().startswith( 'y' ): checkUnusedCEs() yn = raw_input( 'Do you want to update CE details in the CS ? [default yes] [yes|no]: ' ) yn = yn.strip() if yn == '' or yn.lower().startswith( 'y' ): updateSites() if doSEs: yn = raw_input( 'Do you want to check/add new storage elements to CS ? [default yes] [yes|no]: ' ) yn = yn.strip()