def doCommand( self ): ''' It returns the status of a given CE. :return: a dictionary with status of each CE queues, and 'status' and 'reason' of the CE itself ''' ## INPUT PARAMETERS vos = getVOs() if vos[ 'OK' ]: vo = vos['Value'].pop() else: return S_ERROR( "No appropriate VO was found! %s" % vos['Message'] ) if 'ce' not in self.args: return S_ERROR( "No computing element 'ce' has been specified!" ) else: ce = self.args['ce'] host = self.args.get('host') #getting BDII info diracAdmin = DiracAdmin() ceQueues = diracAdmin.getBDIICEState( ce, useVO = vo, host = host ) if not ceQueues['OK']: return S_ERROR( '"CE" not found on BDII' ) elements = ceQueues['Value'] #extracting the list of CE queues and their status result = {} for element in elements: queue = element.get('GlueCEUniqueID','Unknown') #pylint: disable=no-member statusQueue = element.get('GlueCEStateStatus','Unknown') #pylint: disable=no-member result[queue] = statusQueue.capitalize() #establishing the status of the CE itself result['Status'] = 'Production' result['Reason'] = "All queues in 'Production'" for key, value in result.items(): #warning: it may not be the case that all queues for a given CE #show the same status. In case of mismatch, the status of the CE #will be associated to a non-production status if key not in ['Status', 'Reason'] and value != 'Production': result['Status'] = value result['Reason'] = "Queue %s is in status %s" % ( queue, value ) return S_OK( result )
def initialize(self): """agent's initialisation :param self: self reference """ self.log.info("Starting RucioRSSAgent") # CA location self.caCertPath = Locations.getCAsLocation() # configured VOs res = getVOs() if res["OK"]: voList = getVOs().get("Value", []) else: return S_ERROR(res["Message"]) # configHelper test if isinstance(voList, str): voList = [voList] self.clientConfig = configHelper(voList) self.log.debug(" VO-specific Rucio Client config parameters: ", self.clientConfig) return S_OK()
def initialize(self): """ Gets run paramaters from the configuration """ self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs) self.host = self.am_getOption('Host', self.host) self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs) self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(':')[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.processCEs = self.am_getOption('ProcessCEs', self.processCEs) self.processSEs = self.am_getOption('ProcessSEs', self.processSEs) self.selectedSites = self.am_getOption('SelectedSites', []) self.dryRun = self.am_getOption('DryRun', self.dryRun) self.voName = self.am_getOption('VirtualOrganization', self.voName) if not self.voName: self.voName = self.am_getOption('VO', []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def getGridVOs(): """ Get all the VOMS VO names served by this DIRAC service """ voNames = [] result = getVOs() if not result['OK']: return result else: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: voNames.append(vomsVO) return S_OK(voNames)
def doCommand(self): ''' It returns the status of a given CE. :return: a dictionary with status of each CE queues, and 'status' and 'reason' of the CE itself ''' ## INPUT PARAMETERS vos = getVOs() if vos['OK']: vo = vos['Value'].pop() else: return S_ERROR("No appropriate VO was found! %s" % vos['Message']) if 'ce' not in self.args: return S_ERROR("No computing element 'ce' has been specified!") else: ce = self.args['ce'] host = self.args.get('host') #getting BDII info diracAdmin = DiracAdmin() ceQueues = diracAdmin.getBDIICEState(ce, useVO=vo, host=host) if not ceQueues['OK']: return S_ERROR('"CE" not found on BDII') elements = ceQueues['Value'] #extracting the list of CE queues and their status result = {} for element in elements: queue = element.get('GlueCEUniqueID', 'Unknown') #pylint: disable=no-member statusQueue = element.get('GlueCEStateStatus', 'Unknown') #pylint: disable=no-member result[queue] = statusQueue.capitalize() #establishing the status of the CE itself result['Status'] = 'Production' result['Reason'] = "All queues in 'Production'" for key, value in result.items(): #warning: it may not be the case that all queues for a given CE #show the same status. In case of mismatch, the status of the CE #will be associated to a non-production status if key not in ['Status', 'Reason'] and value != 'Production': result['Status'] = value result['Reason'] = "Queue %s is in status %s" % (queue, value) return S_OK(result)
def initialize(self): """Gets run paramaters from the configuration""" self.addressTo = self.am_getOption("MailTo", self.addressTo) self.addressFrom = self.am_getOption("MailFrom", self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs", self.alternativeBDIIs) self.host = self.am_getOption("Host", self.host) self.injectSingleCoreQueues = self.am_getOption( "InjectSingleCoreQueues", self.injectSingleCoreQueues) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(":")[-1].isdigit(): self.alternativeBDIIs[index] += ":2170" if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.processCEs = self.am_getOption("ProcessCEs", self.processCEs) self.selectedSites = self.am_getOption("SelectedSites", []) self.dryRun = self.am_getOption("DryRun", self.dryRun) self.voName = self.am_getOption("VirtualOrganization", self.voName) if not self.voName: self.voName = self.am_getOption("VO", []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == "all"): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result["OK"]: vos = result["Value"] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def getGridVOs(): """Get all the VOMS VO names served by this DIRAC service :return: S_OK(list)/S_ERROR() """ voNames = [] result = getVOs() if not result["OK"]: return result else: vos = result["Value"] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: voNames.append(vomsVO) return S_OK(voNames)
def initialize(self): self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', []) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(':')[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.subject = "Bdii2CSAgent" self.processCEs = self.am_getOption('ProcessCEs', True) self.processSEs = self.am_getOption('ProcessSEs', False) self.voName = self.am_getOption('VirtualOrganization', []) if not self.voName: self.voName = self.am_getOption('VO', []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.voBdiiCEDict = {} self.voBdiiSEDict = {} self.csAPI = CSAPI() return self.csAPI.initialize()
def initialize( self ): self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" self.processCEs = self.am_getOption( 'ProcessCEs', True ) self.processSEs = self.am_getOption( 'ProcessSEs', False ) self.voName = self.am_getOption( 'VirtualOrganization', [] ) if not self.voName: self.voName = self.am_getOption( 'VO', [] ) if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption( vo, "VOMSName" ) if vomsVO: self.voName.append( vomsVO ) if self.voName: self.log.info( "Agent will manage VO(s) %s" % self.voName ) else: self.log.fatal( "VirtualOrganization option not defined for agent" ) return S_ERROR() self.voBdiiCEDict = {} self.voBdiiSEDict = {} self.csAPI = CSAPI() return self.csAPI.initialize()
def parseEduperson(self, claimDict): """Parse eduperson claims :return: dict """ vos = {} credDict = {} attributes = { "eduperson_unique_id": "^(?P<ID>.*)", "eduperson_entitlement": "%s:%s" % ( "^(?P<NAMESPACE>[A-z,.,_,-,:]+):(group:registry|group)", "(?P<VO>[A-z,.,_,-]+):role=(?P<VORole>[A-z,.,_,-]+)[:#].*", ), } self.log.debug("Try to parse eduperson claims..") # Parse eduperson claims resDict = claimParser(claimDict, attributes) if resDict.get("eduperson_unique_id"): self.log.debug("Found eduperson_unique_id claim:", pprint.pformat(resDict["eduperson_unique_id"])) credDict["ID"] = resDict["eduperson_unique_id"]["ID"] if resDict.get("eduperson_entitlement"): self.log.debug("Found eduperson_entitlement claim:", pprint.pformat(resDict["eduperson_entitlement"])) for voDict in resDict["eduperson_entitlement"]: if voDict["VO"] not in vos: vos[voDict["VO"]] = {"VORoles": []} if voDict["VORole"] not in vos[voDict["VO"]]["VORoles"]: vos[voDict["VO"]]["VORoles"].append(voDict["VORole"]) allowedVOs = getVOs()["Value"] # Always return S_OK() # Search DIRAC groups for vo in vos: # Skip VO if it absent in Registry if vo in allowedVOs and (result := getVOMSRoleGroupMapping(vo))["OK"]: for role in vos[vo]["VORoles"]: groups = result["Value"]["VOMSDIRAC"].get("/%s/%s" % (vo, role)) if groups: credDict["DIRACGroups"] = list(set(credDict.get("DIRACGroups", []) + groups))
def getSRMUpdates(vo, bdiiInfo=None): changeSet = set() def addToChangeSet(entry, changeSet): _section, _option, value, new_value = entry if new_value and new_value != value: changeSet.add(entry) result = getGridSRMs(vo, bdiiInfo=bdiiInfo) if not result['OK']: return result srmBdiiDict = result['Value'] result = getSEsFromCS() if not result['OK']: return result seDict = result['Value'] result = getVOs() if result['OK']: csVOs = set(result['Value']) else: csVOs = set([vo]) for seHost, diracSE in seDict.items(): seSection = '/Resources/StorageElements/%s' % diracSE[0] # Look up existing values first description = gConfig.getValue(cfgPath(seSection, 'Description'), 'Unknown') backend = gConfig.getValue(cfgPath(seSection, 'BackendType'), 'Unknown') vos = gConfig.getValue(cfgPath(seSection, 'VO'), 'Unknown').replace(' ', '') size = gConfig.getValue(cfgPath(seSection, 'TotalSize'), 'Unknown') # Look up current BDII values srmDict = {} seBdiiDict = {} for site in srmBdiiDict: if seHost in srmBdiiDict[site]: srmDict = srmBdiiDict[site][seHost]['SRM'] seBdiiDict = srmBdiiDict[site][seHost]['SE'] break if not srmDict or not seBdiiDict: continue newDescription = seBdiiDict.get('GlueSEName', 'Unknown') newBackend = seBdiiDict.get('GlueSEImplementationName', 'Unknown') newSize = seBdiiDict.get('GlueSESizeTotal', 'Unknown') addToChangeSet((seSection, 'Description', description, newDescription), changeSet) addToChangeSet((seSection, 'BackendType', backend, newBackend), changeSet) addToChangeSet((seSection, 'TotalSize', size, newSize), changeSet) # Evaluate VOs if no space token defined, otherwise this is VO specific spaceToken = '' for i in range(1, 10): protocol = gConfig.getValue(cfgPath(seSection, 'AccessProtocol.%d' % i, 'Protocol'), '') if protocol.lower() == 'srm': spaceToken = gConfig.getValue(cfgPath(seSection, 'AccessProtocol.%d' % i, 'SpaceToken'), '') break if not spaceToken: bdiiVOs = srmDict.get('GlueServiceAccessControlBaseRule', []) bdiiVOs = set([re.sub('^VO:', '', rule) for rule in bdiiVOs]) seVOs = csVOs.intersection(bdiiVOs) newVOs = ','.join(seVOs) addToChangeSet((seSection, 'VO', vos, newVOs), changeSet) return S_OK(changeSet)
def checkUnusedSEs(): global vo, dry result = getGridSRMs(vo, unUsed=True) if not result['OK']: gLogger.error('Failed to look up SRMs in BDII', result['Message']) siteSRMDict = result['Value'] # Evaluate VOs result = getVOs() if result['OK']: csVOs = set(result['Value']) else: csVOs = {vo} changeSetFull = set() for site in siteSRMDict: for gridSE in siteSRMDict[site]: changeSet = set() seDict = siteSRMDict[site][gridSE]['SE'] srmDict = siteSRMDict[site][gridSE]['SRM'] # Check the SRM version version = srmDict.get('GlueServiceVersion', '') if not (version and version.startswith('2')): gLogger.debug('Skipping SRM service with version %s' % version) continue result = getDIRACSiteName(site) if not result['OK']: gLogger.notice('Unused se %s is detected at unused site %s' % (gridSE, site)) gLogger.notice('Consider adding site %s to the DIRAC CS' % site) continue diracSites = result['Value'] yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % (gridSE, str(diracSites))) if not yn or yn.lower().startswith('y'): if len(diracSites) > 1: prompt = 'Which DIRAC site the new SE should be attached to ?' for i, s in enumerate(diracSites): prompt += '\n[%d] %s' % (i, s) prompt += '\nEnter your choice number: ' inp = raw_input(prompt) try: ind = int(inp) except BaseException: gLogger.notice('Can not interpret your choice: %s, try again later' % inp) continue diracSite = diracSites[ind] else: diracSite = diracSites[0] domain, siteName, country = diracSite.split('.') recName = '%s-disk' % siteName inp = raw_input('Give a DIRAC name to the grid SE %s, default %s : ' % (gridSE, recName)) diracSEName = inp if not inp: diracSEName = recName gLogger.notice('Adding new SE %s at site %s' % (diracSEName, diracSite)) seSection = cfgPath('/Resources/StorageElements', diracSEName) changeSet.add((seSection, 'BackendType', seDict.get('GlueSEImplementationName', 'Unknown'))) changeSet.add((seSection, 'Description', seDict.get('GlueSEName', 'Unknown'))) bdiiVOs = set([re.sub('^VO:', '', rule) for rule in srmDict.get('GlueServiceAccessControlBaseRule', [])]) seVOs = csVOs.intersection(bdiiVOs) changeSet.add((seSection, 'VO', ','.join(seVOs))) accessSection = cfgPath(seSection, 'AccessProtocol.1') changeSet.add((accessSection, 'Protocol', 'srm')) changeSet.add((accessSection, 'PluginName', 'SRM2')) endPoint = srmDict.get('GlueServiceEndpoint', '') host = urlparse(endPoint).hostname port = urlparse(endPoint).port changeSet.add((accessSection, 'Host', host)) changeSet.add((accessSection, 'Port', port)) changeSet.add((accessSection, 'Access', 'remote')) voPathSection = cfgPath(accessSection, 'VOPath') if 'VOPath' in seDict: path = seDict['VOPath'] voFromPath = os.path.basename(path) if voFromPath != diracVO: gLogger.notice('\n!!! Warning: non-conventional VO path: %s\n' % path) changeSet.add((voPathSection, diracVO, path)) path = os.path.dirname(path) else: # Try to guess the Path domain = '.'.join(host.split('.')[-2:]) path = '/dpm/%s/home' % domain changeSet.add((accessSection, 'Path', path)) changeSet.add((accessSection, 'SpaceToken', '')) changeSet.add((accessSection, 'WSUrl', '/srm/managerv2?SFN=')) gLogger.notice('SE %s will be added with the following parameters' % diracSEName) changeList = sorted(changeSet) for entry in changeList: gLogger.notice(entry) yn = raw_input('Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName) if not yn or yn.lower().startswith('y'): changeSetFull = changeSetFull.union(changeSet) if dry: if changeSetFull: gLogger.notice('Skipping commit of the new SE data in a dry run') else: gLogger.notice("No new SE to be added") return S_OK() if changeSetFull: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error('Failed to initialize CSAPI object', result['Message']) DIRACExit(-1) changeList = sorted(changeSetFull) for section, option, value in changeList: csAPI.setOption(cfgPath(section, option), value) yn = raw_input('New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ') if not yn or yn.lower().startswith('y'): result = csAPI.commit() if not result['OK']: gLogger.error("Error while commit to CS", result['Message']) else: gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull)) else: gLogger.notice("No new SE to be added") return S_OK()
def checkUnusedSEs(): global vo, dry result = getGridSRMs( vo, unUsed = True ) if not result['OK']: gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] ) siteSRMDict = result['Value'] # Evaluate VOs result = getVOs() if result['OK']: csVOs = set( result['Value'] ) else: csVOs = set( [vo] ) changeSetFull = set() for site in siteSRMDict: for gridSE in siteSRMDict[site]: changeSet = set() seDict = siteSRMDict[site][gridSE]['SE'] srmDict = siteSRMDict[site][gridSE]['SRM'] # Check the SRM version version = srmDict.get( 'GlueServiceVersion', '' ) if not ( version and version.startswith( '2' ) ): gLogger.debug( 'Skipping SRM service with version %s' % version ) continue result = getDIRACSiteName( site ) if not result['OK']: gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) ) gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site ) continue diracSites = result['Value'] yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) ) if not yn or yn.lower().startswith( 'y' ): if len( diracSites ) > 1: prompt = 'Which DIRAC site the new SE should be attached to ?' for i, s in enumerate( diracSites ): prompt += '\n[%d] %s' % ( i, s ) prompt += '\nEnter your choice number: ' inp = raw_input( prompt ) try: ind = int( inp ) except: gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp ) continue diracSite = diracSites[ind] else: diracSite = diracSites[0] domain, siteName, country = diracSite.split( '.' ) recName = '%s-disk' % siteName inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) ) diracSEName = inp if not inp: diracSEName = recName gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) ) seSection = cfgPath( '/Resources/StorageElements', diracSEName ) changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) ) changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) ) bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] ) seVOs = csVOs.intersection( bdiiVOs ) changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) ) accessSection = cfgPath( seSection, 'AccessProtocol.1' ) changeSet.add( ( accessSection, 'Protocol', 'srm' ) ) changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) ) endPoint = srmDict.get( 'GlueServiceEndpoint', '' ) result = pfnparse( endPoint ) if not result['OK']: gLogger.error( 'Can not get the SRM service end point. Skipping ...' ) continue host = result['Value']['Host'] port = result['Value']['Port'] changeSet.add( ( accessSection, 'Host', host ) ) changeSet.add( ( accessSection, 'Port', port ) ) changeSet.add( ( accessSection, 'Access', 'remote' ) ) voPathSection = cfgPath( accessSection, 'VOPath' ) if 'VOPath' in seDict: path = seDict['VOPath'] voFromPath = os.path.basename( path ) if voFromPath != diracVO: gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path ) changeSet.add( ( voPathSection, diracVO, path ) ) path = os.path.dirname( path ) else: # Try to guess the Path domain = '.'.join( host.split( '.' )[-2:] ) path = '/dpm/%s/home' % domain changeSet.add( ( accessSection, 'Path', path ) ) changeSet.add( ( accessSection, 'SpaceToken', '' ) ) changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) ) gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName ) changeList = list( changeSet ) changeList.sort() for entry in changeList: gLogger.notice( entry ) yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName ) if not yn or yn.lower().startswith( 'y' ): changeSetFull = changeSetFull.union( changeSet ) if dry: if changeSetFull: gLogger.notice( 'Skipping commit of the new SE data in a dry run' ) else: gLogger.notice( "No new SE to be added" ) return S_OK() if changeSetFull: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error( 'Failed to initialize CSAPI object', result['Message'] ) DIRACExit( -1 ) changeList = list( changeSetFull ) changeList.sort() for section, option, value in changeList: csAPI.setOption( cfgPath( section, option ), value ) yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' ) if not yn or yn.lower().startswith( 'y' ): result = csAPI.commit() if not result['OK']: gLogger.error( "Error while commit to CS", result['Message'] ) else: gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) ) else: gLogger.notice( "No new SE to be added" ) return S_OK()
def getSRMUpdates(vo, bdiiInfo=None): changeSet = set() def addToChangeSet(entry, changeSet): _section, _option, value, new_value = entry if new_value and new_value != value: changeSet.add(entry) result = getGridSRMs(vo, bdiiInfo=bdiiInfo) if not result['OK']: return result srmBdiiDict = result['Value'] result = getSEsFromCS() if not result['OK']: return result seDict = result['Value'] result = getVOs() if result['OK']: csVOs = set(result['Value']) else: csVOs = set([vo]) for seHost, diracSE in seDict.items(): seSection = '/Resources/StorageElements/%s' % diracSE[0] # Look up existing values first description = gConfig.getValue(cfgPath(seSection, 'Description'), 'Unknown') backend = gConfig.getValue(cfgPath(seSection, 'BackendType'), 'Unknown') vos = gConfig.getValue(cfgPath(seSection, 'VO'), 'Unknown').replace(' ', '') size = gConfig.getValue(cfgPath(seSection, 'TotalSize'), 'Unknown') # Look up current BDII values srmDict = {} seBdiiDict = {} for site in srmBdiiDict: if seHost in srmBdiiDict[site]: srmDict = srmBdiiDict[site][seHost]['SRM'] seBdiiDict = srmBdiiDict[site][seHost]['SE'] break if not srmDict or not seBdiiDict: continue newDescription = seBdiiDict.get('GlueSEName', 'Unknown') newBackend = seBdiiDict.get('GlueSEImplementationName', 'Unknown') newSize = seBdiiDict.get('GlueSESizeTotal', 'Unknown') addToChangeSet((seSection, 'Description', description, newDescription), changeSet) addToChangeSet((seSection, 'BackendType', backend, newBackend), changeSet) addToChangeSet((seSection, 'TotalSize', size, newSize), changeSet) # Evaluate VOs if no space token defined, otherwise this is VO specific spaceToken = '' for i in range(1, 10): protocol = gConfig.getValue( cfgPath(seSection, 'AccessProtocol.%d' % i, 'Protocol'), '') if protocol.lower() == 'srm': spaceToken = gConfig.getValue( cfgPath(seSection, 'AccessProtocol.%d' % i, 'SpaceToken'), '') break if not spaceToken: bdiiVOs = srmDict.get('GlueServiceAccessControlBaseRule', []) bdiiVOs = set([re.sub('^VO:', '', rule) for rule in bdiiVOs]) seVOs = csVOs.intersection(bdiiVOs) newVOs = ','.join(seVOs) addToChangeSet((seSection, 'VO', vos, newVOs), changeSet) return S_OK(changeSet)
def checkUnusedSEs(): global vo, dry result = getGridSRMs(vo, unUsed=True) if not result["OK"]: gLogger.error("Failed to look up SRMs in BDII", result["Message"]) siteSRMDict = result["Value"] # Evaluate VOs result = getVOs() if result["OK"]: csVOs = set(result["Value"]) else: csVOs = {vo} changeSetFull = set() for site in siteSRMDict: for gridSE in siteSRMDict[site]: changeSet = set() seDict = siteSRMDict[site][gridSE]["SE"] srmDict = siteSRMDict[site][gridSE]["SRM"] # Check the SRM version version = srmDict.get("GlueServiceVersion", "") if not (version and version.startswith("2")): gLogger.debug("Skipping SRM service with version %s" % version) continue result = getDIRACSiteName(site) if not result["OK"]: gLogger.notice("Unused se %s is detected at unused site %s" % (gridSE, site)) gLogger.notice("Consider adding site %s to the DIRAC CS" % site) continue diracSites = result["Value"] yn = raw_input( "\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites)) ) if not yn or yn.lower().startswith("y"): if len(diracSites) > 1: prompt = "Which DIRAC site the new SE should be attached to ?" for i, s in enumerate(diracSites): prompt += "\n[%d] %s" % (i, s) prompt += "\nEnter your choice number: " inp = raw_input(prompt) try: ind = int(inp) except: gLogger.notice("Can not interpret your choice: %s, try again later" % inp) continue diracSite = diracSites[ind] else: diracSite = diracSites[0] domain, siteName, country = diracSite.split(".") recName = "%s-disk" % siteName inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName)) diracSEName = inp if not inp: diracSEName = recName gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite)) seSection = cfgPath("/Resources/StorageElements", diracSEName) changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown"))) changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown"))) bdiiVOs = set( [re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])] ) seVOs = csVOs.intersection(bdiiVOs) changeSet.add((seSection, "VO", ",".join(seVOs))) accessSection = cfgPath(seSection, "AccessProtocol.1") changeSet.add((accessSection, "Protocol", "srm")) changeSet.add((accessSection, "PluginName", "SRM2")) endPoint = srmDict.get("GlueServiceEndpoint", "") host = urlparse(endPoint).hostname port = urlparse(endPoint).port changeSet.add((accessSection, "Host", host)) changeSet.add((accessSection, "Port", port)) changeSet.add((accessSection, "Access", "remote")) voPathSection = cfgPath(accessSection, "VOPath") if "VOPath" in seDict: path = seDict["VOPath"] voFromPath = os.path.basename(path) if voFromPath != diracVO: gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path) changeSet.add((voPathSection, diracVO, path)) path = os.path.dirname(path) else: # Try to guess the Path domain = ".".join(host.split(".")[-2:]) path = "/dpm/%s/home" % domain changeSet.add((accessSection, "Path", path)) changeSet.add((accessSection, "SpaceToken", "")) changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN=")) gLogger.notice("SE %s will be added with the following parameters" % diracSEName) changeList = list(changeSet) changeList.sort() for entry in changeList: gLogger.notice(entry) yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName) if not yn or yn.lower().startswith("y"): changeSetFull = changeSetFull.union(changeSet) if dry: if changeSetFull: gLogger.notice("Skipping commit of the new SE data in a dry run") else: gLogger.notice("No new SE to be added") return S_OK() if changeSetFull: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result["OK"]: gLogger.error("Failed to initialize CSAPI object", result["Message"]) DIRACExit(-1) changeList = list(changeSetFull) changeList.sort() for section, option, value in changeList: csAPI.setOption(cfgPath(section, option), value) yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ") if not yn or yn.lower().startswith("y"): result = csAPI.commit() if not result["OK"]: gLogger.error("Error while commit to CS", result["Message"]) else: gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull)) else: gLogger.notice("No new SE to be added") return S_OK()