def _syncSites(self): """ Sync DB content with sites that are in the CS """ # sites in the DB now sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName']) sitesIn = [s[0] for s in sitesIn] # sites in CS now sitesList = getSites()['Value'] try: sitesList.remove('LCG.Dummy.ch') except ValueError: pass # remove sites from the DB not more in the CS for site in sitesIn: if site not in sitesList: self.rsDB.removeSite(site) # add to DB what is in CS now and wasn't before for site in sitesList: if site not in sitesIn: # DIRAC Tier tier = getSiteTier(site)['Value'][0] if tier == 0 or tier == '0': t = 'T0' elif tier == 1 or tier == '1': t = 'T1' elif tier == 3 or tier == '3': t = 'T3' else: t = 'T2' #Grid Name of the site gridSiteName = getGOCSiteName(site) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] #Grid Tier (with a workaround!) DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName) if not DIRACSitesOfGridSites['OK']: raise RSSException, DIRACSitesOfGridSites['Message'] DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len(DIRACSitesOfGridSites) == 1: gt = t else: gt = self.__getGOCTier(DIRACSitesOfGridSites) self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) self.rsDB.addOrModifyGridSite(gridSiteName, gt) sitesIn.append(site)
def _syncSites( self ): """ Sync DB content with sites that are in the CS """ # sites in the DB now sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] ) sitesIn = [s[0] for s in sitesIn] # sites in CS now sitesList = getSites()['Value'] try: sitesList.remove( 'LCG.Dummy.ch' ) except ValueError: pass # remove sites from the DB not more in the CS for site in sitesIn: if site not in sitesList: self.rsDB.removeSite( site ) # add to DB what is in CS now and wasn't before for site in sitesList: if site not in sitesIn: # DIRAC Tier tier = getSiteTier( site )['Value'][0] if tier == 0 or tier == '0': t = 'T0' elif tier == 1 or tier == '1': t = 'T1' elif tier == 3 or tier == '3': t = 'T3' else: t = 'T2' #Grid Name of the site gridSiteName = getGOCSiteName( site ) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] #Grid Tier (with a workaround!) DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName ) if not DIRACSitesOfGridSites['OK']: raise RSSException, DIRACSitesOfGridSites['Message'] DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len( DIRACSitesOfGridSites ) == 1: gt = t else: gt = self.__getGOCTier( DIRACSitesOfGridSites ) self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) self.rsDB.addOrModifyGridSite( gridSiteName, gt ) sitesIn.append( site )
def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site="NULL"): nodesToUpdate = NodeInCS - resourcesInDB if len(nodesToUpdate) > 0: gLogger.debug(str(NodeInCS)) gLogger.debug(str(nodesToUpdate)) # Update Service table siteInGOCDB = [ self.__getServiceEndpointInfo(node) for node in nodesToUpdate ] siteInGOCDB = Utils.list_sanitize(siteInGOCDB) #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB] sites = [] for sInGOCDB in siteInGOCDB: siteName = getDIRACSiteName(sInGOCDB[0]['SITENAME']) if not siteName['OK']: gLogger.error(siteName['Message']) return siteName sites.append(siteName['Value']) sites = Utils.list_sanitize(Utils.list_flatten(sites)) _ = [self.__updateService(s, serviceType) for s in sites] # Update Resource table for node in NodeInCS: if serviceType == "Computing": resourceType = CS.getCEType(site, node) if node not in resourcesInDB and node is not None: try: siteInGOCDB = self.__getServiceEndpointInfo( node)[0]['SITENAME'] except IndexError: # No INFO in GOCDB: Node does not exist gLogger.warn( "Node %s is not in GOCDB!! Considering that it does not exists!" % node) continue assert (type(siteInGOCDB) == str) #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB ) res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB) if not res['OK']: gLogger.error(res['Message']) return res resourcesInDB.add(node)
def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"): nodesToUpdate = NodeInCS - resourcesInDB if len(nodesToUpdate) > 0: gLogger.debug(str(NodeInCS)) gLogger.debug(str(nodesToUpdate)) # Update Service table siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate] siteInGOCDB = Utils.list_sanitize(siteInGOCDB) #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB] sites = [] for sInGOCDB in siteInGOCDB: siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] ) if not siteName[ 'OK' ]: gLogger.error( siteName[ 'Message' ] ) return siteName sites.append( siteName[ 'Value' ] ) sites = Utils.list_sanitize( Utils.list_flatten( sites ) ) _ = [ self.__updateService(s, serviceType) for s in sites ] # Update Resource table for node in NodeInCS: if serviceType == "Computing": resourceType = CS.getCEType(site, node) if node not in resourcesInDB and node is not None: try: siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME'] except IndexError: # No INFO in GOCDB: Node does not exist gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node) continue assert(type(siteInGOCDB) == str) #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB ) res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res resourcesInDB.add( node )
def _getInfo( self, requestParams ): gLogger.info( requestParams ) if not requestParams[ 'name' ]: gLogger.warn( 'No name given' ) self.finish( { 'success': 'false', 'error': 'We need a Site Name to generate an Overview' } ) elementName = requestParams[ 'name' ][ 0 ] pub = RPCClient( 'ResourceStatus/Publisher' ) elementStatuses = pub.getElementStatuses( 'Site', str( elementName ), None, 'all', None, None ) if not elementStatuses[ 'OK' ]: gLogger.error( elementStatuses[ 'Message' ] ) self.finish( { 'success': 'false', 'error': 'Error getting ElementStatus information' } ) if not elementStatuses[ 'Value' ]: gLogger.error( 'element "%s" not found' % elementName ) self.finish( { 'success' : 'false', 'error' : 'element "%s" not found' % elementName } ) elementStatus = [ dict( zip( elementStatuses[ 'Columns' ], element ) ) for element in elementStatuses[ 'Value' ] ][ 0 ] elementStatus[ 'DateEffective' ] = str( elementStatus[ 'DateEffective' ] ) elementStatus[ 'LastCheckTime' ] = str( elementStatus[ 'LastCheckTime' ] ) elementStatus[ 'TokenExpiration' ] = str( elementStatus[ 'TokenExpiration' ] ) gocdb_name = getGOCSiteName( elementName ) if not gocdb_name[ 'OK' ]: gLogger.error( gocdb_name[ 'Message' ] ) elementStatus[ 'GOCDB' ] = "" gocdb_name = '' else: gocdb_name = gocdb_name[ 'Value' ] elementStatus[ 'GOCDB' ] = '<a href="https://goc.egi.eu/portal/index.php?Page_Type=Submit_Search&SearchString=%s" target="_blank">%s</a>' % ( gocdb_name, gocdb_name ) dirac_names = getDIRACSiteName( gocdb_name ) if not dirac_names[ 'OK' ]: gLogger.error( dirac_names[ 'Message' ] ) dirac_names = [] else: elementStatus[ 'GOCDB' ] += "(" for i in dirac_names['Value']: elementStatus[ 'GOCDB' ] += "%s " % i elementStatus[ 'GOCDB' ] += ")" elementStatus["GGUS"] = '<a href="https://ggus.eu/ws/ticket_search.php?show_columns_check[]=REQUEST_ID&show_columns_check[]=TICKET_TYPE&show_columns_check[]=AFFECTED_VO&' elementStatus["GGUS"] += 'show_columns_check[]=AFFECTED_SITE&show_columns_check[]=PRIORITY&show_columns_check[]=RESPONSIBLE_UNIT&show_columns_check[]=STATUS&show_columns_check[]=DATE_OF_CREATION&' elementStatus["GGUS"] += 'show_columns_check[]=LAST_UPDATE&show_columns_check[]=TYPE_OF_PROBLEM&show_columns_check[]=SUBJECT&ticket=&supportunit=all&su_hierarchy=all&vo=all&user=&keyword=&involvedsupporter=&assignto=&' elementStatus["GGUS"] += 'affectedsite=%s&specattrib=0&status=open&priority=all&typeofproblem=all&ticketcategory=&mouarea=&technology_provider=&date_type=creation+date&radiotf=1&timeframe=any&from_date=&to_date=&' % gocdb_name elementStatus["GGUS"] += 'untouched_date=&orderticketsby=GHD_INT_REQUEST_ID&orderhow=descending" target="_blank"> %s tickets</a>' % gocdb_name convertName = {'CERN-PROD':'CERN', 'INFN-T1':'CNAF', 'FZK-LCG2':'GridKa', 'IN2P3-CC':'IN2P3', 'NIKHEF-ELPROD':'NIKHEF', 'pic':'PIC', 'RAL-LCG2':'RAL', 'SARA-MATRIX':'SARA'} elog = convertName.get( gocdb_name, "" ); elementStatus['Elog'] = '<a href="https://lblogbook.cern.ch/Operations/?Site=^' + elog + '%24&mode=summary" target="_blank">' + elog + '</a>' return { 'success' : 'true', 'result' : elementStatus, 'total' : len( elementStatus ) }
def checkUnusedCEs(): global vo, dry, ceBdiiDict gLogger.notice( 'looking for new computing resources in the BDII database...' ) result = getCEsFromCS() if not result['OK']: gLogger.error( 'ERROR: failed to get CEs from CS', result['Message'] ) DIRACExit( -1 ) knownCEs = result['Value'] result = getGridCEs( vo, ceBlackList = knownCEs ) if not result['OK']: gLogger.error( 'ERROR: failed to get CEs from BDII', result['Message'] ) DIRACExit( -1 ) ceBdiiDict = result['BdiiInfo'] siteDict = result['Value'] if siteDict: gLogger.notice( 'New resources available:\n' ) for site in siteDict: diracSite = 'Unknown' result = getDIRACSiteName( site ) if result['OK']: diracSite = ','.join( result['Value'] ) ces = siteDict[site].keys() if ces: gLogger.notice( " %s, DIRAC site %s" % ( site, diracSite) ) for ce in ces: gLogger.notice( ' '*4+ce ) gLogger.notice( ' %s, %s' % ( siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System'] ) ) else: gLogger.notice( 'No new resources available, exiting' ) DIRACExit( 0 ) inp = raw_input( "\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if not inp and inp.lower().startswith( 'n' ): gLogger.notice( 'Nothing else to be done, exiting' ) DIRACExit( 0 ) gLogger.notice( '\nAdding new sites/CEs interactively\n' ) sitesAdded = [] for site in siteDict: # Get the country code: country = '' ces = siteDict[site].keys() for ce in ces: country = ce.strip().split('.')[-1].lower() if len( country ) == 2: break if country == 'gov': country = 'us' break if not country or len( country ) != 2: country = 'xx' result = getDIRACSiteName( site ) if not result['OK']: gLogger.notice( '\nThe site %s is not yet in the CS, give it a name' % site ) diracSite = raw_input( '[help|skip|<domain>.<name>.%s]: ' % country ) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice( '%s site details:' % site ) for k,v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice( '%s\t%s' % (k,v) ) gLogger.notice( '\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country ) diracSite = raw_input( '[<domain>.<name>.%s]: ' % country ) try: domain,siteName,country = diracSite.split('.') except Exception, x: gLogger.error( 'ERROR: DIRAC site name does not follow convention: %s' % diracSite ) continue diracSites = [diracSite] else: diracSites = result['Value'] if len( diracSites ) > 1: gLogger.notice( 'Attention! GOC site %s corresponds to more than one DIRAC sites:' % site ) gLogger.notice( str( diracSites ) ) gLogger.notice( 'Please, pay attention which DIRAC site the new CEs will join\n' ) newCEs = {} addedCEs = [] for ce in ces: ceType = siteDict[site][ce]['CEType'] for diracSite in diracSites: if ce in addedCEs: continue yn = raw_input( "Add CE %s of type %s to %s? [default yes] [yes|no]: " % ( ce, ceType, diracSite ) ) if yn == '' or yn.lower() == 'y': newCEs.setdefault( diracSite, [] ) newCEs[diracSite].append( ce ) addedCEs.append( ce ) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % ( diracSite, site, ' '.join( newCEs[diracSite] ) ) gLogger.notice( "\nNew site/CEs will be added with command:\n%s" % cmd ) yn = raw_input( "Add it ? [default yes] [yes|no]: " ) if not ( yn == '' or yn.lower() == 'y' ) : continue if dry: gLogger.notice( "Command is skipped in the dry run" ) else: result = shellCall( 0, cmd ) if not result['OK']: gLogger.error( 'Error while executing dirac-admin-add-site command' ) yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith( 'n' ): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:' ) for site, diracSite in sitesAdded: gLogger.notice( "%s\t%s" % ( site, diracSite ) ) DIRACExit( 0 ) else: exitStatus, stdData, errData = result[ 'Value' ] if exitStatus: gLogger.error( 'Error while executing dirac-admin-add-site command\n', '\n'.join( [stdData, errData] ) ) yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith( 'n' ): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:' ) for site, diracSite in sitesAdded: gLogger.notice( "%s\t%s" % ( site, diracSite ) ) DIRACExit( 0 ) else: sitesAdded.append( ( site, diracSite ) ) gLogger.notice( stdData )
def checkUnusedSEs(): global vo, dry result = getGridSRMs( vo, unUsed = True ) if not result['OK']: gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] ) siteSRMDict = result['Value'] # Evaluate VOs result = getVOs() if result['OK']: csVOs = set( result['Value'] ) else: csVOs = set( [vo] ) changeSetFull = set() for site in siteSRMDict: for gridSE in siteSRMDict[site]: changeSet = set() seDict = siteSRMDict[site][gridSE]['SE'] srmDict = siteSRMDict[site][gridSE]['SRM'] # Check the SRM version version = srmDict.get( 'GlueServiceVersion', '' ) if not ( version and version.startswith( '2' ) ): gLogger.debug( 'Skipping SRM service with version %s' % version ) continue result = getDIRACSiteName( site ) if not result['OK']: gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) ) gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site ) continue diracSites = result['Value'] yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) ) if not yn or yn.lower().startswith( 'y' ): if len( diracSites ) > 1: prompt = 'Which DIRAC site the new SE should be attached to ?' for i, s in enumerate( diracSites ): prompt += '\n[%d] %s' % ( i, s ) prompt += '\nEnter your choice number: ' inp = raw_input( prompt ) try: ind = int( inp ) except: gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp ) continue diracSite = diracSites[ind] else: diracSite = diracSites[0] domain, siteName, country = diracSite.split( '.' ) recName = '%s-disk' % siteName inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) ) diracSEName = inp if not inp: diracSEName = recName gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) ) seSection = cfgPath( '/Resources/StorageElements', diracSEName ) changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) ) changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) ) bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] ) seVOs = csVOs.intersection( bdiiVOs ) changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) ) accessSection = cfgPath( seSection, 'AccessProtocol.1' ) changeSet.add( ( accessSection, 'Protocol', 'srm' ) ) changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) ) endPoint = srmDict.get( 'GlueServiceEndpoint', '' ) result = pfnparse( endPoint ) if not result['OK']: gLogger.error( 'Can not get the SRM service end point. Skipping ...' ) continue host = result['Value']['Host'] port = result['Value']['Port'] changeSet.add( ( accessSection, 'Host', host ) ) changeSet.add( ( accessSection, 'Port', port ) ) changeSet.add( ( accessSection, 'Access', 'remote' ) ) voPathSection = cfgPath( accessSection, 'VOPath' ) if 'VOPath' in seDict: path = seDict['VOPath'] voFromPath = os.path.basename( path ) if voFromPath != diracVO: gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path ) changeSet.add( ( voPathSection, diracVO, path ) ) path = os.path.dirname( path ) else: # Try to guess the Path domain = '.'.join( host.split( '.' )[-2:] ) path = '/dpm/%s/home' % domain changeSet.add( ( accessSection, 'Path', path ) ) changeSet.add( ( accessSection, 'SpaceToken', '' ) ) changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) ) gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName ) changeList = list( changeSet ) changeList.sort() for entry in changeList: gLogger.notice( entry ) yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName ) if not yn or yn.lower().startswith( 'y' ): changeSetFull = changeSetFull.union( changeSet ) if dry: if changeSetFull: gLogger.notice( 'Skipping commit of the new SE data in a dry run' ) else: gLogger.notice( "No new SE to be added" ) return S_OK() if changeSetFull: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error( 'Failed to initialize CSAPI object', result['Message'] ) DIRACExit( -1 ) changeList = list( changeSetFull ) changeList.sort() for section, option, value in changeList: csAPI.setOption( cfgPath( section, option ), value ) yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' ) if not yn or yn.lower().startswith( 'y' ): result = csAPI.commit() if not result['OK']: gLogger.error( "Error while commit to CS", result['Message'] ) else: gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) ) else: gLogger.notice( "No new SE to be added" ) return S_OK()
def checkUnusedCEs(): global vo, dry, ceBdiiDict gLogger.notice("looking for new computing resources in the BDII database...") result = getCEsFromCS() if not result["OK"]: gLogger.error("ERROR: failed to get CEs from CS", result["Message"]) DIRACExit(-1) knownCEs = result["Value"] result = getGridCEs(vo, ceBlackList=knownCEs) if not result["OK"]: gLogger.error("ERROR: failed to get CEs from BDII", result["Message"]) DIRACExit(-1) ceBdiiDict = result["BdiiInfo"] siteDict = result["Value"] if siteDict: gLogger.notice("New resources available:\n") for site in siteDict: diracSite = "Unknown" result = getDIRACSiteName(site) if result["OK"]: diracSite = ",".join(result["Value"]) ces = siteDict[site].keys() if ces: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in ces: gLogger.notice(" " * 4 + ce) gLogger.notice( " %s, %s" % (siteDict[site][ce]["CEType"], "%s_%s_%s" % siteDict[site][ce]["System"]) ) else: gLogger.notice("No new resources available, exiting") DIRACExit(0) inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if not inp and inp.lower().startswith("n"): gLogger.notice("Nothing else to be done, exiting") DIRACExit(0) gLogger.notice("\nAdding new sites/CEs interactively\n") sitesAdded = [] for site in siteDict: # Get the country code: country = "" ces = siteDict[site].keys() for ce in ces: country = ce.strip().split(".")[-1].lower() if len(country) == 2: break if country == "gov": country = "us" break if not country or len(country) != 2: country = "xx" result = getDIRACSiteName(site) if not result["OK"]: gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site) diracSite = raw_input("[help|skip|<domain>.<name>.%s]: " % country) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice("%s site details:" % site) for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice("%s\t%s" % (k, v)) gLogger.notice("\nEnter DIRAC site name in the form <domain>.<name>.%s\n" % country) diracSite = raw_input("[<domain>.<name>.%s]: " % country) try: _, _, _ = diracSite.split(".") except ValueError: gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite) continue diracSites = [diracSite] else: diracSites = result["Value"] if len(diracSites) > 1: gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site) gLogger.notice(str(diracSites)) gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n") newCEs = {} addedCEs = [] for ce in ces: ceType = siteDict[site][ce]["CEType"] for diracSite in diracSites: if ce in addedCEs: continue yn = raw_input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite)) if yn == "" or yn.lower() == "y": newCEs.setdefault(diracSite, []) newCEs[diracSite].append(ce) addedCEs.append(ce) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, " ".join(newCEs[diracSite])) gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd) yn = raw_input("Add it ? [default yes] [yes|no]: ") if not (yn == "" or yn.lower() == "y"): continue if dry: gLogger.notice("Command is skipped in the dry run") else: result = shellCall(0, cmd) if not result["OK"]: gLogger.error("Error while executing dirac-admin-add-site command") yn = raw_input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: exitStatus, stdData, errData = result["Value"] if exitStatus: gLogger.error( "Error while executing dirac-admin-add-site command\n", "\n".join([stdData, errData]) ) yn = raw_input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: sitesAdded.append((site, diracSite)) gLogger.notice(stdData) if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) else: gLogger.notice("No new CEs were added this time")
csAPI = CSAPI() if len(args) < 3: Script.showHelp() DIRACExit(-1) diracSiteName = args[0] gridSiteName = args[1] ces = args[2:] try: diracGridType, place, country = diracSiteName.split(".") except ValueError: gLogger.error("The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch") DIRACExit(-1) result = getDIRACSiteName(gridSiteName) newSite = True if result["OK"]: if result["Value"]: if len(result["Value"]) > 1: gLogger.notice("%s GOC site name is associated with several DIRAC sites:" % gridSiteName) for i, dSite in enumerate(result["Value"]): gLogger.notice("%d: %s" % (i, dSite)) inp = raw_input("Enter your choice number: ") try: inp = int(inp) except ValueError: gLogger.error("You should enter an integer number") DIRACExit(-1) if 0 <= inp < len(result["Value"]): diracCSSite = result["Value"][inp]
def doCommand( self, sites = None ): """ Returns downtimes information for all the sites in input. :params: :attr:`sites`: list of site names (when not given, take every site) :returns: {'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK', 'StartDate': 'aDate', ...} ... } """ if self.client is None: from DIRAC.Core.LCG.GOCDBClient import GOCDBClient self.client = GOCDBClient() if sites is None: # from DIRAC.Core.DISET.RPCClient import RPCClient RPC = RPCClient( "ResourceStatus/ResourceStatus" ) GOC_sites = RPC.getGridSitesList() if not GOC_sites['OK']: raise RSSException, where( self, self.doCommand ) + " " + sites['Message'] else: GOC_sites = GOC_sites['Value'] else: GOC_sites = [getGOCSiteName( x )['Value'] for x in sites] try: res = self.client.getStatus( 'Site', GOC_sites, None, 120 ) except: gLogger.exception( "Exception when calling GOCDBClient." ) return {} if not res['OK']: raise RSSException, where( self, self.doCommand ) + " " + res['Message'] else: res = res['Value'] if res == None: return {} resToReturn = {} for dt_ID in res: try: dt = {} dt['ID'] = dt_ID dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE'] dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE'] dt['Severity'] = res[dt_ID]['SEVERITY'] dt['Description'] = res[dt_ID]['DESCRIPTION'].replace( '\'', '' ) dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL'] DIRACnames = getDIRACSiteName( res[dt_ID]['SITENAME'] ) if not DIRACnames['OK']: raise RSSException, DIRACnames['Message'] DIRACnames = DIRACnames['Value'] for DIRACname in DIRACnames: resToReturn[dt_ID.split()[0] + ' ' + DIRACname] = dt except KeyError: continue return resToReturn
def _syncResources( self ): # resources in the DB now resourcesIn = self.rsDB.getMonitoredsList( 'Resource', paramsList = ['ResourceName'] ) resourcesIn = [r[0] for r in resourcesIn] # services in the DB now servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] ) servicesIn = [s[0] for s in servicesIn] # Site-CE mapping in CS now siteCE = getSiteCEMapping( 'LCG' )['Value'] # Site-SE mapping in CS now siteSE = getSiteSEMapping( 'LCG' )['Value'] # CEs in CS now CEList = [] for i in siteCE.values(): for ce in i: if ce is None: continue CEList.append( ce ) # SEs in CS now SEList = [] for i in siteSE.values(): for x in i: SEList.append( x ) # SE Nodes in CS now SENodeList = [] for SE in SEList: node = getSENodes( SE )['Value'][0] if node is None: continue if node not in SENodeList: SENodeList.append( node ) # LFC Nodes in CS now LFCNodeList_L = [] LFCNodeList_C = [] for site in getLFCSites()['Value']: for readable in ( 'ReadOnly', 'ReadWrite' ): LFCNode = getLFCNode( site, readable )['Value'] if LFCNode is None or LFCNode == []: continue LFCNode = LFCNode[0] if readable == 'ReadWrite': if LFCNode not in LFCNodeList_C: LFCNodeList_C.append( LFCNode ) elif readable == 'ReadOnly': if LFCNode not in LFCNodeList_L: LFCNodeList_L.append( LFCNode ) # FTS Nodes in CS now FTSNodeList = [] sitesWithFTS = getFTSSites() for site in sitesWithFTS['Value']: fts = getFTSEndpoint( site )['Value'] if fts is None or fts == []: continue fts = fts[0] if fts not in FTSNodeList: FTSNodeList.append( fts ) # VOMS Nodes in CS now VOMSNodeList = getVOMSEndpoints()['Value'] # complete list of resources in CS now resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList # list of services in CS now (to be done) servicesList = [] #remove resources no more in the CS for res in resourcesIn: if res not in resourcesList: self.rsDB.removeResource( res ) self.rsDB.removeStorageElement( resourceName = res ) # add to DB what is in CS now and wasn't before # CEs for site in siteCE.keys(): if site == 'LCG.Dummy.ch': continue for ce in siteCE[site]: if ce is None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( ce )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue serviceType = 'Computing' service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if ce not in resourcesIn: CEType = getCEType( site, ce )['Value'] ceType = 'CE' if CEType == 'CREAM': ceType = 'CREAMCE' self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( ce ) # SRMs for srm in SENodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( srm )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if srm not in resourcesIn and srm is not None: self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( srm ) # LFC_C for lfc in LFCNodeList_C: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( lfc )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( lfc ) # LFC_L for lfc in LFCNodeList_L: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( lfc )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( lfc ) # FTSs for fts in FTSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( fts )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if fts not in resourcesIn and fts is not None: self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( fts ) # VOMSs for voms in VOMSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( voms )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] site = siteInDIRAC['Value'] serviceType = 'VOMS' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if voms not in resourcesIn and voms is not None: self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( voms ) #remove services no more in the CS for ser in servicesIn: if ser not in servicesList: serType = ser.split( '@' )[0] if serType != 'VO-BOX': self.rsDB.removeService( ser ) self.rsDB.removeResource( serviceName = ser ) site = ser.split( '@' )[1] if serType == 'Storage': self.rsDB.removeStorageElement( siteName = site )
def doCommand(self, sites=None): """ Returns downtimes information for all the sites in input. :params: :attr:`sites`: list of site names (when not given, take every site) :returns: {'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK', 'StartDate': 'aDate', ...} ... } """ self.APIs = initAPIs(self.__APIs__, self.APIs) try: if sites is None: GOC_sites = self.APIs['ResourceStatusClient'].getGridSite( meta={'columns': 'GridSiteName'}) if not GOC_sites['OK']: return {'Result': GOC_sites} GOC_sites = [gs[0] for gs in GOC_sites['Value']] else: GOC_sites = [getGOCSiteName(x)['Value'] for x in sites] resGOC = self.APIs['GOCDBClient'].getStatus( 'Site', GOC_sites, None, 120) if not resGOC['OK']: return {'Result': resGOC} resGOC = resGOC['Value'] if resGOC == None: resGOC = [] res = {} for dt_ID in resGOC: try: dt = {} dt['ID'] = dt_ID dt['StartDate'] = resGOC[dt_ID]['FORMATED_START_DATE'] dt['EndDate'] = resGOC[dt_ID]['FORMATED_END_DATE'] dt['Severity'] = resGOC[dt_ID]['SEVERITY'] dt['Description'] = resGOC[dt_ID]['DESCRIPTION'].replace( '\'', '') dt['Link'] = resGOC[dt_ID]['GOCDB_PORTAL_URL'] DIRACnames = getDIRACSiteName(res[dt_ID]['SITENAME']) if not DIRACnames['OK']: return {'Result': DIRACnames} for DIRACname in DIRACnames['Value']: res[dt_ID.split()[0] + ' ' + DIRACname] = dt except KeyError: continue res = S_OK(res) except Exception, e: _msg = '%s (%s): %s' % (self.__class__.__name__, self.args, e) gLogger.exception(_msg) return {'Result': S_ERROR(_msg)}
def _syncResources(self): # resources in the DB now resourcesIn = self.rsDB.getMonitoredsList('Resource', paramsList=['ResourceName']) resourcesIn = [r[0] for r in resourcesIn] # services in the DB now servicesIn = self.rsDB.getMonitoredsList('Service', paramsList=['ServiceName']) servicesIn = [s[0] for s in servicesIn] # Site-CE mapping in CS now siteCE = getSiteCEMapping('LCG')['Value'] # Site-SE mapping in CS now siteSE = getSiteSEMapping('LCG')['Value'] # CEs in CS now CEList = [] for i in siteCE.values(): for ce in i: if ce is None: continue CEList.append(ce) # SEs in CS now SEList = [] for i in siteSE.values(): for x in i: SEList.append(x) # SE Nodes in CS now SENodeList = [] for SE in SEList: node = getSENodes(SE)['Value'][0] if node is None: continue if node not in SENodeList: SENodeList.append(node) # LFC Nodes in CS now LFCNodeList_L = [] LFCNodeList_C = [] for site in getLFCSites()['Value']: for readable in ('ReadOnly', 'ReadWrite'): LFCNode = getLFCNode(site, readable)['Value'] if LFCNode is None or LFCNode == []: continue LFCNode = LFCNode[0] if readable == 'ReadWrite': if LFCNode not in LFCNodeList_C: LFCNodeList_C.append(LFCNode) elif readable == 'ReadOnly': if LFCNode not in LFCNodeList_L: LFCNodeList_L.append(LFCNode) # FTS Nodes in CS now FTSNodeList = [] sitesWithFTS = getFTSSites() for site in sitesWithFTS['Value']: fts = getFTSEndpoint(site)['Value'] if fts is None or fts == []: continue fts = fts[0] if fts not in FTSNodeList: FTSNodeList.append(fts) # VOMS Nodes in CS now VOMSNodeList = getVOMSEndpoints()['Value'] # complete list of resources in CS now resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList # list of services in CS now (to be done) servicesList = [] #remove resources no more in the CS for res in resourcesIn: if res not in resourcesList: self.rsDB.removeResource(res) self.rsDB.removeStorageElement(resourceName=res) # add to DB what is in CS now and wasn't before # CEs for site in siteCE.keys(): if site == 'LCG.Dummy.ch': continue for ce in siteCE[site]: if ce is None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(ce)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue serviceType = 'Computing' service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if ce not in resourcesIn: CEType = getCEType(site, ce)['Value'] ceType = 'CE' if CEType == 'CREAM': ceType = 'CREAMCE' self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(ce) # SRMs for srm in SENodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(srm)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if srm not in resourcesIn and srm is not None: self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(srm) # LFC_C for lfc in LFCNodeList_C: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(lfc)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(lfc) # LFC_L for lfc in LFCNodeList_L: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(lfc)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(lfc) # FTSs for fts in FTSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(fts)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if fts not in resourcesIn and fts is not None: self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(fts) # VOMSs for voms in VOMSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(voms)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] site = siteInDIRAC['Value'] serviceType = 'VOMS' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if voms not in resourcesIn and voms is not None: self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(voms) #remove services no more in the CS for ser in servicesIn: if ser not in servicesList: serType = ser.split('@')[0] if serType != 'VO-BOX': self.rsDB.removeService(ser) self.rsDB.removeResource(serviceName=ser) site = ser.split('@')[1] if serType == 'Storage': self.rsDB.removeStorageElement(siteName=site)
def _getInfo( self, requestParams ): gLogger.info( requestParams ) if not requestParams[ 'name' ]: gLogger.warn( 'No name given' ) self.finish( { 'success': 'false', 'error': 'We need a Site Name to generate an Overview' } ) elementName = requestParams[ 'name' ][ 0 ] pub = RPCClient( 'ResourceStatus/Publisher' ) elementStatuses = pub.getElementStatuses( 'Site', str( elementName ), None, 'all', None, None ) if not elementStatuses[ 'OK' ]: gLogger.error( elementStatuses[ 'Message' ] ) self.finish( { 'success': 'false', 'error': 'Error getting ElementStatus information' } ) if not elementStatuses[ 'Value' ]: gLogger.error( 'element "%s" not found' % elementName ) self.finish( { 'success' : 'false', 'error' : 'element "%s" not found' % elementName } ) elementStatus = [ dict( zip( elementStatuses[ 'Columns' ], element ) ) for element in elementStatuses[ 'Value' ] ][ 0 ] elementStatus[ 'DateEffective' ] = str( elementStatus[ 'DateEffective' ] ) elementStatus[ 'LastCheckTime' ] = str( elementStatus[ 'LastCheckTime' ] ) elementStatus[ 'TokenExpiration' ] = str( elementStatus[ 'TokenExpiration' ] ) gocdb_name = getGOCSiteName( elementName ) if not gocdb_name[ 'OK' ]: gLogger.error( gocdb_name[ 'Message' ] ) elementStatus[ 'GOCDB' ] = "" gocdb_name = '' else: gocdb_name = gocdb_name[ 'Value' ] elementStatus[ 'GOCDB' ] = '<a href="https://goc.egi.eu/portal/index.php?Page_Type=Submit_Search&SearchString=%s" target="_blank">%s</a>' % ( gocdb_name, gocdb_name ) dirac_names = getDIRACSiteName( gocdb_name ) if not dirac_names[ 'OK' ]: gLogger.error( dirac_names[ 'Message' ] ) dirac_names = [] else: elementStatus[ 'GOCDB' ] += "(" for i in dirac_names['Value']: elementStatus[ 'GOCDB' ] += "%s " % i elementStatus[ 'GOCDB' ] += ")" elementStatus["GGUS"] = '<a href="https://ggus.eu/ws/ticket_search.php?show_columns_check[]=REQUEST_ID&show_columns_check[]=TICKET_TYPE&show_columns_check[]=AFFECTED_VO&' elementStatus["GGUS"] += 'show_columns_check[]=AFFECTED_SITE&show_columns_check[]=PRIORITY&show_columns_check[]=RESPONSIBLE_UNIT&show_columns_check[]=STATUS&show_columns_check[]=DATE_OF_CREATION&' elementStatus["GGUS"] += 'show_columns_check[]=LAST_UPDATE&show_columns_check[]=TYPE_OF_PROBLEM&show_columns_check[]=SUBJECT&ticket=&supportunit=all&su_hierarchy=all&vo=all&user=&keyword=&involvedsupporter=&assignto=&' elementStatus["GGUS"] += 'affectedsite=%s&specattrib=0&status=open&priority=all&typeofproblem=all&ticketcategory=&mouarea=&technology_provider=&date_type=creation+date&radiotf=1&timeframe=any&from_date=&to_date=&' % gocdb_name elementStatus["GGUS"] += 'untouched_date=&orderticketsby=GHD_INT_REQUEST_ID&orderhow=descending" target="_blank"> %s tickets</a>' % gocdb_name convertName = {'CERN-PROD':'CERN', 'INFN-T1':'CNAF', 'FZK-LCG2':'GridKa', 'IN2P3-CC':'IN2P3', 'NIKHEF-ELPROD':'NIKHEF', 'pic':'PIC', 'RAL-LCG2':'RAL', 'SARA-MATRIX':'SARA'} elog = convertName.get( gocdb_name, "" ); elementStatus['Elog'] = '<a href="https://lblogbook.cern.ch/Operations/?Site=^' + elog + '%24&mode=summary" target="_blank">' + elog + '</a>' return { 'success' : 'true', 'result' : elementStatus, 'total' : len( elementStatus ) }
def getSiteUpdates(vo, bdiiInfo=None, log=None): """ Get all the necessary updates for the already defined sites and CEs """ def addToChangeSet(entry, changeSet): """ Inner function to update changeSet with entry (a tuple) :param tuple entry: entry to add to changeSet :param set changeSet: set collecting stuff to change """ _section, _option, value, new_value = entry if new_value and new_value != value: changeSet.add(entry) if log is None: log = gLogger ceBdiiDict = bdiiInfo if bdiiInfo is None: result = getBdiiCEInfo(vo) if not result['OK']: return result ceBdiiDict = result['Value'] changeSet = set() for site in ceBdiiDict: result = getDIRACSiteName(site) if not result['OK']: continue siteNames = result['Value'] for siteName in siteNames: siteSection = cfgPath('/Resources', 'Sites', siteName.split('.')[0], siteName) result = gConfig.getOptionsDict(siteSection) if not result['OK']: continue siteDict = result['Value'] # Current CS values coor = siteDict.get('Coordinates', 'Unknown') mail = siteDict.get('Mail', 'Unknown').replace(' ', '') description = siteDict.get('Description', 'Unknown') description = description.replace(' ,', ',') longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip() latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip() # Current BDII value newcoor = '' if longitude and latitude: newcoor = "%s:%s" % (longitude, latitude) newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact', '').replace('mailto:', '').strip() newdescription = ceBdiiDict[site].get('GlueSiteDescription', '').strip() newdescription = ", ".join([line.strip() for line in newdescription.split(",")]) # Adding site data to the changes list addToChangeSet((siteSection, 'Coordinates', coor, newcoor), changeSet) addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet) addToChangeSet((siteSection, 'Description', description, newdescription), changeSet) ces = gConfig.getValue(cfgPath(siteSection, 'CE'), []) for ce in ces: ceSection = cfgPath(siteSection, 'CEs', ce) ceDict = {} result = gConfig.getOptionsDict(ceSection) if result['OK']: ceDict = result['Value'] else: if ceBdiiDict[site]['CEs'].get(ce, None): log.notice("Adding new CE", "%s to site %s/%s" % (ce, siteName, site)) ceInfo = ceBdiiDict[site]['CEs'].get(ce, None) if ceInfo is None: ceType = ceDict.get('CEType', '') continue # Current CS CE info arch = ceDict.get('architecture', 'Unknown') OS = ceDict.get('OS', 'Unknown') si00 = ceDict.get('SI00', 'Unknown') ceType = ceDict.get('CEType', 'Unknown') ram = ceDict.get('MaxRAM', 'Unknown') submissionMode = ceDict.get('SubmissionMode', 'Unknown') # Current BDII CE info newarch = ceBdiiDict[site]['CEs'][ce].get('GlueHostArchitecturePlatformType', '').strip() systemName = ceInfo.get('GlueHostOperatingSystemName', '').strip() systemVersion = ceInfo.get('GlueHostOperatingSystemVersion', '').strip() systemRelease = ceInfo.get('GlueHostOperatingSystemRelease', '').strip() newOS = '' if systemName and systemVersion and systemRelease: newOS = '_'.join((systemName, systemVersion, systemRelease)) newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip() newCEType = 'Unknown' for queue in ceInfo['Queues']: queueDict = ceInfo['Queues'][queue] newCEType = queueDict.get('GlueCEImplementationName', '').strip() if newCEType: break if newCEType == 'ARC-CE': newCEType = 'ARC' newSubmissionMode = None if newCEType in ['ARC', 'CREAM']: newSubmissionMode = "Direct" newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip() # Protect from unreasonable values if newRAM and int(newRAM) > 150000: newRAM = '' # Adding CE data to the change list addToChangeSet((ceSection, 'architecture', arch, newarch), changeSet) addToChangeSet((ceSection, 'OS', OS, newOS), changeSet) addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet) addToChangeSet((ceSection, 'CEType', ceType, newCEType), changeSet) addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet) if submissionMode == "Unknown" and newSubmissionMode: addToChangeSet((ceSection, 'SubmissionMode', submissionMode, newSubmissionMode), changeSet) queues = ceInfo['Queues'].keys() for queue in queues: queueInfo = ceInfo['Queues'][queue] queueStatus = queueInfo['GlueCEStateStatus'] queueSection = cfgPath(ceSection, 'Queues', queue) queueDict = {} result = gConfig.getOptionsDict(queueSection) if result['OK']: queueDict = result['Value'] else: if queueStatus.lower() == "production": log.notice("Adding new queue", "%s to CE %s" % (queue, ce)) else: continue # Current CS queue info maxCPUTime = queueDict.get('maxCPUTime', 'Unknown') si00 = queueDict.get('SI00', 'Unknown') maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown') # Current BDII queue info newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '') if newMaxCPUTime == "4" * len(newMaxCPUTime) or newMaxCPUTime == "9" * len(newMaxCPUTime): newMaxCPUTime = '' wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime', '') if wallTime == "4" * len(wallTime) or wallTime == "9" * len(wallTime): wallTime = '' if wallTime and int(wallTime) > 0: if not newMaxCPUTime: newMaxCPUTime = str(int(0.8 * int(wallTime))) else: if int(wallTime) <= int(newMaxCPUTime): newMaxCPUTime = str(int(0.8 * int(wallTime))) newSI00 = '' caps = queueInfo.get('GlueCECapability', []) if isinstance(caps, basestring): caps = [caps] for cap in caps: if 'CPUScalingReferenceSI00' in cap: newSI00 = cap.split('=')[-1] # Adding queue info to the CS addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime, newMaxCPUTime), changeSet) addToChangeSet((queueSection, 'SI00', si00, newSI00), changeSet) if maxTotalJobs == "Unknown": newTotalJobs = min(1000, int(int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) / 2)) newWaitingJobs = max(2, int(newTotalJobs * 0.1)) newTotalJobs = str(newTotalJobs) newWaitingJobs = str(newWaitingJobs) addToChangeSet((queueSection, 'MaxTotalJobs', '', newTotalJobs), changeSet) addToChangeSet((queueSection, 'MaxWaitingJobs', '', newWaitingJobs), changeSet) # Updating eligible VO list VOs = set() if queueDict.get('VO', ''): VOs = set([q.strip() for q in queueDict.get('VO', '').split(',') if q]) if vo not in VOs: VOs.add(vo) VOs = list(VOs) newVOs = ','.join(VOs) addToChangeSet((queueSection, 'VO', '', newVOs), changeSet) return S_OK(changeSet)
if len(args) < 3: Script.showHelp() DIRACExit(-1) diracSiteName = args[0] gridSiteName = args[1] ces = args[2:] try: diracGridType, place, country = diracSiteName.split('.') except ValueError: gLogger.error( "The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch" ) DIRACExit(-1) result = getDIRACSiteName(gridSiteName) newSite = True if result['OK']: if result['Value']: if len(result['Value']) > 1: gLogger.notice( '%s GOC site name is associated with several DIRAC sites:' % gridSiteName) for i, dSite in enumerate(result['Value']): gLogger.notice('%d: %s' % (i, dSite)) inp = raw_input('Enter your choice number: ') try: inp = int(inp) except ValueError: gLogger.error('You should enter an integer number') DIRACExit(-1)
def doCommand( self, sites = None ): """ Returns downtimes information for all the sites in input. :params: :attr:`sites`: list of site names (when not given, take every site) :returns: {'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK', 'StartDate': 'aDate', ...} ... } """ self.APIs = initAPIs( self.__APIs__, self.APIs ) try: if sites is None: GOC_sites = self.APIs[ 'ResourceStatusClient' ].getGridSite( meta = { 'columns' : 'GridSiteName' }) if not GOC_sites['OK']: return { 'Result' : GOC_sites } GOC_sites = [ gs[0] for gs in GOC_sites['Value'] ] else: GOC_sites = [ getGOCSiteName( x )['Value'] for x in sites ] resGOC = self.APIs[ 'GOCDBClient' ].getStatus( 'Site', GOC_sites, None, 120 ) if not resGOC['OK']: return { 'Result' : resGOC } resGOC = resGOC['Value'] if resGOC == None: resGOC = [] res = {} for dt_ID in resGOC: try: dt = {} dt['ID'] = dt_ID dt['StartDate'] = resGOC[dt_ID]['FORMATED_START_DATE'] dt['EndDate'] = resGOC[dt_ID]['FORMATED_END_DATE'] dt['Severity'] = resGOC[dt_ID]['SEVERITY'] dt['Description'] = resGOC[dt_ID]['DESCRIPTION'].replace( '\'', '' ) dt['Link'] = resGOC[dt_ID]['GOCDB_PORTAL_URL'] DIRACnames = getDIRACSiteName( res[dt_ID]['SITENAME'] ) if not DIRACnames['OK']: return { 'Result' : DIRACnames } for DIRACname in DIRACnames['Value']: res[dt_ID.split()[0] + ' ' + DIRACname] = dt except KeyError: continue res = S_OK( res ) except Exception, e: _msg = '%s (%s): %s' % ( self.__class__.__name__, self.args, e ) gLogger.exception( _msg ) return { 'Result' : S_ERROR( _msg ) }
def doCommand(self, sites=None): """ Returns downtimes information for all the sites in input. :params: :attr:`sites`: list of site names (when not given, take every site) :returns: {'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK', 'StartDate': 'aDate', ...} ... } """ if self.client is None: from DIRAC.Core.LCG.GOCDBClient import GOCDBClient self.client = GOCDBClient() if sites is None: # from DIRAC.Core.DISET.RPCClient import RPCClient RPC = RPCClient("ResourceStatus/ResourceStatus") GOC_sites = RPC.getGridSitesList() if not GOC_sites['OK']: raise RSSException, where( self, self.doCommand) + " " + sites['Message'] else: GOC_sites = GOC_sites['Value'] else: GOC_sites = [getGOCSiteName(x)['Value'] for x in sites] try: res = self.client.getStatus('Site', GOC_sites, None, 120) except: gLogger.exception("Exception when calling GOCDBClient.") return {} if not res['OK']: raise RSSException, where(self, self.doCommand) + " " + res['Message'] else: res = res['Value'] if res == None: return {} resToReturn = {} for dt_ID in res: try: dt = {} dt['ID'] = dt_ID dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE'] dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE'] dt['Severity'] = res[dt_ID]['SEVERITY'] dt['Description'] = res[dt_ID]['DESCRIPTION'].replace('\'', '') dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL'] DIRACnames = getDIRACSiteName(res[dt_ID]['SITENAME']) if not DIRACnames['OK']: raise RSSException, DIRACnames['Message'] DIRACnames = DIRACnames['Value'] for DIRACname in DIRACnames: resToReturn[dt_ID.split()[0] + ' ' + DIRACname] = dt except KeyError: continue return resToReturn
def _syncSites( self ): """ Sync DB content with sites that are in the CS """ def getGOCTier(sitesList): return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)])) # sites in the DB now #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite()))) sites = self.rsClient.getSite() if not sites[ 'OK' ]: gLogger.error( sites[ 'Message' ] ) return sites sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] ) # sites in CS now sitesCS = set( CS.getSites() ) gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB))) # remove sites and associated resources, services, and storage # elements from the DB that are not in the CS: for s in sitesDB - sitesCS: gLogger.info("Purging Site %s (not in CS anymore)" % s) self.__purge_site(s) # add to DB what is missing gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB)) for site in sitesCS - sitesDB: siteType = site.split(".")[0] # DIRAC Tier tier = "T" + str(CS.getSiteTier( site )) if siteType == "LCG": # Grid Name of the site #gridSiteName = Utils.unpack(getGOCSiteName(site)) gridSiteName = getGOCSiteName( site ) if not gridSiteName[ 'OK' ]: gLogger.error( gridSiteName[ 'Message' ] ) return gridSiteName gridSiteName = gridSiteName[ 'Value' ] # Grid Tier (with a workaround!) #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName)) DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName ) if not DIRACSitesOfGridSites[ 'OK' ]: gLogger.error( DIRACSitesOfGridSites[ 'Message' ] ) return DIRACSitesOfGridSites DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ] if len( DIRACSitesOfGridSites ) == 1: gt = tier else: gt = getGOCTier( DIRACSitesOfGridSites ) #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt) res = self.rsClient.addOrModifyGridSite( gridSiteName, gt ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName ) res = self.rsClient.addOrModifySite( site, tier, gridSiteName ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res elif siteType == "DIRAC": #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" ) res = self.rsClient.addOrModifySite( site, tier, "NULL" ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res
def checkUnusedSEs(): global vo, dry result = getGridSRMs(vo, unUsed=True) if not result['OK']: gLogger.error('Failed to look up SRMs in BDII', result['Message']) siteSRMDict = result['Value'] # Evaluate VOs result = getVOs() if result['OK']: csVOs = set(result['Value']) else: csVOs = {vo} changeSetFull = set() for site in siteSRMDict: for gridSE in siteSRMDict[site]: changeSet = set() seDict = siteSRMDict[site][gridSE]['SE'] srmDict = siteSRMDict[site][gridSE]['SRM'] # Check the SRM version version = srmDict.get('GlueServiceVersion', '') if not (version and version.startswith('2')): gLogger.debug('Skipping SRM service with version %s' % version) continue result = getDIRACSiteName(site) if not result['OK']: gLogger.notice('Unused se %s is detected at unused site %s' % (gridSE, site)) gLogger.notice('Consider adding site %s to the DIRAC CS' % site) continue diracSites = result['Value'] yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % (gridSE, str(diracSites))) if not yn or yn.lower().startswith('y'): if len(diracSites) > 1: prompt = 'Which DIRAC site the new SE should be attached to ?' for i, s in enumerate(diracSites): prompt += '\n[%d] %s' % (i, s) prompt += '\nEnter your choice number: ' inp = raw_input(prompt) try: ind = int(inp) except BaseException: gLogger.notice('Can not interpret your choice: %s, try again later' % inp) continue diracSite = diracSites[ind] else: diracSite = diracSites[0] domain, siteName, country = diracSite.split('.') recName = '%s-disk' % siteName inp = raw_input('Give a DIRAC name to the grid SE %s, default %s : ' % (gridSE, recName)) diracSEName = inp if not inp: diracSEName = recName gLogger.notice('Adding new SE %s at site %s' % (diracSEName, diracSite)) seSection = cfgPath('/Resources/StorageElements', diracSEName) changeSet.add((seSection, 'BackendType', seDict.get('GlueSEImplementationName', 'Unknown'))) changeSet.add((seSection, 'Description', seDict.get('GlueSEName', 'Unknown'))) bdiiVOs = set([re.sub('^VO:', '', rule) for rule in srmDict.get('GlueServiceAccessControlBaseRule', [])]) seVOs = csVOs.intersection(bdiiVOs) changeSet.add((seSection, 'VO', ','.join(seVOs))) accessSection = cfgPath(seSection, 'AccessProtocol.1') changeSet.add((accessSection, 'Protocol', 'srm')) changeSet.add((accessSection, 'PluginName', 'SRM2')) endPoint = srmDict.get('GlueServiceEndpoint', '') host = urlparse(endPoint).hostname port = urlparse(endPoint).port changeSet.add((accessSection, 'Host', host)) changeSet.add((accessSection, 'Port', port)) changeSet.add((accessSection, 'Access', 'remote')) voPathSection = cfgPath(accessSection, 'VOPath') if 'VOPath' in seDict: path = seDict['VOPath'] voFromPath = os.path.basename(path) if voFromPath != diracVO: gLogger.notice('\n!!! Warning: non-conventional VO path: %s\n' % path) changeSet.add((voPathSection, diracVO, path)) path = os.path.dirname(path) else: # Try to guess the Path domain = '.'.join(host.split('.')[-2:]) path = '/dpm/%s/home' % domain changeSet.add((accessSection, 'Path', path)) changeSet.add((accessSection, 'SpaceToken', '')) changeSet.add((accessSection, 'WSUrl', '/srm/managerv2?SFN=')) gLogger.notice('SE %s will be added with the following parameters' % diracSEName) changeList = sorted(changeSet) for entry in changeList: gLogger.notice(entry) yn = raw_input('Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName) if not yn or yn.lower().startswith('y'): changeSetFull = changeSetFull.union(changeSet) if dry: if changeSetFull: gLogger.notice('Skipping commit of the new SE data in a dry run') else: gLogger.notice("No new SE to be added") return S_OK() if changeSetFull: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error('Failed to initialize CSAPI object', result['Message']) DIRACExit(-1) changeList = sorted(changeSetFull) for section, option, value in changeList: csAPI.setOption(cfgPath(section, option), value) yn = raw_input('New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ') if not yn or yn.lower().startswith('y'): result = csAPI.commit() if not result['OK']: gLogger.error("Error while commit to CS", result['Message']) else: gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull)) else: gLogger.notice("No new SE to be added") return S_OK()
def _syncSites(self): """ Sync DB content with sites that are in the CS """ def getGOCTier(sitesList): return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)])) # sites in the DB now #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite()))) sites = self.rsClient.getSite() if not sites['OK']: gLogger.error(sites['Message']) return sites sitesDB = set([site[0] for site in sites['Value']]) # sites in CS now sitesCS = set(CS.getSites()) gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB))) # remove sites and associated resources, services, and storage # elements from the DB that are not in the CS: for s in sitesDB - sitesCS: gLogger.info("Purging Site %s (not in CS anymore)" % s) self.__purge_site(s) # add to DB what is missing gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB)) for site in sitesCS - sitesDB: siteType = site.split(".")[0] # DIRAC Tier tier = "T" + str(CS.getSiteTier(site)) if siteType == "LCG": # Grid Name of the site #gridSiteName = Utils.unpack(getGOCSiteName(site)) gridSiteName = getGOCSiteName(site) if not gridSiteName['OK']: gLogger.error(gridSiteName['Message']) return gridSiteName gridSiteName = gridSiteName['Value'] # Grid Tier (with a workaround!) #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName)) DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName) if not DIRACSitesOfGridSites['OK']: gLogger.error(DIRACSitesOfGridSites['Message']) return DIRACSitesOfGridSites DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len(DIRACSitesOfGridSites) == 1: gt = tier else: gt = getGOCTier(DIRACSitesOfGridSites) #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt) res = self.rsClient.addOrModifyGridSite(gridSiteName, gt) if not res['OK']: gLogger.error(res['Message']) return res #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName ) res = self.rsClient.addOrModifySite(site, tier, gridSiteName) if not res['OK']: gLogger.error(res['Message']) return res elif siteType == "DIRAC": #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" ) res = self.rsClient.addOrModifySite(site, tier, "NULL") if not res['OK']: gLogger.error(res['Message']) return res
def checkUnusedCEs(): global vo, dry, ceBdiiDict, hostURL, glue2 gLogger.notice('looking for new computing resources in the BDII database...') result = getCEsFromCS() if not result['OK']: gLogger.error('ERROR: failed to get CEs from CS', result['Message']) DIRACExit(-1) knownCEs = result['Value'] result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL, glue2=glue2) if not result['OK']: gLogger.error('ERROR: failed to get CEs from BDII', result['Message']) DIRACExit(-1) ceBdiiDict = result['BdiiInfo'] siteDict = result['Value'] if siteDict: gLogger.notice('New resources available:\n') for site in siteDict: diracSite = 'Unknown' result = getDIRACSiteName(site) if result['OK']: diracSite = ','.join(result['Value']) ces = siteDict[site].keys() # pylint: disable=no-member if ces: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in ces: gLogger.notice(' ' * 4 + ce) gLogger.notice(' %s, %s' % (siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System'])) else: gLogger.notice('No new resources available, exiting') DIRACExit(0) inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if not inp and inp.lower().startswith('n'): gLogger.notice('Nothing else to be done, exiting') DIRACExit(0) gLogger.notice('\nAdding new sites/CEs interactively\n') sitesAdded = [] for site in siteDict: # Get the country code: country = '' ces = siteDict[site].keys() # pylint: disable=no-member for ce in ces: country = ce.strip().split('.')[-1].lower() if len(country) == 2: break if country == 'gov': country = 'us' break if not country or len(country) != 2: country = 'xx' result = getDIRACSiteName(site) if not result['OK']: gLogger.notice('\nThe site %s is not yet in the CS, give it a name' % site) diracSite = raw_input('[help|skip|<domain>.<name>.%s]: ' % country) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice('%s site details:' % site) for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice('%s\t%s' % (k, v)) gLogger.notice('\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country) diracSite = raw_input('[<domain>.<name>.%s]: ' % country) try: _, _, _ = diracSite.split('.') except ValueError: gLogger.error('ERROR: DIRAC site name does not follow convention: %s' % diracSite) continue diracSites = [diracSite] else: diracSites = result['Value'] if len(diracSites) > 1: gLogger.notice('Attention! GOC site %s corresponds to more than one DIRAC sites:' % site) gLogger.notice(str(diracSites)) gLogger.notice('Please, pay attention which DIRAC site the new CEs will join\n') newCEs = {} addedCEs = [] for ce in ces: ceType = siteDict[site][ce]['CEType'] for diracSite in diracSites: if ce in addedCEs: continue yn = raw_input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite)) if yn == '' or yn.lower() == 'y': newCEs.setdefault(diracSite, []) newCEs[diracSite].append(ce) addedCEs.append(ce) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, ' '.join(newCEs[diracSite])) gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd) yn = raw_input("Add it ? [default yes] [yes|no]: ") if not (yn == '' or yn.lower() == 'y'): continue if dry: gLogger.notice("Command is skipped in the dry run") else: result = systemCall(0, shlex.split(cmd)) if not result['OK']: gLogger.error('Error while executing dirac-admin-add-site command') yn = raw_input("Do you want to continue ? [default no] [yes|no]: ") if yn == '' or yn.lower().startswith('n'): if sitesAdded: gLogger.notice('CEs were added at the following sites:') for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: exitStatus, stdData, errData = result['Value'] if exitStatus: gLogger.error('Error while executing dirac-admin-add-site command\n', '\n'.join([stdData, errData])) yn = raw_input("Do you want to continue ? [default no] [yes|no]: ") if yn == '' or yn.lower().startswith('n'): if sitesAdded: gLogger.notice('CEs were added at the following sites:') for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: sitesAdded.append((site, diracSite)) gLogger.notice(stdData) if sitesAdded: gLogger.notice('CEs were added at the following sites:') for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) else: gLogger.notice('No new CEs were added this time')
def getSiteUpdates(vo, bdiiInfo=None, log=None): """ Get all the necessary updates for the already defined sites and CEs """ def addToChangeSet(entry, changeSet): """ Inner function to update changeSet with entry (a tuple) :param tuple entry: entry to add to changeSet :param set changeSet: set collecting stuff to change """ _section, _option, value, new_value = entry if new_value and new_value != value: changeSet.add(entry) if log is None: log = gLogger ceBdiiDict = bdiiInfo if bdiiInfo is None: result = getBdiiCEInfo(vo) if not result['OK']: return result ceBdiiDict = result['Value'] changeSet = set() for site in ceBdiiDict: result = getDIRACSiteName(site) if not result['OK']: continue siteNames = result['Value'] for siteName in siteNames: siteSection = cfgPath('/Resources', 'Sites', siteName.split('.')[0], siteName) result = gConfig.getOptionsDict(siteSection) if not result['OK']: continue siteDict = result['Value'] # Current CS values coor = siteDict.get('Coordinates', 'Unknown') mail = siteDict.get('Mail', 'Unknown').replace(' ', '') description = siteDict.get('Description', 'Unknown') description = description.replace(' ,', ',') longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip() latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip() # Current BDII value newcoor = '' if longitude and latitude: newcoor = "%s:%s" % (longitude, latitude) newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact', '').replace('mailto:', '').strip() newdescription = ceBdiiDict[site].get('GlueSiteDescription', '').strip() newdescription = ", ".join( [line.strip() for line in newdescription.split(",")]) # Adding site data to the changes list addToChangeSet((siteSection, 'Coordinates', coor, newcoor), changeSet) addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet) addToChangeSet( (siteSection, 'Description', description, newdescription), changeSet) ces = gConfig.getValue(cfgPath(siteSection, 'CE'), []) for ce in ces: ceSection = cfgPath(siteSection, 'CEs', ce) ceDict = {} result = gConfig.getOptionsDict(ceSection) if result['OK']: ceDict = result['Value'] else: if ceBdiiDict[site]['CEs'].get(ce, None): log.notice("Adding new CE", "%s to site %s/%s" % (ce, siteName, site)) ceInfo = ceBdiiDict[site]['CEs'].get(ce, None) if ceInfo is None: ceType = ceDict.get('CEType', '') continue # Current CS CE info arch = ceDict.get('architecture', 'Unknown') OS = ceDict.get('OS', 'Unknown') si00 = ceDict.get('SI00', 'Unknown') ceType = ceDict.get('CEType', 'Unknown') ram = ceDict.get('MaxRAM', 'Unknown') submissionMode = ceDict.get('SubmissionMode', 'Unknown') # Current BDII CE info newarch = ceBdiiDict[site]['CEs'][ce].get( 'GlueHostArchitecturePlatformType', '').strip() systemName = ceInfo.get('GlueHostOperatingSystemName', '').strip() systemVersion = ceInfo.get('GlueHostOperatingSystemVersion', '').strip() systemRelease = ceInfo.get('GlueHostOperatingSystemRelease', '').strip() newOS = '' if systemName and systemVersion and systemRelease: newOS = '_'.join( (systemName, systemVersion, systemRelease)) newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip() newCEType = 'Unknown' for queue in ceInfo['Queues']: queueDict = ceInfo['Queues'][queue] newCEType = queueDict.get('GlueCEImplementationName', '').strip() if newCEType: break if newCEType == 'ARC-CE': newCEType = 'ARC' newSubmissionMode = None if newCEType in ['ARC', 'CREAM']: newSubmissionMode = "Direct" newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip() # Protect from unreasonable values if newRAM and int(newRAM) > 150000: newRAM = '' # Adding CE data to the change list addToChangeSet((ceSection, 'architecture', arch, newarch), changeSet) addToChangeSet((ceSection, 'OS', OS, newOS), changeSet) addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet) addToChangeSet((ceSection, 'CEType', ceType, newCEType), changeSet) addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet) if submissionMode == "Unknown" and newSubmissionMode: addToChangeSet((ceSection, 'SubmissionMode', submissionMode, newSubmissionMode), changeSet) queues = ceInfo['Queues'].keys() for queue in queues: queueInfo = ceInfo['Queues'][queue] queueStatus = queueInfo['GlueCEStateStatus'] queueSection = cfgPath(ceSection, 'Queues', queue) queueDict = {} result = gConfig.getOptionsDict(queueSection) if result['OK']: queueDict = result['Value'] else: if queueStatus.lower() == "production": log.notice("Adding new queue", "%s to CE %s" % (queue, ce)) else: continue # Current CS queue info maxCPUTime = queueDict.get('maxCPUTime', 'Unknown') si00 = queueDict.get('SI00', 'Unknown') maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown') # Current BDII queue info newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '') if newMaxCPUTime == "4" * len( newMaxCPUTime) or newMaxCPUTime == "9" * len( newMaxCPUTime): newMaxCPUTime = '' wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime', '') if wallTime == "4" * len( wallTime) or wallTime == "9" * len(wallTime): wallTime = '' if wallTime and int(wallTime) > 0: if not newMaxCPUTime: newMaxCPUTime = str(int(0.8 * int(wallTime))) else: if int(wallTime) <= int(newMaxCPUTime): newMaxCPUTime = str(int(0.8 * int(wallTime))) newSI00 = '' caps = queueInfo.get('GlueCECapability', []) if isinstance(caps, basestring): caps = [caps] for cap in caps: if 'CPUScalingReferenceSI00' in cap: newSI00 = cap.split('=')[-1] # Adding queue info to the CS addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime, newMaxCPUTime), changeSet) addToChangeSet((queueSection, 'SI00', si00, newSI00), changeSet) if maxTotalJobs == "Unknown": newTotalJobs = min( 1000, int( int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) / 2)) newWaitingJobs = max(2, int(newTotalJobs * 0.1)) newTotalJobs = str(newTotalJobs) newWaitingJobs = str(newWaitingJobs) addToChangeSet( (queueSection, 'MaxTotalJobs', '', newTotalJobs), changeSet) addToChangeSet((queueSection, 'MaxWaitingJobs', '', newWaitingJobs), changeSet) # Updating eligible VO list VOs = set() if queueDict.get('VO', ''): VOs = set([ q.strip() for q in queueDict.get('VO', '').split(',') if q ]) if vo not in VOs: VOs.add(vo) VOs = list(VOs) newVOs = ','.join(VOs) addToChangeSet((queueSection, 'VO', '', newVOs), changeSet) return S_OK(changeSet)
def checkUnusedSEs(): global vo, dry result = getGridSRMs(vo, unUsed=True) if not result["OK"]: gLogger.error("Failed to look up SRMs in BDII", result["Message"]) siteSRMDict = result["Value"] # Evaluate VOs result = getVOs() if result["OK"]: csVOs = set(result["Value"]) else: csVOs = {vo} changeSetFull = set() for site in siteSRMDict: for gridSE in siteSRMDict[site]: changeSet = set() seDict = siteSRMDict[site][gridSE]["SE"] srmDict = siteSRMDict[site][gridSE]["SRM"] # Check the SRM version version = srmDict.get("GlueServiceVersion", "") if not (version and version.startswith("2")): gLogger.debug("Skipping SRM service with version %s" % version) continue result = getDIRACSiteName(site) if not result["OK"]: gLogger.notice("Unused se %s is detected at unused site %s" % (gridSE, site)) gLogger.notice("Consider adding site %s to the DIRAC CS" % site) continue diracSites = result["Value"] yn = raw_input( "\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites)) ) if not yn or yn.lower().startswith("y"): if len(diracSites) > 1: prompt = "Which DIRAC site the new SE should be attached to ?" for i, s in enumerate(diracSites): prompt += "\n[%d] %s" % (i, s) prompt += "\nEnter your choice number: " inp = raw_input(prompt) try: ind = int(inp) except: gLogger.notice("Can not interpret your choice: %s, try again later" % inp) continue diracSite = diracSites[ind] else: diracSite = diracSites[0] domain, siteName, country = diracSite.split(".") recName = "%s-disk" % siteName inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName)) diracSEName = inp if not inp: diracSEName = recName gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite)) seSection = cfgPath("/Resources/StorageElements", diracSEName) changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown"))) changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown"))) bdiiVOs = set( [re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])] ) seVOs = csVOs.intersection(bdiiVOs) changeSet.add((seSection, "VO", ",".join(seVOs))) accessSection = cfgPath(seSection, "AccessProtocol.1") changeSet.add((accessSection, "Protocol", "srm")) changeSet.add((accessSection, "PluginName", "SRM2")) endPoint = srmDict.get("GlueServiceEndpoint", "") host = urlparse(endPoint).hostname port = urlparse(endPoint).port changeSet.add((accessSection, "Host", host)) changeSet.add((accessSection, "Port", port)) changeSet.add((accessSection, "Access", "remote")) voPathSection = cfgPath(accessSection, "VOPath") if "VOPath" in seDict: path = seDict["VOPath"] voFromPath = os.path.basename(path) if voFromPath != diracVO: gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path) changeSet.add((voPathSection, diracVO, path)) path = os.path.dirname(path) else: # Try to guess the Path domain = ".".join(host.split(".")[-2:]) path = "/dpm/%s/home" % domain changeSet.add((accessSection, "Path", path)) changeSet.add((accessSection, "SpaceToken", "")) changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN=")) gLogger.notice("SE %s will be added with the following parameters" % diracSEName) changeList = list(changeSet) changeList.sort() for entry in changeList: gLogger.notice(entry) yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName) if not yn or yn.lower().startswith("y"): changeSetFull = changeSetFull.union(changeSet) if dry: if changeSetFull: gLogger.notice("Skipping commit of the new SE data in a dry run") else: gLogger.notice("No new SE to be added") return S_OK() if changeSetFull: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result["OK"]: gLogger.error("Failed to initialize CSAPI object", result["Message"]) DIRACExit(-1) changeList = list(changeSetFull) changeList.sort() for section, option, value in changeList: csAPI.setOption(cfgPath(section, option), value) yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ") if not yn or yn.lower().startswith("y"): result = csAPI.commit() if not result["OK"]: gLogger.error("Error while commit to CS", result["Message"]) else: gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull)) else: gLogger.notice("No new SE to be added") return S_OK()