def __lookForNewCEs( self ): """ Look up BDII for CEs not yet present in the DIRAC CS """ bannedCEs = self.am_getOption( 'BannedCEs', [] ) result = getCEsFromCS() if not result['OK']: return result knownCEs = set( result['Value'] ) knownCEs = knownCEs.union( set( bannedCEs ) ) for vo in self.voName: result = self.__getBdiiCEInfo( vo ) if not result['OK']: continue bdiiInfo = result['Value'] result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs ) if not result['OK']: self.log.error( 'Failed to get unused CEs', result['Message'] ) siteDict = result['Value'] body = '' for site in siteDict: newCEs = set( siteDict[site].keys() ) if not newCEs: continue ceString = '' for ce in newCEs: queueString = '' ceInfo = bdiiInfo[site]['CEs'][ce] ceString = "CE: %s, GOCDB Site Name: %s" % ( ce, site ) systemTuple = siteDict[site][ce]['System'] osString = "%s_%s_%s" % ( systemTuple ) newCEString = "\n%s\n%s\n" % ( ceString, osString ) for queue in ceInfo['Queues']: queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' ) if 'production' in queueStatus.lower(): ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' ) queueString += " %s %s %s\n" % ( queue, queueStatus, ceType ) if queueString: ceString = newCEString ceString += "Queues:\n" ceString += queueString if ceString: body += ceString if body: body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n" body += "Add new Sites/CEs for vo %s with the command:\n" % vo body += "dirac-admin-add-resources --vo %s --ce\n" % vo self.log.info( body ) if self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) if not result['OK']: self.log.error( 'Can not send new site notification mail', result['Message'] ) return S_OK()
def __lookForNewCEs(self): """ Look up BDII for CEs not yet present in the DIRAC CS """ bannedCEs = self.am_getOption('BannedCEs', []) for vo in self.voName: # get the known CEs for a given VO, so we can know the unknowns, or no longer supported, # for a VO res = getQueues(community=vo) if not res['OK']: return res knownCEs = set() for _site, ces in res['Value'].items(): knownCEs.update(ces) knownCEs.update(bannedCEs) result = self.__getBdiiCEInfo(vo) if not result['OK']: continue bdiiInfo = result['Value'] result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs) if not result['OK']: self.log.error('Failed to get unused CEs', result['Message']) continue # next VO siteDict = result['Value'] unknownCEs = set(result['UnknownCEs']) - set(bannedCEs) body = '' for site in siteDict: newCEs = set(siteDict[site]) # pylint: disable=no-member if not newCEs: continue ceString = '' for ce in newCEs: queueString = '' ceInfo = bdiiInfo[site]['CEs'][ce] newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site) systemTuple = siteDict[site][ce]['System'] osString = "%s_%s_%s" % (systemTuple) newCEString = "\n%s\n%s\n" % (newCEString, osString) for queue in ceInfo['Queues']: queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus') if 'production' in queueStatus.lower(): ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '') queueString += " %s %s %s\n" % (queue, queueStatus, ceType) if queueString: ceString += newCEString ceString += "Queues:\n" ceString += queueString if ceString: body += ceString if siteDict: body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n" body += "Add new Sites/CEs for vo %s with the command:\n" % vo body += "dirac-admin-add-resources --vo %s --ce\n" % vo if unknownCEs: body += '\n\n' body += 'There is no (longer) information about the following CEs for the %s VO.\n' % vo body += '\n'.join(sorted(unknownCEs)) body += '\n\n' if body: self.log.info(body) if self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False, avoidSpam=True) if not result['OK']: self.log.error('Can not send new site notification mail', result['Message']) return S_OK()
def checkUnusedCEs(): global vo, dry, ceBdiiDict, hostURL, glue2 gLogger.notice( 'looking for new computing resources in the BDII database...') res = getQueues(community=vo) if not res['OK']: gLogger.error('ERROR: failed to get CEs from CS', res['Message']) DIRACExit(-1) knownCEs = set() for _site, ces in res['Value'].items(): knownCEs.update(ces) result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL, glue2=glue2) if not result['OK']: gLogger.error('ERROR: failed to get CEs from BDII', result['Message']) DIRACExit(-1) ceBdiiDict = result['BdiiInfo'] unknownCEs = result['UnknownCEs'] if unknownCEs: gLogger.notice( 'There is no (longer) information about the following CEs for the %s VO:' % vo) gLogger.notice('\n'.join(sorted(unknownCEs))) siteDict = result['Value'] if siteDict: gLogger.notice('New resources available:') for site in siteDict: diracSite = 'Unknown' result = getDIRACSiteName(site) if result['OK']: diracSite = ','.join(result['Value']) if siteDict[site]: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in siteDict[site]: gLogger.notice(' ' * 4 + ce) gLogger.notice(' %s, %s' % (siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System'])) else: gLogger.notice('No new resources available, exiting') return inp = six.moves.input( "\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if not inp and inp.lower().startswith('n'): return gLogger.notice('\nAdding new sites/CEs interactively\n') sitesAdded = [] for site in siteDict: # Get the country code: country = '' for ce in siteDict[site]: country = ce.strip().split('.')[-1].lower() if len(country) == 2: break if country == 'gov': country = 'us' break if not country or len(country) != 2: country = 'xx' result = getDIRACSiteName(site) if not result['OK']: gLogger.notice( '\nThe site %s is not yet in the CS, give it a name' % site) diracSite = six.moves.input('[help|skip|<domain>.<name>.%s]: ' % country) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice('%s site details:' % site) for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice('%s\t%s' % (k, v)) gLogger.notice( '\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country) diracSite = six.moves.input('[<domain>.<name>.%s]: ' % country) try: _, _, _ = diracSite.split('.') except ValueError: gLogger.error( 'ERROR: DIRAC site name does not follow convention: %s' % diracSite) continue diracSites = [diracSite] else: diracSites = result['Value'] if len(diracSites) > 1: gLogger.notice( 'Attention! GOC site %s corresponds to more than one DIRAC sites:' % site) gLogger.notice(str(diracSites)) gLogger.notice( 'Please, pay attention which DIRAC site the new CEs will join\n' ) newCEs = {} addedCEs = [] for ce in siteDict[site]: ceType = siteDict[site][ce]['CEType'] for diracSite in diracSites: if ce in addedCEs: continue yn = six.moves.input( "Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite)) if yn == '' or yn.lower().startswith('y'): newCEs.setdefault(diracSite, []) newCEs[diracSite].append(ce) addedCEs.append(ce) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % ( diracSite, site, ' '.join(newCEs[diracSite])) gLogger.notice( "\nNew site/CEs will be added with command:\n%s" % cmd) yn = six.moves.input("Add it ? [default yes] [yes|no]: ") if not (yn == '' or yn.lower().startswith('y')): continue if dry: gLogger.notice("Command is skipped in the dry run") else: result = systemCall(0, shlex.split(cmd)) if not result['OK']: gLogger.error( 'Error while executing dirac-admin-add-site command' ) yn = six.moves.input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith('n'): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:') for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: exitStatus, stdData, errData = result['Value'] if exitStatus: gLogger.error( 'Error while executing dirac-admin-add-site command\n', '\n'.join([stdData, errData])) yn = six.moves.input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith('n'): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:' ) for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: sitesAdded.append((site, diracSite)) gLogger.notice(stdData) if sitesAdded: gLogger.notice('CEs were added at the following sites:') for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) else: gLogger.notice('No new CEs were added this time')
def checkUnusedCEs(): global vo, dry, ceBdiiDict gLogger.notice( 'looking for new computing resources in the BDII database...' ) result = getCEsFromCS() if not result['OK']: gLogger.error( 'ERROR: failed to get CEs from CS', result['Message'] ) DIRACExit( -1 ) knownCEs = result['Value'] result = getGridCEs( vo, ceBlackList = knownCEs ) if not result['OK']: gLogger.error( 'ERROR: failed to get CEs from BDII', result['Message'] ) DIRACExit( -1 ) ceBdiiDict = result['BdiiInfo'] siteDict = result['Value'] if siteDict: gLogger.notice( 'New resources available:\n' ) for site in siteDict: diracSite = 'Unknown' result = getDIRACSiteName( site ) if result['OK']: diracSite = ','.join( result['Value'] ) ces = siteDict[site].keys() if ces: gLogger.notice( " %s, DIRAC site %s" % ( site, diracSite) ) for ce in ces: gLogger.notice( ' '*4+ce ) gLogger.notice( ' %s, %s' % ( siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System'] ) ) else: gLogger.notice( 'No new resources available, exiting' ) DIRACExit( 0 ) inp = raw_input( "\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if not inp and inp.lower().startswith( 'n' ): gLogger.notice( 'Nothing else to be done, exiting' ) DIRACExit( 0 ) gLogger.notice( '\nAdding new sites/CEs interactively\n' ) sitesAdded = [] for site in siteDict: # Get the country code: country = '' ces = siteDict[site].keys() for ce in ces: country = ce.strip().split('.')[-1].lower() if len( country ) == 2: break if country == 'gov': country = 'us' break if not country or len( country ) != 2: country = 'xx' result = getDIRACSiteName( site ) if not result['OK']: gLogger.notice( '\nThe site %s is not yet in the CS, give it a name' % site ) diracSite = raw_input( '[help|skip|<domain>.<name>.%s]: ' % country ) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice( '%s site details:' % site ) for k,v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice( '%s\t%s' % (k,v) ) gLogger.notice( '\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country ) diracSite = raw_input( '[<domain>.<name>.%s]: ' % country ) try: domain,siteName,country = diracSite.split('.') except Exception, x: gLogger.error( 'ERROR: DIRAC site name does not follow convention: %s' % diracSite ) continue diracSites = [diracSite] else: diracSites = result['Value'] if len( diracSites ) > 1: gLogger.notice( 'Attention! GOC site %s corresponds to more than one DIRAC sites:' % site ) gLogger.notice( str( diracSites ) ) gLogger.notice( 'Please, pay attention which DIRAC site the new CEs will join\n' ) newCEs = {} addedCEs = [] for ce in ces: ceType = siteDict[site][ce]['CEType'] for diracSite in diracSites: if ce in addedCEs: continue yn = raw_input( "Add CE %s of type %s to %s? [default yes] [yes|no]: " % ( ce, ceType, diracSite ) ) if yn == '' or yn.lower() == 'y': newCEs.setdefault( diracSite, [] ) newCEs[diracSite].append( ce ) addedCEs.append( ce ) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % ( diracSite, site, ' '.join( newCEs[diracSite] ) ) gLogger.notice( "\nNew site/CEs will be added with command:\n%s" % cmd ) yn = raw_input( "Add it ? [default yes] [yes|no]: " ) if not ( yn == '' or yn.lower() == 'y' ) : continue if dry: gLogger.notice( "Command is skipped in the dry run" ) else: result = shellCall( 0, cmd ) if not result['OK']: gLogger.error( 'Error while executing dirac-admin-add-site command' ) yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith( 'n' ): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:' ) for site, diracSite in sitesAdded: gLogger.notice( "%s\t%s" % ( site, diracSite ) ) DIRACExit( 0 ) else: exitStatus, stdData, errData = result[ 'Value' ] if exitStatus: gLogger.error( 'Error while executing dirac-admin-add-site command\n', '\n'.join( [stdData, errData] ) ) yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith( 'n' ): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:' ) for site, diracSite in sitesAdded: gLogger.notice( "%s\t%s" % ( site, diracSite ) ) DIRACExit( 0 ) else: sitesAdded.append( ( site, diracSite ) ) gLogger.notice( stdData )
def checkUnusedCEs(): global vo, dry, ceBdiiDict, hostURL gLogger.notice("looking for new computing resources in the BDII database...") res = getQueues(community=vo) if not res["OK"]: gLogger.error("ERROR: failed to get CEs from CS", res["Message"]) DIRACExit(-1) knownCEs = set() for _site, ces in res["Value"].items(): knownCEs.update(ces) result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL) if not result["OK"]: gLogger.error("ERROR: failed to get CEs from BDII", result["Message"]) DIRACExit(-1) ceBdiiDict = result["BdiiInfo"] unknownCEs = result["UnknownCEs"] if unknownCEs: gLogger.notice("There is no (longer) information about the following CEs for the %s VO:" % vo) gLogger.notice("\n".join(sorted(unknownCEs))) siteDict = result["Value"] if siteDict: gLogger.notice("New resources available:") for site in siteDict: diracSite = "Unknown" result = getDIRACSiteName(site) if result["OK"]: diracSite = ",".join(result["Value"]) if siteDict[site]: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in siteDict[site]: gLogger.notice(" " * 4 + ce) gLogger.notice( " %s, %s" % (siteDict[site][ce]["CEType"], "%s_%s_%s" % siteDict[site][ce]["System"]) ) else: gLogger.notice("No new resources available, exiting") return inp = six.moves.input("\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if inp and inp.lower().startswith("n"): return gLogger.notice("\nAdding new sites/CEs interactively\n") sitesAdded = [] for site in siteDict: # Get the country code: country = "" for ce in siteDict[site]: country = ce.strip().split(".")[-1].lower() if len(country) == 2: break if country == "gov": country = "us" break if not country or len(country) != 2: country = "xx" result = getDIRACSiteName(site) if not result["OK"]: gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site) diracSite = six.moves.input("[help|skip|<domain>.<name>.%s]: " % country) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice("%s site details:" % site) for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice("%s\t%s" % (k, v)) gLogger.notice("\nEnter DIRAC site name in the form <domain>.<name>.%s\n" % country) diracSite = six.moves.input("[<domain>.<name>.%s]: " % country) try: _, _, _ = diracSite.split(".") except ValueError: gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite) continue diracSites = [diracSite] else: diracSites = result["Value"] if len(diracSites) > 1: gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site) gLogger.notice(str(diracSites)) gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n") newCEs = {} addedCEs = [] for ce in siteDict[site]: ceType = siteDict[site][ce]["CEType"] for diracSite in diracSites: if ce in addedCEs: continue yn = six.moves.input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite)) if yn == "" or yn.lower().startswith("y"): newCEs.setdefault(diracSite, []) newCEs[diracSite].append(ce) addedCEs.append(ce) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, " ".join(newCEs[diracSite])) gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd) yn = six.moves.input("Add it ? [default yes] [yes|no]: ") if not (yn == "" or yn.lower().startswith("y")): continue if dry: gLogger.notice("Command is skipped in the dry run") else: result = systemCall(0, shlex.split(cmd)) if not result["OK"]: gLogger.error("Error while executing dirac-admin-add-site command") yn = six.moves.input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: exitStatus, stdData, errData = result["Value"] if exitStatus: gLogger.error( "Error while executing dirac-admin-add-site command\n", "\n".join([stdData, errData]) ) yn = six.moves.input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: sitesAdded.append((site, diracSite)) gLogger.notice(stdData) if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) else: gLogger.notice("No new CEs were added this time")
def checkUnusedCEs(): global vo, dry, ceBdiiDict gLogger.notice("looking for new computing resources in the BDII database...") result = getCEsFromCS() if not result["OK"]: gLogger.error("ERROR: failed to get CEs from CS", result["Message"]) DIRACExit(-1) knownCEs = result["Value"] result = getGridCEs(vo, ceBlackList=knownCEs) if not result["OK"]: gLogger.error("ERROR: failed to get CEs from BDII", result["Message"]) DIRACExit(-1) ceBdiiDict = result["BdiiInfo"] siteDict = result["Value"] if siteDict: gLogger.notice("New resources available:\n") for site in siteDict: diracSite = "Unknown" result = getDIRACSiteName(site) if result["OK"]: diracSite = ",".join(result["Value"]) ces = siteDict[site].keys() if ces: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in ces: gLogger.notice(" " * 4 + ce) gLogger.notice( " %s, %s" % (siteDict[site][ce]["CEType"], "%s_%s_%s" % siteDict[site][ce]["System"]) ) else: gLogger.notice("No new resources available, exiting") DIRACExit(0) inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if not inp and inp.lower().startswith("n"): gLogger.notice("Nothing else to be done, exiting") DIRACExit(0) gLogger.notice("\nAdding new sites/CEs interactively\n") sitesAdded = [] for site in siteDict: # Get the country code: country = "" ces = siteDict[site].keys() for ce in ces: country = ce.strip().split(".")[-1].lower() if len(country) == 2: break if country == "gov": country = "us" break if not country or len(country) != 2: country = "xx" result = getDIRACSiteName(site) if not result["OK"]: gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site) diracSite = raw_input("[help|skip|<domain>.<name>.%s]: " % country) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice("%s site details:" % site) for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice("%s\t%s" % (k, v)) gLogger.notice("\nEnter DIRAC site name in the form <domain>.<name>.%s\n" % country) diracSite = raw_input("[<domain>.<name>.%s]: " % country) try: _, _, _ = diracSite.split(".") except ValueError: gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite) continue diracSites = [diracSite] else: diracSites = result["Value"] if len(diracSites) > 1: gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site) gLogger.notice(str(diracSites)) gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n") newCEs = {} addedCEs = [] for ce in ces: ceType = siteDict[site][ce]["CEType"] for diracSite in diracSites: if ce in addedCEs: continue yn = raw_input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite)) if yn == "" or yn.lower() == "y": newCEs.setdefault(diracSite, []) newCEs[diracSite].append(ce) addedCEs.append(ce) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, " ".join(newCEs[diracSite])) gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd) yn = raw_input("Add it ? [default yes] [yes|no]: ") if not (yn == "" or yn.lower() == "y"): continue if dry: gLogger.notice("Command is skipped in the dry run") else: result = shellCall(0, cmd) if not result["OK"]: gLogger.error("Error while executing dirac-admin-add-site command") yn = raw_input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: exitStatus, stdData, errData = result["Value"] if exitStatus: gLogger.error( "Error while executing dirac-admin-add-site command\n", "\n".join([stdData, errData]) ) yn = raw_input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: sitesAdded.append((site, diracSite)) gLogger.notice(stdData) if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) else: gLogger.notice("No new CEs were added this time")