def main(vo): # thanks to Stephane for suggesting this fix! #res1 = gConfig.getSections( 'Resources/Sites/LCG/', listOrdered = True ) res = getQueues() if not res['OK']: gLogger.error(res['Message']) gLogger.error("Cannot obtain Queues") dexit(1) sites = res['Value'].keys() values = [res['Value'][key].keys() for key in sites] sites_ce = dict(zip(sites,values)) vo_ces = ldapCEs(vo) final_dict = {} for site in sites_ce: final_dict[site]={"Tags":[],"CE":[]} ces_current_site = sites_ce[site] for ce in ces_current_site: if ce in vo_ces: curr_ces = final_dict[site]["CE"] curr_ces.append(ce) final_dict[site].update({"Tags":ldapTag(ce,vo),"CE":curr_ces}) #final_dict[site]={"Tags":ldapTag(ce,vo),"CE":[ce]} ret_dict = {} for key in final_dict: if len(final_dict[key]['CE'])!=0: ret_dict[key]=final_dict[key] return ret_dict
def main(vo): # thanks to Stephane for suggesting this fix! #res1 = gConfig.getSections( 'Resources/Sites/LCG/', listOrdered = True ) res = getQueues() if not res['OK']: gLogger.error(res['Message']) gLogger.error("Cannot obtain Queues") dexit(1) sites = res['Value'].keys() values = [res['Value'][key].keys() for key in sites] sites_ce = dict(zip(sites, values)) vo_ces = ldapCEs(vo) final_dict = {} for site in sites_ce: final_dict[site] = {"Tags": [], "CE": []} ces_current_site = sites_ce[site] for ce in ces_current_site: if ce in vo_ces: curr_ces = final_dict[site]["CE"] curr_ces.append(ce) final_dict[site].update({ "Tags": ldapTag(ce, vo), "CE": curr_ces }) #final_dict[site]={"Tags":ldapTag(ce,vo),"CE":[ce]} ret_dict = {} for key in final_dict: if len(final_dict[key]['CE']) != 0: ret_dict[key] = final_dict[key] return ret_dict
def getNodes(): """ Gets all nodes """ nodes = [] queues = getQueues() if queues['OK']: nodes = nodes + queues['Value'] return S_OK(nodes)
def getSiteElements(siteName): """ Gets all the computing and storage elements for a given site """ res = DMSHelpers().getSiteSEMapping() if not res["OK"]: return res resources = res["Value"][1].get(siteName, []) res = getQueues(siteName) if not res["OK"]: return res resources = list(resources) + list(res["Value"].get(siteName, [])) return S_OK(resources)
def getSiteElements(siteName): """ Gets all the computing and storage elements for a given site """ res = DMSHelpers().getSiteSEMapping() if not res['OK']: return res resources = res['Value'][1].get(siteName, []) res = getQueues(siteName) if not res['OK']: return res resources = list(resources) + list(res['Value'][siteName]) return S_OK(resources)
def testAndProbeSites(): """submits jobs to test sites""" clip = Params() clip.registerSwitches() Script.parseCommandLine() from DIRAC import gLogger, exit as dexit from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import CheckWNs from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueues res = getQueues(siteList=clip.site, ceList=clip.ce) if not res['OK']: gLogger.error("Failed getting the queues", res['Message']) dexit(1) sitedict = res['Value'] CEs = [] for ces in sitedict.values(): CEs.extend(ces.keys()) gLogger.notice("Found %s CEs to look at." % len(CEs)) d = DiracILC(True, "SiteProbe.rep") for CE in CEs: j = UserJob() j.setDestinationCE(CE) c = CheckWNs() res = j.append(c) if not res['OK']: gLogger.error(res['Message']) continue j.setOutputSandbox("*.log") j.setCPUTime(30000) j.dontPromptMe() res = j.submit(d) if not res['OK']: gLogger.error("Failed to submit job, aborting") dexit(1) dexit(0)
def __getSiteForCEs(self, ces): """ We want to get the site for a given CE because that's what the job expects """ res = getQueues(community=self.vo) if not res["OK"]: return S_ERROR("Could not get site for CE") sitedict = res["Value"] final_sdict = {} for site, s_ces in sitedict.items(): for ce in ces: if ce in s_ces: if not site in final_sdict: final_sdict[site] = [] final_sdict[site].append(ce) return S_OK(final_sdict)
def __getSiteForCEs(self, ces): """ We want to get the site for a given CE because that's what the job expects """ res = getQueues(community=self.vo) if not res['OK']: return S_ERROR("Could not get site for CE") sitedict = res['Value'] final_sdict = {} for site, s_ces in sitedict.items(): for ce in ces: if ce in s_ces: if not site in final_sdict: final_sdict[site] = [] final_sdict[site].append(ce) return S_OK(final_sdict)
def getSiteForCEs(ces, vo="glast.org"): """ We want to get the site for a given CE because that's what the job expects copy of private DB method. """ res = getQueues(community=vo) if not res["OK"]: return S_ERROR("Could not get site for CE") sitedict = res["Value"] final_sdict = {} for site, s_ces in sitedict.items(): for ce in ces: if ce in s_ces: if not site in final_sdict: final_sdict[site] = [] final_sdict[site].append(ce) return S_OK(final_sdict)
def __getCESforSite(self, site): """ As the name suggests, get all the CEs for a given site """ slist = None if site != 'ALL': slist = [site] res = getQueues(siteList=slist, community=self.vo) if not res['OK']: return S_ERROR("Could not get CEs for site") if not res['Value']: return S_ERROR("No CEs for site %s" % site) ces = [] for key, items in res['Value'].items(): if site != 'ALL': if key != site: continue ces.extend(items.keys()) if not ces: return S_ERROR("No CEs for site %s" % site) return S_OK(ces)
def __getCESforSite(self, site): """ As the name suggests, get all the CEs for a given site """ slist = None if site != "ALL": slist = [site] res = getQueues(siteList=slist, community=self.vo) if not res["OK"]: return S_ERROR("Could not get CEs for site") if not res["Value"]: return S_ERROR("No CEs for site %s" % site) ces = [] for key, items in res["Value"].items(): if site != "ALL": if key != site: continue ces.extend(items.keys()) if not ces: return S_ERROR("No CEs for site %s" % site) return S_OK(ces)
def getSiteForCEs(ces, vo="glast.org"): """ We want to get the site for a given CE because that's what the job expects copy of private DB method. """ res = getVO("glast.org") if not res['OK']: return S_ERROR("Could not get VO") vo = res["Value"] res = getQueues(community=vo) if not res['OK']: return S_ERROR("Could not get site for CE") sitedict = res['Value'] final_sdict = {} for site, s_ces in sitedict.items(): for ce in ces: if ce in s_ces: if not site in final_sdict: final_sdict[site] = [] final_sdict[site].append(ce) return S_OK(final_sdict)
def __lookForNewCEs(self): """ Look up BDII for CEs not yet present in the DIRAC CS """ bannedCEs = self.am_getOption('BannedCEs', []) for vo in self.voName: # get the known CEs for a given VO, so we can know the unknowns, or no longer supported, # for a VO res = getQueues(community=vo) if not res['OK']: return res knownCEs = set() for _site, ces in res['Value'].items(): knownCEs.update(ces) knownCEs.update(bannedCEs) result = self.__getBdiiCEInfo(vo) if not result['OK']: continue bdiiInfo = result['Value'] result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs) if not result['OK']: self.log.error('Failed to get unused CEs', result['Message']) continue # next VO siteDict = result['Value'] unknownCEs = set(result['UnknownCEs']) - set(bannedCEs) body = '' for site in siteDict: newCEs = set(siteDict[site]) # pylint: disable=no-member if not newCEs: continue ceString = '' for ce in newCEs: queueString = '' ceInfo = bdiiInfo[site]['CEs'][ce] newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site) systemTuple = siteDict[site][ce]['System'] osString = "%s_%s_%s" % (systemTuple) newCEString = "\n%s\n%s\n" % (newCEString, osString) for queue in ceInfo['Queues']: queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus') if 'production' in queueStatus.lower(): ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '') queueString += " %s %s %s\n" % (queue, queueStatus, ceType) if queueString: ceString += newCEString ceString += "Queues:\n" ceString += queueString if ceString: body += ceString if siteDict: body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n" body += "Add new Sites/CEs for vo %s with the command:\n" % vo body += "dirac-admin-add-resources --vo %s --ce\n" % vo if unknownCEs: body += '\n\n' body += 'There is no (longer) information about the following CEs for the %s VO.\n' % vo body += '\n'.join(sorted(unknownCEs)) body += '\n\n' if body: self.log.info(body) if self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False, avoidSpam=True) if not result['OK']: self.log.error('Can not send new site notification mail', result['Message']) return S_OK()
gLogger.notice("Found the tags:", res['Value']) #Get tags with f****d up status res = sw.getTagsWithStatus("NewSomething") if not res['OK']: gLogger.error(res['Message']) else: gLogger.notice("Found the tags:", res['Value']) res = sw.updateStatus(mytag, mysite, "Probing") if not res['OK']: gLogger.error(res['Message']) else: gLogger.notice("Updated %s at %s to %s" % (mytag, mysite, "Probing")) res = getQueues(siteList=[mysite]) if not res['OK']: gLogger.error(res['Message']) dexit(0) cetest = res['Value'][mysite].keys()[0] res = sw.updateCEStatus(mytag, cetest, "Valid") if not res['OK']: gLogger.error(res['Message']) else: gLogger.notice("Updated %s to %s at %s" % (mytag, "Valid", cetest)) #try again now that at least one CE as a Valid tag res = sw.getSitesForTag(mytag) if not res['OK']: gLogger.error(res['Message'])
def checkUnusedCEs(): global vo, dry, ceBdiiDict, hostURL gLogger.notice("looking for new computing resources in the BDII database...") res = getQueues(community=vo) if not res["OK"]: gLogger.error("ERROR: failed to get CEs from CS", res["Message"]) DIRACExit(-1) knownCEs = set() for _site, ces in res["Value"].items(): knownCEs.update(ces) result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL) if not result["OK"]: gLogger.error("ERROR: failed to get CEs from BDII", result["Message"]) DIRACExit(-1) ceBdiiDict = result["BdiiInfo"] unknownCEs = result["UnknownCEs"] if unknownCEs: gLogger.notice("There is no (longer) information about the following CEs for the %s VO:" % vo) gLogger.notice("\n".join(sorted(unknownCEs))) siteDict = result["Value"] if siteDict: gLogger.notice("New resources available:") for site in siteDict: diracSite = "Unknown" result = getDIRACSiteName(site) if result["OK"]: diracSite = ",".join(result["Value"]) if siteDict[site]: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in siteDict[site]: gLogger.notice(" " * 4 + ce) gLogger.notice( " %s, %s" % (siteDict[site][ce]["CEType"], "%s_%s_%s" % siteDict[site][ce]["System"]) ) else: gLogger.notice("No new resources available, exiting") return inp = six.moves.input("\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if inp and inp.lower().startswith("n"): return gLogger.notice("\nAdding new sites/CEs interactively\n") sitesAdded = [] for site in siteDict: # Get the country code: country = "" for ce in siteDict[site]: country = ce.strip().split(".")[-1].lower() if len(country) == 2: break if country == "gov": country = "us" break if not country or len(country) != 2: country = "xx" result = getDIRACSiteName(site) if not result["OK"]: gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site) diracSite = six.moves.input("[help|skip|<domain>.<name>.%s]: " % country) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice("%s site details:" % site) for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice("%s\t%s" % (k, v)) gLogger.notice("\nEnter DIRAC site name in the form <domain>.<name>.%s\n" % country) diracSite = six.moves.input("[<domain>.<name>.%s]: " % country) try: _, _, _ = diracSite.split(".") except ValueError: gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite) continue diracSites = [diracSite] else: diracSites = result["Value"] if len(diracSites) > 1: gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site) gLogger.notice(str(diracSites)) gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n") newCEs = {} addedCEs = [] for ce in siteDict[site]: ceType = siteDict[site][ce]["CEType"] for diracSite in diracSites: if ce in addedCEs: continue yn = six.moves.input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite)) if yn == "" or yn.lower().startswith("y"): newCEs.setdefault(diracSite, []) newCEs[diracSite].append(ce) addedCEs.append(ce) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, " ".join(newCEs[diracSite])) gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd) yn = six.moves.input("Add it ? [default yes] [yes|no]: ") if not (yn == "" or yn.lower().startswith("y")): continue if dry: gLogger.notice("Command is skipped in the dry run") else: result = systemCall(0, shlex.split(cmd)) if not result["OK"]: gLogger.error("Error while executing dirac-admin-add-site command") yn = six.moves.input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: exitStatus, stdData, errData = result["Value"] if exitStatus: gLogger.error( "Error while executing dirac-admin-add-site command\n", "\n".join([stdData, errData]) ) yn = six.moves.input("Do you want to continue ? [default no] [yes|no]: ") if yn == "" or yn.lower().startswith("n"): if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: sitesAdded.append((site, diracSite)) gLogger.notice(stdData) if sitesAdded: gLogger.notice("CEs were added at the following sites:") for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) else: gLogger.notice("No new CEs were added this time")
gLogger.notice("Found the tags:",res['Value']) #Get tags with f****d up status res = sw.getTagsWithStatus("NewSomething") if not res['OK']: gLogger.error(res['Message']) else: gLogger.notice("Found the tags:", res['Value']) res = sw.updateStatus(mytag, mysite, "Probing") if not res['OK']: gLogger.error(res['Message']) else: gLogger.notice("Updated %s at %s to %s" % (mytag, mysite, "Probing")) res = getQueues(siteList = [mysite]) if not res['OK']: gLogger.error(res['Message']) dexit(0) cetest = res['Value'][mysite].keys()[0] res = sw.updateCEStatus(mytag, cetest, "Valid") if not res['OK']: gLogger.error(res['Message']) else: gLogger.notice("Updated %s to %s at %s" %(mytag, "Valid", cetest)) #try again now that at least one CE as a Valid tag res = sw.getSitesForTag(mytag) if not res['OK']: gLogger.error(res['Message'])
def checkUnusedCEs(): global vo, dry, ceBdiiDict, hostURL, glue2 gLogger.notice( 'looking for new computing resources in the BDII database...') res = getQueues(community=vo) if not res['OK']: gLogger.error('ERROR: failed to get CEs from CS', res['Message']) DIRACExit(-1) knownCEs = set() for _site, ces in res['Value'].items(): knownCEs.update(ces) result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL, glue2=glue2) if not result['OK']: gLogger.error('ERROR: failed to get CEs from BDII', result['Message']) DIRACExit(-1) ceBdiiDict = result['BdiiInfo'] unknownCEs = result['UnknownCEs'] if unknownCEs: gLogger.notice( 'There is no (longer) information about the following CEs for the %s VO:' % vo) gLogger.notice('\n'.join(sorted(unknownCEs))) siteDict = result['Value'] if siteDict: gLogger.notice('New resources available:') for site in siteDict: diracSite = 'Unknown' result = getDIRACSiteName(site) if result['OK']: diracSite = ','.join(result['Value']) if siteDict[site]: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in siteDict[site]: gLogger.notice(' ' * 4 + ce) gLogger.notice(' %s, %s' % (siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System'])) else: gLogger.notice('No new resources available, exiting') return inp = six.moves.input( "\nDo you want to add sites ? [default=yes] [yes|no]: ") inp = inp.strip() if not inp and inp.lower().startswith('n'): return gLogger.notice('\nAdding new sites/CEs interactively\n') sitesAdded = [] for site in siteDict: # Get the country code: country = '' for ce in siteDict[site]: country = ce.strip().split('.')[-1].lower() if len(country) == 2: break if country == 'gov': country = 'us' break if not country or len(country) != 2: country = 'xx' result = getDIRACSiteName(site) if not result['OK']: gLogger.notice( '\nThe site %s is not yet in the CS, give it a name' % site) diracSite = six.moves.input('[help|skip|<domain>.<name>.%s]: ' % country) if diracSite.lower() == "skip": continue if diracSite.lower() == "help": gLogger.notice('%s site details:' % site) for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice('%s\t%s' % (k, v)) gLogger.notice( '\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country) diracSite = six.moves.input('[<domain>.<name>.%s]: ' % country) try: _, _, _ = diracSite.split('.') except ValueError: gLogger.error( 'ERROR: DIRAC site name does not follow convention: %s' % diracSite) continue diracSites = [diracSite] else: diracSites = result['Value'] if len(diracSites) > 1: gLogger.notice( 'Attention! GOC site %s corresponds to more than one DIRAC sites:' % site) gLogger.notice(str(diracSites)) gLogger.notice( 'Please, pay attention which DIRAC site the new CEs will join\n' ) newCEs = {} addedCEs = [] for ce in siteDict[site]: ceType = siteDict[site][ce]['CEType'] for diracSite in diracSites: if ce in addedCEs: continue yn = six.moves.input( "Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite)) if yn == '' or yn.lower().startswith('y'): newCEs.setdefault(diracSite, []) newCEs[diracSite].append(ce) addedCEs.append(ce) for diracSite in diracSites: if diracSite in newCEs: cmd = "dirac-admin-add-site %s %s %s" % ( diracSite, site, ' '.join(newCEs[diracSite])) gLogger.notice( "\nNew site/CEs will be added with command:\n%s" % cmd) yn = six.moves.input("Add it ? [default yes] [yes|no]: ") if not (yn == '' or yn.lower().startswith('y')): continue if dry: gLogger.notice("Command is skipped in the dry run") else: result = systemCall(0, shlex.split(cmd)) if not result['OK']: gLogger.error( 'Error while executing dirac-admin-add-site command' ) yn = six.moves.input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith('n'): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:') for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: exitStatus, stdData, errData = result['Value'] if exitStatus: gLogger.error( 'Error while executing dirac-admin-add-site command\n', '\n'.join([stdData, errData])) yn = six.moves.input( "Do you want to continue ? [default no] [yes|no]: " ) if yn == '' or yn.lower().startswith('n'): if sitesAdded: gLogger.notice( 'CEs were added at the following sites:' ) for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) DIRACExit(0) else: sitesAdded.append((site, diracSite)) gLogger.notice(stdData) if sitesAdded: gLogger.notice('CEs were added at the following sites:') for site, diracSite in sitesAdded: gLogger.notice("%s\t%s" % (site, diracSite)) else: gLogger.notice('No new CEs were added this time')