def checkUnusedCEs():
global vo, dry, ceBdiiDict, hostURL, glue2
gLogger.notice(
'looking for new computing resources in the BDII database...')
res = getQueues(community=vo)
if not res['OK']:
gLogger.error('ERROR: failed to get CEs from CS', res['Message'])
DIRACExit(-1)
knownCEs = set()
for _site, ces in res['Value'].items():
knownCEs.update(ces)
result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL, glue2=glue2)
if not result['OK']:
gLogger.error('ERROR: failed to get CEs from BDII', result['Message'])
DIRACExit(-1)
ceBdiiDict = result['BdiiInfo']
unknownCEs = result['UnknownCEs']
if unknownCEs:
gLogger.notice(
'There is no (longer) information about the following CEs for the %s VO:'
% vo)
gLogger.notice('\n'.join(sorted(unknownCEs)))
siteDict = result['Value']
if siteDict:
gLogger.notice('New resources available:')
for site in siteDict:
diracSite = 'Unknown'
result = getDIRACSiteName(site)
if result['OK']:
diracSite = ','.join(result['Value'])
if siteDict[site]:
gLogger.notice(" %s, DIRAC site %s" % (site, diracSite))
for ce in siteDict[site]:
gLogger.notice(' ' * 4 + ce)
gLogger.notice(' %s, %s' %
(siteDict[site][ce]['CEType'],
'%s_%s_%s' % siteDict[site][ce]['System']))
else:
gLogger.notice('No new resources available, exiting')
return
inp = six.moves.input(
"\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith('n'):
return
gLogger.notice('\nAdding new sites/CEs interactively\n')
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ''
for ce in siteDict[site]:
country = ce.strip().split('.')[-1].lower()
if len(country) == 2:
break
if country == 'gov':
country = 'us'
break
if not country or len(country) != 2:
country = 'xx'
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice(
'\nThe site %s is not yet in the CS, give it a name' % site)
diracSite = six.moves.input('[help|skip|<domain>.<name>.%s]: ' %
country)
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice('%s site details:' % site)
for k, v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice('%s\t%s' % (k, v))
gLogger.notice(
'\nEnter DIRAC site name in the form <domain>.<name>.%s\n'
% country)
diracSite = six.moves.input('[<domain>.<name>.%s]: ' % country)
try:
_, _, _ = diracSite.split('.')
except ValueError:
gLogger.error(
'ERROR: DIRAC site name does not follow convention: %s' %
diracSite)
continue
diracSites = [diracSite]
else:
diracSites = result['Value']
if len(diracSites) > 1:
gLogger.notice(
'Attention! GOC site %s corresponds to more than one DIRAC sites:'
% site)
gLogger.notice(str(diracSites))
gLogger.notice(
'Please, pay attention which DIRAC site the new CEs will join\n'
)
newCEs = {}
addedCEs = []
for ce in siteDict[site]:
ceType = siteDict[site][ce]['CEType']
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = six.moves.input(
"Add CE %s of type %s to %s? [default yes] [yes|no]: " %
(ce, ceType, diracSite))
if yn == '' or yn.lower().startswith('y'):
newCEs.setdefault(diracSite, [])
newCEs[diracSite].append(ce)
addedCEs.append(ce)
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % (
diracSite, site, ' '.join(newCEs[diracSite]))
gLogger.notice(
"\nNew site/CEs will be added with command:\n%s" % cmd)
yn = six.moves.input("Add it ? [default yes] [yes|no]: ")
if not (yn == '' or yn.lower().startswith('y')):
continue
if dry:
gLogger.notice("Command is skipped in the dry run")
else:
result = systemCall(0, shlex.split(cmd))
if not result['OK']:
gLogger.error(
'Error while executing dirac-admin-add-site command'
)
yn = six.moves.input(
"Do you want to continue ? [default no] [yes|no]: "
)
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice(
'CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" %
(site, diracSite))
DIRACExit(0)
else:
exitStatus, stdData, errData = result['Value']
if exitStatus:
gLogger.error(
'Error while executing dirac-admin-add-site command\n',
'\n'.join([stdData, errData]))
yn = six.moves.input(
"Do you want to continue ? [default no] [yes|no]: "
)
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice(
'CEs were added at the following sites:'
)
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" %
(site, diracSite))
DIRACExit(0)
else:
sitesAdded.append((site, diracSite))
gLogger.notice(stdData)
if sitesAdded:
gLogger.notice('CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
else:
gLogger.notice('No new CEs were added this time')
def getSiteUpdates(vo, bdiiInfo=None, log=None, onecore=False):
"""Get all the necessary updates for the already defined sites and CEs
:param str vo: VO name
:param dict bdiiInfo: information from DBII
:param object log: logger
:param bool onecore: whether to add single core copies of multicore queues, see the documentation about :ref:`CE`
and the :mod:`~DIRAC.ConfigurationSystem.Agent.Bdii2CSAgent` configuration for details
:result: S_OK(set)/S_ERROR()
"""
def addToChangeSet(entry, changeSet):
"""Inner function to update changeSet with entry (a tuple)
:param tuple entry: entry to add to changeSet
:param set changeSet: set collecting stuff to change
"""
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
def dropTag(tags, tagToDrop):
"""Remove tag from a comma-separated string of tags.
:param str tags: the string of current tags
:param str tagToDrop: the tag to potentially remove
:return: string of comma separated tags
"""
return ",".join(sorted(set(tags.split(",")).difference({tagToDrop
}))).strip(",")
def addTag(tags, tagToAdd):
"""Add tag to a comma-separated string of tags.
:param str tags: the string of current tags
:param str tagToAdd: the tag to potentially add
:return: string of comma separated tags
"""
return ",".join(sorted(set(tags.split(",")).union({tagToAdd
}))).strip(",")
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getGlue2CEInfo(vo)
if not result["OK"]:
return result
ceBdiiDict = result["Value"]
if onecore:
# If enabled this creates a copy of the queue with multiple processors and sets NumberOfProcessors to 1 for ARC
# and HTCondorCE entries
def makeNewQueueName(queueName, ceType):
"""Create a new queueName for single core queues."""
if ceType == "HTCondorCE":
return queueName + "1core"
# we should have only ARC left, we add 1core to the middle part
queueNameSplit = queueName.split("-", 2)
queueNameSplit[1] = queueNameSplit[1] + "1core"
return "-".join(queueNameSplit)
for siteName, ceDict in ceBdiiDict.items():
for _ceName, ceInfo in ceDict["CEs"].items():
newQueues = dict()
for queueName, queueDict in ceInfo["Queues"].items():
if (queueDict["GlueCEImplementationName"]
not in ("ARC", "HTCondorCE") or int(
queueDict.get("NumberOfProcessors", 1)) == 1):
continue
newQueueName = makeNewQueueName(
queueName, queueDict["GlueCEImplementationName"])
newQueueDict = deepcopy(queueDict)
newQueueDict["NumberOfProcessors"] = 1
newQueues[newQueueName] = newQueueDict
ceInfo["Queues"].update(newQueues)
changeSet = set()
for site in ceBdiiDict:
result = getDIRACSiteName(site)
if not result["OK"]:
continue
siteNames = result["Value"]
for siteName in siteNames:
siteSection = cfgPath("/Resources", "Sites",
siteName.split(".")[0], siteName)
result = gConfig.getOptionsDict(siteSection)
if not result["OK"]:
continue
siteDict = result["Value"]
# Current CS values
coor = siteDict.get("Coordinates", "Unknown")
mail = siteDict.get("Mail", "Unknown").replace(" ", "")
description = siteDict.get("Description", "Unknown")
description = description.replace(" ,", ",")
longitude = ceBdiiDict[site].get("GlueSiteLongitude", "").strip()
latitude = ceBdiiDict[site].get("GlueSiteLatitude", "").strip()
# Current BDII value
newcoor = ""
if longitude and latitude:
newcoor = "%s:%s" % (longitude, latitude)
newmail = ceBdiiDict[site].get("GlueSiteSysAdminContact",
"").replace("mailto:", "").strip()
newdescription = ceBdiiDict[site].get("GlueSiteDescription",
"").strip()
newdescription = ", ".join(
[line.strip() for line in newdescription.split(",")])
# Adding site data to the changes list
addToChangeSet((siteSection, "Coordinates", coor, newcoor),
changeSet)
addToChangeSet((siteSection, "Mail", mail, newmail), changeSet)
addToChangeSet(
(siteSection, "Description", description, newdescription),
changeSet)
ces = gConfig.getSections(cfgPath(siteSection, "CEs"))
for ce in ces.get("Value", []):
ceSection = cfgPath(siteSection, "CEs", ce)
ceDict = {}
result = gConfig.getOptionsDict(ceSection)
if result["OK"]:
ceDict = result["Value"]
else:
if ceBdiiDict[site]["CEs"].get(ce, None):
log.notice("Adding new CE",
"%s to site %s/%s" % (ce, siteName, site))
ceInfo = ceBdiiDict[site]["CEs"].get(ce, None)
if ceInfo is None:
ceType = ceDict.get("CEType", "")
continue
# Current CS CE info
arch = ceDict.get("architecture", "Unknown")
OS = ceDict.get("OS", "Unknown")
si00 = ceDict.get("SI00", "Unknown")
ceType = ceDict.get("CEType", "Unknown")
ram = ceDict.get("MaxRAM", "Unknown")
submissionMode = ceDict.get("SubmissionMode", "Unknown")
# Current BDII CE info
newarch = ceBdiiDict[site]["CEs"][ce].get(
"GlueHostArchitecturePlatformType", "").strip()
systemName = ceInfo.get("GlueHostOperatingSystemName",
"").strip()
systemVersion = ceInfo.get("GlueHostOperatingSystemVersion",
"").strip()
systemRelease = ceInfo.get("GlueHostOperatingSystemRelease",
"").strip()
newOS = ""
if systemName and systemVersion and systemRelease:
newOS = "_".join(
(systemName, systemVersion, systemRelease))
newsi00 = ceInfo.get("GlueHostBenchmarkSI00", "").strip()
newCEType = "Unknown"
for queue in ceInfo["Queues"]:
queueDict = ceInfo["Queues"][queue]
newCEType = queueDict.get("GlueCEImplementationName",
"").strip()
if newCEType:
break
if newCEType == "ARC-CE":
newCEType = "ARC"
newSubmissionMode = None
if newCEType in ["ARC", "CREAM"]:
newSubmissionMode = "Direct"
newRAM = ceInfo.get("GlueHostMainMemoryRAMSize", "").strip()
# Protect from unreasonable values
if newRAM and int(newRAM) > 150000:
newRAM = ""
# Adding CE data to the change list
addToChangeSet((ceSection, "architecture", arch, newarch),
changeSet)
addToChangeSet((ceSection, "OS", OS, newOS), changeSet)
addToChangeSet((ceSection, "SI00", si00, newsi00), changeSet)
addToChangeSet((ceSection, "CEType", ceType, newCEType),
changeSet)
addToChangeSet((ceSection, "MaxRAM", ram, newRAM), changeSet)
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet((ceSection, "SubmissionMode",
submissionMode, newSubmissionMode),
changeSet)
for queue, queueInfo in ceInfo["Queues"].items():
queueStatus = queueInfo["GlueCEStateStatus"]
queueSection = cfgPath(ceSection, "Queues", queue)
queueDict = {}
result = gConfig.getOptionsDict(queueSection)
if result["OK"]:
queueDict = result["Value"]
else:
if queueStatus.lower() == "production":
log.notice("Adding new queue",
"%s to CE %s" % (queue, ce))
else:
continue
# Current CS queue info
maxCPUTime = queueDict.get("maxCPUTime", "Unknown")
si00 = queueDict.get("SI00", "Unknown")
maxTotalJobs = queueDict.get("MaxTotalJobs", "Unknown")
# Current BDII queue info
newMaxCPUTime = queueInfo.get("GlueCEPolicyMaxCPUTime", "")
if newMaxCPUTime == "4" * len(
newMaxCPUTime) or newMaxCPUTime == "9" * len(
newMaxCPUTime):
newMaxCPUTime = ""
wallTime = queueInfo.get("GlueCEPolicyMaxWallClockTime",
"")
if wallTime == "4" * len(
wallTime) or wallTime == "9" * len(wallTime):
wallTime = ""
if wallTime and int(wallTime) > 0:
if not newMaxCPUTime:
newMaxCPUTime = str(int(0.8 * int(wallTime)))
else:
if int(wallTime) <= int(newMaxCPUTime):
newMaxCPUTime = str(int(0.8 * int(wallTime)))
newSI00 = ""
caps = queueInfo.get("GlueCECapability", [])
if isinstance(caps, str):
caps = [caps]
for cap in caps:
if "CPUScalingReferenceSI00" in cap:
newSI00 = cap.split("=")[-1]
# tags, processors, localCEType
tag = queueDict.get("Tag", "")
reqTag = queueDict.get("RequiredTag", "")
# LocalCEType can be empty (equivalent to "InProcess")
# or "Pool", "Singularity", but also "Pool/Singularity"
localCEType = queueDict.get("LocalCEType", "")
try:
localCEType_inner = localCEType.split("/")[1]
except IndexError:
localCEType_inner = ""
numberOfProcessors = int(
queueDict.get("NumberOfProcessors", 0))
newNOP = int(queueInfo.get("NumberOfProcessors", 1))
# Adding queue info to the CS
addToChangeSet((queueSection, "maxCPUTime", maxCPUTime,
newMaxCPUTime), changeSet)
addToChangeSet((queueSection, "SI00", si00, newSI00),
changeSet)
# add RequiredTag if onecore is enabled, do this here for previously created MultiCore queues
if newNOP > 1 and onecore:
addToChangeSet((queueSection, "RequiredTag", reqTag,
addTag(reqTag, "MultiProcessor")),
changeSet)
if newNOP != numberOfProcessors:
addToChangeSet((queueSection, "NumberOfProcessors",
numberOfProcessors, newNOP), changeSet)
if newNOP > 1:
# if larger than one, add MultiProcessor to site tags, and LocalCEType=Pool
addToChangeSet((queueSection, "Tag", tag,
addTag(tag, "MultiProcessor")),
changeSet)
if localCEType_inner:
newLocalCEType = "Pool/" + localCEType_inner
else:
newLocalCEType = "Pool"
addToChangeSet((queueSection, "LocalCEType",
localCEType, newLocalCEType),
changeSet)
else:
# if not larger than one, drop MultiProcessor Tag.
# Here we do not change the LocalCEType as Pool CE would still be perfectly valid.
changeSet.add((queueSection, "Tag", tag,
dropTag(tag, "MultiProcessor")))
if onecore:
changeSet.add(
(queueSection, "RequiredTag", reqTag,
dropTag(reqTag, "MultiProcessor")))
if maxTotalJobs == "Unknown":
newTotalJobs = min(
1000,
int(
int(queueInfo.get("GlueCEInfoTotalCPUs", 0)) /
2))
newWaitingJobs = max(2, int(newTotalJobs * 0.1))
newTotalJobs = str(newTotalJobs)
newWaitingJobs = str(newWaitingJobs)
addToChangeSet(
(queueSection, "MaxTotalJobs", "", newTotalJobs),
changeSet)
addToChangeSet((queueSection, "MaxWaitingJobs", "",
newWaitingJobs), changeSet)
# Updating eligible VO list
VOs = set()
if queueDict.get("VO", ""):
VOs = set([
q.strip()
for q in queueDict.get("VO", "").split(",") if q
])
if vo not in VOs:
VOs.add(vo)
VOs = list(VOs)
newVOs = ",".join(VOs)
addToChangeSet((queueSection, "VO", "", newVOs),
changeSet)
return S_OK(changeSet)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"DIRACSiteName: Name of the site for DIRAC in the form GRID.LOCATION.COUNTRY "
"(ie: LCG.CERN.ch)")
Script.registerArgument(
"GridSiteName: Name of the site in the Grid (ie: CERN-PROD)")
Script.registerArgument([
"CE: Name of the CE to be included in the site (ie: ce111.cern.ch)"
])
Script.parseCommandLine(ignoreErrors=True)
diracSiteName, gridSiteName, ces = Script.getPositionalArgs(group=True)
try:
diracGridType, place, country = diracSiteName.split(".")
except ValueError:
gLogger.error(
"The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch"
)
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result["OK"] and result["Value"]:
if len(result["Value"]) > 1:
gLogger.notice(
"%s GOC site name is associated with several DIRAC sites:" %
gridSiteName)
for i, dSite in enumerate(result["Value"]):
gLogger.notice("%d: %s" % (i, dSite))
inp = six.moves.input("Enter your choice number: ")
try:
inp = int(inp)
except ValueError:
gLogger.error("You should enter an integer number")
DIRACExit(-1)
if 0 <= inp < len(result["Value"]):
diracCSSite = result["Value"][inp]
else:
gLogger.error("Number out of range: %d" % inp)
DIRACExit(-1)
else:
diracCSSite = result["Value"][0]
if diracCSSite == diracSiteName:
gLogger.notice("Site with GOC name %s is already defined as %s" %
(gridSiteName, diracSiteName))
newSite = False
else:
gLogger.error(
"ERROR: Site with GOC name %s is already defined as %s" %
(gridSiteName, diracCSSite))
DIRACExit(-1)
else:
gLogger.error("ERROR getting DIRAC site name of %s" % gridSiteName,
result.get("Message"))
csAPI = CSAPI()
if newSite:
gLogger.notice("Site to CS: %s" % diracSiteName)
res = csAPI.addSite(diracSiteName, {"Name": gridSiteName})
if not res["OK"]:
gLogger.error("Failed adding site to CS", res["Message"])
DIRACExit(1)
res = csAPI.commit()
if not res["OK"]:
gLogger.error("Failure committing to CS", res["Message"])
DIRACExit(3)
for ce in ces:
gLogger.notice("Adding CE %s" % ce)
res = csAPI.addCEtoSite(diracSiteName, ce)
if not res["OK"]:
gLogger.error("Failed adding CE %s to CS" % ce, res["Message"])
DIRACExit(2)
res = csAPI.commit()
if not res["OK"]:
gLogger.error("Failure committing to CS", res["Message"])
DIRACExit(3)
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
Script.showHelp(exitCode=1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split('.')
except ValueError:
gLogger.error(
"The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch"
)
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result['OK'] and result['Value']:
if len(result['Value']) > 1:
gLogger.notice(
'%s GOC site name is associated with several DIRAC sites:' %
gridSiteName)
for i, dSite in enumerate(result['Value']):
gLogger.notice('%d: %s' % (i, dSite))
inp = six.moves.input('Enter your choice number: ')
try:
inp = int(inp)
except ValueError:
gLogger.error('You should enter an integer number')
DIRACExit(-1)
if 0 <= inp < len(result['Value']):
diracCSSite = result['Value'][inp]
else:
gLogger.error('Number out of range: %d' % inp)
DIRACExit(-1)
else:
diracCSSite = result['Value'][0]
if diracCSSite == diracSiteName:
gLogger.notice('Site with GOC name %s is already defined as %s' %
(gridSiteName, diracSiteName))
newSite = False
else:
gLogger.error(
'ERROR: Site with GOC name %s is already defined as %s' %
(gridSiteName, diracCSSite))
DIRACExit(-1)
else:
gLogger.error("ERROR getting DIRAC site name of %s" % gridSiteName,
result.get('Message'))
csAPI = CSAPI()
if newSite:
gLogger.notice("Site to CS: %s" % diracSiteName)
res = csAPI.addSite(diracSiteName, {"Name": gridSiteName})
if not res['OK']:
gLogger.error("Failed adding site to CS", res['Message'])
DIRACExit(1)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Failure committing to CS", res['Message'])
DIRACExit(3)
for ce in ces:
gLogger.notice("Adding CE %s" % ce)
res = csAPI.addCEtoSite(diracSiteName, ce)
if not res['OK']:
gLogger.error("Failed adding CE %s to CS" % ce, res['Message'])
DIRACExit(2)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Failure committing to CS", res['Message'])
DIRACExit(3)
def getSiteUpdates(vo, bdiiInfo=None, log=None, glue2=True):
""" Get all the necessary updates for the already defined sites and CEs
:param str vo: VO name
:param dict bdiiInfo: information from DBII
:param object log: logger
:result: S_OK(set)/S_ERROR()
"""
def addToChangeSet(entry, changeSet):
""" Inner function to update changeSet with entry (a tuple)
:param tuple entry: entry to add to changeSet
:param set changeSet: set collecting stuff to change
"""
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo, glue2=glue2)
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
for site in ceBdiiDict:
result = getDIRACSiteName(site)
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath('/Resources', 'Sites',
siteName.split('.')[0], siteName)
result = gConfig.getOptionsDict(siteSection)
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get('Coordinates', 'Unknown')
mail = siteDict.get('Mail', 'Unknown').replace(' ', '')
description = siteDict.get('Description', 'Unknown')
description = description.replace(' ,', ',')
longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip()
latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % (longitude, latitude)
newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact',
'').replace('mailto:', '').strip()
newdescription = ceBdiiDict[site].get('GlueSiteDescription',
'').strip()
newdescription = ", ".join(
[line.strip() for line in newdescription.split(",")])
# Adding site data to the changes list
addToChangeSet((siteSection, 'Coordinates', coor, newcoor),
changeSet)
addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet)
addToChangeSet(
(siteSection, 'Description', description, newdescription),
changeSet)
ces = gConfig.getSections(cfgPath(siteSection, 'CEs'))
for ce in ces.get('Value', []):
ceSection = cfgPath(siteSection, 'CEs', ce)
ceDict = {}
result = gConfig.getOptionsDict(ceSection)
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get(ce, None):
log.notice("Adding new CE",
"%s to site %s/%s" % (ce, siteName, site))
ceInfo = ceBdiiDict[site]['CEs'].get(ce, None)
if ceInfo is None:
ceType = ceDict.get('CEType', '')
continue
# Current CS CE info
arch = ceDict.get('architecture', 'Unknown')
OS = ceDict.get('OS', 'Unknown')
si00 = ceDict.get('SI00', 'Unknown')
ceType = ceDict.get('CEType', 'Unknown')
ram = ceDict.get('MaxRAM', 'Unknown')
submissionMode = ceDict.get('SubmissionMode', 'Unknown')
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get(
'GlueHostArchitecturePlatformType', '').strip()
systemName = ceInfo.get('GlueHostOperatingSystemName',
'').strip()
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion',
'').strip()
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease',
'').strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join(
(systemName, systemVersion, systemRelease))
newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get('GlueCEImplementationName',
'').strip()
if newCEType:
break
if newCEType == 'ARC-CE':
newCEType = 'ARC'
newSubmissionMode = None
if newCEType in ['ARC', 'CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip()
# Protect from unreasonable values
if newRAM and int(newRAM) > 150000:
newRAM = ''
# Adding CE data to the change list
addToChangeSet((ceSection, 'architecture', arch, newarch),
changeSet)
addToChangeSet((ceSection, 'OS', OS, newOS), changeSet)
addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet)
addToChangeSet((ceSection, 'CEType', ceType, newCEType),
changeSet)
addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet)
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet((ceSection, 'SubmissionMode',
submissionMode, newSubmissionMode),
changeSet)
for queue, queueInfo in ceInfo['Queues'].items():
queueStatus = queueInfo['GlueCEStateStatus']
queueSection = cfgPath(ceSection, 'Queues', queue)
queueDict = {}
result = gConfig.getOptionsDict(queueSection)
if result['OK']:
queueDict = result['Value']
else:
if queueStatus.lower() == "production":
log.notice("Adding new queue",
"%s to CE %s" % (queue, ce))
else:
continue
# Current CS queue info
maxCPUTime = queueDict.get('maxCPUTime', 'Unknown')
si00 = queueDict.get('SI00', 'Unknown')
maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown')
# Current BDII queue info
newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '')
if newMaxCPUTime == "4" * len(
newMaxCPUTime) or newMaxCPUTime == "9" * len(
newMaxCPUTime):
newMaxCPUTime = ''
wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime',
'')
if wallTime == "4" * len(
wallTime) or wallTime == "9" * len(wallTime):
wallTime = ''
if wallTime and int(wallTime) > 0:
if not newMaxCPUTime:
newMaxCPUTime = str(int(0.8 * int(wallTime)))
else:
if int(wallTime) <= int(newMaxCPUTime):
newMaxCPUTime = str(int(0.8 * int(wallTime)))
newSI00 = ''
caps = queueInfo.get('GlueCECapability', [])
if isinstance(caps, six.string_types):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split('=')[-1]
# tags, processors, localCEType
tag = queueDict.get('Tag', '')
# LocalCEType can be empty (equivalent to "InProcess")
# or "Pool", "Singularity", but also "Pool/Singularity"
localCEType = queueDict.get('LocalCEType', '')
try:
localCEType_inner = localCEType.split('/')[1]
except IndexError:
localCEType_inner = ''
numberOfProcessors = int(
queueDict.get('NumberOfProcessors', 0))
newNOP = int(queueInfo.get('NumberOfProcessors', 1))
# Adding queue info to the CS
addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime,
newMaxCPUTime), changeSet)
addToChangeSet((queueSection, 'SI00', si00, newSI00),
changeSet)
if newNOP != numberOfProcessors:
addToChangeSet((queueSection, 'NumberOfProcessors',
numberOfProcessors, newNOP), changeSet)
if newNOP > 1:
# if larger than one, add MultiProcessor to site tags, and LocalCEType=Pool
newTag = ','.join(
sorted(
set(tag.split(',')).union(
{'MultiProcessor'}))).strip(',')
addToChangeSet((queueSection, 'Tag', tag, newTag),
changeSet)
if localCEType_inner:
newLocalCEType = 'Pool/' + localCEType_inner
else:
newLocalCEType = 'Pool'
addToChangeSet((queueSection, 'LocalCEType',
localCEType, newLocalCEType),
changeSet)
else:
# if not larger than one, drop MultiProcessor Tag.
# Here we do not change the LocalCEType as Pool CE would still be perfectly valid.
newTag = ','.join(
sorted(
set(tag.split(',')).difference(
{'MultiProcessor'}))).strip(',')
changeSet.add((queueSection, 'Tag', tag, newTag))
if maxTotalJobs == "Unknown":
newTotalJobs = min(
1000,
int(
int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) /
2))
newWaitingJobs = max(2, int(newTotalJobs * 0.1))
newTotalJobs = str(newTotalJobs)
newWaitingJobs = str(newWaitingJobs)
addToChangeSet(
(queueSection, 'MaxTotalJobs', '', newTotalJobs),
changeSet)
addToChangeSet((queueSection, 'MaxWaitingJobs', '',
newWaitingJobs), changeSet)
# Updating eligible VO list
VOs = set()
if queueDict.get('VO', ''):
VOs = set([
q.strip()
for q in queueDict.get('VO', '').split(',') if q
])
if vo not in VOs:
VOs.add(vo)
VOs = list(VOs)
newVOs = ','.join(VOs)
addToChangeSet((queueSection, 'VO', '', newVOs),
changeSet)
return S_OK(changeSet)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
Script.showHelp(exitCode=1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split('.')
except ValueError:
gLogger.error("The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch")
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result['OK'] and result['Value']:
if len(result['Value']) > 1:
gLogger.notice('%s GOC site name is associated with several DIRAC sites:' % gridSiteName)
for i, dSite in enumerate(result['Value']):
gLogger.notice('%d: %s' % (i, dSite))
inp = raw_input('Enter your choice number: ')
try:
inp = int(inp)
except ValueError:
gLogger.error('You should enter an integer number')
DIRACExit(-1)
if 0 <= inp < len(result['Value']):
diracCSSite = result['Value'][inp]
else:
def checkUnusedCEs():
global vo, dry, ceBdiiDict, hostURL
gLogger.notice("looking for new computing resources in the BDII database...")
res = getQueues(community=vo)
if not res["OK"]:
gLogger.error("ERROR: failed to get CEs from CS", res["Message"])
DIRACExit(-1)
knownCEs = set()
for _site, ces in res["Value"].items():
knownCEs.update(ces)
result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL)
if not result["OK"]:
gLogger.error("ERROR: failed to get CEs from BDII", result["Message"])
DIRACExit(-1)
ceBdiiDict = result["BdiiInfo"]
unknownCEs = result["UnknownCEs"]
if unknownCEs:
gLogger.notice("There is no (longer) information about the following CEs for the %s VO:" % vo)
gLogger.notice("\n".join(sorted(unknownCEs)))
siteDict = result["Value"]
if siteDict:
gLogger.notice("New resources available:")
for site in siteDict:
diracSite = "Unknown"
result = getDIRACSiteName(site)
if result["OK"]:
diracSite = ",".join(result["Value"])
if siteDict[site]:
gLogger.notice(" %s, DIRAC site %s" % (site, diracSite))
for ce in siteDict[site]:
gLogger.notice(" " * 4 + ce)
gLogger.notice(
" %s, %s" % (siteDict[site][ce]["CEType"], "%s_%s_%s" % siteDict[site][ce]["System"])
)
else:
gLogger.notice("No new resources available, exiting")
return
inp = six.moves.input("\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if inp and inp.lower().startswith("n"):
return
gLogger.notice("\nAdding new sites/CEs interactively\n")
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ""
for ce in siteDict[site]:
country = ce.strip().split(".")[-1].lower()
if len(country) == 2:
break
if country == "gov":
country = "us"
break
if not country or len(country) != 2:
country = "xx"
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site)
diracSite = six.moves.input("[help|skip|<domain>.<name>.%s]: " % country)
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice("%s site details:" % site)
for k, v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice("%s\t%s" % (k, v))
gLogger.notice("\nEnter DIRAC site name in the form <domain>.<name>.%s\n" % country)
diracSite = six.moves.input("[<domain>.<name>.%s]: " % country)
try:
_, _, _ = diracSite.split(".")
except ValueError:
gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite)
continue
diracSites = [diracSite]
else:
diracSites = result["Value"]
if len(diracSites) > 1:
gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site)
gLogger.notice(str(diracSites))
gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n")
newCEs = {}
addedCEs = []
for ce in siteDict[site]:
ceType = siteDict[site][ce]["CEType"]
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = six.moves.input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite))
if yn == "" or yn.lower().startswith("y"):
newCEs.setdefault(diracSite, [])
newCEs[diracSite].append(ce)
addedCEs.append(ce)
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, " ".join(newCEs[diracSite]))
gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd)
yn = six.moves.input("Add it ? [default yes] [yes|no]: ")
if not (yn == "" or yn.lower().startswith("y")):
continue
if dry:
gLogger.notice("Command is skipped in the dry run")
else:
result = systemCall(0, shlex.split(cmd))
if not result["OK"]:
gLogger.error("Error while executing dirac-admin-add-site command")
yn = six.moves.input("Do you want to continue ? [default no] [yes|no]: ")
if yn == "" or yn.lower().startswith("n"):
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
exitStatus, stdData, errData = result["Value"]
if exitStatus:
gLogger.error(
"Error while executing dirac-admin-add-site command\n", "\n".join([stdData, errData])
)
yn = six.moves.input("Do you want to continue ? [default no] [yes|no]: ")
if yn == "" or yn.lower().startswith("n"):
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
sitesAdded.append((site, diracSite))
gLogger.notice(stdData)
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
else:
gLogger.notice("No new CEs were added this time")