def __lookForNewCEs( self ):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption( 'BannedCEs', [] )
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set( result['Value'] )
knownCEs = knownCEs.union( set( bannedCEs ) )
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs )
if not result['OK']:
self.log.error( 'Failed to get unused CEs', result['Message'] )
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set( siteDict[site].keys() )
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
ceString = "CE: %s, GOCDB Site Name: %s" % ( ce, site )
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % ( systemTuple )
newCEString = "\n%s\n%s\n" % ( ceString, osString )
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' )
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' )
queueString += " %s %s %s\n" % ( queue, queueStatus, ceType )
if queueString:
ceString = newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def checkUnusedCEs():
global vo, dry, ceBdiiDict, hostURL, glue2
gLogger.notice('looking for new computing resources in the BDII database...')
result = getCEsFromCS()
if not result['OK']:
gLogger.error('ERROR: failed to get CEs from CS', result['Message'])
DIRACExit(-1)
knownCEs = result['Value']
result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL, glue2=glue2)
if not result['OK']:
gLogger.error('ERROR: failed to get CEs from BDII', result['Message'])
DIRACExit(-1)
ceBdiiDict = result['BdiiInfo']
siteDict = result['Value']
if siteDict:
gLogger.notice('New resources available:\n')
for site in siteDict:
diracSite = 'Unknown'
result = getDIRACSiteName(site)
if result['OK']:
diracSite = ','.join(result['Value'])
ces = siteDict[site].keys() # pylint: disable=no-member
if ces:
gLogger.notice(" %s, DIRAC site %s" % (site, diracSite))
for ce in ces:
gLogger.notice(' ' * 4 + ce)
gLogger.notice(' %s, %s' % (siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System']))
else:
gLogger.notice('No new resources available, exiting')
DIRACExit(0)
inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith('n'):
gLogger.notice('Nothing else to be done, exiting')
DIRACExit(0)
gLogger.notice('\nAdding new sites/CEs interactively\n')
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ''
ces = siteDict[site].keys() # pylint: disable=no-member
for ce in ces:
country = ce.strip().split('.')[-1].lower()
if len(country) == 2:
break
if country == 'gov':
country = 'us'
break
if not country or len(country) != 2:
country = 'xx'
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice('\nThe site %s is not yet in the CS, give it a name' % site)
diracSite = raw_input('[help|skip|<domain>.<name>.%s]: ' % country)
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice('%s site details:' % site)
for k, v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice('%s\t%s' % (k, v))
gLogger.notice('\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country)
diracSite = raw_input('[<domain>.<name>.%s]: ' % country)
try:
_, _, _ = diracSite.split('.')
except ValueError:
gLogger.error('ERROR: DIRAC site name does not follow convention: %s' % diracSite)
continue
diracSites = [diracSite]
else:
diracSites = result['Value']
if len(diracSites) > 1:
gLogger.notice('Attention! GOC site %s corresponds to more than one DIRAC sites:' % site)
gLogger.notice(str(diracSites))
gLogger.notice('Please, pay attention which DIRAC site the new CEs will join\n')
newCEs = {}
addedCEs = []
for ce in ces:
ceType = siteDict[site][ce]['CEType']
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = raw_input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite))
if yn == '' or yn.lower() == 'y':
newCEs.setdefault(diracSite, [])
newCEs[diracSite].append(ce)
addedCEs.append(ce)
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, ' '.join(newCEs[diracSite]))
gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd)
yn = raw_input("Add it ? [default yes] [yes|no]: ")
if not (yn == '' or yn.lower() == 'y'):
continue
if dry:
gLogger.notice("Command is skipped in the dry run")
else:
result = systemCall(0, shlex.split(cmd))
if not result['OK']:
gLogger.error('Error while executing dirac-admin-add-site command')
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice('CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
exitStatus, stdData, errData = result['Value']
if exitStatus:
gLogger.error('Error while executing dirac-admin-add-site command\n', '\n'.join([stdData, errData]))
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice('CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
sitesAdded.append((site, diracSite))
gLogger.notice(stdData)
if sitesAdded:
gLogger.notice('CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
else:
gLogger.notice('No new CEs were added this time')
def checkUnusedCEs():
global vo, dry, ceBdiiDict
gLogger.notice( 'looking for new computing resources in the BDII database...' )
result = getCEsFromCS()
if not result['OK']:
gLogger.error( 'ERROR: failed to get CEs from CS', result['Message'] )
DIRACExit( -1 )
knownCEs = result['Value']
result = getGridCEs( vo, ceBlackList = knownCEs )
if not result['OK']:
gLogger.error( 'ERROR: failed to get CEs from BDII', result['Message'] )
DIRACExit( -1 )
ceBdiiDict = result['BdiiInfo']
siteDict = result['Value']
if siteDict:
gLogger.notice( 'New resources available:\n' )
for site in siteDict:
diracSite = 'Unknown'
result = getDIRACSiteName( site )
if result['OK']:
diracSite = ','.join( result['Value'] )
ces = siteDict[site].keys()
if ces:
gLogger.notice( " %s, DIRAC site %s" % ( site, diracSite) )
for ce in ces:
gLogger.notice( ' '*4+ce )
gLogger.notice( ' %s, %s' % ( siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System'] ) )
else:
gLogger.notice( 'No new resources available, exiting' )
DIRACExit( 0 )
inp = raw_input( "\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith( 'n' ):
gLogger.notice( 'Nothing else to be done, exiting' )
DIRACExit( 0 )
gLogger.notice( '\nAdding new sites/CEs interactively\n' )
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ''
ces = siteDict[site].keys()
for ce in ces:
country = ce.strip().split('.')[-1].lower()
if len( country ) == 2:
break
if country == 'gov':
country = 'us'
break
if not country or len( country ) != 2:
country = 'xx'
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( '\nThe site %s is not yet in the CS, give it a name' % site )
diracSite = raw_input( '[help|skip|<domain>.<name>.%s]: ' % country )
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice( '%s site details:' % site )
for k,v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice( '%s\t%s' % (k,v) )
gLogger.notice( '\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country )
diracSite = raw_input( '[<domain>.<name>.%s]: ' % country )
try:
domain,siteName,country = diracSite.split('.')
except Exception, x:
gLogger.error( 'ERROR: DIRAC site name does not follow convention: %s' % diracSite )
continue
diracSites = [diracSite]
else:
diracSites = result['Value']
if len( diracSites ) > 1:
gLogger.notice( 'Attention! GOC site %s corresponds to more than one DIRAC sites:' % site )
gLogger.notice( str( diracSites ) )
gLogger.notice( 'Please, pay attention which DIRAC site the new CEs will join\n' )
newCEs = {}
addedCEs = []
for ce in ces:
ceType = siteDict[site][ce]['CEType']
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = raw_input( "Add CE %s of type %s to %s? [default yes] [yes|no]: " % ( ce, ceType, diracSite ) )
if yn == '' or yn.lower() == 'y':
newCEs.setdefault( diracSite, [] )
newCEs[diracSite].append( ce )
addedCEs.append( ce )
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % ( diracSite, site, ' '.join( newCEs[diracSite] ) )
gLogger.notice( "\nNew site/CEs will be added with command:\n%s" % cmd )
yn = raw_input( "Add it ? [default yes] [yes|no]: " )
if not ( yn == '' or yn.lower() == 'y' ) :
continue
if dry:
gLogger.notice( "Command is skipped in the dry run" )
else:
result = shellCall( 0, cmd )
if not result['OK']:
gLogger.error( 'Error while executing dirac-admin-add-site command' )
yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " )
if yn == '' or yn.lower().startswith( 'n' ):
if sitesAdded:
gLogger.notice( 'CEs were added at the following sites:' )
for site, diracSite in sitesAdded:
gLogger.notice( "%s\t%s" % ( site, diracSite ) )
DIRACExit( 0 )
else:
exitStatus, stdData, errData = result[ 'Value' ]
if exitStatus:
gLogger.error( 'Error while executing dirac-admin-add-site command\n', '\n'.join( [stdData, errData] ) )
yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " )
if yn == '' or yn.lower().startswith( 'n' ):
if sitesAdded:
gLogger.notice( 'CEs were added at the following sites:' )
for site, diracSite in sitesAdded:
gLogger.notice( "%s\t%s" % ( site, diracSite ) )
DIRACExit( 0 )
else:
sitesAdded.append( ( site, diracSite ) )
gLogger.notice( stdData )
def checkUnusedCEs():
global vo, dry, ceBdiiDict
gLogger.notice("looking for new computing resources in the BDII database...")
result = getCEsFromCS()
if not result["OK"]:
gLogger.error("ERROR: failed to get CEs from CS", result["Message"])
DIRACExit(-1)
knownCEs = result["Value"]
result = getGridCEs(vo, ceBlackList=knownCEs)
if not result["OK"]:
gLogger.error("ERROR: failed to get CEs from BDII", result["Message"])
DIRACExit(-1)
ceBdiiDict = result["BdiiInfo"]
siteDict = result["Value"]
if siteDict:
gLogger.notice("New resources available:\n")
for site in siteDict:
diracSite = "Unknown"
result = getDIRACSiteName(site)
if result["OK"]:
diracSite = ",".join(result["Value"])
ces = siteDict[site].keys()
if ces:
gLogger.notice(" %s, DIRAC site %s" % (site, diracSite))
for ce in ces:
gLogger.notice(" " * 4 + ce)
gLogger.notice(
" %s, %s" % (siteDict[site][ce]["CEType"], "%s_%s_%s" % siteDict[site][ce]["System"])
)
else:
gLogger.notice("No new resources available, exiting")
DIRACExit(0)
inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith("n"):
gLogger.notice("Nothing else to be done, exiting")
DIRACExit(0)
gLogger.notice("\nAdding new sites/CEs interactively\n")
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ""
ces = siteDict[site].keys()
for ce in ces:
country = ce.strip().split(".")[-1].lower()
if len(country) == 2:
break
if country == "gov":
country = "us"
break
if not country or len(country) != 2:
country = "xx"
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site)
diracSite = raw_input("[help|skip|<domain>.<name>.%s]: " % country)
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice("%s site details:" % site)
for k, v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice("%s\t%s" % (k, v))
gLogger.notice("\nEnter DIRAC site name in the form <domain>.<name>.%s\n" % country)
diracSite = raw_input("[<domain>.<name>.%s]: " % country)
try:
_, _, _ = diracSite.split(".")
except ValueError:
gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite)
continue
diracSites = [diracSite]
else:
diracSites = result["Value"]
if len(diracSites) > 1:
gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site)
gLogger.notice(str(diracSites))
gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n")
newCEs = {}
addedCEs = []
for ce in ces:
ceType = siteDict[site][ce]["CEType"]
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = raw_input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite))
if yn == "" or yn.lower() == "y":
newCEs.setdefault(diracSite, [])
newCEs[diracSite].append(ce)
addedCEs.append(ce)
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, " ".join(newCEs[diracSite]))
gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd)
yn = raw_input("Add it ? [default yes] [yes|no]: ")
if not (yn == "" or yn.lower() == "y"):
continue
if dry:
gLogger.notice("Command is skipped in the dry run")
else:
result = shellCall(0, cmd)
if not result["OK"]:
gLogger.error("Error while executing dirac-admin-add-site command")
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == "" or yn.lower().startswith("n"):
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
exitStatus, stdData, errData = result["Value"]
if exitStatus:
gLogger.error(
"Error while executing dirac-admin-add-site command\n", "\n".join([stdData, errData])
)
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == "" or yn.lower().startswith("n"):
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
sitesAdded.append((site, diracSite))
gLogger.notice(stdData)
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
else:
gLogger.notice("No new CEs were added this time")
