def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get(
'GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get(
'GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def __lookForNewSEs( self ):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption( 'BannedSEs', [] )
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set( result['Value'] )
knownSEs = knownSEs.union( set( bannedSEs ) )
for vo in self.voName:
result = self.__getBdiiSEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs( vo, bdiiInfo = bdiiInfo, srmBlackList = knownSEs )
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set( siteDict[site].keys() )
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % ( se, site )
backend = siteDict[site][se]['SE'].get( 'GlueSEImplementationName', 'Unknown' )
size = siteDict[site][se]['SE'].get( 'GlueSESizeTotal', 'Unknown' )
body += ' Backend %s, Size %s' % ( backend, size )
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result['OK']:
gLogger.error('Failed to look up SRMs in BDII', result['Message'])
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set(result['Value'])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get('GlueServiceVersion', '')
if not (version and version.startswith('2')):
gLogger.debug('Skipping SRM service with version %s' % version)
continue
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice('Unused se %s is detected at unused site %s' % (gridSE, site))
gLogger.notice('Consider adding site %s to the DIRAC CS' % site)
continue
diracSites = result['Value']
yn = raw_input(
'\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' %
(gridSE, str(diracSites)))
if not yn or yn.lower().startswith('y'):
if len(diracSites) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate(diracSites):
prompt += '\n[%d] %s' % (i, s)
prompt += '\nEnter your choice number: '
inp = raw_input(prompt)
try:
ind = int(inp)
except BaseException:
gLogger.notice('Can not interpret your choice: %s, try again later' % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split('.')
recName = '%s-disk' % siteName
inp = raw_input('Give a DIRAC name to the grid SE %s, default %s : ' % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice('Adding new SE %s at site %s' % (diracSEName, diracSite))
seSection = cfgPath('/Resources/StorageElements', diracSEName)
changeSet.add((seSection, 'BackendType', seDict.get('GlueSEImplementationName', 'Unknown')))
changeSet.add((seSection, 'Description', seDict.get('GlueSEName', 'Unknown')))
bdiiVOs = set([re.sub('^VO:', '', rule) for rule in srmDict.get('GlueServiceAccessControlBaseRule', [])])
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, 'VO', ','.join(seVOs)))
accessSection = cfgPath(seSection, 'AccessProtocol.1')
changeSet.add((accessSection, 'Protocol', 'srm'))
changeSet.add((accessSection, 'PluginName', 'SRM2'))
endPoint = srmDict.get('GlueServiceEndpoint', '')
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, 'Host', host))
changeSet.add((accessSection, 'Port', port))
changeSet.add((accessSection, 'Access', 'remote'))
voPathSection = cfgPath(accessSection, 'VOPath')
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice('\n!!! Warning: non-conventional VO path: %s\n' % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = '.'.join(host.split('.')[-2:])
path = '/dpm/%s/home' % domain
changeSet.add((accessSection, 'Path', path))
changeSet.add((accessSection, 'SpaceToken', ''))
changeSet.add((accessSection, 'WSUrl', '/srm/managerv2?SFN='))
gLogger.notice('SE %s will be added with the following parameters' % diracSEName)
changeList = sorted(changeSet)
for entry in changeList:
gLogger.notice(entry)
yn = raw_input('Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName)
if not yn or yn.lower().startswith('y'):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice('Skipping commit of the new SE data in a dry run')
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object', result['Message'])
DIRACExit(-1)
changeList = sorted(changeSetFull)
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input('New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ')
if not yn or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs( vo, unUsed = True )
if not result['OK']:
gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] )
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set( result['Value'] )
else:
csVOs = set( [vo] )
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get( 'GlueServiceVersion', '' )
if not ( version and version.startswith( '2' ) ):
gLogger.debug( 'Skipping SRM service with version %s' % version )
continue
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) )
gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site )
continue
diracSites = result['Value']
yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
if not yn or yn.lower().startswith( 'y' ):
if len( diracSites ) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate( diracSites ):
prompt += '\n[%d] %s' % ( i, s )
prompt += '\nEnter your choice number: '
inp = raw_input( prompt )
try:
ind = int( inp )
except:
gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split( '.' )
recName = '%s-disk' % siteName
inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )
seSection = cfgPath( '/Resources/StorageElements', diracSEName )
changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )
bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
seVOs = csVOs.intersection( bdiiVOs )
changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
accessSection = cfgPath( seSection, 'AccessProtocol.1' )
changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
result = pfnparse( endPoint )
if not result['OK']:
gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
continue
host = result['Value']['Host']
port = result['Value']['Port']
changeSet.add( ( accessSection, 'Host', host ) )
changeSet.add( ( accessSection, 'Port', port ) )
changeSet.add( ( accessSection, 'Access', 'remote' ) )
voPathSection = cfgPath( accessSection, 'VOPath' )
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename( path )
if voFromPath != diracVO:
gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
changeSet.add( ( voPathSection, diracVO, path ) )
path = os.path.dirname( path )
else:
# Try to guess the Path
domain = '.'.join( host.split( '.' )[-2:] )
path = '/dpm/%s/home' % domain
changeSet.add( ( accessSection, 'Path', path ) )
changeSet.add( ( accessSection, 'SpaceToken', '' ) )
changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) )
gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
changeList = list( changeSet )
changeList.sort()
for entry in changeList:
gLogger.notice( entry )
yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )
if not yn or yn.lower().startswith( 'y' ):
changeSetFull = changeSetFull.union( changeSet )
if dry:
if changeSetFull:
gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
changeList = list( changeSetFull )
changeList.sort()
for section, option, value in changeList:
csAPI.setOption( cfgPath( section, option ), value )
yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
if not yn or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result["OK"]:
gLogger.error("Failed to look up SRMs in BDII", result["Message"])
siteSRMDict = result["Value"]
# Evaluate VOs
result = getVOs()
if result["OK"]:
csVOs = set(result["Value"])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]["SE"]
srmDict = siteSRMDict[site][gridSE]["SRM"]
# Check the SRM version
version = srmDict.get("GlueServiceVersion", "")
if not (version and version.startswith("2")):
gLogger.debug("Skipping SRM service with version %s" % version)
continue
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("Unused se %s is detected at unused site %s" % (gridSE, site))
gLogger.notice("Consider adding site %s to the DIRAC CS" % site)
continue
diracSites = result["Value"]
yn = raw_input(
"\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites))
)
if not yn or yn.lower().startswith("y"):
if len(diracSites) > 1:
prompt = "Which DIRAC site the new SE should be attached to ?"
for i, s in enumerate(diracSites):
prompt += "\n[%d] %s" % (i, s)
prompt += "\nEnter your choice number: "
inp = raw_input(prompt)
try:
ind = int(inp)
except:
gLogger.notice("Can not interpret your choice: %s, try again later" % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split(".")
recName = "%s-disk" % siteName
inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite))
seSection = cfgPath("/Resources/StorageElements", diracSEName)
changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown")))
changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown")))
bdiiVOs = set(
[re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])]
)
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, "VO", ",".join(seVOs)))
accessSection = cfgPath(seSection, "AccessProtocol.1")
changeSet.add((accessSection, "Protocol", "srm"))
changeSet.add((accessSection, "PluginName", "SRM2"))
endPoint = srmDict.get("GlueServiceEndpoint", "")
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, "Host", host))
changeSet.add((accessSection, "Port", port))
changeSet.add((accessSection, "Access", "remote"))
voPathSection = cfgPath(accessSection, "VOPath")
if "VOPath" in seDict:
path = seDict["VOPath"]
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = ".".join(host.split(".")[-2:])
path = "/dpm/%s/home" % domain
changeSet.add((accessSection, "Path", path))
changeSet.add((accessSection, "SpaceToken", ""))
changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN="))
gLogger.notice("SE %s will be added with the following parameters" % diracSEName)
changeList = list(changeSet)
changeList.sort()
for entry in changeList:
gLogger.notice(entry)
yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName)
if not yn or yn.lower().startswith("y"):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice("Skipping commit of the new SE data in a dry run")
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result["OK"]:
gLogger.error("Failed to initialize CSAPI object", result["Message"])
DIRACExit(-1)
changeList = list(changeSetFull)
changeList.sort()
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ")
if not yn or yn.lower().startswith("y"):
result = csAPI.commit()
if not result["OK"]:
gLogger.error("Error while commit to CS", result["Message"])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
