def doCommand( self ):
'''
It returns the status of a given CE.
:return:
a dictionary with status of each CE queues,
and 'status' and 'reason' of the CE itself
'''
## INPUT PARAMETERS
vos = getVOs()
if vos[ 'OK' ]:
vo = vos['Value'].pop()
else:
return S_ERROR( "No appropriate VO was found! %s" % vos['Message'] )
if 'ce' not in self.args:
return S_ERROR( "No computing element 'ce' has been specified!" )
else:
ce = self.args['ce']
host = self.args.get('host')
#getting BDII info
diracAdmin = DiracAdmin()
ceQueues = diracAdmin.getBDIICEState( ce,
useVO = vo,
host = host )
if not ceQueues['OK']:
return S_ERROR( '"CE" not found on BDII' )
elements = ceQueues['Value']
#extracting the list of CE queues and their status
result = {}
for element in elements:
queue = element.get('GlueCEUniqueID','Unknown') #pylint: disable=no-member
statusQueue = element.get('GlueCEStateStatus','Unknown') #pylint: disable=no-member
result[queue] = statusQueue.capitalize()
#establishing the status of the CE itself
result['Status'] = 'Production'
result['Reason'] = "All queues in 'Production'"
for key, value in result.items():
#warning: it may not be the case that all queues for a given CE
#show the same status. In case of mismatch, the status of the CE
#will be associated to a non-production status
if key not in ['Status', 'Reason'] and value != 'Production':
result['Status'] = value
result['Reason'] = "Queue %s is in status %s" % ( queue, value )
return S_OK( result )
def initialize(self):
"""agent's initialisation
:param self: self reference
"""
self.log.info("Starting RucioRSSAgent")
# CA location
self.caCertPath = Locations.getCAsLocation()
# configured VOs
res = getVOs()
if res["OK"]:
voList = getVOs().get("Value", [])
else:
return S_ERROR(res["Message"])
# configHelper test
if isinstance(voList, str):
voList = [voList]
self.clientConfig = configHelper(voList)
self.log.debug(" VO-specific Rucio Client config parameters: ",
self.clientConfig)
return S_OK()
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs',
self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def getGridVOs():
""" Get all the VOMS VO names served by this DIRAC service
"""
voNames = []
result = getVOs()
if not result['OK']:
return result
else:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
voNames.append(vomsVO)
return S_OK(voNames)
def doCommand(self):
'''
It returns the status of a given CE.
:return:
a dictionary with status of each CE queues,
and 'status' and 'reason' of the CE itself
'''
## INPUT PARAMETERS
vos = getVOs()
if vos['OK']:
vo = vos['Value'].pop()
else:
return S_ERROR("No appropriate VO was found! %s" % vos['Message'])
if 'ce' not in self.args:
return S_ERROR("No computing element 'ce' has been specified!")
else:
ce = self.args['ce']
host = self.args.get('host')
#getting BDII info
diracAdmin = DiracAdmin()
ceQueues = diracAdmin.getBDIICEState(ce, useVO=vo, host=host)
if not ceQueues['OK']:
return S_ERROR('"CE" not found on BDII')
elements = ceQueues['Value']
#extracting the list of CE queues and their status
result = {}
for element in elements:
queue = element.get('GlueCEUniqueID', 'Unknown') #pylint: disable=no-member
statusQueue = element.get('GlueCEStateStatus', 'Unknown') #pylint: disable=no-member
result[queue] = statusQueue.capitalize()
#establishing the status of the CE itself
result['Status'] = 'Production'
result['Reason'] = "All queues in 'Production'"
for key, value in result.items():
#warning: it may not be the case that all queues for a given CE
#show the same status. In case of mismatch, the status of the CE
#will be associated to a non-production status
if key not in ['Status', 'Reason'] and value != 'Production':
result['Status'] = value
result['Reason'] = "Queue %s is in status %s" % (queue, value)
return S_OK(result)
def getGridVOs():
""" Get all the VOMS VO names served by this DIRAC service
"""
voNames = []
result = getVOs()
if not result['OK']:
return result
else:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
voNames.append(vomsVO)
return S_OK(voNames)
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def initialize(self):
"""Gets run paramaters from the configuration"""
self.addressTo = self.am_getOption("MailTo", self.addressTo)
self.addressFrom = self.am_getOption("MailFrom", self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs",
self.alternativeBDIIs)
self.host = self.am_getOption("Host", self.host)
self.injectSingleCoreQueues = self.am_getOption(
"InjectSingleCoreQueues", self.injectSingleCoreQueues)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(":")[-1].isdigit():
self.alternativeBDIIs[index] += ":2170"
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption("ProcessCEs", self.processCEs)
self.selectedSites = self.am_getOption("SelectedSites", [])
self.dryRun = self.am_getOption("DryRun", self.dryRun)
self.voName = self.am_getOption("VirtualOrganization", self.voName)
if not self.voName:
self.voName = self.am_getOption("VO", [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == "all"):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result["OK"]:
vos = result["Value"]
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def getGridVOs():
"""Get all the VOMS VO names served by this DIRAC service
:return: S_OK(list)/S_ERROR()
"""
voNames = []
result = getVOs()
if not result["OK"]:
return result
else:
vos = result["Value"]
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
voNames.append(vomsVO)
return S_OK(voNames)
def initialize(self):
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', [])
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "Bdii2CSAgent"
self.processCEs = self.am_getOption('ProcessCEs', True)
self.processSEs = self.am_getOption('ProcessSEs', False)
self.voName = self.am_getOption('VirtualOrganization', [])
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = CSAPI()
return self.csAPI.initialize()
def initialize( self ):
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
self.processCEs = self.am_getOption( 'ProcessCEs', True )
self.processSEs = self.am_getOption( 'ProcessSEs', False )
self.voName = self.am_getOption( 'VirtualOrganization', [] )
if not self.voName:
self.voName = self.am_getOption( 'VO', [] )
if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption( vo, "VOMSName" )
if vomsVO:
self.voName.append( vomsVO )
if self.voName:
self.log.info( "Agent will manage VO(s) %s" % self.voName )
else:
self.log.fatal( "VirtualOrganization option not defined for agent" )
return S_ERROR()
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = CSAPI()
return self.csAPI.initialize()
def parseEduperson(self, claimDict):
"""Parse eduperson claims
:return: dict
"""
vos = {}
credDict = {}
attributes = {
"eduperson_unique_id": "^(?P<ID>.*)",
"eduperson_entitlement": "%s:%s"
% (
"^(?P<NAMESPACE>[A-z,.,_,-,:]+):(group:registry|group)",
"(?P<VO>[A-z,.,_,-]+):role=(?P<VORole>[A-z,.,_,-]+)[:#].*",
),
}
self.log.debug("Try to parse eduperson claims..")
# Parse eduperson claims
resDict = claimParser(claimDict, attributes)
if resDict.get("eduperson_unique_id"):
self.log.debug("Found eduperson_unique_id claim:", pprint.pformat(resDict["eduperson_unique_id"]))
credDict["ID"] = resDict["eduperson_unique_id"]["ID"]
if resDict.get("eduperson_entitlement"):
self.log.debug("Found eduperson_entitlement claim:", pprint.pformat(resDict["eduperson_entitlement"]))
for voDict in resDict["eduperson_entitlement"]:
if voDict["VO"] not in vos:
vos[voDict["VO"]] = {"VORoles": []}
if voDict["VORole"] not in vos[voDict["VO"]]["VORoles"]:
vos[voDict["VO"]]["VORoles"].append(voDict["VORole"])
allowedVOs = getVOs()["Value"] # Always return S_OK()
# Search DIRAC groups
for vo in vos:
# Skip VO if it absent in Registry
if vo in allowedVOs and (result := getVOMSRoleGroupMapping(vo))["OK"]:
for role in vos[vo]["VORoles"]:
groups = result["Value"]["VOMSDIRAC"].get("/%s/%s" % (vo, role))
if groups:
credDict["DIRACGroups"] = list(set(credDict.get("DIRACGroups", []) + groups))
def getSRMUpdates(vo, bdiiInfo=None):
changeSet = set()
def addToChangeSet(entry, changeSet):
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
result = getGridSRMs(vo, bdiiInfo=bdiiInfo)
if not result['OK']:
return result
srmBdiiDict = result['Value']
result = getSEsFromCS()
if not result['OK']:
return result
seDict = result['Value']
result = getVOs()
if result['OK']:
csVOs = set(result['Value'])
else:
csVOs = set([vo])
for seHost, diracSE in seDict.items():
seSection = '/Resources/StorageElements/%s' % diracSE[0]
# Look up existing values first
description = gConfig.getValue(cfgPath(seSection, 'Description'), 'Unknown')
backend = gConfig.getValue(cfgPath(seSection, 'BackendType'), 'Unknown')
vos = gConfig.getValue(cfgPath(seSection, 'VO'), 'Unknown').replace(' ', '')
size = gConfig.getValue(cfgPath(seSection, 'TotalSize'), 'Unknown')
# Look up current BDII values
srmDict = {}
seBdiiDict = {}
for site in srmBdiiDict:
if seHost in srmBdiiDict[site]:
srmDict = srmBdiiDict[site][seHost]['SRM']
seBdiiDict = srmBdiiDict[site][seHost]['SE']
break
if not srmDict or not seBdiiDict:
continue
newDescription = seBdiiDict.get('GlueSEName', 'Unknown')
newBackend = seBdiiDict.get('GlueSEImplementationName', 'Unknown')
newSize = seBdiiDict.get('GlueSESizeTotal', 'Unknown')
addToChangeSet((seSection, 'Description', description, newDescription), changeSet)
addToChangeSet((seSection, 'BackendType', backend, newBackend), changeSet)
addToChangeSet((seSection, 'TotalSize', size, newSize), changeSet)
# Evaluate VOs if no space token defined, otherwise this is VO specific
spaceToken = ''
for i in range(1, 10):
protocol = gConfig.getValue(cfgPath(seSection, 'AccessProtocol.%d' % i, 'Protocol'), '')
if protocol.lower() == 'srm':
spaceToken = gConfig.getValue(cfgPath(seSection, 'AccessProtocol.%d' % i, 'SpaceToken'), '')
break
if not spaceToken:
bdiiVOs = srmDict.get('GlueServiceAccessControlBaseRule', [])
bdiiVOs = set([re.sub('^VO:', '', rule) for rule in bdiiVOs])
seVOs = csVOs.intersection(bdiiVOs)
newVOs = ','.join(seVOs)
addToChangeSet((seSection, 'VO', vos, newVOs), changeSet)
return S_OK(changeSet)
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result['OK']:
gLogger.error('Failed to look up SRMs in BDII', result['Message'])
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set(result['Value'])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get('GlueServiceVersion', '')
if not (version and version.startswith('2')):
gLogger.debug('Skipping SRM service with version %s' % version)
continue
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice('Unused se %s is detected at unused site %s' % (gridSE, site))
gLogger.notice('Consider adding site %s to the DIRAC CS' % site)
continue
diracSites = result['Value']
yn = raw_input(
'\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' %
(gridSE, str(diracSites)))
if not yn or yn.lower().startswith('y'):
if len(diracSites) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate(diracSites):
prompt += '\n[%d] %s' % (i, s)
prompt += '\nEnter your choice number: '
inp = raw_input(prompt)
try:
ind = int(inp)
except BaseException:
gLogger.notice('Can not interpret your choice: %s, try again later' % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split('.')
recName = '%s-disk' % siteName
inp = raw_input('Give a DIRAC name to the grid SE %s, default %s : ' % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice('Adding new SE %s at site %s' % (diracSEName, diracSite))
seSection = cfgPath('/Resources/StorageElements', diracSEName)
changeSet.add((seSection, 'BackendType', seDict.get('GlueSEImplementationName', 'Unknown')))
changeSet.add((seSection, 'Description', seDict.get('GlueSEName', 'Unknown')))
bdiiVOs = set([re.sub('^VO:', '', rule) for rule in srmDict.get('GlueServiceAccessControlBaseRule', [])])
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, 'VO', ','.join(seVOs)))
accessSection = cfgPath(seSection, 'AccessProtocol.1')
changeSet.add((accessSection, 'Protocol', 'srm'))
changeSet.add((accessSection, 'PluginName', 'SRM2'))
endPoint = srmDict.get('GlueServiceEndpoint', '')
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, 'Host', host))
changeSet.add((accessSection, 'Port', port))
changeSet.add((accessSection, 'Access', 'remote'))
voPathSection = cfgPath(accessSection, 'VOPath')
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice('\n!!! Warning: non-conventional VO path: %s\n' % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = '.'.join(host.split('.')[-2:])
path = '/dpm/%s/home' % domain
changeSet.add((accessSection, 'Path', path))
changeSet.add((accessSection, 'SpaceToken', ''))
changeSet.add((accessSection, 'WSUrl', '/srm/managerv2?SFN='))
gLogger.notice('SE %s will be added with the following parameters' % diracSEName)
changeList = sorted(changeSet)
for entry in changeList:
gLogger.notice(entry)
yn = raw_input('Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName)
if not yn or yn.lower().startswith('y'):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice('Skipping commit of the new SE data in a dry run')
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object', result['Message'])
DIRACExit(-1)
changeList = sorted(changeSetFull)
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input('New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ')
if not yn or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs( vo, unUsed = True )
if not result['OK']:
gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] )
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set( result['Value'] )
else:
csVOs = set( [vo] )
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get( 'GlueServiceVersion', '' )
if not ( version and version.startswith( '2' ) ):
gLogger.debug( 'Skipping SRM service with version %s' % version )
continue
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) )
gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site )
continue
diracSites = result['Value']
yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
if not yn or yn.lower().startswith( 'y' ):
if len( diracSites ) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate( diracSites ):
prompt += '\n[%d] %s' % ( i, s )
prompt += '\nEnter your choice number: '
inp = raw_input( prompt )
try:
ind = int( inp )
except:
gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split( '.' )
recName = '%s-disk' % siteName
inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )
seSection = cfgPath( '/Resources/StorageElements', diracSEName )
changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )
bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
seVOs = csVOs.intersection( bdiiVOs )
changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
accessSection = cfgPath( seSection, 'AccessProtocol.1' )
changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
result = pfnparse( endPoint )
if not result['OK']:
gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
continue
host = result['Value']['Host']
port = result['Value']['Port']
changeSet.add( ( accessSection, 'Host', host ) )
changeSet.add( ( accessSection, 'Port', port ) )
changeSet.add( ( accessSection, 'Access', 'remote' ) )
voPathSection = cfgPath( accessSection, 'VOPath' )
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename( path )
if voFromPath != diracVO:
gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
changeSet.add( ( voPathSection, diracVO, path ) )
path = os.path.dirname( path )
else:
# Try to guess the Path
domain = '.'.join( host.split( '.' )[-2:] )
path = '/dpm/%s/home' % domain
changeSet.add( ( accessSection, 'Path', path ) )
changeSet.add( ( accessSection, 'SpaceToken', '' ) )
changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) )
gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
changeList = list( changeSet )
changeList.sort()
for entry in changeList:
gLogger.notice( entry )
yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )
if not yn or yn.lower().startswith( 'y' ):
changeSetFull = changeSetFull.union( changeSet )
if dry:
if changeSetFull:
gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
changeList = list( changeSetFull )
changeList.sort()
for section, option, value in changeList:
csAPI.setOption( cfgPath( section, option ), value )
yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
if not yn or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
def getSRMUpdates(vo, bdiiInfo=None):
changeSet = set()
def addToChangeSet(entry, changeSet):
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
result = getGridSRMs(vo, bdiiInfo=bdiiInfo)
if not result['OK']:
return result
srmBdiiDict = result['Value']
result = getSEsFromCS()
if not result['OK']:
return result
seDict = result['Value']
result = getVOs()
if result['OK']:
csVOs = set(result['Value'])
else:
csVOs = set([vo])
for seHost, diracSE in seDict.items():
seSection = '/Resources/StorageElements/%s' % diracSE[0]
# Look up existing values first
description = gConfig.getValue(cfgPath(seSection, 'Description'),
'Unknown')
backend = gConfig.getValue(cfgPath(seSection, 'BackendType'),
'Unknown')
vos = gConfig.getValue(cfgPath(seSection, 'VO'),
'Unknown').replace(' ', '')
size = gConfig.getValue(cfgPath(seSection, 'TotalSize'), 'Unknown')
# Look up current BDII values
srmDict = {}
seBdiiDict = {}
for site in srmBdiiDict:
if seHost in srmBdiiDict[site]:
srmDict = srmBdiiDict[site][seHost]['SRM']
seBdiiDict = srmBdiiDict[site][seHost]['SE']
break
if not srmDict or not seBdiiDict:
continue
newDescription = seBdiiDict.get('GlueSEName', 'Unknown')
newBackend = seBdiiDict.get('GlueSEImplementationName', 'Unknown')
newSize = seBdiiDict.get('GlueSESizeTotal', 'Unknown')
addToChangeSet((seSection, 'Description', description, newDescription),
changeSet)
addToChangeSet((seSection, 'BackendType', backend, newBackend),
changeSet)
addToChangeSet((seSection, 'TotalSize', size, newSize), changeSet)
# Evaluate VOs if no space token defined, otherwise this is VO specific
spaceToken = ''
for i in range(1, 10):
protocol = gConfig.getValue(
cfgPath(seSection, 'AccessProtocol.%d' % i, 'Protocol'), '')
if protocol.lower() == 'srm':
spaceToken = gConfig.getValue(
cfgPath(seSection, 'AccessProtocol.%d' % i, 'SpaceToken'),
'')
break
if not spaceToken:
bdiiVOs = srmDict.get('GlueServiceAccessControlBaseRule', [])
bdiiVOs = set([re.sub('^VO:', '', rule) for rule in bdiiVOs])
seVOs = csVOs.intersection(bdiiVOs)
newVOs = ','.join(seVOs)
addToChangeSet((seSection, 'VO', vos, newVOs), changeSet)
return S_OK(changeSet)
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result["OK"]:
gLogger.error("Failed to look up SRMs in BDII", result["Message"])
siteSRMDict = result["Value"]
# Evaluate VOs
result = getVOs()
if result["OK"]:
csVOs = set(result["Value"])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]["SE"]
srmDict = siteSRMDict[site][gridSE]["SRM"]
# Check the SRM version
version = srmDict.get("GlueServiceVersion", "")
if not (version and version.startswith("2")):
gLogger.debug("Skipping SRM service with version %s" % version)
continue
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("Unused se %s is detected at unused site %s" % (gridSE, site))
gLogger.notice("Consider adding site %s to the DIRAC CS" % site)
continue
diracSites = result["Value"]
yn = raw_input(
"\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites))
)
if not yn or yn.lower().startswith("y"):
if len(diracSites) > 1:
prompt = "Which DIRAC site the new SE should be attached to ?"
for i, s in enumerate(diracSites):
prompt += "\n[%d] %s" % (i, s)
prompt += "\nEnter your choice number: "
inp = raw_input(prompt)
try:
ind = int(inp)
except:
gLogger.notice("Can not interpret your choice: %s, try again later" % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split(".")
recName = "%s-disk" % siteName
inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite))
seSection = cfgPath("/Resources/StorageElements", diracSEName)
changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown")))
changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown")))
bdiiVOs = set(
[re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])]
)
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, "VO", ",".join(seVOs)))
accessSection = cfgPath(seSection, "AccessProtocol.1")
changeSet.add((accessSection, "Protocol", "srm"))
changeSet.add((accessSection, "PluginName", "SRM2"))
endPoint = srmDict.get("GlueServiceEndpoint", "")
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, "Host", host))
changeSet.add((accessSection, "Port", port))
changeSet.add((accessSection, "Access", "remote"))
voPathSection = cfgPath(accessSection, "VOPath")
if "VOPath" in seDict:
path = seDict["VOPath"]
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = ".".join(host.split(".")[-2:])
path = "/dpm/%s/home" % domain
changeSet.add((accessSection, "Path", path))
changeSet.add((accessSection, "SpaceToken", ""))
changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN="))
gLogger.notice("SE %s will be added with the following parameters" % diracSEName)
changeList = list(changeSet)
changeList.sort()
for entry in changeList:
gLogger.notice(entry)
yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName)
if not yn or yn.lower().startswith("y"):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice("Skipping commit of the new SE data in a dry run")
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result["OK"]:
gLogger.error("Failed to initialize CSAPI object", result["Message"])
DIRACExit(-1)
changeList = list(changeSetFull)
changeList.sort()
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ")
if not yn or yn.lower().startswith("y"):
result = csAPI.commit()
if not result["OK"]:
gLogger.error("Error while commit to CS", result["Message"])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
