def __init__( self, useProxy=False ):
self.rootConfigPath = '/Resources/StorageElements'
self.valid = True
self.proxy = False
self.proxy = useProxy
self.resourceStatus = ResourceStatus()
def setUp(self):
result = getProxyInfo()
if result['Value']['group'] not in ['lhcb_user', 'dirac_user']:
print "GET A USER GROUP"
exit(1)
result = ResourceStatus().getElementStatus('PIC-USER',
'StorageElement',
'WriteAccess')
if result['Value']['PIC-USER']['WriteAccess'].lower() != 'banned':
print "BAN PIC-USER in writing! and then restart this test"
exit(1)
res = DataManager().getReplicas([
'/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt',
'/lhcb/user/f/fstagni/test/testInputFile.txt'
])
if not res['OK']:
print "DATAMANAGER.getRepicas failure: %s" % res['Message']
exit(1)
if res['Value']['Failed']:
print "DATAMANAGER.getRepicas failed for something: %s" % res[
'Value']['Failed']
exit(1)
replicas = res['Value']['Successful']
if replicas['/lhcb/user/f/fstagni/test/testInputFile.txt'].keys() != [
'CERN-USER', 'IN2P3-USER'
]:
print "/lhcb/user/f/fstagni/test/testInputFile.txt locations are not correct"
if replicas[
'/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt'].keys(
) != ['CERN-USER']:
print "/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt locations are not correct"
class InputDataAgent( OptimizerModule ):
"""
The specific Optimizer must provide the following methods:
- initializeOptimizer() before each execution cycle
- checkJob() - the main method called for each job
"""
#############################################################################
def initializeOptimizer( self ):
"""Initialize specific parameters for JobSanityAgent.
"""
self.failedMinorStatus = self.am_getOption( '/FailedJobStatus', 'Input Data Not Available' )
#this will ignore failover SE files
self.checkFileMetadata = self.am_getOption( 'CheckFileMetadata', True )
#Define the shifter proxy needed
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'ProductionManager' )
try:
self.replicaManager = ReplicaManager()
except Exception, e:
msg = 'Failed to create ReplicaManager'
self.log.exception( msg )
return S_ERROR( msg + str( e ) )
self.resourceStatus = ResourceStatus()
self.seToSiteMapping = {}
self.lastCScheck = 0
self.cacheLength = 600
return S_OK()
def __init__(self, useProxy=False, vo=None):
self.valid = True
self.proxy = False
self.proxy = useProxy
self.resourceStatus = ResourceStatus()
self.resourcesHelper = Resources(vo=vo)
def rssClient(cls):
"""ResourceStatusClient getter"""
if not cls.__rssClient:
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
cls.__rssClient = ResourceStatus()
return cls.__rssClient
def __init__(self):
self.rootConfigPath = '/Resources/StorageElements'
self.valid = True
self.proxy = False
res = gConfig.getOption("%s/UseProxy" % self.rootConfigPath)
if res['OK'] and (res['Value'] == 'True'):
self.proxy = True
self.resourceStatus = ResourceStatus()
def __init__(self):
"""
Constructor, initializes the rssClient.
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.rssConfig = RssConfiguration()
self.__opHelper = Operations()
self.rssFlag = ResourceStatus().rssFlag
self.rsClient = ResourceStatusClient()
def __init__( self, csPath = None, ftsHistoryViews = None ):
"""
Nothing special done here
:param csPath : path of the CS
:param ftsHistoryViews : history view of the db (useful for FTS2)
"""
self.csPath = csPath
self.ftsHistoryViews = ftsHistoryViews
self.rssStatus = ResourceStatus()
self.log = gLogger.getSubLogger( 'FTSAbstractPlacement', True )
def init_on_load(self):
""" This method initializes some attributes.
It is called by sqlalchemy (which does not call __init__)
"""
self._vo = None
self.dManager = DataManager()
self.rssClient = ResourceStatus()
opID = getattr(self, 'operationID', None)
loggerName = '%s/' % opID if opID else ''
loggerName += 'req_%s/op_%s' % (self.rmsReqID, self.rmsOpID)
self._log = gLogger.getSubLogger(loggerName, True)
def __init__(self):
"""Internal initialization of the DIRAC Admin API."""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + "/LogLevel", "DEBUG") == "DEBUG":
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + "/ScratchDir",
"/tmp")
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
def __init__(self):
"""
Constructor, initializes the rssClient.
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.rssConfig = RssConfiguration()
self.__opHelper = Operations()
self.rssFlag = ResourceStatus().rssFlag
self.rsClient = ResourceStatusClient()
cacheLifeTime = int(self.rssConfig.getConfigCache())
# RSSCache only affects the calls directed to RSS, if using the CS it is not used.
self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache)
def __init__(self, csPath=None, ftsHistoryViews=None):
"""
Call the init of the parent, and initialize the list of FTS3 servers
"""
self.log = gLogger.getSubLogger("FTS3Placement")
super(FTS3Placement, self).__init__(csPath=csPath,
ftsHistoryViews=ftsHistoryViews)
srvList = getFTS3Servers()
if not srvList['OK']:
self.log.error(srvList['Message'])
self.__serverList = srvList.get('Value', [])
self.maxAttempts = len(self.__serverList)
self.rssClient = ResourceStatus()
def initializeOptimizer(self):
"""Initialize specific parameters for JobSanityAgent.
"""
self.failedMinorStatus = self.am_getOption('/FailedJobStatus',
'Input Data Not Available')
#this will ignore failover SE files
self.checkFileMetadata = self.am_getOption('CheckFileMetadata', True)
self.dataManager = DataManager()
self.resourceStatus = ResourceStatus()
self.fc = FileCatalog()
self.seToSiteMapping = {}
self.lastCScheck = 0
self.cacheLength = 600
return S_OK()
def __init__(self, useProxy=False, vo=None):
self.proxy = False
self.proxy = useProxy
self.resourceStatus = ResourceStatus()
self.vo = vo
if self.vo is None:
result = getVOfromProxyGroup()
if result['OK']:
self.vo = result['Value']
else:
RuntimeError("Can not get the current VO context")
self.remotePlugins = []
self.localPlugins = []
self.name = ''
self.options = {}
self.protocols = {}
self.storages = []
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + '/ScratchDir',
'/tmp')
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
self._siteSet = set(getSites().get('Value', []))
def printCEInfo(voName):
resultQueues = Resources.getQueues(community=voName)
if not resultQueues['OK']:
gLogger.error('Failed to get CE information')
DIRACExit(-1)
fields = ("Site", 'CE', 'CEType', 'Queue', 'Status')
records = []
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask['OK']:
return resultMask
siteMaskList = resultMask.get('Value', [])
rssClient = ResourceStatus()
for site in resultQueues['Value']:
siteStatus = "Active" if site in siteMaskList else "InActive"
siteNew = True
for ce in resultQueues['Value'][site]:
ceStatus = siteStatus
if rssClient.rssFlag:
result = rssClient.getElementStatus(ce, "ComputingElement")
if result['OK']:
ceStatus = result['Value'][ce]['all']
ceNew = True
for queue in resultQueues['Value'][site][ce]['Queues']:
pSite = site if siteNew else ''
pCE = ''
ceType = ''
if ceNew:
pCE = ce
ceType = resultQueues['Value'][site][ce]['CEType']
records.append((pSite, pCE, ceType, queue, ceStatus))
ceNew = False
siteNew = False
gLogger.notice(
printTable(fields, records, printOut=False, columnSeparator=' '))
return S_OK()
def printCEInfo(voName):
resultQueues = Resources.getQueues(community=voName)
if not resultQueues["OK"]:
gLogger.error("Failed to get CE information")
DIRACExit(-1)
fields = ("Site", "CE", "CEType", "Queue", "Status")
records = []
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask["OK"]:
return resultMask
siteMaskList = resultMask.get("Value", [])
rssClient = ResourceStatus()
for site in resultQueues["Value"]:
siteStatus = "Active" if site in siteMaskList else "InActive"
siteNew = True
for ce in resultQueues["Value"][site]:
ceStatus = siteStatus
if rssClient.rssFlag:
result = rssClient.getElementStatus(ce, "ComputingElement")
if result["OK"]:
ceStatus = result["Value"][ce]["all"]
ceNew = True
for queue in resultQueues["Value"][site][ce]["Queues"]:
pSite = site if siteNew else ""
pCE = ""
ceType = ""
if ceNew:
pCE = ce
ceType = resultQueues["Value"][site][ce]["CEType"]
records.append((pSite, pCE, ceType, queue, ceStatus))
ceNew = False
siteNew = False
gLogger.notice(
printTable(fields, records, printOut=False, columnSeparator=" "))
return S_OK()
def __init__(self, serverDict, serverPolicy="Random"):
"""
Call the init of the parent, and initialize the list of FTS3 servers
"""
self.log = gLogger.getSubLogger("FTS3ServerPolicy")
self._serverDict = serverDict
self._serverList = serverDict.keys()
self._maxAttempts = len(self._serverList)
self._nextServerID = 0
self._resourceStatus = ResourceStatus()
methName = "_%sServerPolicy" % serverPolicy.lower()
if not hasattr(self, methName):
self.log.error('Unknown server policy %s. Using Random instead' % serverPolicy)
methName = "_randomServerPolicy"
self._policyMethod = getattr(self, methName)
def init_on_load(self):
"""This method initializes some attributes.
It is called by sqlalchemy (which does not call __init__)
"""
self._vo = None
# Note that in the case of an FTS3Operation created from an RMS
# object, the members here will probably be "wrong" in the sense
# that the VO will not be known by then.
# It does not really matter however, since we do not perform anything
# on an operation created this way, it's just to be then serialized
# in the DB.
self.dManager = DataManager()
self.rssClient = ResourceStatus()
self.fts3Plugin = FTS3Utilities.getFTS3Plugin(vo=self.vo)
opID = getattr(self, "operationID", None)
loggerName = "%s/" % opID if opID else ""
loggerName += "req_%s/op_%s" % (self.rmsReqID, self.rmsOpID)
self._log = gLogger.getSubLogger(loggerName)
def checkSEs(self, printOutput=False): # pylint: disable=no-self-use
"""Check the status of read and write operations in the DIRAC SE mask.
Example usage:
>>> print dirac.checkSEs()
{'OK': True, 'Value': {<LFN>:{'<Name>':'<Value>',...},...}}
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
res = gConfig.getSections('/Resources/StorageElements', True)
if not res['OK']:
self.log.error('Failed to get storage element information', res['Message'])
return res
if printOutput:
self.log.notice('%s %s %s' % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15)))
seList = sorted(res['Value'])
result = {}
rss = ResourceStatus()
for se in seList:
res = rss.getElementStatus(se, 'StorageElement')
if not res['OK']:
self.log.error("Failed to get StorageElement status for %s" % se)
else:
readState = res['Value'].get('ReadAccess', 'Active')
writeState = res['Value'].get('WriteAccess', 'Active')
result[se] = {'ReadStatus': readState, 'WriteStatus': writeState}
if printOutput:
self.log.notice('%s %s %s' % (se.ljust(25), readState.rjust(15), writeState.rjust(15)))
return S_OK(result)
def rssClient( self ):
""" RSS client getter """
if not self.__rssClient:
self.__rssClient = ResourceStatus()
return self.__rssClient
DIRAC.exit(-1)
if site not in res['Value']:
gLogger.error('The provided site (%s) is not known.' % site)
DIRAC.exit(-1)
ses.extend(res['Value']['SE'].replace(' ', '').split(','))
if not ses:
gLogger.error('There were no SEs provided')
DIRAC.exit(-1)
readBanned = []
writeBanned = []
checkBanned = []
removeBanned = []
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res['OK']:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-ban-se by %s' % userName
for se, seOptions in res['Value'].items():
resW = resC = resR = {'OK': False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and 'ReadAccess' in seOptions:
def rssClient(cls):
""" RSS client getter """
if not cls.__rssClient:
cls.__rssClient = ResourceStatus()
return cls.__rssClient
def main():
global vo
global noVOFlag
global allVOsFlag
Script.registerSwitch("V:", "vo=", "Virtual Organization", setVO)
Script.registerSwitch("a", "all", "All Virtual Organizations flag",
setAllVO)
Script.registerSwitch("n", "noVO",
"No Virtual Organizations assigned flag", setNoVO)
Script.parseCommandLine()
from DIRAC import gConfig, gLogger
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.Core.Utilities.PrettyPrint import printTable
from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
storageCFGBase = "/Resources/StorageElements"
res = gConfig.getSections(storageCFGBase, True)
if not res['OK']:
gLogger.error('Failed to get storage element info')
gLogger.error(res['Message'])
DIRACexit(1)
gLogger.info("%s %s %s" %
('Storage Element'.ljust(25), 'Read Status'.rjust(15),
'Write Status'.rjust(15)))
seList = sorted(res['Value'])
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(seList, "StorageElement")
if not res['OK']:
gLogger.error("Failed to get StorageElement status for %s" %
str(seList))
DIRACexit(1)
fields = ['SE', 'ReadAccess', 'WriteAccess', 'RemoveAccess', 'CheckAccess']
records = []
if vo is None and not allVOsFlag:
result = getVOfromProxyGroup()
if not result['OK']:
gLogger.error('Failed to determine the user VO')
DIRACexit(1)
vo = result['Value']
for se, statusDict in res['Value'].items():
# Check if the SE is allowed for the user VO
if not allVOsFlag:
voList = gConfig.getValue('/Resources/StorageElements/%s/VO' % se,
[])
if noVOFlag and voList:
continue
if voList and vo not in voList:
continue
record = [se]
for status in fields[1:]:
value = statusDict.get(status, 'Unknown')
record.append(value)
records.append(record)
printTable(fields, records, numbering=False, sortField='SE')
DIRACexit(0)
def main():
read = True
write = True
check = True
remove = True
site = ''
mute = False
Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element")
Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element")
Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element")
Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element")
Script.registerSwitch("a", "All", " Ban all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch(
"S:",
"Site=",
" Ban all SEs associate to site (note that if writing is allowed, check is always allowed)")
Script.parseCommandLine(ignoreErrors=True)
ses = Script.getPositionalArgs()
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() in ("r", "banread"):
write = False
check = False
remove = False
if switch[0].lower() in ("w", "banwrite"):
read = False
check = False
remove = False
if switch[0].lower() in ("k", "bancheck"):
read = False
write = False
remove = False
if switch[0].lower() in ("v", "banremove"):
read = False
write = False
check = False
if switch[0].lower() in ("a", "all"):
pass
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
site = switch[1]
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
DIRAC.exit(2)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
DIRAC.exit(2)
if site:
res = getSites()
if not res['OK']:
gLogger.error(res['Message'])
DIRAC.exit(-1)
if site not in res['Value']:
gLogger.error('The provided site (%s) is not known.' % site)
DIRAC.exit(-1)
ses.extend(res['Value']['SE'].replace(' ', '').split(','))
if not ses:
gLogger.error('There were no SEs provided')
DIRAC.exit(-1)
readBanned = []
writeBanned = []
checkBanned = []
removeBanned = []
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res['OK']:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-ban-se by %s' % userName
for se, seOptions in res['Value'].items():
resW = resC = resR = {'OK': False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and 'ReadAccess' in seOptions:
if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Read option for %s is %s, instead of %s' %
(se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" )
if not resR['OK']:
gLogger.error('Failed to update %s read access to Banned' % se)
else:
gLogger.notice('Successfully updated %s read access to Banned' % se)
readBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if write and 'WriteAccess' in seOptions:
if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Write option for %s is %s, instead of %s' %
(se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" )
if not resW['OK']:
gLogger.error("Failed to update %s write access to Banned" % se)
else:
gLogger.notice("Successfully updated %s write access to Banned" % se)
writeBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if check and 'CheckAccess' in seOptions:
if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Check option for %s is %s, instead of %s' %
(se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC['OK']:
gLogger.error("Failed to update %s check access to Banned" % se)
else:
gLogger.notice("Successfully updated %s check access to Banned" % se)
checkBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if remove and 'RemoveAccess' in seOptions:
if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Remove option for %s is %s, instead of %s' %
(se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC['OK']:
gLogger.error("Failed to update %s remove access to Banned" % se)
else:
gLogger.notice("Successfully updated %s remove access to Banned" % se)
removeBanned.append(se)
if not(resR['OK'] or resW['OK'] or resC['OK']):
DIRAC.exit(-1)
if not (writeBanned or readBanned or checkBanned or removeBanned):
gLogger.notice("No storage elements were banned")
DIRAC.exit(-1)
if mute:
gLogger.notice('Email is muted by script switch')
DIRAC.exit(0)
subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = ''
if read:
body = "%s\n\nThe following storage elements were banned for reading:" % body
for se in readBanned:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were banned for writing:" % body
for se in writeBanned:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were banned for check access:" % body
for se in checkBanned:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were banned for remove access:" % body
for se in removeBanned:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice('Notifying %s' % address)
if res['OK']:
gLogger.notice(res['Value'])
else:
gLogger.notice(res['Message'])
DIRAC.exit(0)
DIRACExit(-1)
siteDict = resultQueues['Value']
result = getQueuesResolved(siteDict)
if not resultQueues['OK']:
gLogger.error('Failed to get CE information')
DIRACExit(-1)
queueDict = result['Value']
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask['OK']:
gLogger.error('Failed to get Site mask information')
DIRACExit(-1)
siteMaskList = resultMask.get('Value', [])
rssClient = ResourceStatus()
fields = ('Site', 'CE', 'Queue', 'Status', 'Match', 'Reason')
records = []
for queue, queueInfo in queueDict.iteritems():
site = queueInfo['Site']
ce = queueInfo['CEName']
siteStatus = "Active" if site in siteMaskList else "InActive"
ceStatus = siteStatus
if rssClient.rssFlag:
result = rssClient.getElementStatus(ce, "ComputingElement")
if result['OK']:
ceStatus = result['Value'][ce]['all']
result = matchQueue(jdl, queueInfo, fullMatch=fullMatch)
def main():
read = True
write = True
check = True
remove = True
sites = []
mute = False
Script.registerSwitch("r", "BanRead",
" Ban only reading from the storage element")
Script.registerSwitch("w", "BanWrite",
" Ban writing to the storage element")
Script.registerSwitch("k", "BanCheck",
" Ban check access to the storage element")
Script.registerSwitch("v", "BanRemove",
" Ban remove access to the storage element")
Script.registerSwitch("a", "All",
" Ban all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch(
"S:", "Site=",
" Ban all SEs associate to site (note that if writing is allowed, check is always allowed)"
)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
["seGroupList: list of SEs or comma-separated SEs"])
switches, ses = Script.parseCommandLine(ignoreErrors=True)
for switch in switches:
if switch[0].lower() in ("r", "banread"):
write = False
check = False
remove = False
if switch[0].lower() in ("w", "banwrite"):
read = False
check = False
remove = False
if switch[0].lower() in ("k", "bancheck"):
read = False
write = False
remove = False
if switch[0].lower() in ("v", "banremove"):
read = False
write = False
check = False
if switch[0].lower() in ("a", "all"):
pass
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
sites = switch[1].split(",")
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
DIRAC.exit(2)
res = getProxyInfo()
if not res["OK"]:
gLogger.error("Failed to get proxy information", res["Message"])
DIRAC.exit(2)
userName = res["Value"].get("username")
if not userName:
gLogger.error("Failed to get username for proxy")
DIRAC.exit(2)
for site in sites:
res = DMSHelpers().getSEsForSite(site)
if not res["OK"]:
gLogger.error(res["Message"], site)
DIRAC.exit(-1)
ses.extend(res["Value"])
if not ses:
gLogger.error("There were no SEs provided")
DIRAC.exit(-1)
readBanned = []
writeBanned = []
checkBanned = []
removeBanned = []
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res["OK"]:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = "Forced with dirac-admin-ban-se by %s" % userName
for se, seOptions in res["Value"].items():
resW = resC = resR = {"OK": False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and "ReadAccess" in seOptions:
if seOptions["ReadAccess"] == "Banned":
gLogger.notice("Read access already banned", se)
resR["OK"] = True
elif not seOptions["ReadAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Read option for %s is %s, instead of %s" %
(se, seOptions["ReadAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resR = resourceStatus.setElementStatus(se, "StorageElement",
"ReadAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" )
if not resR["OK"]:
gLogger.error("Failed to update %s read access to Banned" %
se)
else:
gLogger.notice(
"Successfully updated %s read access to Banned" % se)
readBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if write and "WriteAccess" in seOptions:
if seOptions["WriteAccess"] == "Banned":
gLogger.notice("Write access already banned", se)
resW["OK"] = True
elif not seOptions["WriteAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Write option for %s is %s, instead of %s" %
(se, seOptions["WriteAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resW = resourceStatus.setElementStatus(se, "StorageElement",
"WriteAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" )
if not resW["OK"]:
gLogger.error(
"Failed to update %s write access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s write access to Banned" % se)
writeBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if check and "CheckAccess" in seOptions:
if seOptions["CheckAccess"] == "Banned":
gLogger.notice("Check access already banned", se)
resC["OK"] = True
elif not seOptions["CheckAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Check option for %s is %s, instead of %s" %
(se, seOptions["CheckAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resC = resourceStatus.setElementStatus(se, "StorageElement",
"CheckAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC["OK"]:
gLogger.error(
"Failed to update %s check access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s check access to Banned" % se)
checkBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if remove and "RemoveAccess" in seOptions:
if seOptions["RemoveAccess"] == "Banned":
gLogger.notice("Remove access already banned", se)
resC["OK"] = True
elif not seOptions["RemoveAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Remove option for %s is %s, instead of %s" %
(se, seOptions["RemoveAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resC = resourceStatus.setElementStatus(se, "StorageElement",
"RemoveAccess",
"Banned", reason,
userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC["OK"]:
gLogger.error(
"Failed to update %s remove access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s remove access to Banned" % se)
removeBanned.append(se)
if not (resR["OK"] or resW["OK"] or resC["OK"]):
DIRAC.exit(-1)
if not (writeBanned or readBanned or checkBanned or removeBanned):
gLogger.notice("No storage elements were banned")
DIRAC.exit(-1)
if mute:
gLogger.notice("Email is muted by script switch")
DIRAC.exit(0)
subject = "%s storage elements banned for use" % len(
writeBanned + readBanned + checkBanned + removeBanned)
addressPath = "EMail/Production"
address = Operations().getValue(addressPath, "")
body = ""
if read:
body = "%s\n\nThe following storage elements were banned for reading:" % body
for se in readBanned:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were banned for writing:" % body
for se in writeBanned:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were banned for check access:" % body
for se in checkBanned:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were banned for remove access:" % body
for se in removeBanned:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice("Notifying %s" % address)
if res["OK"]:
gLogger.notice(res["Value"])
else:
gLogger.notice(res["Message"])
DIRAC.exit(0)
def __init__(self,
configSection,
channels=None,
bandwidths=None,
failedFiles=None):
"""c'tor
:param self: self reference
:param str configSection: path on CS to ReplicationScheduler agent
:param bandwithds: observed throughput on active channels
:param channels: active channels
:param int failedFiles: max number of distinct failed files to allow scheduling
"""
## save config section
self.configSection = configSection + "/" + self.__class__.__name__
##
## sublogger
self.log = gLogger.getSubLogger("StrategyHandler", child=True)
self.log.setLevel(
gConfig.getValue(self.configSection + "/LogLevel", "DEBUG"))
self.supportedStrategies = [
'Simple', 'DynamicThroughput', 'Swarm', 'MinimiseTotalWait'
]
self.log.info("Supported strategies = %s" %
", ".join(self.supportedStrategies))
self.sigma = gConfig.getValue(self.configSection + '/HopSigma', 0.0)
self.log.info("HopSigma = %s" % self.sigma)
self.schedulingType = gConfig.getValue(
self.configSection + '/SchedulingType', 'File')
self.log.info("SchedulingType = %s" % self.schedulingType)
self.activeStrategies = gConfig.getValue(
self.configSection + '/ActiveStrategies', ['MinimiseTotalWait'])
self.log.info("ActiveStrategies = %s" %
", ".join(self.activeStrategies))
self.numberOfStrategies = len(self.activeStrategies)
self.log.info("Number of active strategies = %s" %
self.numberOfStrategies)
self.acceptableFailureRate = gConfig.getValue(
self.configSection + '/AcceptableFailureRate', 75)
self.log.info("AcceptableFailureRate = %s" %
self.acceptableFailureRate)
self.acceptableFailedFiles = gConfig.getValue(
self.configSection + "/AcceptableFailedFiles", 5)
self.log.info("AcceptableFailedFiles = %s" %
self.acceptableFailedFiles)
self.rwUpdatePeriod = gConfig.getValue(
self.configSection + "/RssRWUpdatePeriod", 600)
self.log.info("RSSUpdatePeriod = %s s" % self.rwUpdatePeriod)
self.rwUpdatePeriod = datetime.timedelta(seconds=self.rwUpdatePeriod)
## bandwithds
self.bandwidths = bandwidths if bandwidths else {}
## channels
self.channels = channels if channels else {}
## distinct failed files per channel
self.failedFiles = failedFiles if failedFiles else {}
## chosen strategy
self.chosenStrategy = 0
## fts graph
self.ftsGraph = None
## timestamp for last update
self.lastRssUpdate = datetime.datetime.now()
# dispatcher
self.strategyDispatcher = {
"MinimiseTotalWait": self.minimiseTotalWait,
"DynamicThroughput": self.dynamicThroughput,
"Simple": self.simple,
"Swarm": self.swarm
}
## own RSS client
self.resourceStatus = ResourceStatus()
## create fts graph
ftsGraph = self.setup(self.channels, self.bandwidths, self.failedFiles)
if not ftsGraph["OK"]:
raise SHGraphCreationError(ftsGraph["Message"])
self.log.info("%s has been constructed" % self.__class__.__name__)
def main():
read = False
write = False
check = False
remove = False
site = ''
mute = False
Script.registerSwitch("r", "AllowRead",
" Allow only reading from the storage element")
Script.registerSwitch("w", "AllowWrite",
" Allow only writing to the storage element")
Script.registerSwitch(
"k", "AllowCheck",
" Allow only check access to the storage element")
Script.registerSwitch(
"v", "AllowRemove",
" Allow only remove access to the storage element")
Script.registerSwitch("a", "All",
" Allow all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch("S:", "Site=",
" Allow all SEs associated to site")
Script.parseCommandLine(ignoreErrors=True)
ses = Script.getPositionalArgs()
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() in ("r", "allowread"):
read = True
if switch[0].lower() in ("w", "allowwrite"):
write = True
if switch[0].lower() in ("k", "allowcheck"):
check = True
if switch[0].lower() in ("v", "allowremove"):
remove = True
if switch[0].lower() in ("a", "all"):
read = True
write = True
check = True
remove = True
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
site = switch[1]
# imports
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
if not (read or write or check or remove):
# No switch was specified, means we need all of them
gLogger.notice(
"No option given, all accesses will be allowed if they were not")
read = True
write = True
check = True
remove = True
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
DIRAC.exit(2)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
DIRAC.exit(2)
if site:
res = getSites()
if not res['OK']:
gLogger.error(res['Message'])
DIRAC.exit(-1)
if site not in res['Value']:
gLogger.error('The provided site (%s) is not known.' % site)
DIRAC.exit(-1)
ses.extend(res['Value']['SE'].replace(' ', '').split(','))
if not ses:
gLogger.error('There were no SEs provided')
DIRAC.exit()
STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"]
ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"]
statusAllowedDict = {}
for statusType in STATUS_TYPES:
statusAllowedDict[statusType] = []
statusFlagDict = {}
statusFlagDict['ReadAccess'] = read
statusFlagDict['WriteAccess'] = write
statusFlagDict['CheckAccess'] = check
statusFlagDict['RemoveAccess'] = remove
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res['OK']:
gLogger.error('Storage Element %s does not exist' % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-allow-se by %s' % userName
for se, seOptions in res['Value'].items():
# InActive is used on the CS model, Banned is the equivalent in RSS
for statusType in STATUS_TYPES:
if statusFlagDict[statusType]:
if seOptions.get(statusType) == "Active":
gLogger.notice('%s status of %s is already Active' %
(statusType, se))
continue
if statusType in seOptions:
if not seOptions[statusType] in ALLOWED_STATUSES:
gLogger.notice(
'%s option for %s is %s, instead of %s' %
(statusType, se, seOptions['ReadAccess'],
ALLOWED_STATUSES))
gLogger.notice('Try specifying the command switches')
else:
resR = resourceStatus.setElementStatus(
se, "StorageElement", statusType, 'Active', reason,
userName)
if not resR['OK']:
gLogger.fatal(
"Failed to update %s %s to Active, exit -" %
(se, statusType), resR['Message'])
DIRAC.exit(-1)
else:
gLogger.notice(
"Successfully updated %s %s to Active" %
(se, statusType))
statusAllowedDict[statusType].append(se)
totalAllowed = 0
totalAllowedSEs = []
for statusType in STATUS_TYPES:
totalAllowed += len(statusAllowedDict[statusType])
totalAllowedSEs += statusAllowedDict[statusType]
totalAllowedSEs = list(set(totalAllowedSEs))
if not totalAllowed:
gLogger.info("No storage elements were allowed")
DIRAC.exit(-1)
if mute:
gLogger.notice('Email is muted by script switch')
DIRAC.exit(0)
subject = '%s storage elements allowed for use' % len(totalAllowedSEs)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = ''
if read:
body = "%s\n\nThe following storage elements were allowed for reading:" % body
for se in statusAllowedDict['ReadAccess']:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were allowed for writing:" % body
for se in statusAllowedDict['WriteAccess']:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were allowed for checking:" % body
for se in statusAllowedDict['CheckAccess']:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were allowed for removing:" % body
for se in statusAllowedDict['RemoveAccess']:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice('Notifying %s' % address)
if res['OK']:
gLogger.notice(res['Value'])
else:
gLogger.notice(res['Message'])
DIRAC.exit(0)
def __init__(self,
configSection,
bandwidths=None,
channels=None,
failedFiles=None):
"""c'tor
:param self: self reference
:param str configSection: path on CS to ReplicationScheduler agent
:param bandwithds: observed throughput on active channels
:param channels: active channels
:param int failedFiles: max number of distinct failed files to allow scheduling
"""
## save config section
self.configSection = configSection + "/" + self.__class__.__name__
## sublogger
self.log = gLogger.getSubLogger("StrategyHandler", child=True)
self.log.setLevel(
gConfig.getValue(self.configSection + "/LogLevel", "DEBUG"))
self.supportedStrategies = [
'Simple', 'DynamicThroughput', 'Swarm', 'MinimiseTotalWait'
]
self.log.debug("Supported strategies = %s" %
", ".join(self.supportedStrategies))
self.sigma = gConfig.getValue(self.configSection + '/HopSigma', 0.0)
self.log.debug("HopSigma = %s" % self.sigma)
self.schedulingType = gConfig.getValue(
self.configSection + '/SchedulingType', 'File')
self.log.debug("SchedulingType = %s" % self.schedulingType)
self.activeStrategies = gConfig.getValue(
self.configSection + '/ActiveStrategies', ['MinimiseTotalWait'])
self.log.debug("ActiveStrategies = %s" %
", ".join(self.activeStrategies))
self.numberOfStrategies = len(self.activeStrategies)
self.log.debug("Number of active strategies = %s" %
self.numberOfStrategies)
self.acceptableFailureRate = gConfig.getValue(
self.configSection + '/AcceptableFailureRate', 75)
self.log.debug("AcceptableFailureRate = %s" %
self.acceptableFailureRate)
self.acceptableFailedFiles = gConfig.getValue(
self.configSection + "/AcceptableFailedFiles", 5)
self.log.debug("AcceptableFailedFiles = %s" %
self.acceptableFailedFiles)
self.bandwidths = bandwidths if bandwidths else {}
self.channels = channels if channels else {}
self.failedFiles = failedFiles if failedFiles else {}
self.chosenStrategy = 0
# dispatcher
self.strategyDispatcher = {
re.compile("MinimiseTotalWait"): self.__minimiseTotalWait,
re.compile("DynamicThroughput"): self.__dynamicThroughput,
re.compile("Simple"): self.__simple,
re.compile("Swarm"): self.__swarm
}
self.resourceStatus = ResourceStatus()
self.log.debug("strategyDispatcher entries:")
for key, value in self.strategyDispatcher.items():
self.log.debug("%s : %s" % (key.pattern, value.__name__))
self.log.debug("%s has been constructed" % self.__class__.__name__)
