def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ':' if short else '',
longOption + '=', doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp(exitCode=1)
ops = Operations()
if not ops.getValue('DataManagement/ArchiveFiles/Enabled', False):
sLog.error(
'The "ArchiveFiles" operation is not enabled, contact your administrator!'
)
DIRAC.exit(1)
for _short, longOption, _doc in self.options:
defaultValue = ops.getValue(
'DataManagement/ArchiveFiles/%s' % longOption, None)
if defaultValue:
sLog.verbose(
'Found default value in the CS for %r with value %r' %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for _short, longOption, _doc in self.flags:
defaultValue = ops.getValue(
'DataManagement/ArchiveFiles/%s' % longOption, False)
if defaultValue:
sLog.verbose(
'Found default value in the CS for %r with value %r' %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose('Found switch %r with value %r' %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches['DryRun'] = not self.switches.get('Execute', False)
self.switches['SourceSE'] = self.switches.get('SourceSE',
'').split(',')
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
TransformationAgentsUtilities.__init__(self)
# few parameters
self.pluginLocation = self.am_getOption(
'PluginLocation',
'DIRAC.TransformationSystem.Agent.TransformationPlugin')
self.transformationStatus = self.am_getOption(
'transformationStatus', ['Active', 'Completing', 'Flush'])
self.maxFiles = self.am_getOption('MaxFiles', 5000)
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = sortList(agentTSTypes)
else:
dataProc = Operations().getValue('Transformations/DataProcessing',
['MCSimulation', 'Merge'])
dataManip = Operations().getValue(
'Transformations/DataManipulation', ['Replication', 'Removal'])
self.transformationTypes = sortList(dataProc + dataManip)
# clients
self.transfClient = TransformationClient()
# for the threading
self.transQueue = Queue.Queue()
self.transInQueue = []
# for caching using a pickle file
self.workDirectory = self.am_getWorkDirectory()
self.cacheFile = os.path.join(self.workDirectory, 'ReplicaCache.pkl')
self.dateWriteCache = datetime.datetime.utcnow()
# Validity of the cache
self.replicaCache = None
self.replicaCacheValidity = self.am_getOption('ReplicaCacheValidity',
2)
self.writingCache = False
self.noUnusedDelay = self.am_getOption('NoUnusedDelay', 6)
self.unusedFiles = {}
self.unusedTimeStamp = {}
def initialize( self ):
""" standard initialize
"""
# few parameters
self.pluginLocation = self.am_getOption( 'PluginLocation',
'DIRAC.TransformationSystem.Agent.TransformationPlugin' )
self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush'] )
self.maxFiles = self.am_getOption( 'MaxFiles', 5000 )
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sorted( agentTSTypes )
else:
dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
self.transformationTypes = sorted( dataProc + dataManip )
# clients
self.transfClient = TransformationClient()
# for caching using a pickle file
self.workDirectory = self.am_getWorkDirectory()
self.cacheFile = os.path.join( self.workDirectory, 'ReplicaCache.pkl' )
self.controlDirectory = self.am_getControlDirectory()
# remember the offset if any in TS
self.lastFileOffset = {}
# Validity of the cache
self.replicaCache = {}
self.replicaCacheValidity = self.am_getOption( 'ReplicaCacheValidity', 2 )
self.noUnusedDelay = self.am_getOption( 'NoUnusedDelay', 6 )
# Get it threaded
maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.log.info( "Multithreaded with %d threads" % maxNumberOfThreads )
for i in xrange( maxNumberOfThreads ):
threadPool.generateJobAndQueueIt( self._execute, [i] )
self.log.info( "Will treat the following transformation types: %s" % str( self.transformationTypes ) )
return S_OK()
def __init__(self):
monitoringType = "DataOperation"
# Will use the `MonitoringBackends/Default` value as monitoring backend unless a flag for `MonitoringBackends/DataOperation` is set.
self.monitoringOptions = Operations().getMonitoringBackends(
monitoringType)
if "Monitoring" in self.monitoringOptions:
self.dataOperationReporter = MonitoringReporter(monitoringType)
if "Accounting" in self.monitoringOptions:
self.dataOp = DataOperation()
def checkCVMFS(platform, app):
""" Check the existence of the CVMFS path for given platform and application
:param string platform: platform
:param tuple app: tuple off application name and version
:returns: S_OK of tuple of path and environmen stript
"""
name, version = app
csPath = "/AvailableTarBalls/%s/%s/%s" % (platform, name, version)
cvmfspath = Operations().getValue(csPath + "/CVMFSPath", "")
envScript = Operations().getValue(csPath + "/CVMFSEnvScript", "")
if cvmfspath and os.path.exists(cvmfspath):
return S_OK((cvmfspath, envScript))
message = "Package %s version %s not found natively on cvmfs," % (name,
version)
DIRAC.gLogger.info(message + " will try to use shared area instead: " +
str(Operations().getOptionsDict(csPath)))
return S_ERROR('Missing CVMFS!')
def __init__(self, opsHelper):
if not opsHelper:
opsHelper = Operations()
self.__opsHelper = opsHelper
self.__log = gLogger.getSubLogger("SharesCorrector")
self.__shareCorrectors = {}
self.__correctorsOrder = []
self.__baseCS = "JobScheduling/ShareCorrections"
self.__objLoader = ObjectLoader.ObjectLoader()
def __init__(self, **kwargs):
""" Simple constructor
"""
Client.__init__(self, **kwargs)
opsH = Operations()
self.maxResetCounter = opsH.getValue('Productions/ProductionFilesMaxResetCounter', 10)
self.setServer('Transformation/TransformationManager')
def __init__(self, **kwargs):
"""Simple constructor"""
super(TransformationClient, self).__init__(**kwargs)
opsH = Operations()
self.maxResetCounter = opsH.getValue(
"Transformations/FilesMaxResetCounter", 10)
self.setServer("Transformation/TransformationManager")
def initialize(self):
self.transClient = TransformationClient()
self.bkClient = BookkeepingClient()
self.notifyClient = NotificationClient()
self.operations = Operations()
self.email = self.am_getOption("MailTo", '')
return S_OK()
def __checkChoiceVarInDescription(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__description:
varValue = Operations().getValue(
"JobDescription/Default%s" % varName, defaultVal)
else:
varValue = self.__description[varName]
initialVal = varValue
if varValue not in Operations().getValue(
"JobDescription/Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" %
(varValue, varName))
if initialVal != varValue:
self.__description.setOption(varName, varValue)
return S_OK(varValue)
def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults,
clients=None, url=None):
super(SlackAction, self).__init__(name, decisionParams, enforcementResult,
singlePolicyResults, clients)
if url is not None:
self.url = url
else:
self.url = Operations().getValue('ResourceStatus/Config/Slack')
def main():
Script.registerSwitch("N:", "NumberOfProcessors=", "Run n parallel copies of the benchmark")
Script.registerSwitch("U", "Update", "Update dirac.cfg with the resulting value")
Script.registerSwitch("R:", "Reconfig=", "Update given configuration file with the resulting value")
Script.parseCommandLine(ignoreErrors=True)
update = False
configFile = None
numberOfProcessors = 0
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("U", "Update"):
update = True
elif unprocSw[0] in ("R", "Reconfig"):
configFile = unprocSw[1]
elif unprocSw[0] in ("N", "NumberOfProcessors"):
try:
numberOfProcessors = int(unprocSw[1])
except ValueError:
gLogger.warn("Cannot make benchmark measurements: NumberOfProcessors is not a number")
# if numberOfProcessors has not been provided, try to get it from the configuration
if not numberOfProcessors:
numberOfProcessors = gConfig.getValue("/Resources/Computing/CEDefaults/NumberOfProcessors", 1)
gLogger.info("Computing benchmark measurements on", "%d processor(s)..." % numberOfProcessors)
# we want to get the logs coming from db12
gLogger.enableLogsFromExternalLibs()
# multiprocessor allocations generally have a CPU Power lower than single core one.
# in order to avoid having wrong estimations, we run multiple copies of the benchmark simultaneously
result = multiple_dirac_benchmark(numberOfProcessors)
if result is None:
gLogger.error("Cannot make benchmark measurements")
DIRAC.exit(1)
# we take a conservative approach and use the minimum value returned as the CPU Power
db12Result = min(result["raw"])
# because hardware is continuously evolving, original benchmark scores might need a correction
corr = Operations().getValue("JobScheduling/CPUNormalizationCorrection", 1.0)
gLogger.info("Applying a correction on the CPU power:", corr)
cpuPower = round(db12Result / corr, 1)
gLogger.notice("Estimated CPU power is %.1f HS06" % cpuPower)
if update:
gConfig.setOptionValue("/LocalSite/CPUNormalizationFactor", cpuPower)
if configFile:
gConfig.dumpLocalCFGToFile(configFile)
else:
gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath)
DIRAC.exit()
def initialize(self):
"""Sets defaults
"""
self.am_setOption('shifterProxy', 'ProductionManager')
self.transClient = TransformationClient()
self.reqClient = ReqClient()
self.consChecks = ConsistencyChecks(interactive=False,
transClient=self.transClient)
transformationTypes = Operations().getValue(
'Transformations/DataProcessing', [])
extendableTTypes = Operations().getValue(
'Transformations/ExtendableTransfTypes', ['MCSimulation'])
self.transformationTypes = list(
set(transformationTypes) - set(extendableTTypes))
return S_OK()
def initialize( self ):
''' Make the necessary initializations
'''
gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM )
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sorted( agentTSTypes )
else:
dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
self.transformationTypes = sorted( dataProc + dataManip )
extendables = Operations().getValue( 'Transformations/ExtendableTransfTypes', [])
if extendables:
for extendable in extendables:
if extendable in self.transformationTypes:
self.transformationTypes.remove(extendable)
#This is because the Extendables do not use this Agent (have no Input data query)
return S_OK()
def __init__(self, withRepo=False, repoLocation=''):
"""Internal initialization of the ExtDIRAC API.
"""
#self.dirac = Dirac(WithRepo=WithRepo, RepoLocation=RepoLocation)
super(Dirac, self).__init__(withRepo, repoLocation)
#Dirac.__init__(self, withRepo = withRepo, repoLocation = repoLocation)
self.log = gLogger
self.software_versions = {}
self.checked = False
self.ops = Operations()
def __init__(self):
"""
Constructor, initializes the rssClient.
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.rssConfig = RssConfiguration()
self.__opHelper = Operations()
self.rssFlag = ResourceStatus().rssFlag
self.rsClient = ResourceStatusClient()
def getServiceURLs(system, service=None, setup=False, failover=False):
"""Generate url.
:param str system: system name or full name e.g.: Framework/ProxyManager
:param str service: service name, like 'ProxyManager'.
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:param bool failover: to add failover URLs to end of result list
:return: list -- complete urls. e.g. [dips://some-domain:3424/Framework/Service]
"""
system, service = divideFullName(system, service)
resList = []
mainServers = None
systemSection = getSystemSection(system, setup=setup)
# Add failover URLs at the end of the list
failover = "Failover" if failover else ""
for fURLs in ["", "Failover"] if failover else [""]:
urlList = []
urls = List.fromChar(
gConfigurationData.extractOptionFromCFG(
"%s/%sURLs/%s" % (systemSection, fURLs, service)))
# Be sure that urls not None
for url in urls or []:
# Trying if we are refering to the list of main servers
# which would be like dips://$MAINSERVERS$:1234/System/Component
if "$MAINSERVERS$" in url:
if not mainServers:
# Operations cannot be imported at the beginning because of a bootstrap problem
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
mainServers = Operations(setup=setup).getValue(
"MainServers", [])
if not mainServers:
raise Exception("No Main servers defined")
for srv in mainServers:
_url = checkComponentURL(url.replace("$MAINSERVERS$", srv),
system,
service,
pathMandatory=True)
if _url not in urlList:
urlList.append(_url)
continue
_url = checkComponentURL(url, system, service, pathMandatory=True)
if _url not in urlList:
urlList.append(_url)
# Randomize list if needed
resList.extend(List.randomize(urlList))
return resList
def __init__( self, transID = 0, transClient = None ):
""" c'tor
"""
super( Transformation, self ).__init__()
self.paramTypes = { 'TransformationID' : [types.IntType, types.LongType],
'TransformationName' : types.StringTypes,
'Status' : types.StringTypes,
'Description' : types.StringTypes,
'LongDescription' : types.StringTypes,
'Type' : types.StringTypes,
'Plugin' : types.StringTypes,
'AgentType' : types.StringTypes,
'FileMask' : types.StringTypes,
'TransformationGroup' : types.StringTypes,
'GroupSize' : [types.IntType, types.LongType, types.FloatType],
'InheritedFrom' : [types.IntType, types.LongType],
'Body' : types.StringTypes,
'MaxNumberOfTasks' : [types.IntType, types.LongType],
'EventsPerTask' : [types.IntType, types.LongType]}
self.paramValues = { 'TransformationID' : 0,
'TransformationName' : '',
'Status' : 'New',
'Description' : '',
'LongDescription' : '',
'Type' : '',
'Plugin' : 'Standard',
'AgentType' : 'Manual',
'FileMask' : '',
'TransformationGroup' : 'General',
'GroupSize' : 1,
'InheritedFrom' : 0,
'Body' : '',
'MaxNumberOfTasks' : 0,
'EventsPerTask' : 0}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue( 'Transformations/AllowedPlugins',
['Broadcast', 'Standard', 'BySize', 'ByShare'] )
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError( 'TransformationID %d does not exist' % transID )
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal( "Failed to get transformation from database", "%s @ %s" % ( transID,
self.transClient.serverURL ) )
def getValidPolicyTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/PolicyTypes
'''
DEFAULTS = [ 'Resource_PolType', 'Alarm_PolType', 'Collective_PolType', 'RealBan_PolType' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/PolicyTypes' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def initializeHandler(cls, svcInfoDict):
cls.gJobDB = JobDB()
cls.gJobLoggingDB = JobLoggingDB()
cls.gTaskQueueDB = TaskQueueDB()
cls.gElasticJobParametersDB = None
useESForJobParametersFlag = Operations().getValue(
'/Services/JobMonitoring/useESForJobParametersFlag', False)
if useESForJobParametersFlag:
cls.gElasticJobParametersDB = ElasticJobParametersDB()
return S_OK()
def getValidElements():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Granularity
'''
DEFAULTS = [ 'Site', 'Service', 'Resource', 'StorageElement' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/Granularity' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidResourceTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/ResourceType
'''
DEFAULTS = [ 'CE', 'CREAMCE', 'SE', 'LFC_C', 'LFC_L', 'FTS', 'VOMS' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/ResourceType' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def __setupManagerProxies(self):
""" setup grid proxy for all defined managers """
oHelper = Operations()
shifters = oHelper.getSections("Shifter")
if not shifters["OK"]:
self.log.error(shifters["Message"])
return shifters
shifters = shifters["Value"]
for shifter in shifters:
shifterDict = oHelper.getOptionsDict("Shifter/%s" % shifter)
if not shifterDict["OK"]:
self.log.error(shifterDict["Message"])
continue
userName = shifterDict["Value"].get("User", "")
userGroup = shifterDict["Value"].get("Group", "")
userDN = CS.getDNForUsername(userName)
if not userDN["OK"]:
self.log.error(userDN["Message"])
continue
userDN = userDN["Value"][0]
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
self.log.debug(
"getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
getProxy = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
else:
self.log.debug("getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
getProxy = gProxyManager.downloadProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
if not getProxy["OK"]:
self.log.error(getProxy["Message"])
return S_ERROR("unable to setup shifter proxy for %s: %s" %
(shifter, getProxy["Message"]))
chain = getProxy["chain"]
fileName = getProxy["Value"]
self.log.debug("got %s: %s %s" % (shifter, userName, userGroup))
self.__managersDict[shifter] = {
"ShifterDN": userDN,
"ShifterName": userName,
"ShifterGroup": userGroup,
"Chain": chain,
"ProxyFile": fileName
}
return S_OK()
def __init__(self,
transClient=None,
logger=None,
submissionClient=None,
jobMonitoringClient=None,
outputDataModule=None,
jobClass=None,
opsH=None,
destinationPlugin=None):
""" Generates some default objects.
jobClass is by default "DIRAC.Interfaces.API.Job.Job". An extension of it also works:
VOs can pass in their job class extension, if present
"""
if not logger:
logger = gLogger.getSubLogger('WorkflowTasks')
super(WorkflowTasks, self).__init__(transClient, logger)
if not submissionClient:
self.submissionClient = WMSClient()
else:
self.submissionClient = submissionClient
if not jobMonitoringClient:
self.jobMonitoringClient = JobMonitoringClient()
else:
self.jobMonitoringClient = jobMonitoringClient
if not jobClass:
self.jobClass = Job
else:
self.jobClass = jobClass
if not opsH:
self.opsH = Operations()
else:
self.opsH = opsH
if not outputDataModule:
self.outputDataModule = self.opsH.getValue(
"Transformations/OutputDataModule", "")
else:
self.outputDataModule = outputDataModule
if not destinationPlugin:
self.destinationPlugin = self.opsH.getValue(
'Transformations/DestinationPlugin', 'BySE')
else:
self.destinationPlugin = destinationPlugin
self.destinationPlugin_o = None
self.outputDataModule_o = None
def initialize( self ):
credDict = self.getRemoteCredentials()
self.ownerDN = credDict['DN']
self.ownerGroup = credDict['group']
operations = Operations( group = self.ownerGroup )
self.globalJobsInfo = operations.getValue( '/Services/JobMonitoring/GlobalJobsInfo', True )
self.jobPolicy = JobPolicy( self.ownerDN, self.ownerGroup, self.globalJobsInfo )
self.jobPolicy.setJobDB( gJobDB )
return S_OK()
def __init__(self, plugin, operationsHelper=None):
""" plugin name has to be passed in: it will then be executed as one of the functions below, e.g.
plugin = 'BySize' will execute TransformationPlugin('BySize')._BySize()
"""
self.plugin = plugin
self.params = {}
if operationsHelper is None:
self.opsH = Operations()
else:
self.opsH = operationsHelper
def getValidStatus():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Status
'''
DEFAULTS = [ 'Active', 'Bad', 'Probing', 'Banned' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/Status' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidPolicyResult():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/PolicyResult
'''
DEFAULTS = [ 'Error', 'Unknown', 'Banned', 'Probing', 'Bad', 'Active' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/PolicyResult' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidServiceTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/ServiceType
'''
DEFAULTS = [ 'Computing', 'Storage', 'VO-BOX', 'VOMS', 'CondDB' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/ServiceType' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidSiteTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/SiteType
'''
DEFAULTS = [ 'T1', 'T2', 'T3', 'T4' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/SiteType' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
