def getValidStatusTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Resources
'''
DEFAULTS = {
'Site' : { 'StatusType' : "''" },
'Service' : { 'StatusType' : "''" },
'Resource' : { 'StatusType' : "''" },
'StorageElement': { 'StatusType' : [ 'Read', 'Write', 'Remove', 'Check' ] }
}
opHelper = Operations()
sections = opHelper.getSections( 'RSSConfiguration/GeneralConfig/Resources' )
if not sections[ 'OK' ]:
return DEFAULTS
result = {}
for section in sections[ 'Value' ]:
res = opHelper.getValue( 'RSSConfiguration/GeneralConfig/Resources/%s/StatusType' % section )
if res is None:
if DEFAULTS.has_key( section ):
result[ section ] = { 'StatusType' : DEFAULTS[ section ] }
else:
result[ section ] = { 'StatusType' : None }
else:
result[ section ] = { 'StatusType' : Utils.getTypedList( res ) }
return result
def resolveDeps(sysconfig, appli, appversion):
""" Resolve the dependencies
:param str sysconfig: system configuration
:param str appli: application name
:param str appversion: application version
:return: list of dictionaries
"""
log = gLogger.getSubLogger("resolveDeps")
ops = Operations()
deps = ops.getSections('/AvailableTarBalls/%s/%s/%s/Dependencies' % (sysconfig, appli,
appversion), '')
depsarray = []
if deps['OK']:
for dep in deps['Value']:
vers = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' % (sysconfig, appli,
appversion, dep), '')
depvers = ''
if vers:
depvers = vers
else:
log.error("Retrieving dependency version for %s failed, skipping to next !" % (dep))
continue
log.verbose("Found dependency %s %s" % (dep, depvers))
depdict = {}
depdict["app"] = dep
depdict["version"] = depvers
depsarray.append(depdict)
##resolve recursive dependencies
depsofdeps = resolveDeps(sysconfig, dep, depvers)
depsarray.extend(depsofdeps)
else:
log.verbose("Could not find any dependency for %s %s, ignoring" % (appli, appversion))
return depsarray
def web_getLaunchpadSetupWithLFNs(self):
""" Method obtain launchpad setup with pre-selected LFNs as input data parameter,
the caller js client will use setup to open an new Launchpad
"""
# On the fly file catalog for advanced launchpad
if not hasattr(self, 'fc'):
userData = self.getSessionData()
group = str(userData["user"]["group"])
vo = getVOForGroup(group)
self.fc = FileCatalog(vo=vo)
self.set_header('Content-type', 'text/plain')
arguments = self.request.arguments
gLogger.always("submit: incoming arguments %s to getLaunchpadSetupWithLFNs" % arguments)
lfnList = str(arguments['path'][0]).split(',')
# Modified for Eiscat
# Checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder
gLogger.always("submit: checking if some rtg_def.m", arguments)
processed = []
metaDict = {'type': 'info'}
for lfn in lfnList:
pos_relative = lfn.find("/")
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
experiment_lfn = lfn[0:pos_relative]
if experiment_lfn in processed:
continue
processed.append(experiment_lfn)
gLogger.always( "checking rtg_def.m in %s" % experiment_lfn )
result = self.fc.findFilesByMetadata( metaDict, path=str(experiment_lfn) )
if not result['OK'] or not result['Value']:
gLogger.error( "Failed to get type info from $s, %s" % (experiment_lfn,result[ "Message" ]) )
continue
for candidate_lfn in result['Value']:
if candidate_lfn.find('rtg_def.m') > 0:
lfnList.append(candidate_lfn)
# End modified
ptlfn = ''
for lfn in lfnList:
ptlfn += (', ' + lfn) if ptlfn else lfn
params = self.defaultParams.copy()
params["InputData"] = [1, ptlfn]
obj = Operations(vo=vo)
predefinedSets = {}
launchpadSections = obj.getSections("Launchpad")
if launchpadSections['OK']:
for section in launchpadSections["Value"]:
predefinedSets[section] = {}
sectionOptions = obj.getOptionsDict("Launchpad/" + section)
pprint.pprint(sectionOptions)
if sectionOptions['OK']:
predefinedSets[section] = sectionOptions["Value"]
self.write({"success": "true", "result": params, "predefinedSets": predefinedSets})
def web_getLaunchpadOpts(self):
defaultParams = {"JobName" : [1, 'DIRAC'],
"Executable" : [1, "/bin/ls"],
"Arguments" : [1, "-ltrA"],
"OutputSandbox" : [1, "std.out, std.err"],
"InputData" : [0, ""],
"OutputData" : [0, ""],
"OutputSE" : [0, "DIRAC-USER"],
"OutputPath": [0, ""],
"CPUTime" : [0, "86400"],
"Site" : [0, ""],
"BannedSite" : [0, ""],
"Platform" : [0, "Linux_x86_64_glibc-2.5"],
"Priority" : [0, "5"],
"StdError" : [0, "std.err"],
"StdOutput" : [0, "std.out"],
"Parameters" : [0, "0"],
"ParameterStart" : [0, "0"],
"ParameterStep" : [0, "1"]}
delimiter = gConfig.getValue("/Website/Launchpad/ListSeparator" , ',')
options = self.__getOptionsFromCS(delimiter=delimiter)
# platform = self.__getPlatform()
# if platform and options:
# if not options.has_key("Platform"):
# options[ "Platform" ] = platform
# else:
# csPlatform = list(options[ "Platform" ])
# allPlatforms = csPlatform + platform
# platform = uniqueElements(allPlatforms)
# options[ "Platform" ] = platform
gLogger.debug("Combined options from CS: %s" % options)
override = gConfig.getValue("/Website/Launchpad/OptionsOverride" , False)
gLogger.info("end __getLaunchpadOpts")
# Updating the default values from OptionsOverride configuration branch
for key in options:
if key not in defaultParams:
defaultParams[key] = [ 0, "" ]
defaultParams[key][1] = options[key][0]
# Reading of the predefined sets of launchpad parameters values
obj = Operations()
predefinedSets = {}
launchpadSections = obj.getSections("Launchpad")
import pprint
if launchpadSections['OK']:
for section in launchpadSections["Value"]:
predefinedSets[section] = {}
sectionOptions = obj.getOptionsDict("Launchpad/" + section)
pprint.pprint(sectionOptions)
if sectionOptions['OK']:
predefinedSets[section] = sectionOptions["Value"]
self.write({"success":"true", "result":defaultParams, "predefinedSets":predefinedSets})
def __setupManagerProxies(self):
""" setup grid proxy for all defined managers """
oHelper = Operations()
shifters = oHelper.getSections("Shifter")
if not shifters["OK"]:
self.log.error(shifters["Message"])
return shifters
shifters = shifters["Value"]
for shifter in shifters:
shifterDict = oHelper.getOptionsDict("Shifter/%s" % shifter)
if not shifterDict["OK"]:
self.log.error(shifterDict["Message"])
continue
userName = shifterDict["Value"].get("User", "")
userGroup = shifterDict["Value"].get("Group", "")
userDN = CS.getDNForUsername(userName)
if not userDN["OK"]:
self.log.error(userDN["Message"])
continue
userDN = userDN["Value"][0]
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
self.log.debug(
"getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
getProxy = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
else:
self.log.debug("getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
getProxy = gProxyManager.downloadProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
if not getProxy["OK"]:
self.log.error(getProxy["Message"])
return S_ERROR("unable to setup shifter proxy for %s: %s" %
(shifter, getProxy["Message"]))
chain = getProxy["chain"]
fileName = getProxy["Value"]
self.log.debug("got %s: %s %s" % (shifter, userName, userGroup))
self.__managersDict[shifter] = {
"ShifterDN": userDN,
"ShifterName": userName,
"ShifterGroup": userGroup,
"Chain": chain,
"ProxyFile": fileName
}
return S_OK()
def _getCatalogs( self ):
# Get the eligible catalogs first
# First, look in the Operations, if nothing defined look in /Resources for backward compatibility
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
opHelper = Operations( vo = vo )
result = opHelper.getSections( '/Services/FileCatalogs' )
fileCatalogs = []
operationsFlag = False
if result['OK']:
fileCatalogs = result['Value']
operationsFlag = True
else:
res = gConfig.getSections( self.rootConfigPath, listOrdered = True )
if not res['OK']:
errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration."
gLogger.error( errStr, res['Message'] )
return S_ERROR( errStr )
fileCatalogs = res['Value']
# Get the catalogs now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails( catalogName )
if not res['OK']:
return res
catalogConfig = res['Value']
if operationsFlag:
result = opHelper.getOptionsDict( '/Services/FileCatalogs/%s' % catalogName )
if not result['OK']:
return result
catalogConfig.update( result['Value'] )
if catalogConfig['Status'] == 'Active':
res = self._generateCatalogObject( catalogName )
if not res['OK']:
return res
oCatalog = res['Value']
master = catalogConfig['Master']
# If the catalog is read type
if re.search( 'Read', catalogConfig['AccessType'] ):
if master:
self.readCatalogs.insert( 0, ( catalogName, oCatalog, master ) )
else:
self.readCatalogs.append( ( catalogName, oCatalog, master ) )
# If the catalog is write type
if re.search( 'Write', catalogConfig['AccessType'] ):
if master:
self.writeCatalogs.insert( 0, ( catalogName, oCatalog, master ) )
else:
self.writeCatalogs.append( ( catalogName, oCatalog, master ) )
return S_OK()
def __setupManagerProxies( self ):
""" setup grid proxy for all defined managers """
oHelper = Operations()
shifters = oHelper.getSections( "Shifter" )
if not shifters["OK"]:
self.log.error( shifters["Message"] )
return shifters
shifters = shifters["Value"]
for shifter in shifters:
shifterDict = oHelper.getOptionsDict( "Shifter/%s" % shifter )
if not shifterDict["OK"]:
self.log.error( shifterDict["Message"] )
continue
userName = shifterDict["Value"].get( "User", "" )
userGroup = shifterDict["Value"].get( "Group", "" )
userDN = CS.getDNForUsername( userName )
if not userDN["OK"]:
self.log.error( userDN["Message"] )
continue
userDN = userDN["Value"][0]
vomsAttr = CS.getVOMSAttributeForGroup( userGroup )
if vomsAttr:
self.log.debug( "getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName,
userGroup, userDN ) )
getProxy = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredTimeLeft = 1200,
cacheTime = 4 * 43200 )
else:
self.log.debug( "getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) )
getProxy = gProxyManager.downloadProxyToFile( userDN, userGroup,
requiredTimeLeft = 1200,
cacheTime = 4 * 43200 )
if not getProxy["OK"]:
self.log.error( getProxy["Message" ] )
return S_ERROR( "unable to setup shifter proxy for %s: %s" % ( shifter, getProxy["Message"] ) )
chain = getProxy["chain"]
fileName = getProxy["Value" ]
self.log.debug( "got %s: %s %s" % ( shifter, userName, userGroup ) )
self.__managersDict[shifter] = { "ShifterDN" : userDN,
"ShifterName" : userName,
"ShifterGroup" : userGroup,
"Chain" : chain,
"ProxyFile" : fileName }
return S_OK()
def resolveDeps(sysconfig, appli, appversion):
""" Resolve the dependencies
:param string sysconfig: system configuration
:param string appli: application name
:param string appversion: application version
:return: list of dictionaries
"""
log = gLogger.getSubLogger("resolveDeps")
ops = Operations()
deps = ops.getSections(
'/AvailableTarBalls/%s/%s/%s/Dependencies' %
(sysconfig, appli, appversion), '')
depsarray = []
if deps['OK']:
for dep in deps['Value']:
vers = ops.getValue(
'/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' %
(sysconfig, appli, appversion, dep), '')
depvers = ''
if vers:
depvers = vers
else:
log.error(
"Retrieving dependency version for %s failed, skipping to next !"
% (dep))
continue
log.verbose("Found dependency %s %s" % (dep, depvers))
depdict = {}
depdict["app"] = dep
depdict["version"] = depvers
depsarray.append(depdict)
##resolve recursive dependencies
depsofdeps = resolveDeps(sysconfig, dep, depvers)
depsarray.extend(depsofdeps)
else:
log.verbose("Could not find any dependency for %s %s, ignoring" %
(appli, appversion))
return depsarray
def matchQueue(jobJDL, queueDict, fullMatch=False):
"""
Match the job description to the queue definition
:param str job: JDL job description
:param bool fullMatch: test matching on all the criteria
:param dict queueDict: queue parameters dictionary
:return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or
S_ERROR with the reason for no match
"""
# Check the job description validity
job = ClassAd(jobJDL)
if not job.isOK():
return S_ERROR("Invalid job description")
noMatchReasons = []
# Check job requirements to resource
# 1. CPUTime
cpuTime = job.getAttributeInt("CPUTime")
if not cpuTime:
cpuTime = 84600
if cpuTime > int(queueDict.get("CPUTime", 0)):
noMatchReasons.append("Job CPUTime requirement not satisfied")
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 2. Multi-value match requirements
for parameter in ["Site", "GridCE", "Platform", "JobType"]:
if parameter in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression("%ss" % parameter))
queueSet = set(fromChar(queueDict[parameter]))
if valueSet and queueSet and not valueSet.intersection(queueSet):
valueToPrint = ",".join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 3. Banned multi-value match requirements
for par in ["Site", "GridCE", "Platform", "JobType"]:
parameter = "Banned%s" % par
if par in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression("%ss" % parameter))
queueSet = set(fromChar(queueDict[par]))
if valueSet and queueSet and valueSet.issubset(queueSet):
valueToPrint = ",".join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 4. Tags
tags = set(job.getListFromExpression("Tag"))
nProc = job.getAttributeInt("NumberOfProcessors")
if nProc and nProc > 1:
tags.add("MultiProcessor")
wholeNode = job.getAttributeString("WholeNode")
if wholeNode:
tags.add("WholeNode")
queueTags = set(queueDict.get("Tag", []))
if not tags.issubset(queueTags):
noMatchReasons.append("Job Tag %s not satisfied" % ",".join(tags))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 4. MultiProcessor requirements
if nProc and nProc > int(queueDict.get("NumberOfProcessors", 1)):
noMatchReasons.append("Job NumberOfProcessors %d requirement not satisfied" % nProc)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 5. RAM
ram = job.getAttributeInt("RAM")
# If MaxRAM is not specified in the queue description, assume 2GB
if ram and ram > int(queueDict.get("MaxRAM", 2048) / 1024):
noMatchReasons.append("Job RAM %d requirement not satisfied" % ram)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# Check resource requirements to job
# 1. OwnerGroup - rare case but still
if "OwnerGroup" in queueDict:
result = getProxyInfo(disableVOMS=True)
if not result["OK"]:
return S_ERROR("No valid proxy available")
ownerGroup = result["Value"]["group"]
if ownerGroup != queueDict["OwnerGroup"]:
noMatchReasons.append("Resource OwnerGroup %s requirement not satisfied" % queueDict["OwnerGroup"])
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 2. Required tags
requiredTags = set(queueDict.get("RequiredTags", []))
if not requiredTags.issubset(tags):
noMatchReasons.append("Resource RequiredTags %s not satisfied" % ",".join(requiredTags))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 3. RunningLimit
site = queueDict["Site"]
ce = queueDict.get("GridCE")
opsHelper = Operations()
result = opsHelper.getSections("JobScheduling/RunningLimit")
if result["OK"] and site in result["Value"]:
result = opsHelper.getSections("JobScheduling/RunningLimit/%s" % site)
if result["OK"]:
for parameter in result["Value"]:
value = job.getAttributeString(parameter)
if (
value
and (
opsHelper.getValue("JobScheduling/RunningLimit/%s/%s/%s" % (site, parameter, value), 1)
or opsHelper.getValue(
"JobScheduling/RunningLimit/%s/CEs/%s/%s/%s" % (site, ce, parameter, value), 1
)
)
== 0
):
noMatchReasons.append("Resource operational %s requirement not satisfied" % parameter)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
return S_OK({"Match": not bool(noMatchReasons), "Reason": noMatchReasons})
class WorkflowTasks(TaskBase):
""" Handles jobs
"""
def __init__(self,
transClient=None,
logger=None,
submissionClient=None,
jobMonitoringClient=None,
outputDataModule=None,
jobClass=None,
opsH=None,
destinationPlugin=None,
ownerDN=None,
ownerGroup=None):
""" Generates some default objects.
jobClass is by default "DIRAC.Interfaces.API.Job.Job". An extension of it also works:
VOs can pass in their job class extension, if present
"""
if not logger:
logger = gLogger.getSubLogger('WorkflowTasks')
super(WorkflowTasks, self).__init__(transClient, logger)
useCertificates = True if (bool(ownerDN)
and bool(ownerGroup)) else False
if not submissionClient:
self.submissionClient = WMSClient(useCertificates=useCertificates,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup)
else:
self.submissionClient = submissionClient
if not jobMonitoringClient:
self.jobMonitoringClient = JobMonitoringClient()
else:
self.jobMonitoringClient = jobMonitoringClient
if not jobClass:
self.jobClass = Job
else:
self.jobClass = jobClass
if not opsH:
self.opsH = Operations()
else:
self.opsH = opsH
if not outputDataModule:
self.outputDataModule = self.opsH.getValue(
"Transformations/OutputDataModule", "")
else:
self.outputDataModule = outputDataModule
if not destinationPlugin:
self.destinationPlugin = self.opsH.getValue(
'Transformations/DestinationPlugin', 'BySE')
else:
self.destinationPlugin = destinationPlugin
self.destinationPlugin_o = None
self.outputDataModule_o = None
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup='',
ownerDN='',
bulkSubmissionFlag=False):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
jobClass is by default "DIRAC.Interfaces.API.Job.Job". An extension of it also works.
:param str transBody: transformation job template
:param dict taskDict: dictionary of per task parameters
:param str owner: owner of the transformation
:param str ownerGroup: group of the owner of the transformation
:param str ownerDN: DN of the owner of the transformation
:param bool bulkSubmissionFlag: flag for using bulk submission or not
:return: S_OK/S_ERROR with updated taskDict
"""
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
if bulkSubmissionFlag:
return self.__prepareTasksBulk(transBody, taskDict, owner,
ownerGroup, ownerDN)
# not a bulk submission
return self.__prepareTasks(transBody, taskDict, owner, ownerGroup,
ownerDN)
def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup,
ownerDN):
""" Prepare transformation tasks with a single job object for bulk submission
:param str transBody: transformation job template
:param dict taskDict: dictionary of per task parameters
:param str owner: owner of the transformation
:param str ownerGroup: group of the owner of the transformation
:param str ownerDN: DN of the owner of the transformation
:return: S_OK/S_ERROR with updated taskDict
"""
if taskDict:
transID = taskDict.values()[0]['TransformationID']
else:
return S_OK({})
method = '__prepareTasksBulk'
startTime = time.time()
# Prepare the bulk Job object with common parameters
oJob = self.jobClass(transBody)
self._logVerbose('Setting job owner:group to %s:%s' %
(owner, ownerGroup),
transID=transID,
method=method)
oJob.setOwner(owner)
oJob.setOwnerGroup(ownerGroup)
oJob.setOwnerDN(ownerDN)
try:
site = oJob.workflow.findParameter('Site').getValue()
except AttributeError:
site = None
jobType = oJob.workflow.findParameter('JobType').getValue()
transGroup = str(transID).zfill(8)
# Verify that the JOB_ID parameter is added to the workflow
if not oJob.workflow.findParameter('JOB_ID'):
oJob._addParameter(oJob.workflow, 'JOB_ID', 'string', '00000000',
"Initial JOB_ID")
if oJob.workflow.findParameter('PRODUCTION_ID'):
oJob._setParamValue('PRODUCTION_ID', str(transID).zfill(8)) # pylint: disable=protected-access
else:
oJob._addParameter(
oJob.workflow, # pylint: disable=protected-access
'PRODUCTION_ID',
'string',
str(transID).zfill(8),
"Production ID")
oJob.setType(jobType)
self._logVerbose('Adding default transformation group of %s' %
(transGroup),
transID=transID,
method=method)
oJob.setJobGroup(transGroup)
clinicPath = self._checkSickTransformations(transID)
if clinicPath:
self._handleHospital(oJob, clinicPath)
# Collect per job parameters sequences
paramSeqDict = {}
# tasks must be sorted because we use bulk submission and we must find the correspondance
for taskID in sorted(taskDict):
paramsDict = taskDict[taskID]
seqDict = {}
if site is not None:
paramsDict['Site'] = site
paramsDict['JobType'] = jobType
# Handle destination site
sites = self._handleDestination(paramsDict)
if not sites:
self._logError('Could not get a list a sites',
transID=transID,
method=method)
return S_ERROR(ETSUKN, "Can not evaluate destination site")
else:
self._logVerbose('Setting Site: ',
str(sites),
transID=transID,
method=method)
seqDict['Site'] = sites
seqDict['JobName'] = self._transTaskName(transID, taskID)
seqDict['JOB_ID'] = str(taskID).zfill(8)
self._logDebug('TransID: %s, TaskID: %s, paramsDict: %s' %
(transID, taskID, str(paramsDict)),
transID=transID,
method=method)
# Handle Input Data
inputData = paramsDict.get('InputData')
if inputData:
if isinstance(inputData, six.string_types):
inputData = inputData.replace(' ', '').split(';')
self._logVerbose('Setting input data to %s' % inputData,
transID=transID,
method=method)
seqDict['InputData'] = inputData
elif paramSeqDict.get('InputData') is not None:
self._logError(
"Invalid mixture of jobs with and without input data")
return S_ERROR(
ETSDATA,
"Invalid mixture of jobs with and without input data")
for paramName, paramValue in paramsDict.iteritems():
if paramName not in ('InputData', 'Site', 'TargetSE'):
if paramValue:
self._logVerbose('Setting %s to %s' %
(paramName, paramValue),
transID=transID,
method=method)
seqDict[paramName] = paramValue
outputParameterList = []
if self.outputDataModule:
res = self.getOutputData({
'Job': oJob._toXML(), # pylint: disable=protected-access
'TransformationID': transID,
'TaskID': taskID,
'InputData': inputData
})
if not res['OK']:
self._logError("Failed to generate output data",
res['Message'],
transID=transID,
method=method)
continue
for name, output in res['Value'].iteritems():
seqDict[name] = output
outputParameterList.append(name)
if oJob.workflow.findParameter(name):
oJob._setParamValue(name, "%%(%s)s" % name) # pylint: disable=protected-access
else:
oJob._addParameter(
oJob.workflow, # pylint: disable=protected-access
name,
'JDL',
"%%(%s)s" % name,
name)
for pName, seq in seqDict.iteritems():
paramSeqDict.setdefault(pName, []).append(seq)
for paramName, paramSeq in paramSeqDict.iteritems():
if paramName in ['JOB_ID', 'PRODUCTION_ID', 'InputData'
] + outputParameterList:
res = oJob.setParameterSequence(paramName,
paramSeq,
addToWorkflow=paramName)
else:
res = oJob.setParameterSequence(paramName, paramSeq)
if not res['OK']:
return res
if taskDict:
self._logInfo('Prepared %d tasks' % len(taskDict),
transID=transID,
method=method,
reftime=startTime)
taskDict['BulkJobObject'] = oJob
return S_OK(taskDict)
def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN):
""" Prepare transformation tasks with a job object per task
:param str transBody: transformation job template
:param dict taskDict: dictionary of per task parameters
:param owner: owner of the transformation
:param str ownerGroup: group of the owner of the transformation
:param str ownerDN: DN of the owner of the transformation
:return: S_OK/S_ERROR with updated taskDict
"""
if taskDict:
transID = taskDict.values()[0]['TransformationID']
else:
return S_OK({})
method = '__prepareTasks'
startTime = time.time()
oJobTemplate = self.jobClass(transBody)
oJobTemplate.setOwner(owner)
oJobTemplate.setOwnerGroup(ownerGroup)
oJobTemplate.setOwnerDN(ownerDN)
try:
site = oJobTemplate.workflow.findParameter('Site').getValue()
except AttributeError:
site = None
jobType = oJobTemplate.workflow.findParameter('JobType').getValue()
templateOK = False
getOutputDataTiming = 0.
for taskID, paramsDict in taskDict.iteritems():
# Create a job for each task and add it to the taskDict
if not templateOK:
templateOK = True
# Update the template with common information
self._logVerbose('Job owner:group to %s:%s' %
(owner, ownerGroup),
transID=transID,
method=method)
transGroup = str(transID).zfill(8)
self._logVerbose('Adding default transformation group of %s' %
(transGroup),
transID=transID,
method=method)
oJobTemplate.setJobGroup(transGroup)
if oJobTemplate.workflow.findParameter('PRODUCTION_ID'):
oJobTemplate._setParamValue('PRODUCTION_ID',
str(transID).zfill(8))
else:
oJobTemplate._addParameter(oJobTemplate.workflow,
'PRODUCTION_ID', 'string',
str(transID).zfill(8),
"Production ID")
if not oJobTemplate.workflow.findParameter('JOB_ID'):
oJobTemplate._addParameter(oJobTemplate.workflow, 'JOB_ID',
'string', '00000000',
"Initial JOB_ID")
if site is not None:
paramsDict['Site'] = site
paramsDict['JobType'] = jobType
# Now create the job from the template
oJob = copy.deepcopy(oJobTemplate)
constructedName = self._transTaskName(transID, taskID)
self._logVerbose('Setting task name to %s' % constructedName,
transID=transID,
method=method)
oJob.setName(constructedName)
oJob._setParamValue('JOB_ID', str(taskID).zfill(8))
inputData = None
self._logDebug('TransID: %s, TaskID: %s, paramsDict: %s' %
(transID, taskID, str(paramsDict)),
transID=transID,
method=method)
# These helper functions do the real job
sites = self._handleDestination(paramsDict)
if not sites:
self._logError('Could not get a list a sites',
transID=transID,
method=method)
paramsDict['TaskObject'] = ''
continue
else:
self._logDebug('Setting Site: ',
str(sites),
transID=transID,
method=method)
res = oJob.setDestination(sites)
if not res['OK']:
self._logError('Could not set the site: %s' %
res['Message'],
transID=transID,
method=method)
paramsDict['TaskObject'] = ''
continue
self._handleInputs(oJob, paramsDict)
self._handleRest(oJob, paramsDict)
clinicPath = self._checkSickTransformations(transID)
if clinicPath:
self._handleHospital(oJob, clinicPath)
paramsDict['TaskObject'] = ''
if self.outputDataModule:
getOutputDataTiming -= time.time()
res = self.getOutputData({
'Job': oJob._toXML(),
'TransformationID': transID,
'TaskID': taskID,
'InputData': inputData
})
getOutputDataTiming += time.time()
if not res['OK']:
self._logError("Failed to generate output data",
res['Message'],
transID=transID,
method=method)
continue
for name, output in res['Value'].iteritems():
oJob._addJDLParameter(name, ';'.join(output))
paramsDict['TaskObject'] = oJob
if taskDict:
self._logVerbose('Average getOutputData time: %.1f per task' %
(getOutputDataTiming / len(taskDict)),
transID=transID,
method=method)
self._logInfo('Prepared %d tasks' % len(taskDict),
transID=transID,
method=method,
reftime=startTime)
return S_OK(taskDict)
#############################################################################
def _handleDestination(self, paramsDict):
""" Handle Sites and TargetSE in the parameters
"""
try:
sites = ['ANY']
if paramsDict['Site']:
# 'Site' comes from the XML and therefore is ; separated
sites = fromChar(paramsDict['Site'], sepChar=';')
except KeyError:
pass
if self.destinationPlugin_o:
destinationPlugin_o = self.destinationPlugin_o
else:
res = self.__generatePluginObject(self.destinationPlugin)
if not res['OK']:
self._logFatal(
"Could not generate a destination plugin object")
return res
destinationPlugin_o = res['Value']
self.destinationPlugin_o = destinationPlugin_o
destinationPlugin_o.setParameters(paramsDict)
destSites = destinationPlugin_o.run()
if not destSites:
return sites
# Now we need to make the AND with the sites, if defined
if sites != ['ANY']:
# Need to get the AND
destSites &= set(sites)
return list(destSites)
def _handleInputs(self, oJob, paramsDict):
""" set job inputs (+ metadata)
"""
inputData = paramsDict.get('InputData')
transID = paramsDict['TransformationID']
if inputData:
self._logVerbose('Setting input data to %s' % inputData,
transID=transID,
method='_handleInputs')
res = oJob.setInputData(inputData)
if not res['OK']:
self._logError("Could not set the inputs: %s" % res['Message'],
transID=transID,
method='_handleInputs')
def _handleRest(self, oJob, paramsDict):
""" add as JDL parameters all the other parameters that are not for inputs or destination
"""
transID = paramsDict['TransformationID']
for paramName, paramValue in paramsDict.iteritems():
if paramName not in ('InputData', 'Site', 'TargetSE'):
if paramValue:
self._logDebug('Setting %s to %s' %
(paramName, paramValue),
transID=transID,
method='_handleRest')
oJob._addJDLParameter(paramName, paramValue)
def _checkSickTransformations(self, transID):
""" Check if the transformation is in the transformations to be processed at Hospital or Clinic
"""
transID = int(transID)
clinicPath = "Hospital"
if transID in set(
int(x) for x in self.opsH.getValue(
os.path.join(clinicPath, "Transformations"), [])):
return clinicPath
if "Clinics" in self.opsH.getSections("Hospital").get('Value', []):
basePath = os.path.join("Hospital", "Clinics")
clinics = self.opsH.getSections(basePath)['Value']
for clinic in clinics:
clinicPath = os.path.join(basePath, clinic)
if transID in set(
int(x) for x in self.opsH.getValue(
os.path.join(clinicPath, "Transformations"), [])):
return clinicPath
return None
def _handleHospital(self, oJob, clinicPath):
""" Optional handle of hospital/clinic jobs
"""
if not clinicPath:
return
oJob.setInputDataPolicy('download', dataScheduling=False)
# Check first for a clinic, if not it must be the general hospital
hospitalSite = self.opsH.getValue(
os.path.join(clinicPath, "ClinicSite"), '')
hospitalCEs = self.opsH.getValue(os.path.join(clinicPath, "ClinicCE"),
[])
# If not found, get the hospital parameters
if not hospitalSite:
hospitalSite = self.opsH.getValue("Hospital/HospitalSite",
'DIRAC.JobDebugger.ch')
if not hospitalCEs:
hospitalCEs = self.opsH.getValue("Hospital/HospitalCEs", [])
oJob.setDestination(hospitalSite)
if hospitalCEs:
oJob._addJDLParameter('GridCE', hospitalCEs)
def __generatePluginObject(self, plugin):
""" This simply instantiates the TaskManagerPlugin class with the relevant plugin name
"""
method = '__generatePluginObject'
try:
plugModule = __import__(self.pluginLocation, globals(), locals(),
['TaskManagerPlugin'])
except ImportError as e:
self._logException("Failed to import 'TaskManagerPlugin' %s: %s" %
(plugin, e),
method=method)
return S_ERROR()
try:
plugin_o = getattr(plugModule,
'TaskManagerPlugin')('%s' % plugin,
operationsHelper=self.opsH)
return S_OK(plugin_o)
except AttributeError as e:
self._logException("Failed to create %s(): %s." % (plugin, e),
method=method)
return S_ERROR()
#############################################################################
def getOutputData(self, paramDict):
""" Get the list of job output LFNs from the provided plugin
"""
if not self.outputDataModule_o:
# Create the module object
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(self.outputDataModule,
None)
if not moduleInstance['OK']:
return moduleInstance
self.outputDataModule_o = moduleInstance['Value']
# This is the "argument" to the module, set it and then execute
self.outputDataModule_o.paramDict = paramDict
return self.outputDataModule_o.execute()
def submitTransformationTasks(self, taskDict):
""" Submit the tasks
"""
if 'BulkJobObject' in taskDict:
return self.__submitTransformationTasksBulk(taskDict)
return self.__submitTransformationTasks(taskDict)
def __submitTransformationTasksBulk(self, taskDict):
""" Submit jobs in one go with one parametric job
"""
if not taskDict:
return S_OK(taskDict)
startTime = time.time()
method = '__submitTransformationTasksBulk'
oJob = taskDict.pop('BulkJobObject')
# we can only do this, once the job has been popped, or we _might_ crash
transID = taskDict.values()[0]['TransformationID']
if oJob is None:
self._logError('no bulk Job object found',
transID=transID,
method=method)
return S_ERROR(ETSUKN,
'No bulk job object provided for submission')
result = self.submitTaskToExternal(oJob)
if not result['OK']:
self._logError('Failed to submit tasks to external',
transID=transID,
method=method)
return result
jobIDList = result['Value']
if len(jobIDList) != len(taskDict):
for task in taskDict.values():
task['Success'] = False
return S_ERROR(
ETSUKN, 'Submitted less number of jobs than requested tasks')
# Get back correspondance with tasks sorted by ID
for jobID, taskID in zip(jobIDList, sorted(taskDict)):
taskDict[taskID]['ExternalID'] = jobID
taskDict[taskID]['Success'] = True
submitted = len(jobIDList)
self._logInfo('Submitted %d tasks to WMS in %.1f seconds' %
(submitted, time.time() - startTime),
transID=transID,
method=method)
return S_OK(taskDict)
def __submitTransformationTasks(self, taskDict):
""" Submit jobs one by one
"""
method = '__submitTransformationTasks'
submitted = 0
failed = 0
startTime = time.time()
for task in taskDict.itervalues():
transID = task['TransformationID']
if not task['TaskObject']:
task['Success'] = False
failed += 1
continue
res = self.submitTaskToExternal(task['TaskObject'])
if res['OK']:
task['ExternalID'] = res['Value']
task['Success'] = True
submitted += 1
else:
self._logError("Failed to submit task to WMS",
res['Message'],
transID=transID,
method=method)
task['Success'] = False
failed += 1
if submitted:
self._logInfo('Submitted %d tasks to WMS in %.1f seconds' %
(submitted, time.time() - startTime),
transID=transID,
method=method)
if failed:
self._logError('Failed to submit %d tasks to WMS.' % (failed),
transID=transID,
method=method)
return S_OK(taskDict)
def submitTaskToExternal(self, job):
""" Submits a single job (which can be a bulk one) to the WMS.
"""
if isinstance(job, six.string_types):
try:
oJob = self.jobClass(job)
except Exception as x: # pylint: disable=broad-except
self._logException("Failed to create job object", '', x)
return S_ERROR("Failed to create job object")
elif isinstance(job, self.jobClass):
oJob = job
else:
self._logError("No valid job description found")
return S_ERROR("No valid job description found")
workflowFileObject = StringIO.StringIO(oJob._toXML())
jdl = oJob._toJDL(jobDescriptionObject=workflowFileObject)
return self.submissionClient.submitJob(jdl, workflowFileObject)
def updateTransformationReservedTasks(self, taskDicts):
transID = None
jobNames = [
self._transTaskName(taskDict['TransformationID'],
taskDict['TaskID']) for taskDict in taskDicts
]
res = self.jobMonitoringClient.getJobs({'JobName': jobNames})
if not res['OK']:
self._logError("Failed to get task from WMS",
res['Message'],
transID=transID,
method='updateTransformationReservedTasks')
return res
jobNameIDs = {}
for wmsID in res['Value']:
res = self.jobMonitoringClient.getJobPrimarySummary(int(wmsID))
if not res['OK']:
self._logWarn("Failed to get task summary from WMS",
res['Message'],
transID=transID,
method='updateTransformationReservedTasks')
else:
jobNameIDs[res['Value']['JobName']] = int(wmsID)
noTask = list(set(jobNames) - set(jobNameIDs))
return S_OK({'NoTasks': noTask, 'TaskNameIDs': jobNameIDs})
def getSubmittedTaskStatus(self, taskDicts):
"""
Check the status of a list of tasks and return lists of taskIDs for each new status
"""
method = 'getSubmittedTaskStatus'
if taskDicts:
wmsIDs = [
int(taskDict['ExternalID']) for taskDict in taskDicts
if int(taskDict['ExternalID'])
]
transID = taskDicts[0]['TransformationID']
else:
return S_OK({})
res = self.jobMonitoringClient.getJobsStatus(wmsIDs)
if not res['OK']:
self._logWarn("Failed to get job status from the WMS system",
transID=transID,
method=method)
return res
statusDict = res['Value']
updateDict = {}
for taskDict in taskDicts:
taskID = taskDict['TaskID']
wmsID = int(taskDict['ExternalID'])
if not wmsID:
continue
oldStatus = taskDict['ExternalStatus']
newStatus = statusDict.get(wmsID, {}).get('Status', 'Removed')
if oldStatus != newStatus:
if newStatus == "Removed":
self._logVerbose(
'Production/Job %d/%d removed from WMS while it is in %s status'
% (transID, taskID, oldStatus),
transID=transID,
method=method)
newStatus = "Failed"
self._logVerbose(
'Setting job status for Production/Job %d/%d to %s' %
(transID, taskID, newStatus),
transID=transID,
method=method)
updateDict.setdefault(newStatus, []).append(taskID)
return S_OK(updateDict)
def getSubmittedFileStatus(self, fileDicts):
"""
Check the status of a list of files and return the new status of each LFN
"""
if not fileDicts:
return S_OK({})
method = 'getSubmittedFileStatus'
# All files are from the same transformation
transID = fileDicts[0]['TransformationID']
taskFiles = {}
for fileDict in fileDicts:
jobName = self._transTaskName(transID, fileDict['TaskID'])
taskFiles.setdefault(jobName,
{})[fileDict['LFN']] = fileDict['Status']
res = self.updateTransformationReservedTasks(fileDicts)
if not res['OK']:
self._logWarn("Failed to obtain taskIDs for files",
transID=transID,
method=method)
return res
noTasks = res['Value']['NoTasks']
taskNameIDs = res['Value']['TaskNameIDs']
updateDict = {}
for jobName in noTasks:
for lfn, oldStatus in taskFiles[jobName].iteritems():
if oldStatus != 'Unused':
updateDict[lfn] = 'Unused'
res = self.jobMonitoringClient.getJobsStatus(taskNameIDs.values())
if not res['OK']:
self._logWarn("Failed to get job status from the WMS system",
transID=transID,
method=method)
return res
statusDict = res['Value']
for jobName, wmsID in taskNameIDs.iteritems():
jobStatus = statusDict.get(wmsID, {}).get('Status')
newFileStatus = {
'Done': 'Processed',
'Completed': 'Processed',
'Failed': 'Unused'
}.get(jobStatus)
if newFileStatus:
for lfn, oldStatus in taskFiles[jobName].iteritems():
if newFileStatus != oldStatus:
updateDict[lfn] = newFileStatus
return S_OK(updateDict)
def configHelper(voList):
"""
A helper function to gather necessary Rucio client options from the CS.
:param volist: list of VO names, or a VO name (str)
:return: a dictionary of a form {vo: params, vo: params,}
:rtype: dict
"""
log = gLogger.getLocalSubLogger("RucioSynchronizerHelper")
if isinstance(voList, str):
voList = [voList]
clientConfig = {}
log.debug("VO list to consider for synchronization: ", voList)
# locate RucioFileCatalog type in resources first
result = gConfig.getSections("/Resources/FileCatalogs")
catNames = []
if result["OK"]:
catalogs = result["Value"]
log.debug("File catalogs defined in Resources", catalogs)
for catalog in catalogs:
result = gConfig.getOptionsDict(getCatalogPath(catalog))
if result["OK"]:
options = result["Value"]
log.debug("Rucio Catalog candidate options", options)
if options.get("Status", None) == "Active" and options.get(
"CatalogType", None) == "RucioFileCatalog":
catNames.append(catalog)
else:
log.error("No catalogs defined in Resources.")
return S_ERROR("No catalogs defined in Resources.")
log.info(
"Active FileCatalogs candidates of type RucioFileCatalog found in Resources:",
catNames)
# we found (possibly more that one) candidate, now we look for it in Operations
# to find out which one is used by which VO. There can be only one
# Rucio catalog per VO.
for vo in voList:
opHelper = Operations(vo=vo)
result = opHelper.getSections("/Services/Catalogs")
if result["OK"]:
catSections = set(result["Value"])
else:
log.warn("No Services/Catalogs section in Operations, for ",
"VO=%s (skipped)" % vo)
continue
selectedCatalog = list(catSections.intersection(catNames))
if len(selectedCatalog) > 1:
log.error(
"VO %s: Services/Catalogs section mis-configured."
" More that one Rucio file catalog",
"[VO: %s, Catalogs: %s]" % (vo, selectedCatalog),
)
continue
if not selectedCatalog:
log.warn("VO is not using RucioFileCatalog (VO skipped)",
"[VO: %s]" % vo)
continue
# check if the section name is in the catalog list to use.
# if the list is not empty it has to contain the selected catalog.
fileCatalogs = opHelper.getValue("/Services/Catalogs/CatalogList", [])
if fileCatalogs and selectedCatalog[0] not in fileCatalogs:
log.warn(
"VO is not using RucioFileCatalog - it is not in the catalog list",
"[VO: %s]" % vo)
continue
# now collect Rucio specific parameters for the VO
params = {}
result = gConfig.getOptionsDict(getCatalogPath(selectedCatalog[0]))
if result["OK"]:
optDict = result["Value"]
params["rucioHost"] = optDict.get("RucioHost", None)
params["authHost"] = optDict.get("AuthHost", None)
params["privilegedAccount"] = optDict.get("PrivilegedAccount",
"root")
clientConfig[vo] = params
log.info("RSEs and users will be configured in Rucio for the VO:",
vo)
else:
log.error(result["Message"])
return clientConfig
class FileCatalog:
ro_methods = [
"exists",
"isLink",
"readLink",
"isFile",
"getFileMetadata",
"getReplicas",
"getReplicaStatus",
"getFileSize",
"isDirectory",
"getDirectoryReplicas",
"listDirectory",
"getDirectoryMetadata",
"getDirectorySize",
"getDirectoryContents",
"resolveDataset",
"getPathPermissions",
"getLFNForPFN",
"getUsers",
"getGroups",
"getFileUserMetadata",
]
write_methods = [
"createLink",
"removeLink",
"addFile",
"setFileStatus",
"addReplica",
"removeReplica",
"removeFile",
"setReplicaStatus",
"setReplicaHost",
"createDirectory",
"setDirectoryStatus",
"removeDirectory",
"removeDataset",
"removeFileFromDataset",
"createDataset",
]
def __init__(self, catalogs=[], vo=None):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.readCatalogs = []
self.writeCatalogs = []
self.rootConfigPath = "/Resources/FileCatalogs"
self.vo = vo
if not vo:
result = getVOfromProxyGroup()
if not result["OK"]:
return result
self.vo = result["Value"]
self.opHelper = Operations(vo=self.vo)
if type(catalogs) in types.StringTypes:
catalogs = [catalogs]
if catalogs:
res = self._getSelectedCatalogs(catalogs)
else:
res = self._getCatalogs()
if not res["OK"]:
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.valid = False
def isOK(self):
return self.valid
def getReadCatalogs(self):
return self.readCatalogs
def getWriteCatalogs(self):
return self.writeCatalogs
def __getattr__(self, name):
self.call = name
if name in FileCatalog.write_methods:
return self.w_execute
elif name in FileCatalog.ro_methods:
return self.r_execute
else:
raise AttributeError
def __checkArgumentFormat(self, path):
if type(path) in types.StringTypes:
urls = {path: False}
elif type(path) == types.ListType:
urls = {}
for url in path:
urls[url] = False
elif type(path) == types.DictType:
urls = path
else:
return S_ERROR("FileCatalog.__checkArgumentFormat: Supplied path is not of the correct format.")
return S_OK(urls)
def w_execute(self, *parms, **kws):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = self.__checkArgumentFormat(fileInfo)
if not res["OK"]:
return res
fileInfo = res["Value"]
allLfns = fileInfo.keys()
for catalogName, oCatalog, master in self.writeCatalogs:
method = getattr(oCatalog, self.call)
res = method(fileInfo, **kws)
if not res["OK"]:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error(
"FileCatalog.w_execute: Failed to execute %s on master catalog %s." % (self.call, catalogName),
res["Message"],
)
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append((catalogName, res["Message"]))
else:
for lfn, message in res["Value"]["Failed"].items():
# Save the error message for the failed operations
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn)
for lfn, result in res["Value"]["Successful"].items():
# Save the result return for each file for the successful operations
if not successful.has_key(lfn):
successful[lfn] = {}
successful[lfn][catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = errorMessage
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def r_execute(self, *parms, **kws):
""" Read method executor.
"""
successful = {}
failed = {}
for _catalogName, oCatalog, _master in self.readCatalogs:
method = getattr(oCatalog, self.call)
res = method(*parms, **kws)
if res["OK"]:
if "Successful" in res["Value"]:
for key, item in res["Value"]["Successful"].items():
if not successful.has_key(key):
successful[key] = item
if failed.has_key(key):
failed.pop(key)
for key, item in res["Value"]["Failed"].items():
if not successful.has_key(key):
failed[key] = item
if len(failed) == 0:
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
else:
return res
if (len(successful) == 0) and (len(failed) == 0):
return S_ERROR("Failed to perform %s from any catalog" % self.call)
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog(self, catalogName, mode="Write", master=False):
""" Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject(catalogName)
if not result["OK"]:
return result
oCatalog = result["Value"]
if mode.lower().find("read") != -1:
self.readCatalogs.append((catalogName, oCatalog, master))
if mode.lower().find("write") != -1:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def removeCatalog(self, catalogName):
""" Remove the specified catalog from the internal pool
"""
catalog_removed = False
for i in range(len(self.readCatalogs)):
catalog, _object, _master = self.readCatalogs[i]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range(len(self.writeCatalogs)):
catalog, _object, _master = self.writeCatalogs[i]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK("Catalog does not exist")
def _getSelectedCatalogs(self, desiredCatalogs):
for catalogName in desiredCatalogs:
res = self._generateCatalogObject(catalogName)
if not res["OK"]:
return res
oCatalog = res["Value"]
self.readCatalogs.append((catalogName, oCatalog, True))
self.writeCatalogs.append((catalogName, oCatalog, True))
return S_OK()
def _getCatalogs(self):
# Get the eligible catalogs first
# First, look in the Operations, if nothing defined look in /Resources for backward compatibility
result = self.opHelper.getSections("/Services/Catalogs")
fileCatalogs = []
operationsFlag = False
if result["OK"]:
fileCatalogs = result["Value"]
operationsFlag = True
else:
res = gConfig.getSections(self.rootConfigPath, listOrdered=True)
if not res["OK"]:
errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration."
gLogger.error(errStr, res["Message"])
return S_ERROR(errStr)
fileCatalogs = res["Value"]
# Get the catalogs now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails(catalogName)
if not res["OK"]:
return res
catalogConfig = res["Value"]
if operationsFlag:
result = self.opHelper.getOptionsDict("/Services/Catalogs/%s" % catalogName)
if not result["OK"]:
return result
catalogConfig.update(result["Value"])
if catalogConfig["Status"] == "Active":
res = self._generateCatalogObject(catalogName)
if not res["OK"]:
return res
oCatalog = res["Value"]
master = catalogConfig["Master"]
# If the catalog is read type
if re.search("Read", catalogConfig["AccessType"]):
if master:
self.readCatalogs.insert(0, (catalogName, oCatalog, master))
else:
self.readCatalogs.append((catalogName, oCatalog, master))
# If the catalog is write type
if re.search("Write", catalogConfig["AccessType"]):
if master:
self.writeCatalogs.insert(0, (catalogName, oCatalog, master))
else:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName)
res = gConfig.getOptions(catalogConfigPath)
if not res["OK"]:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = {}
for option in res["Value"]:
configPath = "%s/%s" % (catalogConfigPath, option)
optionValue = gConfig.getValue(configPath)
catalogConfig[option] = optionValue
# The 'Status' option should be defined (default = 'Active')
if not catalogConfig.has_key("Status"):
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
gLogger.warn(warnStr, catalogName)
catalogConfig["Status"] = "Active"
# The 'AccessType' option must be defined
if not catalogConfig.has_key("AccessType"):
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
if not catalogConfig.has_key("Master"):
catalogConfig["Master"] = False
elif catalogConfig["Master"] == "True":
catalogConfig["Master"] = True
else:
catalogConfig["Master"] = False
return S_OK(catalogConfig)
def _generateCatalogObject(self, catalogName):
""" Create a file catalog object from its name and CS description
"""
useProxy = gConfig.getValue("/LocalSite/Catalogs/%s/UseProxy" % catalogName, False)
if not useProxy:
useProxy = self.opHelper.getValue("/Services/Catalogs/%s/UseProxy" % catalogName, False)
return FileCatalogFactory().createCatalog(catalogName, useProxy)
def web_getLaunchpadOpts(self):
defaultParams = {"JobName" : [1, 'DIRAC'],
"Executable" : [1, "/bin/ls"],
"Arguments" : [1, "-ltrA"],
"OutputSandbox" : [1, "std.out, std.err"],
"JobGroup" : [0, "Unknown"],
"InputData" : [0, ""],
"OutputData" : [0, ""],
"OutputSE" : [0, "DIRAC-USER"],
"OutputPath": [0, ""],
"CPUTime" : [0, "86400"],
"Site" : [0, ""],
"BannedSite" : [0, ""],
"Platform" : [0, "Linux_x86_64_glibc-2.12"],
"Priority" : [0, "5"],
"StdError" : [0, "std.err"],
"StdOutput" : [0, "std.out"],
"Parameters" : [0, "0"],
"ParameterStart" : [0, "0"],
"ParameterStep" : [0, "1"],
"ParameterFactor": [0, "0"]}
delimiter = gConfig.getValue("/WebApp/Launchpad/ListSeparator" , ',')
options = self.__getOptionsFromCS(delimiter=delimiter)
# platform = self.__getPlatform()
# if platform and options:
# if not options.has_key("Platform"):
# options[ "Platform" ] = platform
# else:
# csPlatform = list(options[ "Platform" ])
# allPlatforms = csPlatform + platform
# platform = uniqueElements(allPlatforms)
# options[ "Platform" ] = platform
gLogger.debug("Combined options from CS: %s" % options)
override = gConfig.getValue("/WebApp/Launchpad/OptionsOverride" , False)
gLogger.info("end __getLaunchpadOpts")
# Updating the default values from OptionsOverride configuration branch
for key in options:
if key not in defaultParams:
defaultParams[key] = [ 0, "" ]
defaultParams[key][1] = options[key][0]
# Reading of the predefined sets of launchpad parameters values
obj = Operations( vo = self.vo )
predefinedSets = {}
launchpadSections = obj.getSections("Launchpad")
import pprint
if launchpadSections['OK']:
for section in launchpadSections["Value"]:
predefinedSets[section] = {}
sectionOptions = obj.getOptionsDict("Launchpad/" + section)
pprint.pprint(sectionOptions)
if sectionOptions['OK']:
predefinedSets[section] = sectionOptions["Value"]
self.write({"success":"true", "result":defaultParams, "predefinedSets":predefinedSets})
class CombinedSoftwareInstallation(object):
""" Combined means that it will try to install in the Shared area and in the LocalArea,
depending on the user's rights
"""
def __init__(self, argumentsDict):
""" Standard constructor
Defines, from dictionary of job parameters passed, a set of members to hold e.g. the
applications and the system config.
Also determines the SharedArea and LocalArea.
"""
self.ops = Operations()
self.job = argumentsDict.get('Job', {})
self.ce = argumentsDict.get('CE', {})
self.source = argumentsDict.get('Source', {})
apps = []
if 'SoftwarePackages' in self.job:
if isinstance( self.job['SoftwarePackages'], basestring):
apps = [self.job['SoftwarePackages']]
elif isinstance( self.job['SoftwarePackages'], list ):
apps = self.job['SoftwarePackages']
self.jobConfig = ''
if 'SystemConfig' in self.job:
self.jobConfig = self.job['SystemConfig']
elif 'Platform' in self.job:
self.jobConfig = self.job['Platform']
else:
self.jobConfig = natOS.CMTSupportedConfig()[0]
self.apps = []
for app in apps:
LOG.verbose('Requested Package %s' % app)
app = tuple(app.split('.'))
if len(app) > 2:
tempapp = app
app = []
app.append(tempapp[0])
app.append(".".join(tempapp[1:]))
deps = resolveDeps(self.jobConfig, app[0], app[1])
for dep in deps:
depapp = (dep['app'], dep['version'])
self.apps.append(depapp)
self.apps.append(app)
self.ceConfigs = []
if 'CompatiblePlatforms' in self.ce:
self.ceConfigs = self.ce['CompatiblePlatforms']
if isinstance( self.ceConfigs, basestring ):
self.ceConfigs = [self.ceConfigs]
#else:
### Use always the list of compatible platform.
self.ceConfigs = natOS.CMTSupportedConfig()
self.sharedArea = getSharedAreaLocation()
LOG.info("SharedArea is %s" % self.sharedArea)
self.localArea = getLocalAreaLocation()
LOG.info("LocalArea is %s" % self.localArea)
def execute(self):
""" Main method of the class executed by DIRAC jobAgent
Executes the following:
- look for the compatible platforms in the CS, see if one matches request
- install the applications, calls :mod:`~ILCDIRAC.Core.Utilities.TARsoft`
:return: S_OK(), S_ERROR()
"""
if not self.apps:
# There is nothing to do
return DIRAC.S_OK()
if not self.jobConfig:
LOG.error('No architecture requested')
return DIRAC.S_ERROR( 'No architecture requested' )
found_config = False
LOG.info("Found CE Configs %s, compatible with system reqs %s" % (",".join(self.ceConfigs),
self.jobConfig))
res = self.ops.getSections('/AvailableTarBalls')
if not res['OK']:
return res
else:
supported_systems = res['Value']
###look in CS if specified platform has software available. Normally consistency check is done at submission time
for ceConfig in self.ceConfigs:
for supp_systems in supported_systems:
if ceConfig == supp_systems:
self.jobConfig = ceConfig
found_config = True
break
if not found_config:
if self.ceConfigs: # redundant check as this is done in the job agent, if locally running option might not be defined
LOG.error('Requested architecture not supported by CE')
return DIRAC.S_ERROR( 'Requested architecture not supported by CE' )
else:
LOG.info('Assume locally running job, will install software in ')
areas = []
###Deal with shared/local area: first try to see if the Shared area exists and if not create it (try to). If it fails, fall back to local area
if not self.sharedArea:
if createSharedArea():
self.sharedArea = getSharedAreaLocation()
if self.sharedArea:
areas.append(self.sharedArea)
LOG.info("Will attempt to install in shared area")
else:
areas.append(self.sharedArea)
areas.append(self.localArea)
for app in self.apps:
res = checkCVMFS(self.jobConfig, app)
if res['OK']:
LOG.notice('Software %s is available on CVMFS, skipping' % ", ".join(app))
continue
## First install the original package in any of the areas
resInstall = installInAnyArea(areas, app, self.jobConfig)
if not resInstall['OK']:
return resInstall
## Then install the dependencies, which can be in a different area
resDeps = installDependencies(app, self.jobConfig, areas)
if not resDeps['OK']:
LOG.error("Failed to install dependencies: ", resDeps['Message'])
return S_ERROR("Failed to install dependencies")
if self.sharedArea:
#List content
listAreaDirectory(self.sharedArea)
return DIRAC.S_OK()
class FileCatalog( object ):
def __init__( self, catalogs = None, vo = None ):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.ro_methods = set()
self.write_methods = set()
self.no_lfn_methods = set()
self.readCatalogs = []
self.writeCatalogs = []
self.rootConfigPath = '/Resources/FileCatalogs'
self.vo = vo if vo else getVOfromProxyGroup().get( 'Value', None )
self.log = gLogger.getSubLogger( "FileCatalog" )
self.opHelper = Operations( vo = self.vo )
catalogList = []
if isinstance( catalogs, basestring ):
catalogList = [catalogs]
elif isinstance( catalogs, ( list, tuple ) ):
catalogList = list( catalogs )
if catalogList:
result = self._getEligibleCatalogs()
if not result['OK']:
self.log.error( "Failed to get eligible catalog" )
return
eligibleFileCatalogs = result['Value']
catalogCheck = True
for catalog in catalogList:
if catalog not in eligibleFileCatalogs:
self.log.error( "Specified catalog is not eligible", catalog )
catalogCheck = False
if catalogCheck:
result = self._getSelectedCatalogs( catalogList )
else:
result = S_ERROR( "Specified catalog is not eligible" )
else:
result = self._getCatalogs()
if not result['OK']:
self.log.error( "Failed to create catalog objects" )
self.valid = False
elif ( len( self.readCatalogs ) == 0 ) and ( len( self.writeCatalogs ) == 0 ):
self.log.error( "No catalog object created" )
self.valid = False
result = self.getMasterCatalogNames()
masterCatalogs = result['Value']
# There can not be more than one master catalog
haveMaster = False
if len( masterCatalogs ) > 1:
self.log.error( "More than one master catalog created" )
self.valid = False
elif len( masterCatalogs ) == 1:
haveMaster = True
# Get the list of write methods
if haveMaster:
# All the write methods must be present in the master
_catalogName, oCatalog, _master = self.writeCatalogs[0]
_roList, writeList, nolfnList = oCatalog.getInterfaceMethods()
self.write_methods.update( writeList )
self.no_lfn_methods.update( nolfnList )
else:
for _catalogName, oCatalog, _master in self.writeCatalogs:
_roList, writeList, nolfnList = oCatalog.getInterfaceMethods()
self.write_methods.update( writeList )
self.no_lfn_methods.update( nolfnList )
# Get the list of read methods
for _catalogName, oCatalog, _master in self.readCatalogs:
roList, _writeList, nolfnList = oCatalog.getInterfaceMethods()
self.ro_methods.update( roList )
self.no_lfn_methods.update( nolfnList )
self.condParser = FCConditionParser( vo = self.vo, ro_methods = self.ro_methods )
def isOK( self ):
return self.valid
def getReadCatalogs( self ):
return self.readCatalogs
def getWriteCatalogs( self ):
return self.writeCatalogs
def getMasterCatalogNames( self ):
""" Returns the list of names of the Master catalogs """
masterNames = [catalogName for catalogName, oCatalog, master in self.writeCatalogs if master]
return S_OK( masterNames )
def __getattr__( self, name ):
self.call = name
if name in self.write_methods:
return self.w_execute
elif name in self.ro_methods:
return self.r_execute
else:
raise AttributeError
def w_execute( self, *parms, **kws ):
""" Write method executor.
If one of the LFNs given as input does not pass a condition defined for the
master catalog, we return S_ERROR without trying anything else
:param fcConditions: either a dict or a string, to be propagated to the FCConditionParser
If it is a string, it is given for all catalogs
If it is a dict, it has to be { catalogName: condition}, and only
the specific condition for the catalog will be given
CAUTION !!! If the method is a write no_lfn method, then the return value are completely different
We only return the result of the master catalog
"""
successful = {}
failed = {}
failedCatalogs = {}
successfulCatalogs = {}
specialConditions = kws.pop( 'fcConditions' ) if 'fcConditions' in kws else None
allLfns = []
lfnMapDict = {}
masterResult = {}
parms1 = []
if self.call not in self.no_lfn_methods:
fileInfo = parms[0]
result = checkArgumentFormat( fileInfo, generateMap = True )
if not result['OK']:
return result
fileInfo, lfnMapDict = result['Value']
# No need to check the LFNs again in the clients
kws['LFNChecking'] = False
allLfns = fileInfo.keys()
parms1 = parms[1:]
for catalogName, oCatalog, master in self.writeCatalogs:
# Skip if the method is not implemented in this catalog
# NOTE: it is impossible for the master since the write method list is populated
# only from the master catalog, and if the method is not there, __getattr__
# would raise an exception
if not oCatalog.hasCatalogMethod( self.call ):
continue
method = getattr( oCatalog, self.call )
if self.call in self.no_lfn_methods:
result = method( *parms, **kws )
else:
if isinstance( specialConditions, dict ):
condition = specialConditions.get( catalogName )
else:
condition = specialConditions
# Check whether this catalog should be used for this method
res = self.condParser( catalogName, self.call, fileInfo, condition = condition )
# condParser never returns S_ERROR
condEvals = res['Value']['Successful']
# For a master catalog, ALL the lfns should be valid
if master:
if any([not valid for valid in condEvals.values()]):
gLogger.error( "The master catalog is not valid for some LFNS", condEvals )
return S_ERROR( "The master catalog is not valid for some LFNS %s" % condEvals )
validLFNs = dict( ( lfn, fileInfo[lfn] ) for lfn in condEvals if condEvals[lfn] )
invalidLFNs = [lfn for lfn in condEvals if not condEvals[lfn]]
if invalidLFNs:
gLogger.debug( "Some LFNs are not valid for operation '%s' on catalog '%s' : %s" % ( self.call, catalogName,
invalidLFNs ) )
result = method( validLFNs, *parms1, **kws )
if master:
masterResult = result
if not result['OK']:
if master:
# If this is the master catalog and it fails we don't want to continue with the other catalogs
self.log.error( "Failed to execute call on master catalog",
"%s on %s: %s" % ( self.call, catalogName, result['Message'] ) )
return result
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs[catalogName] = result['Message']
else:
successfulCatalogs[catalogName] = result['Value']
if allLfns:
if result['OK']:
for lfn, message in result['Value']['Failed'].items():
# Save the error message for the failed operations
failed.setdefault( lfn, {} )[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop( lfn, None )
for lfn, result in result['Value']['Successful'].items():
# Save the result return for each file for the successful operations
successful.setdefault( lfn, {} )[catalogName] = result
if allLfns:
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs.items():
for lfn in allLfns:
failed.setdefault( lfn, {} )[catalogName] = errorMessage
# Restore original lfns if they were changed by normalization
if lfnMapDict:
for lfn in failed.keys():
failed[lfnMapDict.get( lfn, lfn )] = failed.pop( lfn )
for lfn in successful.keys():
successful[lfnMapDict.get( lfn, lfn )] = successful.pop( lfn )
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
else:
# FIXME: Return just master result here. This is temporary as more detailed
# per catalog result needs multiple fixes in various client calls
return masterResult
def r_execute( self, *parms, **kws ):
""" Read method executor.
"""
successful = {}
failed = {}
for _catalogName, oCatalog, _master in self.readCatalogs:
# Skip if the method is not implemented in this catalog
if not oCatalog.hasCatalogMethod( self.call ):
continue
method = getattr( oCatalog, self.call )
res = method( *parms, **kws )
if res['OK']:
if 'Successful' in res['Value']:
for key, item in res['Value']['Successful'].items():
successful.setdefault( key, item )
failed.pop( key, None )
for key, item in res['Value']['Failed'].items():
if key not in successful:
failed[key] = item
else:
return res
if not successful and not failed:
return S_ERROR( DErrno.EFCERR, "Failed to perform %s from any catalog" % self.call )
return S_OK( {'Failed':failed, 'Successful':successful} )
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog( self, catalogName, mode = "Write", master = False ):
""" Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject( catalogName )
if not result['OK']:
return result
oCatalog = result['Value']
if mode.lower().find( "read" ) != -1:
self.readCatalogs.append( ( catalogName, oCatalog, master ) )
if mode.lower().find( "write" ) != -1:
self.writeCatalogs.append( ( catalogName, oCatalog, master ) )
return S_OK()
def removeCatalog( self, catalogName ):
""" Remove the specified catalog from the internal pool
"""
catalog_removed = False
for i in range( len( self.readCatalogs ) ):
catalog, _object, _master = self.readCatalogs[i]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range( len( self.writeCatalogs ) ):
catalog, _object, _master = self.writeCatalogs[i]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK( 'Catalog does not exist' )
def _getSelectedCatalogs( self, desiredCatalogs ):
for catalogName in desiredCatalogs:
result = self._getCatalogConfigDetails( catalogName )
if not result['OK']:
return result
catalogConfig = result['Value']
result = self._generateCatalogObject( catalogName )
if not result['OK']:
return result
oCatalog = result['Value']
if re.search( 'Read', catalogConfig['AccessType'] ):
if catalogConfig['Master']:
self.readCatalogs.insert( 0, ( catalogName, oCatalog, catalogConfig['Master'] ) )
else:
self.readCatalogs.append( ( catalogName, oCatalog, catalogConfig['Master'] ) )
if re.search( 'Write', catalogConfig['AccessType'] ):
if catalogConfig['Master']:
self.writeCatalogs.insert( 0, ( catalogName, oCatalog, catalogConfig['Master'] ) )
else:
self.writeCatalogs.append( ( catalogName, oCatalog, catalogConfig['Master'] ) )
return S_OK()
def _getEligibleCatalogs( self ):
""" Get a list of eligible catalogs
:return: S_OK/S_ERROR, Value - a list of catalog names
"""
# First, look in the Operations, if nothing defined look in /Resources for backward compatibility
fileCatalogs = self.opHelper.getValue( '/Services/Catalogs/CatalogList', [] )
if not fileCatalogs:
result = self.opHelper.getSections( '/Services/Catalogs' )
if result['OK']:
fileCatalogs = result['Value']
else:
res = gConfig.getSections( self.rootConfigPath, listOrdered = True )
if not res['OK']:
errStr = "FileCatalog._getEligibleCatalogs: Failed to get file catalog configuration."
self.log.error( errStr, res['Message'] )
return S_ERROR( errStr )
fileCatalogs = res['Value']
return S_OK( fileCatalogs )
def _getCatalogs( self ):
""" Updates self.readCatalogs and self.writeCatalogs with list of catalog objects as found in the CS
"""
# Get the eligible catalogs first
result = self._getEligibleCatalogs()
if not result['OK']:
return result
fileCatalogs = result['Value']
# Get the catalog objects now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails( catalogName )
if not res['OK']:
return res
catalogConfig = res['Value']
if catalogConfig['Status'] == 'Active':
res = self._generateCatalogObject( catalogName )
if not res['OK']:
return res
oCatalog = res['Value']
master = catalogConfig['Master']
# If the catalog is read type
if re.search( 'Read', catalogConfig['AccessType'] ):
if master:
self.readCatalogs.insert( 0, ( catalogName, oCatalog, master ) )
else:
self.readCatalogs.append( ( catalogName, oCatalog, master ) )
# If the catalog is write type
if re.search( 'Write', catalogConfig['AccessType'] ):
if master:
self.writeCatalogs.insert( 0, ( catalogName, oCatalog, master ) )
else:
self.writeCatalogs.append( ( catalogName, oCatalog, master ) )
return S_OK()
def _getCatalogConfigDetails( self, catalogName ):
# First obtain the options that are available
catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName )
result = gConfig.getOptionsDict( catalogConfigPath )
if not result['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
self.log.error( errStr, catalogName )
return S_ERROR( errStr )
catalogConfig = result['Value']
result = self.opHelper.getOptionsDict( '/Services/Catalogs/%s' % catalogName )
if result['OK']:
catalogConfig.update( result['Value'] )
# The 'Status' option should be defined (default = 'Active')
if 'Status' not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
self.log.warn( warnStr, catalogName )
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if 'AccessType' not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
self.log.error( errStr, catalogName )
return S_ERROR( errStr )
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' )
return S_OK( catalogConfig )
def _generateCatalogObject( self, catalogName ):
""" Create a file catalog object from its name and CS description
"""
useProxy = gConfig.getValue( '/LocalSite/Catalogs/%s/UseProxy' % catalogName, False )
if not useProxy:
useProxy = self.opHelper.getValue( '/Services/Catalogs/%s/UseProxy' % catalogName, False )
return FileCatalogFactory().createCatalog( catalogName, useProxy )
def matchQueue(jobJDL, queueDict, fullMatch=False):
"""
Match the job description to the queue definition
:param str job: JDL job description
:param bool fullMatch: test matching on all the criteria
:param dict queueDict: queue parameters dictionary
:return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or
S_ERROR with the reason for no match
"""
# Check the job description validity
job = ClassAd(jobJDL)
if not job.isOK():
return S_ERROR('Invalid job description')
noMatchReasons = []
# Check job requirements to resource
# 1. CPUTime
cpuTime = job.getAttributeInt('CPUTime')
if not cpuTime:
cpuTime = 84600
if cpuTime > queueDict.get('CPUTime', 0.):
noMatchReasons.append('Job CPUTime requirement not satisfied')
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 2. Multi-value match requirements
for parameter in [
'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType',
'SubmitPool', 'JobType'
]:
if parameter in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression('%ss' % parameter))
queueSet = set(fromChar(queueDict[parameter]))
if valueSet and queueSet and not valueSet.intersection(queueSet):
valueToPrint = ','.join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append('Job %s %s requirement not satisfied' %
(parameter, valueToPrint))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 3. Banned multi-value match requirements
for par in [
'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType',
'SubmitPool', 'JobType'
]:
parameter = "Banned%s" % par
if par in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression('%ss' % parameter))
queueSet = set(fromChar(queueDict[par]))
if valueSet and queueSet and valueSet.issubset(queueSet):
valueToPrint = ','.join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append('Job %s %s requirement not satisfied' %
(parameter, valueToPrint))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 4. Tags
tags = set(job.getListFromExpression('Tag'))
nProc = job.getAttributeInt('NumberOfProcessors')
if nProc and nProc > 1:
tags.add('MultiProcessor')
wholeNode = job.getAttributeString('WholeNode')
if wholeNode:
tags.add('WholeNode')
queueTags = set(queueDict.get('Tags', []))
if not tags.issubset(queueTags):
noMatchReasons.append('Job Tag %s not satisfied' % ','.join(tags))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 4. MultiProcessor requirements
if nProc and nProc > int(queueDict.get('NumberOfProcessors', 1)):
noMatchReasons.append(
'Job NumberOfProcessors %d requirement not satisfied' % nProc)
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 5. RAM
ram = job.getAttributeInt('RAM')
# If MaxRAM is not specified in the queue description, assume 2GB
if ram and ram > int(queueDict.get('MaxRAM', 2048)) / 1024:
noMatchReasons.append('Job RAM %d requirement not satisfied' % ram)
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# Check resource requirements to job
# 1. OwnerGroup - rare case but still
if "OwnerGroup" in queueDict:
result = getProxyInfo(disableVOMS=True)
if not result['OK']:
return S_ERROR('No valid proxy available')
ownerGroup = result['Value']['group']
if ownerGroup != queueDict['OwnerGroup']:
noMatchReasons.append(
'Resource OwnerGroup %s requirement not satisfied' %
queueDict['OwnerGroup'])
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 2. Required tags
requiredTags = set(queueDict.get('RequiredTags', []))
if not requiredTags.issubset(tags):
noMatchReasons.append('Resource RequiredTags %s not satisfied' %
','.join(requiredTags))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 3. RunningLimit
site = queueDict['Site']
opsHelper = Operations()
result = opsHelper.getSections('JobScheduling/RunningLimit')
if result['OK'] and site in result['Value']:
result = opsHelper.getSections('JobScheduling/RunningLimit/%s' % site)
if result['OK']:
for parameter in result['Value']:
value = job.getAttributeString(parameter)
if value and opsHelper.getValue(
'JobScheduling/RunningLimit/%s/%s/%s' %
(site, parameter, value), 1) == 0:
noMatchReasons.append(
'Resource operational %s requirement not satisfied' %
parameter)
if not fullMatch:
return S_OK({
'Match': False,
'Reason': noMatchReasons[0]
})
return S_OK({'Match': not bool(noMatchReasons), 'Reason': noMatchReasons})
class Limiter(object):
def __init__(self, jobDB=None, opsHelper=None):
""" Constructor
"""
self.__runningLimitSection = "JobScheduling/RunningLimit"
self.__matchingDelaySection = "JobScheduling/MatchingDelay"
self.csDictCache = DictCache()
self.condCache = DictCache()
self.delayMem = {}
if jobDB:
self.jobDB = jobDB
else:
self.jobDB = JobDB()
self.log = gLogger.getSubLogger("Limiter")
if opsHelper:
self.__opsHelper = opsHelper
else:
self.__opsHelper = Operations()
def getNegativeCond(self):
""" Get negative condition for ALL sites
"""
orCond = self.condCache.get("GLOBAL")
if orCond:
return orCond
negCond = {}
# Run Limit
result = self.__opsHelper.getSections(self.__runningLimitSection)
sites = []
if result['OK']:
sites = result['Value']
for siteName in sites:
result = self.__getRunningCondition(siteName)
if not result['OK']:
continue
data = result['Value']
if data:
negCond[siteName] = data
# Delay limit
result = self.__opsHelper.getSections(self.__matchingDelaySection)
sites = []
if result['OK']:
sites = result['Value']
for siteName in sites:
result = self.__getDelayCondition(siteName)
if not result['OK']:
continue
data = result['Value']
if not data:
continue
if siteName in negCond:
negCond[siteName] = self.__mergeCond(negCond[siteName], data)
else:
negCond[siteName] = data
orCond = []
for siteName in negCond:
negCond[siteName]['Site'] = siteName
orCond.append(negCond[siteName])
self.condCache.add("GLOBAL", 10, orCond)
return orCond
def getNegativeCondForSite(self, siteName):
""" Generate a negative query based on the limits set on the site
"""
# Check if Limits are imposed onto the site
negativeCond = {}
if self.__opsHelper.getValue("JobScheduling/CheckJobLimits", True):
result = self.__getRunningCondition(siteName)
if result['OK']:
negativeCond = result['Value']
self.log.verbose('Negative conditions for site %s after checking limits are: %s' % (siteName, str(negativeCond)))
if self.__opsHelper.getValue("JobScheduling/CheckMatchingDelay", True):
result = self.__getDelayCondition(siteName)
if result['OK']:
delayCond = result['Value']
self.log.verbose('Negative conditions for site %s after delay checking are: %s' % (siteName, str(delayCond)))
negativeCond = self.__mergeCond(negativeCond, delayCond)
if negativeCond:
self.log.info('Negative conditions for site %s are: %s' % (siteName, str(negativeCond)))
return negativeCond
def __mergeCond(self, negCond, addCond):
""" Merge two negative dicts
"""
# Merge both negative dicts
for attr in addCond:
if attr not in negCond:
negCond[attr] = []
for value in addCond[attr]:
if value not in negCond[attr]:
negCond[attr].append(value)
return negCond
def __extractCSData(self, section):
""" Extract limiting information from the CS in the form:
{ 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
"""
stuffDict = self.csDictCache.get(section)
if stuffDict:
return S_OK(stuffDict)
result = self.__opsHelper.getSections(section)
if not result['OK']:
return result
attribs = result['Value']
stuffDict = {}
for attName in attribs:
result = self.__opsHelper.getOptionsDict("%s/%s" % (section, attName))
if not result['OK']:
return result
attLimits = result['Value']
try:
attLimits = dict([(k, int(attLimits[k])) for k in attLimits])
except Exception as excp:
errMsg = "%s/%s has to contain numbers: %s" % (section, attName, str(excp))
self.log.error(errMsg)
return S_ERROR(errMsg)
stuffDict[attName] = attLimits
self.csDictCache.add(section, 300, stuffDict)
return S_OK(stuffDict)
def __getRunningCondition(self, siteName):
""" Get extra conditions allowing site throttling
"""
siteSection = "%s/%s" % (self.__runningLimitSection, siteName)
result = self.__extractCSData(siteSection)
if not result['OK']:
return result
limitsDict = result['Value']
# limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
if not limitsDict:
return S_OK({})
# Check if the site exceeding the given limits
negCond = {}
for attName in limitsDict:
if attName not in self.jobDB.jobAttributeNames:
self.log.error("Attribute %s does not exist. Check the job limits" % attName)
continue
cK = "Running:%s:%s" % (siteName, attName)
data = self.condCache.get(cK)
if not data:
result = self.jobDB.getCounters(
'Jobs', [attName], {
'Site': siteName, 'Status': [
'Running', 'Matched', 'Stalled']})
if not result['OK']:
return result
data = result['Value']
data = dict([(k[0][attName], k[1]) for k in data])
self.condCache.add(cK, 10, data)
for attValue in limitsDict[attName]:
limit = limitsDict[attName][attValue]
running = data.get(attValue, 0)
if running >= limit:
self.log.verbose('Job Limit imposed at %s on %s/%s=%d,'
' %d jobs already deployed' % (siteName, attName, attValue, limit, running))
if attName not in negCond:
negCond[attName] = []
negCond[attName].append(attValue)
# negCond is something like : {'JobType': ['Merge']}
return S_OK(negCond)
def updateDelayCounters(self, siteName, jid):
# Get the info from the CS
siteSection = "%s/%s" % (self.__matchingDelaySection, siteName)
result = self.__extractCSData(siteSection)
if not result['OK']:
return result
delayDict = result['Value']
# limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
if not delayDict:
return S_OK()
attNames = []
for attName in delayDict:
if attName not in self.jobDB.jobAttributeNames:
self.log.error("Attribute %s does not exist in the JobDB. Please fix it!" % attName)
else:
attNames.append(attName)
result = self.jobDB.getJobAttributes(jid, attNames)
if not result['OK']:
self.log.error("While retrieving attributes coming from %s: %s" % (siteSection, result['Message']))
return result
atts = result['Value']
# Create the DictCache if not there
if siteName not in self.delayMem:
self.delayMem[siteName] = DictCache()
# Update the counters
delayCounter = self.delayMem[siteName]
for attName in atts:
attValue = atts[attName]
if attValue in delayDict[attName]:
delayTime = delayDict[attName][attValue]
self.log.notice("Adding delay for %s/%s=%s of %s secs" % (siteName, attName,
attValue, delayTime))
delayCounter.add((attName, attValue), delayTime)
return S_OK()
def __getDelayCondition(self, siteName):
""" Get extra conditions allowing matching delay
"""
if siteName not in self.delayMem:
return S_OK({})
lastRun = self.delayMem[siteName].getKeys()
negCond = {}
for attName, attValue in lastRun:
if attName not in negCond:
negCond[attName] = []
negCond[attName].append(attValue)
return S_OK(negCond)
class FileCatalog(object):
ro_methods = [
'exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata',
'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory',
'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata',
'getDirectorySize', 'getDirectoryContents', 'resolveDataset',
'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups',
'getLFNForGUID'
]
ro_meta_methods = [
'getFileUserMetadata', 'getMetadataFields', 'findFilesByMetadata',
'getFileUserMetadata', 'findDirectoriesByMetadata',
'getReplicasByMetadata', 'findFilesByMetadataDetailed',
'findFilesByMetadataWeb', 'getCompatibleMetadata', 'getMetadataSet'
]
ro_methods += ro_meta_methods
write_methods = [
'createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica',
'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost',
'setReplicaProblematic', 'createDirectory', 'setDirectoryStatus',
'removeDirectory', 'removeDataset', 'removeFileFromDataset',
'createDataset', 'changePathMode', 'changePathOwner', 'changePathGroup'
]
write_meta_methods = [
'addMetadataField', 'deleteMetadataField', 'setMetadata',
'setMetadataBulk', 'removeMetadata', 'addMetadataSet'
]
write_methods += write_meta_methods
def __init__(self, catalogs=None, vo=None):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.readCatalogs = []
self.writeCatalogs = []
self.metaCatalogs = []
self.rootConfigPath = '/Resources/FileCatalogs'
self.vo = vo if vo else getVOfromProxyGroup().get('Value', None)
self.opHelper = Operations(vo=self.vo)
if catalogs is None:
catalogList = []
elif type(catalogs) in types.StringTypes:
catalogList = [catalogs]
else:
catalogList = catalogs
if catalogList:
res = self._getSelectedCatalogs(catalogList)
else:
res = self._getCatalogs()
if not res['OK']:
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.valid = False
def isOK(self):
return self.valid
def getReadCatalogs(self):
return self.readCatalogs
def getWriteCatalogs(self):
return self.writeCatalogs
def getMasterCatalogNames(self):
""" Returns the list of names of the Master catalogs """
masterNames = [
catalogName for catalogName, oCatalog, master in self.writeCatalogs
if master
]
return S_OK(masterNames)
def __getattr__(self, name):
self.call = name
if name in FileCatalog.write_methods:
return self.w_execute
elif name in FileCatalog.ro_methods:
return self.r_execute
else:
raise AttributeError
def w_execute(self, *parms, **kws):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = checkArgumentFormat(fileInfo)
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
parms = parms[1:]
for catalogName, oCatalog, master in self.writeCatalogs:
# Skip if metadata related method on pure File Catalog
if self.call in FileCatalog.write_meta_methods and not catalogName in self.metaCatalogs:
continue
method = getattr(oCatalog, self.call)
res = method(fileInfo, *parms, **kws)
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error(
"FileCatalog.w_execute: Failed to execute %s on master catalog %s."
% (self.call, catalogName), res['Message'])
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append((catalogName, res['Message']))
else:
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
failed.setdefault(lfn, {})[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn, None)
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
successful.setdefault(lfn, {})[catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
failed.setdefault(lfn, {})[catalogName] = errorMessage
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def r_execute(self, *parms, **kws):
""" Read method executor.
"""
successful = {}
failed = {}
for catalogName, oCatalog, _master in self.readCatalogs:
# Skip if metadata related method on pure File Catalog
if self.call in FileCatalog.ro_meta_methods and not catalogName in self.metaCatalogs:
continue
method = getattr(oCatalog, self.call)
res = method(*parms, **kws)
if res['OK']:
if 'Successful' in res['Value']:
for key, item in res['Value']['Successful'].items():
successful.setdefault(key, item)
failed.pop(key, None)
for key, item in res['Value']['Failed'].items():
if key not in successful:
failed[key] = item
else:
return res
if not successful and not failed:
return S_ERROR("Failed to perform %s from any catalog" % self.call)
return S_OK({'Failed': failed, 'Successful': successful})
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog(self, catalogName, mode="Write", master=False):
""" Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject(catalogName)
if not result['OK']:
return result
oCatalog = result['Value']
if mode.lower().find("read") != -1:
self.readCatalogs.append((catalogName, oCatalog, master))
if mode.lower().find("write") != -1:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def removeCatalog(self, catalogName):
""" Remove the specified catalog from the internal pool
"""
catalog_removed = False
for i in range(len(self.readCatalogs)):
catalog, _object, _master = self.readCatalogs[i]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range(len(self.writeCatalogs)):
catalog, _object, _master = self.writeCatalogs[i]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK('Catalog does not exist')
def _getSelectedCatalogs(self, desiredCatalogs):
for catalogName in desiredCatalogs:
res = self._getCatalogConfigDetails(catalogName)
if not res['OK']:
return res
catalogConfig = res['Value']
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
self.readCatalogs.append((catalogName, oCatalog, True))
self.writeCatalogs.append((catalogName, oCatalog, True))
if catalogConfig.get('MetaCatalog') == 'True':
self.metaCatalogs.append(catalogName)
return S_OK()
def _getCatalogs(self):
# Get the eligible catalogs first
# First, look in the Operations, if nothing defined look in /Resources for backward compatibility
operationsFlag = False
fileCatalogs = self.opHelper.getValue('/Services/Catalogs/CatalogList',
[])
if fileCatalogs:
operationsFlag = True
else:
fileCatalogs = self.opHelper.getSections('/Services/Catalogs')
result = self.opHelper.getSections('/Services/Catalogs')
fileCatalogs = []
operationsFlag = False
if result['OK']:
fileCatalogs = result['Value']
operationsFlag = True
else:
res = gConfig.getSections(self.rootConfigPath,
listOrdered=True)
if not res['OK']:
errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration."
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
fileCatalogs = res['Value']
# Get the catalogs now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails(catalogName)
if not res['OK']:
return res
catalogConfig = res['Value']
if operationsFlag:
result = self.opHelper.getOptionsDict('/Services/Catalogs/%s' %
catalogName)
if not result['OK']:
return result
catalogConfig.update(result['Value'])
if catalogConfig['Status'] == 'Active':
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
master = catalogConfig['Master']
# If the catalog is read type
if re.search('Read', catalogConfig['AccessType']):
if master:
self.readCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.readCatalogs.append(
(catalogName, oCatalog, master))
# If the catalog is write type
if re.search('Write', catalogConfig['AccessType']):
if master:
self.writeCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.writeCatalogs.append(
(catalogName, oCatalog, master))
if catalogConfig.get('MetaCatalog') == 'True':
self.metaCatalogs.append(catalogName)
return S_OK()
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
catalogConfigPath = '%s/%s' % (self.rootConfigPath, catalogName)
res = gConfig.getOptions(catalogConfigPath)
if not res['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = {}
for option in res['Value']:
configPath = '%s/%s' % (catalogConfigPath, option)
optionValue = gConfig.getValue(configPath)
catalogConfig[option] = optionValue
# The 'Status' option should be defined (default = 'Active')
if 'Status' not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
gLogger.warn(warnStr, catalogName)
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if 'AccessType' not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig['Master'] = (catalogConfig.setdefault('Master',
False) == 'True')
return S_OK(catalogConfig)
def _generateCatalogObject(self, catalogName):
""" Create a file catalog object from its name and CS description
"""
useProxy = gConfig.getValue(
'/LocalSite/Catalogs/%s/UseProxy' % catalogName, False)
if not useProxy:
useProxy = self.opHelper.getValue(
'/Services/Catalogs/%s/UseProxy' % catalogName, False)
return FileCatalogFactory().createCatalog(catalogName, useProxy)
class CombinedSoftwareInstallation(object):
""" Combined means that it will try to install in the Shared area and in the LocalArea,
depending on the user's rights
"""
def __init__(self, argumentsDict):
""" Standard constructor
Defines, from dictionary of job parameters passed, a set of members to hold e.g. the
applications and the system config.
Also determines the SharedArea and LocalArea.
"""
self.ops = Operations()
self.job = argumentsDict.get('Job', {})
self.ce = argumentsDict.get('CE', {})
self.source = argumentsDict.get('Source', {})
apps = []
if 'SoftwarePackages' in self.job:
if isinstance(self.job['SoftwarePackages'], basestring):
apps = [self.job['SoftwarePackages']]
elif isinstance(self.job['SoftwarePackages'], list):
apps = self.job['SoftwarePackages']
self.jobConfig = ''
if 'SystemConfig' in self.job:
self.jobConfig = self.job['SystemConfig']
elif 'Platform' in self.job:
self.jobConfig = self.job['Platform']
else:
self.jobConfig = natOS.CMTSupportedConfig()[0]
self.apps = []
for app in apps:
DIRAC.gLogger.verbose('Requested Package %s' % app)
app = tuple(app.split('.'))
if len(app) > 2:
tempapp = app
app = []
app.append(tempapp[0])
app.append(".".join(tempapp[1:]))
deps = resolveDeps(self.jobConfig, app[0], app[1])
for dep in deps:
depapp = (dep['app'], dep['version'])
self.apps.append(depapp)
self.apps.append(app)
self.ceConfigs = []
if 'CompatiblePlatforms' in self.ce:
self.ceConfigs = self.ce['CompatiblePlatforms']
if isinstance(self.ceConfigs, basestring):
self.ceConfigs = [self.ceConfigs]
#else:
### Use always the list of compatible platform.
self.ceConfigs = natOS.CMTSupportedConfig()
self.sharedArea = getSharedAreaLocation()
DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea)
self.localArea = getLocalAreaLocation()
DIRAC.gLogger.info("LocalArea is %s" % self.localArea)
def execute(self):
""" Main method of the class executed by DIRAC jobAgent
Executes the following:
- look for the compatible platforms in the CS, see if one matches request
- install the applications, calls :mod:`~ILCDIRAC.Core.Utilities.TARsoft`
:return: S_OK(), S_ERROR()
"""
if not self.apps:
# There is nothing to do
return DIRAC.S_OK()
if not self.jobConfig:
DIRAC.gLogger.error('No architecture requested')
return DIRAC.S_ERROR('No architecture requested')
found_config = False
DIRAC.gLogger.info(
"Found CE Configs %s, compatible with system reqs %s" %
(",".join(self.ceConfigs), self.jobConfig))
res = self.ops.getSections('/AvailableTarBalls')
if not res['OK']:
return res
else:
supported_systems = res['Value']
###look in CS if specified platform has software available. Normally consistency check is done at submission time
for ceConfig in self.ceConfigs:
for supp_systems in supported_systems:
if ceConfig == supp_systems:
self.jobConfig = ceConfig
found_config = True
break
if not found_config:
if self.ceConfigs: # redundant check as this is done in the job agent, if locally running option might not be defined
DIRAC.gLogger.error(
'Requested architecture not supported by CE')
return DIRAC.S_ERROR(
'Requested architecture not supported by CE')
else:
DIRAC.gLogger.info(
'Assume locally running job, will install software in ')
areas = []
###Deal with shared/local area: first try to see if the Shared area exists and if not create it (try to). If it fails, fall back to local area
if not self.sharedArea:
if createSharedArea():
self.sharedArea = getSharedAreaLocation()
if self.sharedArea:
areas.append(self.sharedArea)
DIRAC.gLogger.info(
"Will attempt to install in shared area")
else:
areas.append(self.sharedArea)
areas.append(self.localArea)
for app in self.apps:
res = checkCVMFS(self.jobConfig, app)
if res['OK']:
DIRAC.gLogger.notice(
'Software %s is available on CVMFS, skipping' %
", ".join(app))
continue
## First install the original package in any of the areas
resInstall = installInAnyArea(areas, app, self.jobConfig)
if not resInstall['OK']:
return resInstall
## Then install the dependencies, which can be in a different area
resDeps = installDependencies(app, self.jobConfig, areas)
if not resDeps['OK']:
DIRAC.gLogger.error("Failed to install dependencies: ",
resDeps['Message'])
return S_ERROR("Failed to install dependencies")
if self.sharedArea:
#List content
listAreaDirectory(self.sharedArea)
return DIRAC.S_OK()
class Limiter(object):
# static variables shared between all instances of this class
csDictCache = DictCache()
condCache = DictCache()
delayMem = {}
def __init__(self, jobDB=None, opsHelper=None):
""" Constructor
"""
self.__runningLimitSection = "JobScheduling/RunningLimit"
self.__matchingDelaySection = "JobScheduling/MatchingDelay"
if jobDB:
self.jobDB = jobDB
else:
self.jobDB = JobDB()
self.log = gLogger.getSubLogger("Limiter")
if opsHelper:
self.__opsHelper = opsHelper
else:
self.__opsHelper = Operations()
def getNegativeCond(self):
""" Get negative condition for ALL sites
"""
orCond = self.condCache.get("GLOBAL")
if orCond:
return orCond
negCond = {}
# Run Limit
result = self.__opsHelper.getSections(self.__runningLimitSection)
sites = []
if result['OK']:
sites = result['Value']
for siteName in sites:
result = self.__getRunningCondition(siteName)
if not result['OK']:
continue
data = result['Value']
if data:
negCond[siteName] = data
# Delay limit
result = self.__opsHelper.getSections(self.__matchingDelaySection)
sites = []
if result['OK']:
sites = result['Value']
for siteName in sites:
result = self.__getDelayCondition(siteName)
if not result['OK']:
continue
data = result['Value']
if not data:
continue
if siteName in negCond:
negCond[siteName] = self.__mergeCond(negCond[siteName], data)
else:
negCond[siteName] = data
orCond = []
for siteName in negCond:
negCond[siteName]['Site'] = siteName
orCond.append(negCond[siteName])
self.condCache.add("GLOBAL", 10, orCond)
return orCond
def getNegativeCondForSite(self, siteName):
""" Generate a negative query based on the limits set on the site
"""
# Check if Limits are imposed onto the site
negativeCond = {}
if self.__opsHelper.getValue("JobScheduling/CheckJobLimits", True):
result = self.__getRunningCondition(siteName)
if result['OK']:
negativeCond = result['Value']
self.log.verbose('Negative conditions for site',
'%s after checking limits are: %s' % (siteName, str(negativeCond)))
if self.__opsHelper.getValue("JobScheduling/CheckMatchingDelay", True):
result = self.__getDelayCondition(siteName)
if result['OK']:
delayCond = result['Value']
self.log.verbose('Negative conditions for site',
'%s after delay checking are: %s' % (siteName, str(delayCond)))
negativeCond = self.__mergeCond(negativeCond, delayCond)
if negativeCond:
self.log.info('Negative conditions for site',
'%s are: %s' % (siteName, str(negativeCond)))
return negativeCond
def __mergeCond(self, negCond, addCond):
""" Merge two negative dicts
"""
# Merge both negative dicts
for attr in addCond:
if attr not in negCond:
negCond[attr] = []
for value in addCond[attr]:
if value not in negCond[attr]:
negCond[attr].append(value)
return negCond
def __extractCSData(self, section):
""" Extract limiting information from the CS in the form:
{ 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
"""
stuffDict = self.csDictCache.get(section)
if stuffDict:
return S_OK(stuffDict)
result = self.__opsHelper.getSections(section)
if not result['OK']:
return result
attribs = result['Value']
stuffDict = {}
for attName in attribs:
result = self.__opsHelper.getOptionsDict("%s/%s" % (section, attName))
if not result['OK']:
return result
attLimits = result['Value']
try:
attLimits = dict([(k, int(attLimits[k])) for k in attLimits])
except Exception as excp:
errMsg = "%s/%s has to contain numbers: %s" % (section, attName, str(excp))
self.log.error(errMsg)
return S_ERROR(errMsg)
stuffDict[attName] = attLimits
self.csDictCache.add(section, 300, stuffDict)
return S_OK(stuffDict)
def __getRunningCondition(self, siteName):
""" Get extra conditions allowing site throttling
"""
siteSection = "%s/%s" % (self.__runningLimitSection, siteName)
result = self.__extractCSData(siteSection)
if not result['OK']:
return result
limitsDict = result['Value']
# limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
if not limitsDict:
return S_OK({})
# Check if the site exceeding the given limits
negCond = {}
for attName in limitsDict:
if attName not in self.jobDB.jobAttributeNames:
self.log.error("Attribute does not exist",
"(%s). Check the job limits" % attName)
continue
cK = "Running:%s:%s" % (siteName, attName)
data = self.condCache.get(cK)
if not data:
result = self.jobDB.getCounters(
'Jobs', [attName], {
'Site': siteName, 'Status': [
'Running', 'Matched', 'Stalled']})
if not result['OK']:
return result
data = result['Value']
data = dict([(k[0][attName], k[1]) for k in data])
self.condCache.add(cK, 10, data)
for attValue in limitsDict[attName]:
limit = limitsDict[attName][attValue]
running = data.get(attValue, 0)
if running >= limit:
self.log.verbose('Job Limit imposed',
'at %s on %s/%s=%d, %d jobs already deployed' % (siteName,
attName, attValue, limit, running))
if attName not in negCond:
negCond[attName] = []
negCond[attName].append(attValue)
# negCond is something like : {'JobType': ['Merge']}
return S_OK(negCond)
def updateDelayCounters(self, siteName, jid):
# Get the info from the CS
siteSection = "%s/%s" % (self.__matchingDelaySection, siteName)
result = self.__extractCSData(siteSection)
if not result['OK']:
return result
delayDict = result['Value']
# limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
if not delayDict:
return S_OK()
attNames = []
for attName in delayDict:
if attName not in self.jobDB.jobAttributeNames:
self.log.error("Attribute does not exist in the JobDB. Please fix it!",
"(%s)" % attName)
else:
attNames.append(attName)
result = self.jobDB.getJobAttributes(jid, attNames)
if not result['OK']:
self.log.error("Error while retrieving attributes",
"coming from %s: %s" % (siteSection, result['Message']))
return result
atts = result['Value']
# Create the DictCache if not there
if siteName not in self.delayMem:
self.delayMem[siteName] = DictCache()
# Update the counters
delayCounter = self.delayMem[siteName]
for attName in atts:
attValue = atts[attName]
if attValue in delayDict[attName]:
delayTime = delayDict[attName][attValue]
self.log.notice("Adding delay for %s/%s=%s of %s secs" % (siteName, attName,
attValue, delayTime))
delayCounter.add((attName, attValue), delayTime)
return S_OK()
def __getDelayCondition(self, siteName):
""" Get extra conditions allowing matching delay
"""
if siteName not in self.delayMem:
return S_OK({})
lastRun = self.delayMem[siteName].getKeys()
negCond = {}
for attName, attValue in lastRun:
if attName not in negCond:
negCond[attName] = []
negCond[attName].append(attValue)
return S_OK(negCond)
class OverlayInput(LCUtilityApplication):
""" Helper call to define Overlay processor/driver inputs.
Example:
>>> over = OverlayInput()
>>> over.setBXOverlay(300)
>>> over.setGGToHadInt(3.2)
>>> over.setNumberOfSignalEventsPerJob(10)
>>> over.setBackgroundType("gghad")
"""
def __init__(self, paramdict=None):
self._ops = Operations()
self.bxToOverlay = None
self.numberOfGGToHadronInteractions = 0
self.numberOfSignalEventsPerJob = 0
self.backgroundEventType = ''
self.prodID = 0
self.machine = 'clic_cdr'
self.detectorModel = ''
self.useEnergyForFileLookup = True
super(OverlayInput, self).__init__(paramdict)
self.version = '1'
self._modulename = "OverlayInput"
self.appname = self._modulename
self._moduledescription = 'Helper call to define Overlay processor/driver inputs'
self.accountInProduction = False
self._paramsToExclude.append('_ops')
self.pathToOverlayFiles = ''
self.processorName = ''
def setMachine(self, machine):
""" Define the machine to use, clic_cdr or ilc_dbd
"""
self._checkArgs({'machine': types.StringTypes})
self.machine = machine
def setProdID(self, pid):
""" Define the prodID to use as input, experts only
"""
self._checkArgs({'pid': types.IntType})
self.prodID = pid
return S_OK()
def setUseEnergyForFileLookup(self, useEnergyForFileLookup):
"""
Sets the flag to use the energy meta data in the search of the background files.
Disable the energy when you want to use files created for a different energy than the signal events
:param bool useEnergyForFileLookup: Use the Energy in the metadata search or not
"""
self._checkArgs({'useEnergyForFileLookup': types.BooleanType})
self.useEnergyForFileLookup = useEnergyForFileLookup
return S_OK()
def setOverlayBXPerSigEvt(self, bxoverlay):
""" Define number bunch crossings to overlay for each signal event.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
Alias for :func:`setBXOverlay`
:param int bxoverlay: Bunch crossings to overlay.
"""
self._checkArgs({'bxoverlay': types.IntType})
self.bxToOverlay = bxoverlay
return S_OK()
def setBXOverlay(self, bxoverlay):
""" Define number bunch crossings to overlay for each signal event.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
:param int bxoverlay: Bunch crossings to overlay.
"""
return self.setOverlayBXPerSigEvt(bxoverlay)
def setOverlayEvtsPerBX(self, ggtohadint):
""" Define the number of overlay events per bunch crossing.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
:param float ggtohadint: optional number of overlay events interactions per bunch crossing
"""
self._checkArgs({'ggtohadint': types.FloatType})
self.numberOfGGToHadronInteractions = ggtohadint
return S_OK()
def setGGToHadInt(self, ggtohadint):
""" Define the number of overlay events per bunch crossing.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
Alias for :func:`setOverlayEvtsPerBX`
:param float ggtohadint: optional number of overlay events interactions per bunch crossing
"""
return self.setOverlayEvtsPerBX(ggtohadint)
def setNbSigEvtsPerJob(self, nbsigevtsperjob):
""" Set the number of signal events per job.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
:param int nbsigevtsperjob: Number of signal events per job
"""
self._checkArgs({'nbsigevtsperjob': types.IntType})
self.numberOfSignalEventsPerJob = nbsigevtsperjob
return S_OK()
def setDetectorModel(self, detectormodel):
""" Set the detector type for the background files.
Files are defined in the ConfigurationSystem: Operations/Overlay/<Accelerator>/<energy>/<Detector>
:param string detectormodel: Detector type. Must be 'CLIC_ILD_CDR' or 'CLIC_SID_CDR' or 'sidloi3' or 'ILD_o1_v05'
"""
self._checkArgs({'detectormodel': types.StringTypes})
self.detectorModel = detectormodel
return S_OK()
def setPathToFiles(self, path):
""" Sets the path to where the overlay files are located.
Setting this option will ignore all other settings!
:param string path: LFN path to the folder containing the overlay files
"""
self._checkArgs({'path': types.StringTypes})
self.pathToOverlayFiles = path
return S_OK()
def setBkgEvtType(self, backgroundEventType):
""" Define the background type.
.. deprecated:: 23r0
Use :func:`setBackgroundType` instead
:param string backgroundEventType: Background type.
"""
self._checkArgs({'backgroundEventType': types.StringTypes})
self.backgroundEventType = backgroundEventType
return S_OK()
def setBackgroundType(self, backgroundType):
"""Define the background type
:param string backgroundType: Background type.
"""
return self.setBkgEvtType(backgroundType)
def setProcessorName(self, processorName):
"""Set the processorName to set the input files for. Necessary if multiple
invocations of the overlay processor happen in marlin for example.
Different processors must use different background types
:param string processorName: Name of the Processor these input files are for
"""
self._checkArgs({'processorName': types.StringTypes})
self.processorName = processorName
return S_OK()
def setNumberOfSignalEventsPerJob(self, numberSignalEvents):
"""Alternative to :func:`setNbSigEvtsPerJob`
Number used to determine the number of background files needed.
:param int numberSignalEvents: Number of signal events per job
"""
return self.setNbSigEvtsPerJob(numberSignalEvents)
# def setProdIDToUse(self,prodid):
# """ Optional parameter: Define the production ID to use as input
#
# :param int prodid: Production ID
# """
# self._checkArgs({"prodid" : types.IntType})
# self.prodid = prodid
# return S_OK()
def _applicationModule(self):
m1 = self._createModuleDefinition()
m1.addParameter(
Parameter("BXOverlay", 0, "float", "", "", False, False,
"Bunch crossings to overlay"))
m1.addParameter(
Parameter(
"ggtohadint", 0, "float", "", "", False, False,
"Optional number of gamma gamma -> hadrons interactions per bunch crossing, default is 3.2"
))
m1.addParameter(
Parameter("NbSigEvtsPerJob", 0, "int", "", "", False, False,
"Number of signal events per job"))
m1.addParameter(
Parameter("prodid", 0, "int", "", "", False, False,
"ProdID to use"))
m1.addParameter(
Parameter("BkgEvtType", "", "string", "", "", False, False,
"Background type."))
m1.addParameter(
Parameter("detectormodel", "", "string", "", "", False, False,
"Detector type."))
m1.addParameter(
Parameter("machine", "", "string", "", "", False, False,
"machine: clic_cdr or ilc_dbd"))
m1.addParameter(
Parameter("useEnergyForFileLookup", True, "bool", "", "", False,
False,
"useEnergy to look for background files: True or False"))
m1.addParameter(
Parameter("pathToOverlayFiles", "", "string", "", "", False, False,
"use overlay files from this path"))
m1.addParameter(
Parameter("processorName", "", "string", "", "", False, False,
"Processor Name"))
m1.addParameter(
Parameter("debug", False, "bool", "", "", False, False,
"debug mode"))
return m1
def _applicationModuleValues(self, moduleinstance):
moduleinstance.setValue("BXOverlay", self.bxToOverlay)
moduleinstance.setValue('ggtohadint',
self.numberOfGGToHadronInteractions)
moduleinstance.setValue('NbSigEvtsPerJob',
self.numberOfSignalEventsPerJob)
moduleinstance.setValue('prodid', self.prodID)
moduleinstance.setValue('BkgEvtType', self.backgroundEventType)
moduleinstance.setValue('detectormodel', self.detectorModel)
moduleinstance.setValue('debug', self.debug)
moduleinstance.setValue('machine', self.machine)
moduleinstance.setValue('useEnergyForFileLookup',
self.useEnergyForFileLookup)
moduleinstance.setValue('pathToOverlayFiles', self.pathToOverlayFiles)
moduleinstance.setValue('processorName', self.processorName)
def _userjobmodules(self, stepdefinition):
res1 = self._setApplicationModuleAndParameters(stepdefinition)
if not res1["OK"]:
return S_ERROR('userjobmodules failed')
return S_OK()
def _prodjobmodules(self, stepdefinition):
res1 = self._setApplicationModuleAndParameters(stepdefinition)
if not res1["OK"]:
return S_ERROR('prodjobmodules failed')
return S_OK()
def _addParametersToStep(self, stepdefinition):
res = self._addBaseParameters(stepdefinition)
if not res["OK"]:
return S_ERROR("Failed to set base parameters")
return S_OK()
def _checkConsistency(self, job=None):
""" Checks that all needed parameters are set
"""
if self.pathToOverlayFiles:
res = FileCatalogClient().findFilesByMetadata(
{}, self.pathToOverlayFiles)
if not res['OK']:
return res
self._log.notice("Found %i files in path %s" %
(len(res['Value']), self.pathToOverlayFiles))
if len(res['Value']) == 0:
return S_ERROR(
"OverlayInput: PathToFiles is specified, but there are no files in that path"
)
if not self.bxToOverlay:
return S_ERROR("Number of overlay bunch crossings not defined")
if not self.numberOfGGToHadronInteractions:
return S_ERROR(
"Number of background events per bunch crossing is not defined"
)
if not self.backgroundEventType:
return S_ERROR(
"Background event type is not defined: Chose one gghad, aa_lowpt, ..."
)
if self._jobtype == 'User':
if not self.numberOfSignalEventsPerJob:
return S_ERROR("Number of signal event per job is not defined")
else:
self.prodparameters['detectorModel'] = self.detectorModel
self.prodparameters['BXOverlay'] = self.bxToOverlay
self.prodparameters[
'GGtoHadInt'] = self.numberOfGGToHadronInteractions
return S_OK()
def _checkFinalConsistency(self):
""" Final check of consistency: the overlay files for the specifed energy must exist
"""
if self.pathToOverlayFiles:
return S_OK() # can ignore other parameter
if not self.energy:
return S_ERROR("Energy MUST be specified for the overlay")
res = self._ops.getSections('/Overlay')
if not res['OK']:
return S_ERROR(
"Could not resolve the CS path to the overlay specifications")
sections = res['Value']
if self.machine not in sections:
return S_ERROR(
"Machine %s does not have overlay data, use any of %s" %
(self.machine, sections))
energytouse = energyWithLowerCaseUnit(self.energy)
res = self._ops.getSections("/Overlay/%s" % self.machine)
if energytouse not in res['Value']:
return S_ERROR("No overlay files corresponding to %s" %
energytouse)
res = self._ops.getSections("/Overlay/%s/%s" %
(self.machine, energytouse))
if not res['OK']:
return S_ERROR("Could not find the detector models")
if self.detectorModel not in res['Value']:
return S_ERROR(
"Detector model specified has no overlay data with that energy and machine"
)
res = allowedBkg(self.backgroundEventType,
energytouse,
detectormodel=self.detectorModel,
machine=self.machine)
if not res['OK']:
return res
if res['Value'] < 0:
return S_ERROR("No proper production ID found")
return S_OK()
class OverlayDB ( DB ):
def __init__( self, maxQueueSize = 10 ):
"""
"""
self.ops = Operations()
self.dbname = 'OverlayDB'
self.logger = gLogger.getSubLogger('OverlayDB')
DB.__init__( self, self.dbname, 'Overlay/OverlayDB', maxQueueSize )
self._createTables( { "OverlayData" : { 'Fields' : { 'Site' : "VARCHAR(256) UNIQUE NOT NULL",
'NumberOfJobs' : "INTEGER DEFAULT 0"
},
'PrimaryKey' : 'Site',
'Indexes': {'Index':['Site']}
}
}
)
limits = self.ops.getValue("/Overlay/MaxConcurrentRunning", 200)
self.limits = {}
self.limits["default"] = limits
res = self.ops.getSections("/Overlay/Sites/")
sites = []
if res['OK']:
sites = res['Value']
for tempsite in sites:
res = self.ops.getValue("/Overlay/Sites/%s/MaxConcurrentRunning" % tempsite, 200)
self.limits[tempsite] = res
self.logger.info("Using the following restrictions : %s" % self.limits)
#####################################################################
# Private methods
def __getConnection( self, connection ):
if connection:
return connection
res = self._getConnection()
if res['OK']:
return res['Value']
gLogger.warn( "Failed to get MySQL connection", res['Message'] )
return connection
def _checkSite(self, site, connection = False ):
""" Check the number of jos running at a given site.
"""
connection = self.__getConnection( connection )
req = "SELECT NumberOfJobs FROM OverlayData WHERE Site='%s';" % (site)
res = self._query( req, connection )
if not res['OK']:
return S_ERROR("Could not get site")
if len(res['Value']):
return res
else:
return S_ERROR("Could not find any site %s"%(site))
def _addSite(self, site, connection = False ):
""" Add a new site to the DB
"""
connection = self.__getConnection( connection )
req = "INSERT INTO OverlayData (Site,NumberOfJobs) VALUES ('%s',1);" % site
res = self._update( req, connection )
if not res['OK']:
return res
return res
def _limitForSite(self, site):
""" Get the current limit of jobs for a given site.
"""
if site in self.limits.keys():
return self.limits[site]
return self.limits['default']
def _addNewJob(self, site, nbjobs, connection = False ):
""" Add a new running job in the DB
"""
connection = self.__getConnection( connection )
nbjobs += 1
req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site)
self._update( req, connection )
return S_OK()
### Methods to fix the site
def getSites(self, connection = False):
""" Return the list of sites known to the service
"""
connection = self.__getConnection( connection )
req = 'SELECT Site From OverlayData;'
res = self._query( req, connection )
if not res['OK']:
return S_ERROR("Could not get sites")
sites = []
for row in res['Value']:
sites.append(row[0])
return S_OK(sites)
def setJobsAtSites(self, sitedict, connection = False):
""" As name suggests: set the number of jobs running at the site.
"""
connection = self.__getConnection( connection )
for site, nbjobs in sitedict.items():
req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (int(nbjobs), site)
res = self._update( req, connection )
if not res['OK']:
return S_ERROR("Could not set nb of jobs at site %s" % site)
return S_OK()
### Useful methods for the users
def getJobsAtSite(self, site, connection = False ):
""" Get the number of jobs currently run
"""
connection = self.__getConnection( connection )
nbjobs = 0
res = self._checkSite(site, connection)
if not res['OK']:
return S_OK(nbjobs)
nbjobs = res['Value'][0][0]
return S_OK(nbjobs)
### Important methods
def canRun(self, site, connection = False ):
""" Can the job run at that site?
"""
connection = self.__getConnection( connection )
res = self._checkSite(site, connection)
nbjobs = 0
if not res['OK']:
self._addSite(site, connection)
nbjobs = 1
else:
nbjobs = res['Value'][0][0]
if nbjobs < self._limitForSite(site):
res = self._addNewJob(site, nbjobs, connection)
if not res['OK']:
return res
return S_OK(True)
else:
return S_OK(False)
def jobDone(self, site, connection = False ):
""" Remove a job from the DB, should not remove a job from the DB
if the Site does not exist, but this should never happen
"""
connection = self.__getConnection( connection )
res = self._checkSite(site, connection)
if not res['OK']:
return res
nbjobs = res['Value'][0][0]
if nbjobs == 1:
return S_OK()
nbjobs -= 1
req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site)
res = self._update( req, connection )
if not res['OK']:
return res
return S_OK()
class FileCatalog:
ro_methods = [
'exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata',
'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory',
'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata',
'getDirectorySize', 'getDirectoryContents', 'resolveDataset',
'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups',
'getFileUserMetadata'
]
write_methods = [
'createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica',
'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost',
'createDirectory', 'setDirectoryStatus', 'removeDirectory',
'removeDataset', 'removeFileFromDataset', 'createDataset'
]
def __init__(self, catalogs=[], vo=None):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.readCatalogs = []
self.writeCatalogs = []
self.vo = vo
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
self.vo = result['Value']
self.opHelper = Operations(vo=self.vo)
self.reHelper = Resources(vo=self.vo)
if type(catalogs) in types.StringTypes:
catalogs = [catalogs]
if catalogs:
res = self._getSelectedCatalogs(catalogs)
else:
res = self._getCatalogs()
if not res['OK']:
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.valid = False
def isOK(self):
return self.valid
def getReadCatalogs(self):
return self.readCatalogs
def getWriteCatalogs(self):
return self.writeCatalogs
def __getattr__(self, name):
self.call = name
if name in FileCatalog.write_methods:
return self.w_execute
elif name in FileCatalog.ro_methods:
return self.r_execute
else:
raise AttributeError
def w_execute(self, *parms, **kws):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = checkArgumentFormat(fileInfo)
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
for catalogName, oCatalog, master in self.writeCatalogs:
method = getattr(oCatalog, self.call)
res = method(fileInfo, **kws)
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error(
"FileCatalog.w_execute: Failed to execute %s on master catalog %s."
% (self.call, catalogName), res['Message'])
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append((catalogName, res['Message']))
else:
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn)
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
if not successful.has_key(lfn):
successful[lfn] = {}
successful[lfn][catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = errorMessage
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def r_execute(self, *parms, **kws):
""" Read method executor.
"""
successful = {}
failed = {}
for _catalogName, oCatalog, _master in self.readCatalogs:
method = getattr(oCatalog, self.call)
res = method(*parms, **kws)
if res['OK']:
if 'Successful' in res['Value']:
for key, item in res['Value']['Successful'].items():
if not successful.has_key(key):
successful[key] = item
if failed.has_key(key):
failed.pop(key)
for key, item in res['Value']['Failed'].items():
if not successful.has_key(key):
failed[key] = item
if len(failed) == 0:
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
else:
return res
if (len(successful) == 0) and (len(failed) == 0):
return S_ERROR("Failed to perform %s from any catalog" % self.call)
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog(self, catalogName, mode="Write", master=False):
""" Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject(catalogName)
if not result['OK']:
return result
oCatalog = result['Value']
if mode.lower().find("read") != -1:
self.readCatalogs.append((catalogName, oCatalog, master))
if mode.lower().find("write") != -1:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def removeCatalog(self, catalogName):
""" Remove the specified catalog from the internal pool
"""
catalog_removed = False
for i in range(len(self.readCatalogs)):
catalog, _object, _master = self.readCatalogs[i]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range(len(self.writeCatalogs)):
catalog, _object, _master = self.writeCatalogs[i]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK('Catalog does not exist')
def _getSelectedCatalogs(self, desiredCatalogs):
for catalogName in desiredCatalogs:
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
self.readCatalogs.append((catalogName, oCatalog, True))
self.writeCatalogs.append((catalogName, oCatalog, True))
return S_OK()
def _getCatalogs(self):
# Get the eligible catalogs first
# First, look in the Operations, if nothing defined look in /Resources
result = self.opHelper.getSections('/Services/Catalogs')
fileCatalogs = []
operationsFlag = False
optCatalogDict = {}
if result['OK']:
fcs = result['Value']
for fc in fcs:
fName = self.opHelper.getValue(
'/Services/Catalogs/%s/CatalogName' % fc, fc)
fileCatalogs.append(fName)
optCatalogDict[fName] = fc
operationsFlag = True
else:
res = self.reHelper.getEligibleResources('Catalog')
if not res['OK']:
errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration."
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
fileCatalogs = res['Value']
# Get the catalogs now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails(catalogName)
if not res['OK']:
return res
catalogConfig = res['Value']
if operationsFlag:
result = self.opHelper.getOptionsDict(
'/Services/Catalogs/%s' % optCatalogDict[catalogName])
if not result['OK']:
return result
catalogConfig.update(result['Value'])
if catalogConfig['Status'] == 'Active':
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
master = catalogConfig['Master']
# If the catalog is read type
if re.search('Read', catalogConfig['AccessType']):
if master:
self.readCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.readCatalogs.append(
(catalogName, oCatalog, master))
# If the catalog is write type
if re.search('Write', catalogConfig['AccessType']):
if master:
self.writeCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.writeCatalogs.append(
(catalogName, oCatalog, master))
return S_OK()
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
result = self.reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options"
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = result['Value']
# The 'Status' option should be defined (default = 'Active')
if not catalogConfig.has_key('Status'):
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined"
gLogger.warn(warnStr, catalogName)
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if not catalogConfig.has_key('AccessType'):
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined"
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
if not catalogConfig.has_key('Master'):
catalogConfig['Master'] = False
elif catalogConfig['Master'] == 'True':
catalogConfig['Master'] = True
else:
catalogConfig['Master'] = False
return S_OK(catalogConfig)
def _generateCatalogObject(self, catalogName):
""" Create a file catalog object from its name and CS description
"""
useProxy = gConfig.getValue(
'/LocalSite/Catalogs/%s/UseProxy' % catalogName, False)
if not useProxy:
useProxy = self.opHelper.getValue(
'/Services/Catalogs/%s/UseProxy' % catalogName, False)
return FileCatalogFactory().createCatalog(catalogName, useProxy)
def addShifter(self, shifters=None):
"""
Adds or modify one or more shifters. Also, adds the shifter section in case this is not present.
Shifter identities are used in several places, mostly for running agents
:param dict shifters: has to be in the form {'ShifterRole':{'User':'******', 'Group':'aDIRACGroup'}}
:return: S_OK/S_ERROR
"""
def getOpsSection():
"""
Where is the shifters section?
"""
vo = CSGlobals.getVO()
setup = CSGlobals.getSetup()
if vo:
res = gConfig.getSections('/Operations/%s/%s/Shifter' %
(vo, setup))
if res['OK']:
return S_OK('/Operations/%s/%s/Shifter' % (vo, setup))
res = gConfig.getSections('/Operations/%s/Defaults/Shifter' %
vo)
if res['OK']:
return S_OK('/Operations/%s/Defaults/Shifter' % vo)
else:
res = gConfig.getSections('/Operations/%s/Shifter' % setup)
if res['OK']:
return S_OK('/Operations/%s/Shifter' % setup)
res = gConfig.getSections('/Operations/Defaults/Shifter')
if res['OK']:
return S_OK('/Operations/Defaults/Shifter')
return S_ERROR("No shifter section")
if shifters is None:
shifters = {}
if not self.__initialized['OK']:
return self.__initialized
# get current shifters
opsH = Operations()
currentShifterRoles = opsH.getSections('Shifter')
if not currentShifterRoles['OK']:
# we assume the shifter section is not present
currentShifterRoles = []
else:
currentShifterRoles = currentShifterRoles['Value']
currentShiftersDict = {}
for currentShifterRole in currentShifterRoles:
currentShifter = opsH.getOptionsDict('Shifter/%s' %
currentShifterRole)
if not currentShifter['OK']:
return currentShifter
currentShifter = currentShifter['Value']
currentShiftersDict[currentShifterRole] = currentShifter
# Removing from shifters what does not need to be changed
for sRole in shifters.keys(): # note the pop below
if sRole in currentShiftersDict:
if currentShiftersDict[sRole] == shifters[sRole]:
shifters.pop(sRole)
# get shifters section to modify
section = getOpsSection()
# Is this section present?
if not section['OK']:
if section['Message'] == "No shifter section":
gLogger.warn(section['Message'])
gLogger.info("Adding shifter section")
vo = CSGlobals.getVO()
if vo:
section = '/Operations/%s/Defaults/Shifter' % vo
else:
section = '/Operations/Defaults/Shifter'
res = self.__csMod.createSection(section)
if not res:
gLogger.error("Section %s not created" % section)
return S_ERROR("Section %s not created" % section)
else:
gLogger.error(section['Message'])
return section
else:
section = section['Value']
# add or modify shifters
for shifter in shifters:
self.__csMod.removeSection(section + '/' + shifter)
self.__csMod.createSection(section + '/' + shifter)
self.__csMod.createSection(section + '/' + shifter + '/' + 'User')
self.__csMod.createSection(section + '/' + shifter + '/' + 'Group')
self.__csMod.setOptionValue(section + '/' + shifter + '/' + 'User',
shifters[shifter]['User'])
self.__csMod.setOptionValue(
section + '/' + shifter + '/' + 'Group',
shifters[shifter]['Group'])
self.csModified = True
return S_OK(True)
class FileCatalog:
ro_methods = ['exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas',
'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas',
'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents',
'resolveDataset', 'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups', 'getFileUserMetadata']
write_methods = ['createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica', 'removeReplica',
'removeFile', 'setReplicaStatus', 'setReplicaHost', 'createDirectory', 'setDirectoryStatus',
'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset']
def __init__( self, catalogs = [], vo = None ):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.readCatalogs = []
self.writeCatalogs = []
self.rootConfigPath = '/Resources/FileCatalogs'
self.vo = vo if vo else getVOfromProxyGroup().get( 'Value', None )
self.opHelper = Operations( vo = self.vo )
if type( catalogs ) in types.StringTypes:
catalogs = [catalogs]
if catalogs:
res = self._getSelectedCatalogs( catalogs )
else:
res = self._getCatalogs()
if not res['OK']:
self.valid = False
elif ( len( self.readCatalogs ) == 0 ) and ( len( self.writeCatalogs ) == 0 ):
self.valid = False
def isOK( self ):
return self.valid
def getReadCatalogs( self ):
return self.readCatalogs
def getWriteCatalogs( self ):
return self.writeCatalogs
def __getattr__( self, name ):
self.call = name
if name in FileCatalog.write_methods:
return self.w_execute
elif name in FileCatalog.ro_methods:
return self.r_execute
else:
raise AttributeError
def w_execute( self, *parms, **kws ):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = checkArgumentFormat( fileInfo )
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
for catalogName, oCatalog, master in self.writeCatalogs:
method = getattr( oCatalog, self.call )
res = method( fileInfo, **kws )
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error( "FileCatalog.w_execute: Failed to execute %s on master catalog %s." % ( self.call, catalogName ), res['Message'] )
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append( ( catalogName, res['Message'] ) )
else:
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
failed.setdefault( lfn, {} )[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop( lfn, None )
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
successful.setdefault( lfn, {} )[catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
failed.setdefault( lfn, {} )[catalogName] = errorMessage
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def r_execute( self, *parms, **kws ):
""" Read method executor.
"""
successful = {}
failed = {}
for _catalogName, oCatalog, _master in self.readCatalogs:
method = getattr( oCatalog, self.call )
res = method( *parms, **kws )
if res['OK']:
if 'Successful' in res['Value']:
for key, item in res['Value']['Successful'].items():
successful.setdefault( key, item )
failed.pop( key, None )
for key, item in res['Value']['Failed'].items():
if key not in successful:
failed[key] = item
else:
return res
if not successful and not failed:
return S_ERROR( "Failed to perform %s from any catalog" % self.call )
return S_OK( {'Failed':failed, 'Successful':successful} )
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog( self, catalogName, mode = "Write", master = False ):
""" Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject( catalogName )
if not result['OK']:
return result
oCatalog = result['Value']
if mode.lower().find( "read" ) != -1:
self.readCatalogs.append( ( catalogName, oCatalog, master ) )
if mode.lower().find( "write" ) != -1:
self.writeCatalogs.append( ( catalogName, oCatalog, master ) )
return S_OK()
def removeCatalog( self, catalogName ):
""" Remove the specified catalog from the internal pool
"""
catalog_removed = False
for i in range( len( self.readCatalogs ) ):
catalog, _object, _master = self.readCatalogs[i]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range( len( self.writeCatalogs ) ):
catalog, _object, _master = self.writeCatalogs[i]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK( 'Catalog does not exist' )
def _getSelectedCatalogs( self, desiredCatalogs ):
for catalogName in desiredCatalogs:
res = self._generateCatalogObject( catalogName )
if not res['OK']:
return res
oCatalog = res['Value']
self.readCatalogs.append( ( catalogName, oCatalog, True ) )
self.writeCatalogs.append( ( catalogName, oCatalog, True ) )
return S_OK()
def _getCatalogs( self ):
# Get the eligible catalogs first
# First, look in the Operations, if nothing defined look in /Resources for backward compatibility
result = self.opHelper.getSections( '/Services/Catalogs' )
fileCatalogs = []
operationsFlag = False
if result['OK']:
fileCatalogs = result['Value']
operationsFlag = True
else:
res = gConfig.getSections( self.rootConfigPath, listOrdered = True )
if not res['OK']:
errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration."
gLogger.error( errStr, res['Message'] )
return S_ERROR( errStr )
fileCatalogs = res['Value']
# Get the catalogs now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails( catalogName )
if not res['OK']:
return res
catalogConfig = res['Value']
if operationsFlag:
result = self.opHelper.getOptionsDict( '/Services/Catalogs/%s' % catalogName )
if not result['OK']:
return result
catalogConfig.update( result['Value'] )
if catalogConfig['Status'] == 'Active':
res = self._generateCatalogObject( catalogName )
if not res['OK']:
return res
oCatalog = res['Value']
master = catalogConfig['Master']
# If the catalog is read type
if re.search( 'Read', catalogConfig['AccessType'] ):
if master:
self.readCatalogs.insert( 0, ( catalogName, oCatalog, master ) )
else:
self.readCatalogs.append( ( catalogName, oCatalog, master ) )
# If the catalog is write type
if re.search( 'Write', catalogConfig['AccessType'] ):
if master:
self.writeCatalogs.insert( 0, ( catalogName, oCatalog, master ) )
else:
self.writeCatalogs.append( ( catalogName, oCatalog, master ) )
return S_OK()
def _getCatalogConfigDetails( self, catalogName ):
# First obtain the options that are available
catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName )
res = gConfig.getOptions( catalogConfigPath )
if not res['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
gLogger.error( errStr, catalogName )
return S_ERROR( errStr )
catalogConfig = {}
for option in res['Value']:
configPath = '%s/%s' % ( catalogConfigPath, option )
optionValue = gConfig.getValue( configPath )
catalogConfig[option] = optionValue
# The 'Status' option should be defined (default = 'Active')
if 'Status' not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
gLogger.warn( warnStr, catalogName )
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if 'AccessType' not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
gLogger.error( errStr, catalogName )
return S_ERROR( errStr )
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' )
return S_OK( catalogConfig )
def _generateCatalogObject( self, catalogName ):
""" Create a file catalog object from its name and CS description
"""
useProxy = gConfig.getValue( '/LocalSite/Catalogs/%s/UseProxy' % catalogName, False )
if not useProxy:
useProxy = self.opHelper.getValue( '/Services/Catalogs/%s/UseProxy' % catalogName, False )
return FileCatalogFactory().createCatalog( catalogName, useProxy )
class OverlayInput(LCUtilityApplication):
""" Helper call to define Overlay processor/driver inputs.
Example:
>>> over = OverlayInput()
>>> over.setBXOverlay(300)
>>> over.setGGToHadInt(3.2)
>>> over.setNumberOfSignalEventsPerJob(10)
>>> over.setBackgroundType("gghad")
"""
def __init__(self, paramdict = None):
self._ops = Operations()
self.bxToOverlay = None
self.numberOfGGToHadronInteractions = 0
self.numberOfSignalEventsPerJob = 0
self.backgroundEventType = ''
self.prodID = 0
self.machine = 'clic_cdr'
self.detectorModel = ''
self.useEnergyForFileLookup = True
super(OverlayInput, self).__init__( paramdict )
self.version = '1'
self._modulename = "OverlayInput"
self.appname = self._modulename
self._moduledescription = 'Helper call to define Overlay processor/driver inputs'
self.accountInProduction = False
self._paramsToExclude.append('_ops')
self.pathToOverlayFiles = ''
self.processorName = ''
def setMachine(self, machine):
""" Define the machine to use, clic_cdr or ilc_dbd
"""
self._checkArgs( { 'machine' : types.StringTypes } )
self.machine = machine
def setProdID(self, pid):
""" Define the prodID to use as input, experts only
"""
self._checkArgs( {'pid': types.IntType})
self.prodID = pid
return S_OK()
def setUseEnergyForFileLookup(self, useEnergyForFileLookup):
"""
Sets the flag to use the energy meta data in the search of the background files.
Disable the energy when you want to use files created for a different energy than the signal events
:param bool useEnergyForFileLookup: Use the Energy in the metadata search or not
"""
self._checkArgs( {'useEnergyForFileLookup': types.BooleanType } )
self.useEnergyForFileLookup = useEnergyForFileLookup
return S_OK()
def setOverlayBXPerSigEvt( self, bxoverlay):
""" Define number bunch crossings to overlay for each signal event.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
Alias for :func:`setBXOverlay`
:param int bxoverlay: Bunch crossings to overlay.
"""
self._checkArgs( { 'bxoverlay' : types.IntType } )
self.bxToOverlay = bxoverlay
return S_OK()
def setBXOverlay(self, bxoverlay):
""" Define number bunch crossings to overlay for each signal event.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
:param int bxoverlay: Bunch crossings to overlay.
"""
return self.setOverlayBXPerSigEvt( bxoverlay )
def setOverlayEvtsPerBX( self, ggtohadint ):
""" Define the number of overlay events per bunch crossing.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
:param float ggtohadint: optional number of overlay events interactions per bunch crossing
"""
self._checkArgs( { 'ggtohadint' : types.FloatType } )
self.numberOfGGToHadronInteractions = ggtohadint
return S_OK()
def setGGToHadInt(self, ggtohadint):
""" Define the number of overlay events per bunch crossing.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
Alias for :func:`setOverlayEvtsPerBX`
:param float ggtohadint: optional number of overlay events interactions per bunch crossing
"""
return self.setOverlayEvtsPerBX( ggtohadint )
def setNbSigEvtsPerJob(self, nbsigevtsperjob):
""" Set the number of signal events per job.
This is used to determine the number of required overlay events.
It does not modify any of the actual application parameters using the overly input.
:param int nbsigevtsperjob: Number of signal events per job
"""
self._checkArgs( { 'nbsigevtsperjob' : types.IntType } )
self.numberOfSignalEventsPerJob = nbsigevtsperjob
return S_OK()
def setDetectorModel(self, detectormodel):
""" Set the detector type for the background files.
Files are defined in the ConfigurationSystem: Operations/Overlay/<Accelerator>/<energy>/<Detector>
:param str detectormodel: Detector type. Must be 'CLIC_ILD_CDR' or 'CLIC_SID_CDR' or 'sidloi3' or 'ILD_o1_v05'
"""
self._checkArgs( { 'detectormodel' : types.StringTypes } )
self.detectorModel = detectormodel
return S_OK()
def setPathToFiles(self, path):
""" Sets the path to where the overlay files are located.
Setting this option will ignore all other settings!
:param str path: LFN path to the folder containing the overlay files
"""
self._checkArgs( { 'path' : types.StringTypes } )
self.pathToOverlayFiles = path
return S_OK()
def setBkgEvtType(self, backgroundEventType):
""" Define the background type.
.. deprecated:: 23r0
Use :func:`setBackgroundType` instead
:param str backgroundEventType: Background type.
"""
self._checkArgs( { 'backgroundEventType' : types.StringTypes } )
self.backgroundEventType = backgroundEventType
return S_OK()
def setBackgroundType(self, backgroundType):
"""Define the background type
:param str backgroundType: Background type.
"""
return self.setBkgEvtType(backgroundType)
def setProcessorName(self, processorName):
"""Set the processorName to set the input files for. Necessary if multiple
invocations of the overlay processor happen in marlin for example.
Different processors must use different background types
:param str processorName: Name of the Processor these input files are for
"""
self._checkArgs( { 'processorName' : types.StringTypes } )
self.processorName = processorName
return S_OK()
def setNumberOfSignalEventsPerJob(self, numberSignalEvents):
"""Alternative to :func:`setNbSigEvtsPerJob`
Number used to determine the number of background files needed.
:param int numberSignalEvents: Number of signal events per job
"""
return self.setNbSigEvtsPerJob(numberSignalEvents)
# def setProdIDToUse(self,prodid):
# """ Optional parameter: Define the production ID to use as input
#
# :param int prodid: Production ID
# """
# self._checkArgs({"prodid" : types.IntType})
# self.prodid = prodid
# return S_OK()
def _applicationModule(self):
m1 = self._createModuleDefinition()
m1.addParameter(Parameter("BXOverlay", 0, "float", "", "", False, False,
"Bunch crossings to overlay"))
m1.addParameter(Parameter("ggtohadint", 0, "float", "", "", False, False,
"Optional number of gamma gamma -> hadrons interactions per bunch crossing, default is 3.2"))
m1.addParameter(Parameter("NbSigEvtsPerJob", 0, "int", "", "", False, False,
"Number of signal events per job"))
m1.addParameter(Parameter("prodid", 0, "int", "", "", False, False,
"ProdID to use"))
m1.addParameter(Parameter("BkgEvtType", "", "string", "", "", False, False,
"Background type."))
m1.addParameter(Parameter("detectormodel", "", "string", "", "", False, False,
"Detector type."))
m1.addParameter(Parameter("machine", "", "string", "", "", False, False,
"machine: clic_cdr or ilc_dbd"))
m1.addParameter(Parameter("useEnergyForFileLookup", True, "bool", "", "", False, False,
"useEnergy to look for background files: True or False"))
m1.addParameter(Parameter("pathToOverlayFiles", "", "string", "", "", False, False,
"use overlay files from this path"))
m1.addParameter(Parameter("processorName", "", "string", "", "", False, False,
"Processor Name"))
m1.addParameter(Parameter("debug", False, "bool", "", "", False, False, "debug mode"))
return m1
def _applicationModuleValues(self, moduleinstance):
moduleinstance.setValue("BXOverlay", self.bxToOverlay)
moduleinstance.setValue('ggtohadint', self.numberOfGGToHadronInteractions)
moduleinstance.setValue('NbSigEvtsPerJob', self.numberOfSignalEventsPerJob)
moduleinstance.setValue('prodid', self.prodID)
moduleinstance.setValue('BkgEvtType', self.backgroundEventType)
moduleinstance.setValue('detectormodel', self.detectorModel)
moduleinstance.setValue('debug', self.debug)
moduleinstance.setValue('machine', self.machine )
moduleinstance.setValue('useEnergyForFileLookup', self.useEnergyForFileLookup )
moduleinstance.setValue('pathToOverlayFiles', self.pathToOverlayFiles )
moduleinstance.setValue('processorName', self.processorName )
def _userjobmodules(self, stepdefinition):
res1 = self._setApplicationModuleAndParameters(stepdefinition)
if not res1["OK"] :
return S_ERROR('userjobmodules failed')
return S_OK()
def _prodjobmodules(self, stepdefinition):
res1 = self._setApplicationModuleAndParameters(stepdefinition)
if not res1["OK"] :
return S_ERROR('prodjobmodules failed')
return S_OK()
def _addParametersToStep(self, stepdefinition):
res = self._addBaseParameters(stepdefinition)
if not res["OK"]:
return S_ERROR("Failed to set base parameters")
return S_OK()
def _checkConsistency(self, job=None):
""" Checks that all needed parameters are set
"""
if self.pathToOverlayFiles:
res = FileCatalogClient().findFilesByMetadata({}, self.pathToOverlayFiles)
if not res['OK']:
return res
LOG.notice("Found %i files in path %s" % (len(res['Value']), self.pathToOverlayFiles))
if not res['Value']:
return S_ERROR("OverlayInput: PathToFiles is specified, but there are no files in that path")
if not self.bxToOverlay :
return S_ERROR("Number of overlay bunch crossings not defined")
if not self.numberOfGGToHadronInteractions :
return S_ERROR("Number of background events per bunch crossing is not defined")
if not self.backgroundEventType :
return S_ERROR("Background event type is not defined: Chose one gghad, aa_lowpt, ...")
if self._jobtype == 'User' :
if not self.numberOfSignalEventsPerJob :
return S_ERROR("Number of signal event per job is not defined")
else:
self.prodparameters['detectorModel'] = self.detectorModel
self.prodparameters['BXOverlay'] = self.bxToOverlay
self.prodparameters['GGtoHadInt'] = self.numberOfGGToHadronInteractions
return S_OK()
def _checkFinalConsistency(self):
""" Final check of consistency: the overlay files for the specifed energy must exist
"""
if self.pathToOverlayFiles:
return S_OK() # can ignore other parameter
if not self.energy:
return S_ERROR("Energy MUST be specified for the overlay")
res = self._ops.getSections('/Overlay')
if not res['OK']:
return S_ERROR("Could not resolve the CS path to the overlay specifications")
sections = res['Value']
if self.machine not in sections:
return S_ERROR("Machine %s does not have overlay data, use any of %s" % (self.machine, sections))
energytouse = energyWithLowerCaseUnit( self.energy )
res = self._ops.getSections("/Overlay/%s" % self.machine)
if energytouse not in res['Value']:
return S_ERROR("No overlay files corresponding to %s" % energytouse)
res = self._ops.getSections("/Overlay/%s/%s" % (self.machine, energytouse))
if not res['OK']:
return S_ERROR("Could not find the detector models")
if self.detectorModel not in res['Value']:
return S_ERROR("Detector model specified has no overlay data with that energy and machine")
res = allowedBkg(self.backgroundEventType, energytouse, detectormodel = self.detectorModel, machine = self.machine)
if not res['OK']:
return res
if res['Value'] < 0:
return S_ERROR("No proper production ID found")
return S_OK()
class OverlayDB ( DB ):
""" DB for OverlaySystem
"""
def __init__( self ):
"""
"""
self.ops = Operations()
self.dbname = 'OverlayDB'
self.logger = gLogger.getSubLogger('OverlayDB')
DB.__init__( self, self.dbname, 'Overlay/OverlayDB' )
self._createTables( { "OverlayData" : { 'Fields' : { 'Site' : "VARCHAR(255) UNIQUE NOT NULL",
'NumberOfJobs' : "INTEGER DEFAULT 0"
},
'PrimaryKey' : 'Site',
'Indexes': {'Index':['Site']}
}
}
)
limits = self.ops.getValue("/Overlay/MaxConcurrentRunning", 200)
self.limits = {}
self.limits["default"] = limits
res = self.ops.getSections("/Overlay/Sites/")
sites = []
if res['OK']:
sites = res['Value']
for tempsite in sites:
res = self.ops.getValue("/Overlay/Sites/%s/MaxConcurrentRunning" % tempsite, 200)
self.limits[tempsite] = res
self.logger.info("Using the following restrictions : %s" % self.limits)
#####################################################################
# Private methods
def __getConnection( self, connection ):
if connection:
return connection
res = self._getConnection()
if res['OK']:
return res['Value']
gLogger.warn( "Failed to get MySQL connection", res['Message'] )
return connection
def _checkSite(self, site, connection = False ):
""" Check the number of jobs running at a given site.
"""
connection = self.__getConnection( connection )
req = "SELECT NumberOfJobs FROM OverlayData WHERE Site='%s';" % (site)
res = self._query( req, connection )
if not res['OK']:
return S_ERROR("Could not get site")
if len(res['Value']):
return res
else:
return S_ERROR("Could not find any site %s"%(site))
def _addSite(self, site, connection = False ):
""" Add a new site to the DB
"""
connection = self.__getConnection( connection )
req = "INSERT INTO OverlayData (Site,NumberOfJobs) VALUES ('%s',1);" % site
res = self._update( req, connection )
if not res['OK']:
return res
return res
def _limitForSite(self, site):
""" Get the current limit of jobs for a given site.
"""
if site in self.limits.keys():
return self.limits[site]
return self.limits['default']
def _addNewJob(self, site, nbjobs, connection = False ):
""" Add a new running job in the DB
"""
connection = self.__getConnection( connection )
nbjobs += 1
req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site)
self._update( req, connection )
return S_OK()
### Methods to fix the site
def getSites(self, connection = False):
""" Return the list of sites known to the service
"""
connection = self.__getConnection( connection )
req = 'SELECT Site From OverlayData;'
res = self._query( req, connection )
if not res['OK']:
return S_ERROR("Could not get sites")
sites = []
for row in res['Value']:
sites.append(row[0])
return S_OK(sites)
def setJobsAtSites(self, sitedict, connection = False):
""" As name suggests: set the number of jobs running at the site.
"""
connection = self.__getConnection( connection )
for site, nbjobs in sitedict.items():
req = "UPDATE OverlayData SET NumberOfJobs=%i WHERE Site='%s';" % (int(nbjobs), site)
res = self._update( req, connection )
if not res['OK']:
return S_ERROR("Could not set number of jobs at site %s" % site)
return S_OK()
### Useful methods for the users
def getJobsAtSite(self, site, connection = False ):
""" Get the number of jobs currently run
"""
connection = self.__getConnection( connection )
nbjobs = 0
res = self._checkSite(site, connection)
if not res['OK']:
return S_OK(nbjobs)
nbjobs = res['Value'][0][0]
return S_OK(nbjobs)
### Important methods
def canRun(self, site, connection = False ):
""" Can the job run at that site?
"""
connection = self.__getConnection( connection )
res = self._checkSite(site, connection)
nbjobs = 0
if not res['OK']:
self._addSite(site, connection)
nbjobs = 1
else:
nbjobs = res['Value'][0][0]
if nbjobs < self._limitForSite(site):
res = self._addNewJob(site, nbjobs, connection)
if not res['OK']:
return res
return S_OK(True)
else:
return S_OK(False)
def jobDone(self, site, connection = False ):
""" Remove a job from the DB, should not remove a job from the DB
if the Site does not exist, but this should never happen
"""
connection = self.__getConnection( connection )
res = self._checkSite(site, connection)
if not res['OK']:
return res
nbjobs = res['Value'][0][0]
if nbjobs == 1:
return S_OK()
nbjobs -= 1
req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site)
res = self._update( req, connection )
if not res['OK']:
return res
return S_OK()
class FileCatalog(object):
def __init__(self, catalogs=None, vo=None):
"""Default constructor"""
self.valid = True
self.timeout = 180
self.ro_methods = set()
self.write_methods = set()
self.no_lfn_methods = set()
self.readCatalogs = []
self.writeCatalogs = []
self.rootConfigPath = "/Resources/FileCatalogs"
self.vo = vo if vo else getVOfromProxyGroup().get("Value", None)
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.opHelper = Operations(vo=self.vo)
catalogList = []
if isinstance(catalogs, six.string_types):
catalogList = [catalogs]
elif isinstance(catalogs, (list, tuple)):
catalogList = list(catalogs)
if catalogList:
result = self._getEligibleCatalogs()
if not result["OK"]:
self.log.error("Failed to get eligible catalog")
return
eligibleFileCatalogs = result["Value"]
catalogCheck = True
for catalog in catalogList:
if catalog not in eligibleFileCatalogs:
self.log.error("Specified catalog is not eligible", catalog)
catalogCheck = False
if catalogCheck:
result = self._getSelectedCatalogs(catalogList)
else:
result = S_ERROR("Specified catalog is not eligible")
else:
result = self._getCatalogs()
if not result["OK"]:
self.log.error("Failed to create catalog objects")
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.log.error("No catalog object created")
self.valid = False
result = self.getMasterCatalogNames()
masterCatalogs = result["Value"]
# There can not be more than one master catalog
haveMaster = False
if len(masterCatalogs) > 1:
self.log.error("More than one master catalog created")
self.valid = False
elif len(masterCatalogs) == 1:
haveMaster = True
# Get the list of write methods
if haveMaster:
# All the write methods must be present in the master
_catalogName, oCatalog, _master = self.writeCatalogs[0]
_roList, writeList, nolfnList = oCatalog.getInterfaceMethods()
self.write_methods.update(writeList)
self.no_lfn_methods.update(nolfnList)
else:
for _catalogName, oCatalog, _master in self.writeCatalogs:
_roList, writeList, nolfnList = oCatalog.getInterfaceMethods()
self.write_methods.update(writeList)
self.no_lfn_methods.update(nolfnList)
# Get the list of read methods
for _catalogName, oCatalog, _master in self.readCatalogs:
roList, _writeList, nolfnList = oCatalog.getInterfaceMethods()
self.ro_methods.update(roList)
self.no_lfn_methods.update(nolfnList)
self.condParser = FCConditionParser(vo=self.vo, ro_methods=self.ro_methods)
def isOK(self):
return self.valid
def getReadCatalogs(self):
return self.readCatalogs
def getWriteCatalogs(self):
return self.writeCatalogs
def getMasterCatalogNames(self):
"""Returns the list of names of the Master catalogs"""
masterNames = [catalogName for catalogName, oCatalog, master in self.writeCatalogs if master]
return S_OK(masterNames)
def __getattr__(self, name):
self.call = name
if name in self.write_methods:
return self.w_execute
elif name in self.ro_methods:
return self.r_execute
else:
raise AttributeError
def w_execute(self, *parms, **kws):
"""Write method executor.
If one of the LFNs given as input does not pass a condition defined for the
master catalog, we return S_ERROR without trying anything else
:param fcConditions: either a dict or a string, to be propagated to the FCConditionParser
* If it is a string, it is given for all catalogs
* If it is a dict, it has to be { catalogName: condition}, and only
the specific condition for the catalog will be given
.. warning ::
If the method is a write no_lfn method, then the return value are completely different.
We only return the result of the master catalog
"""
successful = {}
failed = {}
failedCatalogs = {}
successfulCatalogs = {}
specialConditions = kws.pop("fcConditions") if "fcConditions" in kws else None
allLfns = []
lfnMapDict = {}
masterResult = {}
parms1 = []
if self.call not in self.no_lfn_methods:
fileInfo = parms[0]
result = checkArgumentFormat(fileInfo, generateMap=True)
if not result["OK"]:
return result
fileInfo, lfnMapDict = result["Value"]
# No need to check the LFNs again in the clients
kws["LFNChecking"] = False
allLfns = list(fileInfo)
parms1 = parms[1:]
for catalogName, oCatalog, master in self.writeCatalogs:
# Skip if the method is not implemented in this catalog
# NOTE: it is impossible for the master since the write method list is populated
# only from the master catalog, and if the method is not there, __getattr__
# would raise an exception
if not oCatalog.hasCatalogMethod(self.call):
continue
method = getattr(oCatalog, self.call)
if self.call in self.no_lfn_methods:
result = method(*parms, **kws)
else:
if isinstance(specialConditions, dict):
condition = specialConditions.get(catalogName)
else:
condition = specialConditions
# Check whether this catalog should be used for this method
res = self.condParser(catalogName, self.call, fileInfo, condition=condition)
# condParser never returns S_ERROR
condEvals = res["Value"]["Successful"]
# For a master catalog, ALL the lfns should be valid
if master:
if any([not valid for valid in condEvals.values()]):
gLogger.error("The master catalog is not valid for some LFNS", condEvals)
return S_ERROR("The master catalog is not valid for some LFNS %s" % condEvals)
validLFNs = dict((lfn, fileInfo[lfn]) for lfn in condEvals if condEvals[lfn])
# We can skip the execution without worry,
# since at this level it is for sure not a master catalog
if not validLFNs:
gLogger.debug("No valid LFN, skipping the call")
continue
invalidLFNs = [lfn for lfn in condEvals if not condEvals[lfn]]
if invalidLFNs:
gLogger.debug(
"Some LFNs are not valid for operation '%s' on catalog '%s' : %s"
% (self.call, catalogName, invalidLFNs)
)
result = method(validLFNs, *parms1, **kws)
if master:
masterResult = result
if not result["OK"]:
if master:
# If this is the master catalog and it fails we don't want to continue with the other catalogs
self.log.error(
"Failed to execute call on master catalog",
"%s on %s: %s" % (self.call, catalogName, result["Message"]),
)
return result
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs[catalogName] = result["Message"]
else:
successfulCatalogs[catalogName] = result["Value"]
if allLfns:
if result["OK"]:
for lfn, message in result["Value"]["Failed"].items():
# Save the error message for the failed operations
failed.setdefault(lfn, {})[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn, None)
for lfn, result in result["Value"]["Successful"].items():
# Save the result return for each file for the successful operations
successful.setdefault(lfn, {})[catalogName] = result
if allLfns:
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs.items():
for lfn in allLfns:
failed.setdefault(lfn, {})[catalogName] = errorMessage
# Restore original lfns if they were changed by normalization
if lfnMapDict:
for lfn in list(failed):
failed[lfnMapDict.get(lfn, lfn)] = failed.pop(lfn)
for lfn in list(successful):
successful[lfnMapDict.get(lfn, lfn)] = successful.pop(lfn)
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
else:
# FIXME: Return just master result here. This is temporary as more detailed
# per catalog result needs multiple fixes in various client calls
return masterResult
def r_execute(self, *parms, **kws):
"""Read method executor."""
successful = {}
failed = {}
for _catalogName, oCatalog, _master in self.readCatalogs:
# Skip if the method is not implemented in this catalog
if not oCatalog.hasCatalogMethod(self.call):
continue
method = getattr(oCatalog, self.call)
res = method(*parms, **kws)
if res["OK"]:
if "Successful" in res["Value"]:
for key, item in res["Value"]["Successful"].items():
successful.setdefault(key, item)
failed.pop(key, None)
for key, item in res["Value"]["Failed"].items():
if key not in successful:
failed[key] = item
else:
return res
if not successful and not failed:
return S_ERROR(DErrno.EFCERR, "Failed to perform %s from any catalog" % self.call)
return S_OK({"Failed": failed, "Successful": successful})
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog(self, catalogName, mode="Write", master=False):
"""Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject(catalogName)
if not result["OK"]:
return result
oCatalog = result["Value"]
if mode.lower().find("read") != -1:
self.readCatalogs.append((catalogName, oCatalog, master))
if mode.lower().find("write") != -1:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def removeCatalog(self, catalogName):
"""Remove the specified catalog from the internal pool"""
catalog_removed = False
for i in range(len(self.readCatalogs)):
catalog, _object, _master = self.readCatalogs[i]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range(len(self.writeCatalogs)):
catalog, _object, _master = self.writeCatalogs[i]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK("Catalog does not exist")
def _getSelectedCatalogs(self, desiredCatalogs):
for catalogName in desiredCatalogs:
result = self._getCatalogConfigDetails(catalogName)
if not result["OK"]:
return result
catalogConfig = result["Value"]
result = self._generateCatalogObject(catalogName)
if not result["OK"]:
return result
oCatalog = result["Value"]
if re.search("Read", catalogConfig["AccessType"]):
if catalogConfig["Master"]:
self.readCatalogs.insert(0, (catalogName, oCatalog, catalogConfig["Master"]))
else:
self.readCatalogs.append((catalogName, oCatalog, catalogConfig["Master"]))
if re.search("Write", catalogConfig["AccessType"]):
if catalogConfig["Master"]:
self.writeCatalogs.insert(0, (catalogName, oCatalog, catalogConfig["Master"]))
else:
self.writeCatalogs.append((catalogName, oCatalog, catalogConfig["Master"]))
return S_OK()
def _getEligibleCatalogs(self):
"""Get a list of eligible catalogs
:return: S_OK/S_ERROR, Value - a list of catalog names
"""
# First, look in the Operations, if nothing defined look in /Resources for backward compatibility
fileCatalogs = self.opHelper.getValue("/Services/Catalogs/CatalogList", [])
if not fileCatalogs:
result = self.opHelper.getSections("/Services/Catalogs")
if result["OK"]:
fileCatalogs = result["Value"]
else:
res = gConfig.getSections(self.rootConfigPath, listOrdered=True)
if not res["OK"]:
errStr = "FileCatalog._getEligibleCatalogs: Failed to get file catalog configuration."
self.log.error(errStr, res["Message"])
return S_ERROR(errStr)
fileCatalogs = res["Value"]
return S_OK(fileCatalogs)
def _getCatalogs(self):
"""Updates self.readCatalogs and self.writeCatalogs with list of catalog objects as found in the CS"""
# Get the eligible catalogs first
result = self._getEligibleCatalogs()
if not result["OK"]:
return result
fileCatalogs = result["Value"]
# Get the catalog objects now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails(catalogName)
if not res["OK"]:
return res
catalogConfig = res["Value"]
if catalogConfig["Status"] == "Active":
res = self._generateCatalogObject(catalogName)
if not res["OK"]:
return res
oCatalog = res["Value"]
master = catalogConfig["Master"]
# If the catalog is read type
if re.search("Read", catalogConfig["AccessType"]):
if master:
self.readCatalogs.insert(0, (catalogName, oCatalog, master))
else:
self.readCatalogs.append((catalogName, oCatalog, master))
# If the catalog is write type
if re.search("Write", catalogConfig["AccessType"]):
if master:
self.writeCatalogs.insert(0, (catalogName, oCatalog, master))
else:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName)
result = gConfig.getOptionsDict(catalogConfigPath)
if not result["OK"]:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
self.log.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = result["Value"]
result = self.opHelper.getOptionsDict("/Services/Catalogs/%s" % catalogName)
if result["OK"]:
catalogConfig.update(result["Value"])
# The 'Status' option should be defined (default = 'Active')
if "Status" not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
self.log.warn(warnStr, catalogName)
catalogConfig["Status"] = "Active"
# The 'AccessType' option must be defined
if "AccessType" not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
self.log.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig["Master"] = catalogConfig.setdefault("Master", False) == "True"
return S_OK(catalogConfig)
def _generateCatalogObject(self, catalogName):
"""Create a file catalog object from its name and CS description"""
useProxy = gConfig.getValue("/LocalSite/Catalogs/%s/UseProxy" % catalogName, False)
if not useProxy:
useProxy = self.opHelper.getValue("/Services/Catalogs/%s/UseProxy" % catalogName, False)
return FileCatalogFactory().createCatalog(catalogName, useProxy)
def addShifter( self, shifters = None ):
"""
Adds or modify one or more shifters. Also, adds the shifter section in case this is not present.
Shifter identities are used in several places, mostly for running agents
shifters should be in the form {'ShifterRole':{'User':'******', 'Group':'aDIRACGroup'}}
:return: S_OK/S_ERROR
"""
def getOpsSection():
"""
Where is the shifters section?
"""
vo = CSGlobals.getVO()
setup = CSGlobals.getSetup()
if vo:
res = gConfig.getSections( '/Operations/%s/%s/Shifter' % (vo, setup) )
if res['OK']:
return S_OK( '/Operations/%s/%s/Shifter' % ( vo, setup ) )
res = gConfig.getSections( '/Operations/%s/Defaults/Shifter' % vo )
if res['OK']:
return S_OK( '/Operations/%s/Defaults/Shifter' % vo )
else:
res = gConfig.getSections( '/Operations/%s/Shifter' % setup )
if res['OK']:
return S_OK( '/Operations/%s/Shifter' % setup )
res = gConfig.getSections( '/Operations/Defaults/Shifter' )
if res['OK']:
return S_OK( '/Operations/Defaults/Shifter' )
return S_ERROR( "No shifter section" )
if shifters is None: shifters = {}
if not self.__initialized['OK']:
return self.__initialized
# get current shifters
opsH = Operations( )
currentShifterRoles = opsH.getSections( 'Shifter' )
if not currentShifterRoles['OK']:
# we assume the shifter section is not present
currentShifterRoles = []
else:
currentShifterRoles = currentShifterRoles['Value']
currentShiftersDict = {}
for currentShifterRole in currentShifterRoles:
currentShifter = opsH.getOptionsDict( 'Shifter/%s' % currentShifterRole )
if not currentShifter['OK']:
return currentShifter
currentShifter = currentShifter['Value']
currentShiftersDict[currentShifterRole] = currentShifter
# Removing from shifters what does not need to be changed
for sRole in shifters:
if sRole in currentShiftersDict:
if currentShiftersDict[sRole] == shifters[sRole]:
shifters.pop( sRole )
# get shifters section to modify
section = getOpsSection()
# Is this section present?
if not section['OK']:
if section['Message'] == "No shifter section":
gLogger.warn( section['Message'] )
gLogger.info( "Adding shifter section" )
vo = CSGlobals.getVO()
if vo:
section = '/Operations/%s/Defaults/Shifter' % vo
else:
section = '/Operations/Defaults/Shifter'
res = self.__csMod.createSection( section )
if not res:
gLogger.error( "Section %s not created" % section )
return S_ERROR( "Section %s not created" % section )
else:
gLogger.error( section['Message'] )
return section
else:
section = section['Value']
#add or modify shifters
for shifter in shifters:
self.__csMod.removeSection( section + '/' + shifter )
self.__csMod.createSection( section + '/' + shifter )
self.__csMod.createSection( section + '/' + shifter + '/' + 'User' )
self.__csMod.createSection( section + '/' + shifter + '/' + 'Group' )
self.__csMod.setOptionValue( section + '/' + shifter + '/' + 'User', shifters[shifter]['User'] )
self.__csMod.setOptionValue( section + '/' + shifter + '/' + 'Group', shifters[shifter]['Group'] )
self.__csModified = True
return S_OK( True )
class DMSHelpers(object):
"""
This class is used to get information about sites, SEs and their interrelations
"""
def __init__(self, vo=False):
self.siteSEMapping = {}
self.storageElementSet = set()
self.siteSet = set()
self.__opsHelper = Operations(vo=vo)
self.failoverSEs = None
self.archiveSEs = None
self.notForJobSEs = None
def getSiteSEMapping(self):
"""Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
"""
if self.siteSEMapping:
return S_OK(self.siteSEMapping)
# Get the list of SEs and keep a mapping of those using an Alias or a
# BaseSE
storageElements = gConfig.getSections("Resources/StorageElements")
if not storageElements["OK"]:
gLogger.warn("Problem retrieving storage elements",
storageElements["Message"])
return storageElements
storageElements = storageElements["Value"]
equivalentSEs = {}
for se in storageElements:
for option in ("BaseSE", "Alias"):
originalSE = gConfig.getValue(
"Resources/StorageElements/%s/%s" % (se, option))
if originalSE:
equivalentSEs.setdefault(originalSE, []).append(se)
break
siteSEMapping = {}
gridTypes = gConfig.getSections("Resources/Sites/")
if not gridTypes["OK"]:
gLogger.warn("Problem retrieving sections in /Resources/Sites",
gridTypes["Message"])
return gridTypes
gridTypes = gridTypes["Value"]
gLogger.debug("Grid Types are: %s" % (", ".join(gridTypes)))
# Get a list of sites and their local SEs
siteSet = set()
storageElementSet = set()
siteSEMapping[LOCAL] = {}
for grid in gridTypes:
result = gConfig.getSections("/Resources/Sites/%s" % grid)
if not result["OK"]:
gLogger.warn("Problem retrieving /Resources/Sites/%s section" %
grid)
return result
sites = result["Value"]
siteSet.update(sites)
for site in sites:
candidateSEs = gConfig.getValue(
"/Resources/Sites/%s/%s/SE" % (grid, site), [])
if candidateSEs:
candidateSEs += [
eqSE for se in candidateSEs
for eqSE in equivalentSEs.get(se, [])
]
siteSEMapping[LOCAL].setdefault(site,
set()).update(candidateSEs)
storageElementSet.update(candidateSEs)
# Add Sites from the SiteSEMappingByProtocol in the CS
siteSEMapping[PROTOCOL] = {}
cfgLocalSEPath = cfgPath("SiteSEMappingByProtocol")
result = self.__opsHelper.getOptionsDict(cfgLocalSEPath)
if result["OK"]:
sites = result["Value"]
for site in sites:
candidates = set(
self.__opsHelper.getValue(cfgPath(cfgLocalSEPath, site),
[]))
ses = set(resolveSEGroup(candidates - siteSet)) | (candidates
& siteSet)
# If a candidate is a site, then all local SEs are eligible
for candidate in ses & siteSet:
ses.remove(candidate)
ses.update(siteSEMapping[LOCAL][candidate])
siteSEMapping[PROTOCOL].setdefault(site, set()).update(ses)
# Add Sites from the SiteSEMappingByDownload in the CS, else
# SiteLocalSEMapping (old convention)
siteSEMapping[DOWNLOAD] = {}
cfgLocalSEPath = cfgPath("SiteSEMappingByDownload")
result = self.__opsHelper.getOptionsDict(cfgLocalSEPath)
if not result["OK"]:
cfgLocalSEPath = cfgPath("SiteLocalSEMapping")
result = self.__opsHelper.getOptionsDict(cfgLocalSEPath)
if result["OK"]:
sites = result["Value"]
for site in sites:
candidates = set(
self.__opsHelper.getValue(cfgPath(cfgLocalSEPath, site),
[]))
ses = set(resolveSEGroup(candidates - siteSet)) | (candidates
& siteSet)
# If a candidate is a site, then all local SEs are eligible
for candidate in ses & siteSet:
ses.remove(candidate)
ses.update(siteSEMapping[LOCAL][candidate])
siteSEMapping[DOWNLOAD].setdefault(site, set()).update(ses)
self.siteSEMapping = siteSEMapping
# Add storage elements that may not be associated with a site
result = gConfig.getSections("/Resources/StorageElements")
if not result["OK"]:
gLogger.warn(
"Problem retrieving /Resources/StorageElements section",
result["Message"])
return result
self.storageElementSet = storageElementSet | set(result["Value"])
self.siteSet = siteSet
return S_OK(siteSEMapping)
def getSites(self):
"""Get the list of known sites"""
self.getSiteSEMapping()
return sorted(self.siteSet)
def getTiers(self, withStorage=False, tier=None):
"""Get the list of sites for a given (list of) Tier level"""
sites = sorted(
self.getShortSiteNames(withStorage=withStorage,
tier=tier).values())
if sites and isinstance(sites[0], list):
# List of lists, flatten it
sites = [s for sl in sites for s in sl]
return sites
def getShortSiteNames(self, withStorage=True, tier=None):
"""Create a directory of short site names pointing to full site names"""
siteDict = {}
result = self.getSiteSEMapping()
if result["OK"]:
for site in self.siteSEMapping[
LOCAL] if withStorage else self.siteSet:
grid, shortSite, _country = site.split(".")
if isinstance(tier, six.integer_types) and (
grid != "LCG" or gConfig.getValue(
"/Resources/Sites/%s/%s/MoUTierLevel" %
(grid, site), 999) != tier):
continue
if isinstance(tier,
(list, tuple, dict,
set)) and (grid != "LCG" or gConfig.getValue(
"/Resources/Sites/%s/%s/MoUTierLevel" %
(grid, site), 999) not in tier):
continue
if withStorage or tier is not None:
siteDict[shortSite] = site
else:
siteDict.setdefault(shortSite, []).append(site)
return siteDict
def getStorageElements(self):
"""Get the list of known SEs"""
self.getSiteSEMapping()
return sorted(self.storageElementSet)
def isSEFailover(self, storageElement):
"""Is this SE a failover SE"""
if self.failoverSEs is None:
seList = resolveSEGroup(
self.__opsHelper.getValue("DataManagement/SEsUsedForFailover",
[]))
self.failoverSEs = resolveSEGroup(seList)
# FIXME: remove string test at some point
return storageElement in self.failoverSEs or (
not self.failoverSEs
and isinstance(storageElement, six.string_types)
and "FAILOVER" in storageElement.upper())
def isSEForJobs(self, storageElement, checkSE=True):
"""Is this SE suitable for making jobs"""
if checkSE:
self.getSiteSEMapping()
if storageElement not in self.storageElementSet:
return False
if self.notForJobSEs is None:
seList = resolveSEGroup(
self.__opsHelper.getValue(
"DataManagement/SEsNotToBeUsedForJobs", []))
self.notForJobSEs = resolveSEGroup(seList)
return storageElement not in self.notForJobSEs
def isSEArchive(self, storageElement):
"""Is this SE an archive SE"""
if self.archiveSEs is None:
seList = resolveSEGroup(
self.__opsHelper.getValue("DataManagement/SEsUsedForArchive",
[]))
self.archiveSEs = resolveSEGroup(seList)
# FIXME: remove string test at some point
return storageElement in self.archiveSEs or (
not self.archiveSEs
and isinstance(storageElement, six.string_types)
and "ARCHIVE" in storageElement.upper())
def getSitesForSE(self, storageElement, connectionLevel=None):
"""Get the (list of) sites for a given SE and a given connctivity"""
connectionIndex = _getConnectionIndex(connectionLevel,
default=DOWNLOAD)
if connectionIndex == LOCAL:
return self._getLocalSitesForSE(storageElement)
if connectionIndex == PROTOCOL:
return self.getProtocolSitesForSE(storageElement)
if connectionIndex == DOWNLOAD:
return self.getDownloadSitesForSE(storageElement)
return S_ERROR("Unknown connection level")
def getLocalSiteForSE(self, se):
"""Get the site at which the SE is"""
sites = self._getLocalSitesForSE(se)
if not sites["OK"]:
return sites
if not sites["Value"]:
return S_OK(None)
return S_OK(sites["Value"][0])
def _getLocalSitesForSE(self, se):
"""Extract the list of sites that declare this SE"""
mapping = self.getSiteSEMapping()
if not mapping["OK"]:
return mapping
if se not in self.storageElementSet:
return S_ERROR("Non-existing SE")
mapping = mapping["Value"][LOCAL]
sites = [site for site in mapping if se in mapping[site]]
if len(sites) > 1 and self.__opsHelper.getValue(
"DataManagement/ForceSingleSitePerSE", True):
return S_ERROR("SE is at more than one site")
return S_OK(sites)
def getProtocolSitesForSE(self, se):
"""Get sites that can access the SE by protocol"""
mapping = self.getSiteSEMapping()
if not mapping["OK"]:
return mapping
if se not in self.storageElementSet:
return S_ERROR("Non-existing SE")
mapping = mapping["Value"][PROTOCOL]
sites = self._getLocalSitesForSE(se)
if not sites["OK"]:
return sites
sites = set(sites["Value"])
sites.update([site for site in mapping if se in mapping[site]])
return S_OK(sorted(sites))
def getDownloadSitesForSE(self, se):
"""Get the list of sites that are allowed to download files"""
mapping = self.getSiteSEMapping()
if not mapping["OK"]:
return mapping
if se not in self.storageElementSet:
return S_ERROR("Non-existing SE")
mapping = mapping["Value"][DOWNLOAD]
sites = self.getProtocolSitesForSE(se)
if not sites["OK"]:
return sites
sites = set(sites["Value"])
sites.update([site for site in mapping if se in mapping[site]])
return S_OK(sorted(sites))
def getSEsForSite(self, site, connectionLevel=None):
"""Get all SEs accessible from a site, given a connectivity"""
connectionIndex = _getConnectionIndex(connectionLevel,
default=DOWNLOAD)
if connectionIndex is None:
return S_ERROR("Unknown connection level")
if not self.siteSet:
self.getSiteSEMapping()
if site not in self.siteSet:
siteList = [s for s in self.siteSet if ".%s." % site in s]
else:
siteList = [site]
if not siteList:
return S_ERROR("Unknown site")
return self._getSEsForSItes(siteList, connectionIndex=connectionIndex)
def _getSEsForSItes(self, siteList, connectionIndex):
"""Extract list of SEs for a connectivity"""
mapping = self.getSiteSEMapping()
if not mapping["OK"]:
return mapping
ses = []
for index in range(LOCAL, connectionIndex + 1):
for site in siteList:
ses += mapping["Value"][index].get(site, [])
if not ses:
return S_ERROR("No SE found")
return S_OK(sorted(ses))
def getSEsAtSite(self, site):
"""Get local SEs"""
return self.getSEsForSite(site, connectionLevel=LOCAL)
def isSameSiteSE(self, se1, se2):
"""Are these 2 SEs at the same site"""
res = self.getLocalSiteForSE(se1)
if not res["OK"]:
return res
site1 = res["Value"]
res = self.getLocalSiteForSE(se2)
if not res["OK"]:
return res
site2 = res["Value"]
return S_OK(site1 == site2)
def getSEsAtCountry(self, country, connectionLevel=None):
"""Get all SEs at a given country"""
connectionIndex = _getConnectionIndex(connectionLevel,
default=DOWNLOAD)
if connectionIndex is None:
return S_ERROR("Unknown connection level")
if not self.siteSet:
self.getSiteSEMapping()
siteList = [
site for site in self.siteSet
if siteCountryName(site) == country.lower()
]
if not siteList:
return S_ERROR("No SEs found in country")
return self._getSEsForSItes(siteList, connectionIndex)
def getSEInGroupAtSite(self, seGroup, site):
"""Get the SE in a group or list of SEs that is present at a site"""
seList = self.getAllSEsInGroupAtSite(seGroup, site)
if not seList["OK"] or seList["Value"] is None:
return seList
return S_OK(seList["Value"][0])
def getAllSEsInGroupAtSite(self, seGroup, site):
"""Get all SEs in a group or list of SEs that are present at a site"""
seList = resolveSEGroup(seGroup)
if not seList:
return S_ERROR("SEGroup does not exist")
sesAtSite = self.getSEsAtSite(site)
if not sesAtSite["OK"]:
return sesAtSite
foundSEs = set(seList) & set(sesAtSite["Value"])
if not foundSEs:
gLogger.warn("No SE found at that site",
"in group %s at %s" % (seGroup, site))
return S_OK()
return S_OK(sorted(foundSEs))
def getRegistrationProtocols(self):
"""Returns the Favorite registration protocol defined in the CS, or 'srm' as default"""
return self.__opsHelper.getValue(
"DataManagement/RegistrationProtocols", ["srm", "dips"])
def getThirdPartyProtocols(self):
"""Returns the Favorite third party protocol defined in the CS, or 'srm' as default"""
return self.__opsHelper.getValue("DataManagement/ThirdPartyProtocols",
["srm"])
def getAccessProtocols(self):
"""Returns the Favorite access protocol defined in the CS, or 'srm' as default"""
return self.__opsHelper.getValue("DataManagement/AccessProtocols",
["srm", "dips"])
def getWriteProtocols(self):
"""Returns the Favorite Write protocol defined in the CS, or 'srm' as default"""
return self.__opsHelper.getValue("DataManagement/WriteProtocols",
["srm", "dips"])
def getStageProtocols(self):
"""Returns the Favorite staging protocol defined in the CS. There are no default"""
return self.__opsHelper.getValue("DataManagement/StageProtocols",
list())
def getMultiHopMatrix(self):
"""
Returns the multi-hop matrix described in DataManagement/MultiHopMatrixOfShame.
.. code-block :: python
'Default': { 'Default': 'MultiHopSEUsedForAllTransfer',
'Dst3' : 'MultiHopFromAnySourceToDst3',
}
'Src1' : { 'Default' : 'DefaultMultiHopSEFromSrc1',
'Dst1': 'MultiHopSEFromSrc1ToDst1},
'Src2' : { 'Default' : 'DefaultMultiHopSEFromSrc2',
'Dst2': 'MultiHopSEFromSrc1ToDst2}
:returns: dict of dict for all the source se / dest SE defined. We user defaultdict
to allow for the use of non existing source/dest.
"""
matrixBasePath = "DataManagement/MultiHopMatrixOfShame"
multiHopMatrix = defaultdict(lambda: defaultdict(lambda: None))
allSrcSEs = self.__opsHelper.getSections(matrixBasePath).get(
"Value", [])
for src in allSrcSEs:
srcDst = self.__opsHelper.getOptionsDict(
cfgPath(matrixBasePath, src)).get("Value")
if srcDst:
multiHopMatrix[src].update(srcDst)
return multiHopMatrix
def web_getLaunchpadSetupWithLFNs(self):
#on the fly file catalog for advanced launchpad
if not hasattr(self, 'fc'):
userData = self.getSessionData()
group = str(userData["user"]["group"])
vo = getVOForGroup(group)
self.fc = FileCatalog(vo=vo)
self.set_header('Content-type', 'text/plain')
lfnList = []
arguments = self.request.arguments
gLogger.always(
"submit: incoming arguments %s to getLaunchpadSetupWithLFNs" %
arguments)
lfnStr = str(arguments['path'][0])
lfnList = lfnStr.split(',')
#checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder
gLogger.always("submit: checking if some rtg_def.m" % arguments)
processed = []
metaDict = {'type': 'info'}
for lfn in lfnStr.split(','):
pos_relative = lfn.find("/")
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
experiment_lfn = lfn[0:pos_relative]
if experiment_lfn in processed:
continue
processed.append(experiment_lfn)
gLogger.always("checking rtg_def.m in %s" % experiment_lfn)
result = self.fc.findFilesByMetadata(metaDict,
path=str(experiment_lfn))
print "result"
print result
if not result['OK'] or not result['Value']:
gLogger.error("Failed to get type info from $s, %s" %
(experiment_lfn, result["Message"]))
continue
for candidate_lfn in result['Value']:
if candidate_lfn.find('rtg_def.m') > 0:
lfnList.append(candidate_lfn)
totalfn = len(lfnList)
ptlfn = ''
current = 1
for lfn in lfnList:
ptlfn = ptlfn + lfn
if current < totalfn:
ptlfn = ptlfn + ', '
current = current + 1
defaultParams = {
"JobName": [1, 'Eiscat'],
"Executable": [1, "/bin/ls"],
"Arguments": [1, "-ltrA"],
"OutputSandbox": [1, "std.out, std.err"],
"InputData": [1, ptlfn],
"OutputData": [0, ""],
"OutputSE": [1, "EISCAT-disk"],
"OutputPath": [0, ""],
"CPUTime": [0, "86400"],
"Site": [0, ""],
"BannedSite": [0, ""],
"Platform": [0, "Linux_x86_64_glibc-2.5"],
"Priority": [0, "5"],
"StdError": [0, "std.err"],
"StdOutput": [0, "std.out"],
"Parameters": [0, "0"],
"ParameterStart": [0, "0"],
"ParameterStep": [0, "1"]
}
delimiter = gConfig.getValue("/Website/Launchpad/ListSeparator", ',')
options = self.__getOptionsFromCS(delimiter=delimiter)
# platform = self.__getPlatform()
# if platform and options:
# if not options.has_key("Platform"):
# options[ "Platform" ] = platform
# else:
# csPlatform = list(options[ "Platform" ])
# allPlatforms = csPlatform + platform
# platform = uniqueElements(allPlatforms)
# options[ "Platform" ] = platform
gLogger.debug("Options from CS: %s" % options)
override = gConfig.getValue("/Website/Launchpad/OptionsOverride",
False)
gLogger.info("end __getLaunchpadOpts")
# Updating the default values from OptionsOverride configuration branch,
for key in options:
if key not in defaultParams:
defaultParams[key] = [0, ""]
defaultParams[key][1] = options[key][0]
gLogger.info(
"Default params + override from /Website/Launchpad/OptionsOverride -> %s"
% defaultParams)
# Reading of the predefined sets of launchpad parameters values
obj = Operations()
predefinedSets = {}
launchpadSections = obj.getSections("Launchpad")
import pprint
if launchpadSections['OK']:
for section in launchpadSections["Value"]:
predefinedSets[section] = {}
sectionOptions = obj.getOptionsDict("Launchpad/" + section)
pprint.pprint(sectionOptions)
if sectionOptions['OK']:
predefinedSets[section] = sectionOptions["Value"]
self.write({
"success": "true",
"result": defaultParams,
"predefinedSets": predefinedSets
})
class CombinedSoftwareInstallation(object):
""" Combined means that it will try to install in the Shared area and in the LocalArea,
depending on the user's rights
"""
def __init__(self, argumentsDict):
""" Standard constructor
Defines, from dictionary of job parameters passed, a set of members to hold e.g. the
applications and the system config.
Also determines the SharedArea and LocalArea.
"""
self.ops = Operations()
self.job = {}
if argumentsDict.has_key('Job'):
self.job = argumentsDict['Job']
self.ce = {}
if argumentsDict.has_key('CE'):
self.ce = argumentsDict['CE']
self.source = {}
if argumentsDict.has_key('Source'):
self.source = argumentsDict['Source']
apps = []
if self.job.has_key('SoftwarePackages'):
if type( self.job['SoftwarePackages'] ) == type(''):
apps = [self.job['SoftwarePackages']]
elif type( self.job['SoftwarePackages'] ) == type([]):
apps = self.job['SoftwarePackages']
self.apps = []
for app in apps:
DIRAC.gLogger.verbose( 'Requested Package %s' % app )
app = tuple(app.split('.'))
if len(app) > 2:
tempapp = app
app = []
app.append(tempapp[0])
app.append(string.join(tempapp[1:], "."))
self.apps.append(app)
self.jobConfig = ''
if self.job.has_key( 'SystemConfig' ):
self.jobConfig = self.job['SystemConfig']
else:
self.jobConfig = natOS.CMTSupportedConfig()[0]
self.ceConfigs = []
if self.ce.has_key('CompatiblePlatforms'):
self.ceConfigs = self.ce['CompatiblePlatforms']
if type(self.ceConfigs) == type(''):
self.ceConfigs = [self.ceConfigs]
#else:
### Use always the list of compatible platform.
self.ceConfigs = natOS.CMTSupportedConfig()
self.sharedArea = SharedArea()
DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea)
self.localArea = LocalArea()
DIRAC.gLogger.info("LocalArea is %s" % self.localArea)
def execute(self):
""" Main method of the class executed by DIRAC jobAgent
Executes the following:
- look for the compatible platforms in the CS, see if one matches request
- install the applications, calls L{TARsoft}
@return: S_OK(), S_ERROR()
"""
if not self.apps:
# There is nothing to do
return DIRAC.S_OK()
if not self.jobConfig:
DIRAC.gLogger.error( 'No architecture requested' )
return DIRAC.S_ERROR( 'No architecture requested' )
found_config = False
DIRAC.gLogger.info("Found CE Configs %s, compatible with system reqs %s" % (string.join(self.ceConfigs, ","),
self.jobConfig))
res = self.ops.getSections('/AvailableTarBalls')
if not res['OK']:
return res
else:
supported_systems = res['Value']
###look in CS if specified platform has software available. Normally consistency check is done at submission time
for ceConfig in self.ceConfigs:
for supp_systems in supported_systems:
if ceConfig == supp_systems:
self.jobConfig = ceConfig
found_config = True
break
if not found_config:
if self.ceConfigs: # redundant check as this is done in the job agent, if locally running option might not be defined
DIRAC.gLogger.error( 'Requested architecture not supported by CE' )
return DIRAC.S_ERROR( 'Requested architecture not supported by CE' )
else:
DIRAC.gLogger.info( 'Assume locally running job, will install software in ' )
areas = []
###Deal with shared/local area: first try to see if the Shared area exists and if not create it (try to). If it fails, fall back to local area
if not self.sharedArea:
if CreateSharedArea():
self.sharedArea = SharedArea()
if self.sharedArea:
areas.append(self.sharedArea)
DIRAC.gLogger.info("Will attempt to install in shared area")
else:
areas.append(self.sharedArea)
areas.append(self.localArea)
for app in self.apps:
failed = False
for area in areas:
DIRAC.gLogger.info('Attempting to install %s_%s for %s in %s' % (app[0], app[1], self.jobConfig, area))
res = TARinstall(app, self.jobConfig, area)
if not res['OK']:
DIRAC.gLogger.error('Failed to install software in %s: %s' % (area, res['Message']),
'%s_%s' % (app[0], app[1]))
failed = True
continue
else:
DIRAC.gLogger.info('%s was successfully installed for %s in %s' % (app, self.jobConfig, area))
failed = False
break
if failed:
return DIRAC.S_ERROR("Failed to install software")
if self.sharedArea:
#List content
listAreaDirectory(self.sharedArea)
return DIRAC.S_OK()
