def __checkInstallSoftware(self, jobID, jobParams, resourceParams):
"""Checks software requirement of job and whether this is already present
before installing software locally.
"""
if not jobParams.has_key('SoftwareDistModule'):
msg = 'Job has no software installation requirement'
self.log.verbose(msg)
return S_OK(msg)
self.__report(jobID, 'Matched', 'Installing Software')
softwareDist = jobParams['SoftwareDistModule']
#HACK: Delete when svn repo is in production!
softwareDist = softwareDist.replace("DIRAC.LHCbSystem.",
"LHCbDIRAC.Core.")
#END OF HACK
self.log.verbose('Found VO Software Distribution module: %s' %
(softwareDist))
argumentsDict = {'Job': jobParams, 'CE': resourceParams}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(softwareDist, argumentsDict)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
result = module.execute()
return result
def checkJob( self, job, classAdJob ):
"""This method controls the checking of the job.
"""
jobDesc = JobDescription()
result = jobDesc.loadDescription( classAdJob.asJDL() )
if not result[ 'OK' ]:
self.setFailedJob( job, result['Message'], classAdJob )
return result
self.__syncJobDesc( job, jobDesc, classAdJob )
#Check if job defines a path itself
# FIXME: only some group might be able to overwrite the jobPath
jobPath = classAdJob.get_expression( 'JobPath' ).replace( '"', '' ).replace( 'Unknown', '' )
#jobPath = jobDesc.getVarWithDefault( 'JobPath' ).replace( 'Unknown', '' )
if jobPath:
# HACK: Remove the { and } to ensure we have a simple string
jobPath = jobPath.replace( "{", "" ).replace( "}", "" )
self.log.info( 'Job %s defines its own optimizer chain %s' % ( job, jobPath ) )
return self.processJob( job, List.fromChar( jobPath ) )
#If no path, construct based on JDL and VO path module if present
path = list( self.basePath )
if self.voPlugin:
argumentsDict = {'JobID':job, 'ClassAd':classAdJob, 'ConfigPath':self.am_getModuleParam( "section" )}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule( self.voPlugin, argumentsDict )
if not moduleInstance['OK']:
self.log.error( 'Could not instantiate module:', '%s' % ( self.voPlugin ) )
self.setFailedJob( job, 'Could not instantiate module: %s' % ( self.voPlugin ), classAdJob )
return S_ERROR( 'Holding pending jobs' )
module = moduleInstance['Value']
result = module.execute()
if not result['OK']:
self.log.warn( 'Execution of %s failed' % ( self.voPlugin ) )
return result
extraPath = List.fromChar( result['Value'] )
if extraPath:
path.extend( extraPath )
self.log.verbose( 'Adding extra VO specific optimizers to path: %s' % ( extraPath ) )
else:
self.log.verbose( 'No VO specific plugin module specified' )
#Should only rely on an input data setting in absence of VO plugin
result = self.jobDB.getInputData( job )
if not result['OK']:
self.log.error( 'Failed to get input data from JobDB', job )
self.log.warn( result['Message'] )
return result
if result['Value']:
# if the returned tuple is not empty it will evaluate true
self.log.info( 'Job %s has an input data requirement' % ( job ) )
path.extend( self.inputData )
else:
self.log.info( 'Job %s has no input data requirement' % ( job ) )
path.extend( self.endPath )
self.log.info( 'Constructed path for job %s is: %s' % ( job, path ) )
return self.processJob( job, path )
def getOutputData(self, paramDict, moduleLocation):
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(moduleLocation, paramDict)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
return module.execute()
def getOutputData( self, paramDict, moduleLocation ):
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule( moduleLocation, paramDict )
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
return module.execute()
def getOutputData(self,paramDict):
moduleFactory = ModuleFactory()
moduleLocation = gConfig.getValue("/DIRAC/VOPolicy/OutputDataModule","LHCbDIRAC.Core.Utilities.OutputDataPolicy")
moduleInstance = moduleFactory.getModule(moduleLocation, paramDict)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
return module.execute()
def getOutputData(self,paramDict):
moduleFactory = ModuleFactory()
moduleLocation = gConfig.getValue("/DIRAC/VOPolicy/OutputDataModule","LHCbDIRAC.Core.Utilities.OutputDataPolicy")
moduleInstance = moduleFactory.getModule(moduleLocation, paramDict)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
return module.execute()
def __runModule(self, modulePath, remainingReplicas):
"""This method provides a way to run the modules specified by the VO that
govern the input data access policy for the current site. For LHCb the
standard WMS modules are applied in a different order depending on the site.
"""
self.log.info('Attempting to run %s' % (modulePath))
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(modulePath, self.arguments)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
result = module.execute(remainingReplicas)
return result
def getOutputData(self, paramDict):
""" Get the list of job output LFNs from the provided plugin
"""
if not self.outputDataModule_o:
# Create the module object
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(self.outputDataModule, None)
if not moduleInstance['OK']:
return moduleInstance
self.outputDataModule_o = moduleInstance['Value']
# This is the "argument" to the module, set it and then execute
self.outputDataModule_o.paramDict = paramDict
return self.outputDataModule_o.execute()
def __runModule(self, modulePath, remainingReplicas):
"""This method provides a way to run the modules specified by the VO that
govern the input data access policy for the current site. For LHCb the
standard WMS modules are applied in a different order depending on the site.
"""
self.log.info('Attempting to run %s' % (modulePath))
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(modulePath, self.arguments)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
result = module.execute(remainingReplicas)
return result
def __runModule(self, modulePath, remainingReplicas):
"""This method provides a way to run the modules specified by the VO that
govern the input data access policy for the current site. Using the
InputDataPolicy section from Operations different modules can be defined for
particular sites or for InputDataPolicy defined in the JDL of the jobs.
"""
self.log.info('Attempting to run %s' % (modulePath))
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(modulePath, self.arguments)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
result = module.execute(remainingReplicas)
return result
def __runModule(self, modulePath, remainingReplicas):
"""This method provides a way to run the modules specified by the VO that
govern the input data access policy for the current site. Using the
InputDataPolicy section from Operations different modules can be defined for
particular sites or for InputDataPolicy defined in the JDL of the jobs.
"""
self.log.info('Attempting to run %s' % (modulePath))
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(modulePath, self.arguments)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
result = module.execute(remainingReplicas)
return result
def __checkInstallSoftware( self, jobID, jobParams, resourceParams ):
"""Checks software requirement of job and whether this is already present
before installing software locally.
"""
if not jobParams.has_key( 'SoftwareDistModule' ):
msg = 'Job has no software installation requirement'
self.log.verbose( msg )
return S_OK( msg )
self.__report( jobID, 'Matched', 'Installing Software' )
softwareDist = jobParams['SoftwareDistModule']
self.log.verbose( 'Found VO Software Distribution module: %s' % ( softwareDist ) )
argumentsDict = {'Job':jobParams, 'CE':resourceParams}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule( softwareDist, argumentsDict )
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
return module.execute()
def __checkInstallSoftware(self, jobID, jobParams, resourceParams):
"""Checks software requirement of job and whether this is already present
before installing software locally.
"""
if "SoftwareDistModule" not in jobParams:
msg = "Job has no software installation requirement"
self.log.verbose(msg)
return S_OK(msg)
self.__report(jobID, "Matched", "Installing Software")
softwareDist = jobParams["SoftwareDistModule"]
self.log.verbose("Found VO Software Distribution module: %s" % (softwareDist))
argumentsDict = {"Job": jobParams, "CE": resourceParams}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(softwareDist, argumentsDict)
if not moduleInstance["OK"]:
return moduleInstance
module = moduleInstance["Value"]
return module.execute()
def _checkInstallSoftware(self, jobID, jobParams, resourceParams):
"""Checks software requirement of job and whether this is already present
before installing software locally.
"""
if 'SoftwareDistModule' not in jobParams:
msg = 'Job has no software installation requirement'
self.log.verbose(msg)
return S_OK(msg)
self.__report(jobID, 'Matched', 'Installing Software')
softwareDist = jobParams['SoftwareDistModule']
self.log.verbose('Found VO Software Distribution module',
': %s' % (softwareDist))
argumentsDict = {'Job': jobParams, 'CE': resourceParams}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(softwareDist, argumentsDict)
if not moduleInstance['OK']:
return moduleInstance
module = moduleInstance['Value']
return module.execute()
def _checkInstallSoftware(self, jobID, jobParams, resourceParams,
jobReport):
"""Checks software requirement of job and whether this is already present
before installing software locally.
"""
if "SoftwareDistModule" not in jobParams:
msg = "Job has no software installation requirement"
self.log.verbose(msg)
return S_OK(msg)
jobReport.setJobStatus(minorStatus="Installing Software",
sendFlag=False)
softwareDist = jobParams["SoftwareDistModule"]
self.log.verbose("Found VO Software Distribution module",
": %s" % (softwareDist))
argumentsDict = {"Job": jobParams, "CE": resourceParams}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(softwareDist, argumentsDict)
if not moduleInstance["OK"]:
return moduleInstance
module = moduleInstance["Value"]
return module.execute()
def checkJob(self, job, classAdJob):
"""This method controls the checking of the job.
"""
jobDesc = JobDescription()
result = jobDesc.loadDescription(classAdJob.asJDL())
if not result['OK']:
self.setFailedJob(job, result['Message'], classAdJob)
return result
self.__syncJobDesc(job, jobDesc, classAdJob)
#Check if job defines a path itself
# FIXME: only some group might be able to overwrite the jobPath
jobPath = classAdJob.get_expression('JobPath').replace(
'"', '').replace('Unknown', '')
#jobPath = jobDesc.getVarWithDefault( 'JobPath' ).replace( 'Unknown', '' )
if jobPath:
# HACK: Remove the { and } to ensure we have a simple string
jobPath = jobPath.replace("{", "").replace("}", "")
self.log.info('Job %s defines its own optimizer chain %s' %
(job, jobPath))
return self.processJob(job, List.fromChar(jobPath))
#If no path, construct based on JDL and VO path module if present
path = list(self.basePath)
if self.voPlugin:
argumentsDict = {
'JobID': job,
'ClassAd': classAdJob,
'ConfigPath': self.am_getModuleParam("section")
}
moduleFactory = ModuleFactory()
moduleInstance = moduleFactory.getModule(self.voPlugin,
argumentsDict)
if not moduleInstance['OK']:
self.log.error('Could not instantiate module:',
'%s' % (self.voPlugin))
self.setFailedJob(
job, 'Could not instantiate module: %s' % (self.voPlugin),
classAdJob)
return S_ERROR('Holding pending jobs')
module = moduleInstance['Value']
result = module.execute()
if not result['OK']:
self.log.warn('Execution of %s failed' % (self.voPlugin))
return result
extraPath = List.fromChar(result['Value'])
if extraPath:
path.extend(extraPath)
self.log.verbose(
'Adding extra VO specific optimizers to path: %s' %
(extraPath))
else:
self.log.verbose('No VO specific plugin module specified')
#Should only rely on an input data setting in absence of VO plugin
result = self.jobDB.getInputData(job)
if not result['OK']:
self.log.error('Failed to get input data from JobDB', job)
self.log.warn(result['Message'])
return result
if result['Value']:
# if the returned tuple is not empty it will evaluate true
self.log.info('Job %s has an input data requirement' % (job))
path.extend(self.inputData)
else:
self.log.info('Job %s has no input data requirement' % (job))
path.extend(self.endPath)
self.log.info('Constructed path for job %s is: %s' % (job, path))
return self.processJob(job, path)