def checkAgentOptions(getOptionMock, systemName, agentName,
ignoreOptions=None, extension='DIRAC'):
"""Ensure that all the agent options are properly documented.
:param getOptionMock: Mock object for agentmodule.get_amOption function
:param str systemName: name of the **System**
:param str agentName: name of the **Agent**
:param list ignoreOptions: list of options to ignore
:param str extension: name of the DIRAC **Extension** where the Agent comes from
"""
if ignoreOptions is None:
ignoreOptions = []
# add some options that can be set, see the AgentModule for all of them
ignoreOptions.extend(['PollingTime', 'Status', 'Enabled', 'MaxCycles', 'LogOutputs', 'ControlDirectory'])
ignoreOptions = list(set(ignoreOptions))
config = CFG()
LOG.info("Testing %s/%s, ignoring options %s", systemName, agentName, ignoreOptions)
# get the location where DIRAC is in from basefolder/DIRAC/__ini__.py
configFilePath = os.path.join(os.path.dirname(os.path.dirname(DIRAC.__file__)),
extension, systemName, 'ConfigTemplate.cfg')
config.loadFromFile(configFilePath)
optionsDict = config.getAsDict('Agents/%s' % agentName)
outDict = {}
_parseOption(outDict, optionsDict)
optionsDict = outDict
LOG.info("Calls: %s", pformat(getOptionMock.call_args_list))
LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys()))
# check that values in ConfigTemplate are used
for option, value in optionsDict.iteritems():
if any(ignoreOp in option for ignoreOp in ignoreOptions):
LOG.info("From Agent: ignoring option %r with value %r, (%s)", option, value, type(value))
continue
LOG.info("Looking for call to option %r with value %r, (%s)", option, value, type(value))
if not isinstance(value, bool) and not value: # empty string, list, dict ...
assert any(call(option, null) in getOptionMock.call_args_list for null in ({}, set(), [], '', 0))
else:
assert call(option, value) in getOptionMock.call_args_list or \
call(option, [value]) in getOptionMock.call_args_list
# check that options used in the agent are in the ConfigTemplates
for opCall in getOptionMock.call_args_list:
optionArguments = opCall[0]
if len(optionArguments) != 2:
continue
optionName = optionArguments[0]
optionValue = optionArguments[1]
if optionName in ignoreOptions:
LOG.info("From Template: ignoring option %r with %r", optionName, optionValue)
continue
LOG.info("Checking Template option %r with %r", optionName, optionValue)
assert optionName in optionsDict
if not optionsDict[optionName]:
assert not optionValue
continue
assert optionsDict[optionName] == optionValue or [optionsDict[optionName]] == optionValue
class JobRepository(object):
def __init__(self, repository=None):
self.location = repository
if not self.location:
if "HOME" in os.environ:
self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
else:
self.location = '%s/.dirac.repo.rep' % os.getcwd()
self.repo = CFG()
if os.path.exists(self.location):
self.repo.loadFromFile(self.location)
if not self.repo.existsKey('Jobs'):
self.repo.createNewSection('Jobs')
else:
self.repo.createNewSection('Jobs')
self.OK = True
written = self._writeRepository(self.location)
if not written:
self.OK = False
def isOK(self):
return self.OK
def readRepository(self):
return S_OK(self.repo.getAsDict('Jobs'))
def writeRepository(self, alternativePath=None):
destination = self.location
if alternativePath:
destination = alternativePath
written = self._writeRepository(destination)
if not written:
return S_ERROR("Failed to write repository")
return S_OK(destination)
def resetRepository(self, jobIDs=[]):
if not jobIDs:
jobs = self.readRepository()['Value']
jobIDs = jobs.keys()
paramDict = {'State': 'Submitted',
'Retrieved': 0,
'OutputData': 0}
for jobID in jobIDs:
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def _writeRepository(self, path):
handle, tmpName = tempfile.mkstemp()
written = self.repo.writeToFile(tmpName)
os.close(handle)
if not written:
if os.path.exists(tmpName):
os.remove(tmpName)
return written
if os.path.exists(path):
gLogger.debug("Replacing %s" % path)
try:
shutil.move(tmpName, path)
return True
except Exception as x:
gLogger.error("Failed to overwrite repository.", x)
gLogger.info("If your repository is corrupted a backup can be found %s" % tmpName)
return False
def appendToRepository(self, repoLocation):
if not os.path.exists(repoLocation):
gLogger.error("Secondary repository does not exist", repoLocation)
return S_ERROR("Secondary repository does not exist")
self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo)
self._writeRepository(self.location)
return S_OK()
def addJob(self, jobID, state='Submitted', retrieved=0, outputData=0, update=False):
paramDict = {'State': state,
'Time': self._getTime(),
'Retrieved': int(retrieved),
'OutputData': outputData}
self._writeJob(jobID, paramDict, update)
self._writeRepository(self.location)
return S_OK(jobID)
def updateJob(self, jobID, paramDict):
if self._existsJob(jobID):
paramDict['Time'] = self._getTime()
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def updateJobs(self, jobDict):
for jobID, paramDict in jobDict.items():
if self._existsJob(jobID):
paramDict['Time'] = self._getTime()
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def _getTime(self):
runtime = time.ctime()
return runtime.replace(" ", "_")
def _writeJob(self, jobID, paramDict, update):
jobID = str(jobID)
jobExists = self._existsJob(jobID)
if jobExists and (not update):
gLogger.warn("Job exists and not overwriting")
return S_ERROR("Job exists and not overwriting")
if not jobExists:
self.repo.createNewSection('Jobs/%s' % jobID)
for key, value in paramDict.items():
self.repo.setOption('Jobs/%s/%s' % (jobID, key), value)
return S_OK()
def removeJob(self, jobID):
res = self.repo['Jobs'].deleteKey(str(jobID)) # pylint: disable=no-member
if res:
self._writeRepository(self.location)
return S_OK()
def existsJob(self, jobID):
return S_OK(self._existsJob(jobID))
def _existsJob(self, jobID):
return self.repo.isSection('Jobs/%s' % jobID)
def getLocation(self):
return S_OK(self.location)
def getSize(self):
return S_OK(len(self.repo.getAsDict('Jobs')))
class JobRepository( object ):
def __init__( self, repository = None ):
self.location = repository
if not self.location:
if "HOME" in os.environ:
self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
else:
self.location = '%s/.dirac.repo.rep' % os.getcwd()
self.repo = CFG()
if os.path.exists( self.location ):
self.repo.loadFromFile( self.location )
if not self.repo.existsKey( 'Jobs' ):
self.repo.createNewSection( 'Jobs' )
else:
self.repo.createNewSection( 'Jobs' )
self.OK = True
written = self._writeRepository( self.location )
if not written:
self.OK = False
def isOK( self ):
return self.OK
def readRepository( self ):
return S_OK( self.repo.getAsDict( 'Jobs' ) )
def writeRepository( self, alternativePath = None ):
destination = self.location
if alternativePath:
destination = alternativePath
written = self._writeRepository( destination )
if not written:
return S_ERROR( "Failed to write repository" )
return S_OK( destination )
def resetRepository( self, jobIDs = [] ):
if not jobIDs:
jobs = self.readRepository()['Value']
jobIDs = jobs.keys()
paramDict = {'State' : 'Submitted',
'Retrieved' : 0,
'OutputData' : 0}
for jobID in jobIDs:
self._writeJob( jobID, paramDict, True )
self._writeRepository( self.location )
return S_OK()
def _writeRepository( self, path ):
handle, tmpName = tempfile.mkstemp()
written = self.repo.writeToFile( tmpName )
os.close( handle )
if not written:
if os.path.exists( tmpName ):
os.remove( tmpName )
return written
if os.path.exists( path ):
gLogger.debug( "Replacing %s" % path )
try:
shutil.move( tmpName, path )
return True
except Exception as x:
gLogger.error( "Failed to overwrite repository.", x )
gLogger.info( "If your repository is corrupted a backup can be found %s" % tmpName )
return False
def appendToRepository( self, repoLocation ):
if not os.path.exists( repoLocation ):
gLogger.error( "Secondary repository does not exist", repoLocation )
return S_ERROR( "Secondary repository does not exist" )
self.repo = CFG().loadFromFile( repoLocation ).mergeWith( self.repo )
self._writeRepository( self.location )
return S_OK()
def addJob( self, jobID, state = 'Submitted', retrieved = 0, outputData = 0, update = False ):
paramDict = { 'State' : state,
'Time' : self._getTime(),
'Retrieved' : int( retrieved ),
'OutputData' : outputData}
self._writeJob( jobID, paramDict, update )
self._writeRepository( self.location )
return S_OK( jobID )
def updateJob( self, jobID, paramDict ):
if self._existsJob( jobID ):
paramDict['Time'] = self._getTime()
self._writeJob( jobID, paramDict, True )
self._writeRepository( self.location )
return S_OK()
def updateJobs( self, jobDict ):
for jobID, paramDict in jobDict.items():
if self._existsJob( jobID ):
paramDict['Time'] = self._getTime()
self._writeJob( jobID, paramDict, True )
self._writeRepository( self.location )
return S_OK()
def _getTime( self ):
runtime = time.ctime()
return runtime.replace( " ", "_" )
def _writeJob( self, jobID, paramDict, update ):
jobID = str( jobID )
jobExists = self._existsJob( jobID )
if jobExists and ( not update ):
gLogger.warn( "Job exists and not overwriting" )
return S_ERROR( "Job exists and not overwriting" )
if not jobExists:
self.repo.createNewSection( 'Jobs/%s' % jobID )
for key, value in paramDict.items():
self.repo.setOption( 'Jobs/%s/%s' % ( jobID, key ), value )
return S_OK()
def removeJob( self, jobID ):
res = self.repo['Jobs'].deleteKey( str( jobID ) ) #pylint: disable=no-member
if res:
self._writeRepository( self.location )
return S_OK()
def existsJob( self, jobID ):
return S_OK( self._existsJob( jobID ) )
def _existsJob( self, jobID ):
return self.repo.isSection( 'Jobs/%s' % jobID )
def getLocation( self ):
return S_OK( self.location )
def getSize( self ):
return S_OK( len( self.repo.getAsDict( 'Jobs' ) ) )
def checkAgentOptions(getOptionMock, systemName, agentName,
ignoreOptions=None, extension='DIRAC'):
"""Ensure that all the agent options are properly documented.
:param getOptionMock: Mock object for agentmodule.get_amOption function
:param str systemName: name of the **System**
:param str agentName: name of the **Agent**
:param list ignoreOptions: list of options to ignore
:param str extension: name of the DIRAC **Extension** where the Agent comes from
"""
if ignoreOptions is None:
ignoreOptions = []
# add some options that can be set, see the AgentModule for all of them
ignoreOptions.extend(['PollingTime', 'Status', 'Enabled',
'MaxCycles', 'LogOutputs', 'ControlDirectory',
'shifterProxy'])
ignoreOptions = list(set(ignoreOptions))
config = CFG()
LOG.info("Testing %s/%s, ignoring options %s", systemName, agentName, ignoreOptions)
# get the location where DIRAC is in from basefolder/DIRAC/__ini__.py
configFilePath = os.path.join(os.path.dirname(os.path.dirname(DIRAC.__file__)),
extension, systemName, 'ConfigTemplate.cfg')
config.loadFromFile(configFilePath)
optionsDict = config.getAsDict('Agents/%s' % agentName)
outDict = {}
_parseOption(outDict, optionsDict)
optionsDict = outDict
LOG.info("Calls: %s", pformat(getOptionMock.call_args_list))
LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys()))
# check that values in ConfigTemplate are used
for option, value in optionsDict.iteritems():
if any(ignoreOp in option for ignoreOp in ignoreOptions):
LOG.info("From Agent: ignoring option %r with value %r, (%s)", option, value, type(value))
continue
LOG.info("Looking for call to option %r with value %r, (%s)", option, value, type(value))
if not isinstance(value, bool) and not value: # empty string, list, dict ...
assert any(call(option, null) in getOptionMock.call_args_list for null in ({}, set(), [], '', 0))
else:
assert call(option, value) in getOptionMock.call_args_list or \
call(option, [value]) in getOptionMock.call_args_list
# check that options used in the agent are in the ConfigTemplates
for opCall in getOptionMock.call_args_list:
optionArguments = opCall[0]
if len(optionArguments) != 2:
continue
optionName = optionArguments[0]
optionValue = optionArguments[1]
if optionName in ignoreOptions:
LOG.info("From Template: ignoring option %r with %r", optionName, optionValue)
continue
LOG.info("Checking Template option %r with %r", optionName, optionValue)
assert optionName in optionsDict
if not optionsDict[optionName]:
assert not optionValue
continue
assert optionsDict[optionName] == optionValue or [optionsDict[optionName]] == optionValue