def checkAgentOptions(getOptionMock, systemName, agentName, agentLocation, ignoreOptions=None):
"""Ensure that all the agent options are properly documented.
:param getOptionMock: Mock object for agentmodule.get_amOption function
:param str systemName: name of the **System**
:param str agentName: name of the **Agent**
:param list ignoreOptions: list of options to ignore
"""
if ignoreOptions is None:
ignoreOptions = []
# add some options that can be set, see the AgentModule for all of them
ignoreOptions.extend(
["PollingTime", "Status", "Enabled", "MaxCycles", "LogOutputs", "ControlDirectory", "shifterProxy"]
)
ignoreOptions = list(set(ignoreOptions))
config = CFG()
LOG.info("Testing %s/%s, ignoring options %s", systemName, agentName, ignoreOptions)
# expect the ConfigTemplate one level above the agent module
configFilePath = os.path.join(agentLocation, "..", "ConfigTemplate.cfg")
config.loadFromFile(configFilePath)
optionsDict = config.getAsDict("Agents/%s" % agentName)
outDict = {}
_parseOption(outDict, optionsDict)
optionsDict = outDict
LOG.info("Calls: %s", pformat(getOptionMock.call_args_list))
LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys()))
# check that values in ConfigTemplate are used
for option, value in optionsDict.items():
if any(ignoreOp in option for ignoreOp in ignoreOptions):
LOG.info("From Agent: ignoring option %r with value %r, (%s)", option, value, type(value))
continue
LOG.info("Looking for call to option %r with value %r, (%s)", option, value, type(value))
if not isinstance(value, bool) and not value: # empty string, list, dict ...
assert any(call(option, null) in getOptionMock.call_args_list for null in ({}, set(), [], "", 0, None))
else:
assert (
call(option, value) in getOptionMock.call_args_list
or call(option, [value]) in getOptionMock.call_args_list
)
# check that options used in the agent are in the ConfigTemplates
for opCall in getOptionMock.call_args_list:
optionArguments = opCall[0]
if len(optionArguments) != 2:
continue
optionName = optionArguments[0]
optionValue = optionArguments[1]
if optionName in ignoreOptions:
LOG.info("From Template: ignoring option %r with %r", optionName, optionValue)
continue
LOG.info("Checking Template option %r with %r", optionName, optionValue)
assert optionName in optionsDict
if not optionsDict[optionName]:
assert not optionValue
continue
assert optionsDict[optionName] == optionValue or [optionsDict[optionName]] == optionValue
class JobRepository(object):
def __init__(self, repository=None):
self.location = repository
if not self.location:
if "HOME" in os.environ:
self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
else:
self.location = '%s/.dirac.repo.rep' % os.getcwd()
self.repo = CFG()
if os.path.exists(self.location):
self.repo.loadFromFile(self.location)
if not self.repo.existsKey('Jobs'):
self.repo.createNewSection('Jobs')
else:
self.repo.createNewSection('Jobs')
self.OK = True
written = self._writeRepository(self.location)
if not written:
self.OK = False
def isOK(self):
return self.OK
def readRepository(self):
return S_OK(self.repo.getAsDict('Jobs'))
def writeRepository(self, alternativePath=None):
destination = self.location
if alternativePath:
destination = alternativePath
written = self._writeRepository(destination)
if not written:
return S_ERROR("Failed to write repository")
return S_OK(destination)
def resetRepository(self, jobIDs=[]):
if not jobIDs:
jobs = self.readRepository()['Value']
jobIDs = list(jobs)
paramDict = {'State': 'Submitted', 'Retrieved': 0, 'OutputData': 0}
for jobID in jobIDs:
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def _writeRepository(self, path):
handle, tmpName = tempfile.mkstemp()
written = self.repo.writeToFile(tmpName)
os.close(handle)
if not written:
if os.path.exists(tmpName):
os.remove(tmpName)
return written
if os.path.exists(path):
gLogger.debug("Replacing %s" % path)
try:
shutil.move(tmpName, path)
return True
except Exception as x:
gLogger.error("Failed to overwrite repository.", x)
gLogger.info(
"If your repository is corrupted a backup can be found %s" %
tmpName)
return False
def appendToRepository(self, repoLocation):
if not os.path.exists(repoLocation):
gLogger.error("Secondary repository does not exist", repoLocation)
return S_ERROR("Secondary repository does not exist")
self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo)
self._writeRepository(self.location)
return S_OK()
def addJob(self,
jobID,
state='Submitted',
retrieved=0,
outputData=0,
update=False):
paramDict = {
'State': state,
'Time': self._getTime(),
'Retrieved': int(retrieved),
'OutputData': outputData
}
self._writeJob(jobID, paramDict, update)
self._writeRepository(self.location)
return S_OK(jobID)
def updateJob(self, jobID, paramDict):
if self._existsJob(jobID):
paramDict['Time'] = self._getTime()
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def updateJobs(self, jobDict):
for jobID, paramDict in jobDict.items():
if self._existsJob(jobID):
paramDict['Time'] = self._getTime()
self._writeJob(jobID, paramDict, True)
self._writeRepository(self.location)
return S_OK()
def _getTime(self):
runtime = time.ctime()
return runtime.replace(" ", "_")
def _writeJob(self, jobID, paramDict, update):
jobID = str(jobID)
jobExists = self._existsJob(jobID)
if jobExists and (not update):
gLogger.warn("Job exists and not overwriting")
return S_ERROR("Job exists and not overwriting")
if not jobExists:
self.repo.createNewSection('Jobs/%s' % jobID)
for key, value in paramDict.items():
self.repo.setOption('Jobs/%s/%s' % (jobID, key), value)
return S_OK()
def removeJob(self, jobID):
res = self.repo['Jobs'].deleteKey(str(jobID)) # pylint: disable=no-member
if res:
self._writeRepository(self.location)
return S_OK()
def existsJob(self, jobID):
return S_OK(self._existsJob(jobID))
def _existsJob(self, jobID):
return self.repo.isSection('Jobs/%s' % jobID)
def getLocation(self):
return S_OK(self.location)
def getSize(self):
return S_OK(len(self.repo.getAsDict('Jobs')))
def checkAgentOptions(getOptionMock,
systemName,
agentName,
ignoreOptions=None,
extension='DIRAC'):
"""Ensure that all the agent options are properly documented.
:param getOptionMock: Mock object for agentmodule.get_amOption function
:param str systemName: name of the **System**
:param str agentName: name of the **Agent**
:param list ignoreOptions: list of options to ignore
:param str extension: name of the DIRAC **Extension** where the Agent comes from
"""
if ignoreOptions is None:
ignoreOptions = []
# add some options that can be set, see the AgentModule for all of them
ignoreOptions.extend([
'PollingTime', 'Status', 'Enabled', 'MaxCycles', 'LogOutputs',
'ControlDirectory', 'shifterProxy'
])
ignoreOptions = list(set(ignoreOptions))
config = CFG()
LOG.info("Testing %s/%s, ignoring options %s", systemName, agentName,
ignoreOptions)
# get the location where DIRAC is in from basefolder/DIRAC/__ini__.py
configFilePath = os.path.join(
os.path.dirname(os.path.dirname(DIRAC.__file__)), extension,
systemName, 'ConfigTemplate.cfg')
config.loadFromFile(configFilePath)
optionsDict = config.getAsDict('Agents/%s' % agentName)
outDict = {}
_parseOption(outDict, optionsDict)
optionsDict = outDict
LOG.info("Calls: %s", pformat(getOptionMock.call_args_list))
LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys()))
# check that values in ConfigTemplate are used
for option, value in optionsDict.items():
if any(ignoreOp in option for ignoreOp in ignoreOptions):
LOG.info("From Agent: ignoring option %r with value %r, (%s)",
option, value, type(value))
continue
LOG.info("Looking for call to option %r with value %r, (%s)", option,
value, type(value))
if not isinstance(value,
bool) and not value: # empty string, list, dict ...
assert any(
call(option, null) in getOptionMock.call_args_list
for null in ({}, set(), [], '', 0, None))
else:
assert call(option, value) in getOptionMock.call_args_list or \
call(option, [value]) in getOptionMock.call_args_list
# check that options used in the agent are in the ConfigTemplates
for opCall in getOptionMock.call_args_list:
optionArguments = opCall[0]
if len(optionArguments) != 2:
continue
optionName = optionArguments[0]
optionValue = optionArguments[1]
if optionName in ignoreOptions:
LOG.info("From Template: ignoring option %r with %r", optionName,
optionValue)
continue
LOG.info("Checking Template option %r with %r", optionName,
optionValue)
assert optionName in optionsDict
if not optionsDict[optionName]:
assert not optionValue
continue
assert optionsDict[optionName] == optionValue or [
optionsDict[optionName]
] == optionValue
