def testRequestInfoTaskChain(self):
"""
Test the RequestInfo class with a TaskChain workflow.
"""
# test overwrite of original values
reqArgs = deepcopy(taskChainOriginalArgs)
reqArgs['AgentJobInfo'] = {"agent_a": {}}
from pprint import pprint
pprint(reqArgs)
reqInfo = RequestInfo(reqArgs)
self.assertItemsEqual(reqArgs, reqInfo.data)
# test `andFilterCheck` method
self.assertFalse(reqInfo.andFilterCheck(dict(RequestType="BAD_TYPE")))
self.assertTrue(reqInfo.andFilterCheck(dict(RequestType="TaskChain")))
self.assertFalse(reqInfo.andFilterCheck(dict(IncludeParents="False")))
self.assertFalse(reqInfo.andFilterCheck(dict(IncludeParents="True")))
self.assertFalse(reqInfo.andFilterCheck(dict(IncludeParents=True)))
self.assertTrue(reqInfo.andFilterCheck(dict(KeepOutput=False)))
# NOTE: None value will always fail to match (thus returning False)
self.assertFalse(reqInfo.andFilterCheck(dict(PrepID=None)))
self.assertTrue(reqInfo.andFilterCheck(dict(TaskChain=3)))
self.assertTrue(reqInfo.andFilterCheck(dict(Campaign="MainTask")))
self.assertTrue(reqInfo.andFilterCheck(dict(AgentJobInfo="CLEANED")))
# test `isWorkflowCleaned` method
self.assertTrue(reqInfo.isWorkflowCleaned())
# finally, test the `get` method
self.assertIsNone(reqInfo.get(prop="NonExistent", default=None))
self.assertIsNone(reqInfo.get(prop="NonExistent"))
self.assertEqual(
reqInfo.get(prop="NonExistent", default="diff_default"),
"diff_default")
# PrepID is not inside any Step dict, that's why a plain integer...
self.assertEqual(reqInfo.get(prop="PrepID"), None)
self.assertEqual(reqInfo.get(prop="RequestName"), "test_taskchain")
self.assertEqual(reqInfo.get(prop="LumisPerJob"), [10])
self.assertEqual(reqInfo.get(prop="AcquisitionEra"), ["ACQERA"])
# NOTE: inner task properties have precedence over the top level ones.
self.assertEqual(reqInfo.get(prop="ProcessingVersion"), [2])
self.assertItemsEqual(reqInfo.get(prop="BlockWhitelist"), ["A", "B"])
self.assertItemsEqual(
reqInfo.get(prop="ScramArch"),
['slc6_amd64_gcc481', 'slc6_amd64_gcc493', 'slc7_amd64_gcc630'])
def testRequestInfoReReco(self):
"""
Test the RequestInfo class with a ReReco workflow.
"""
# test overwrite of original values
reqArgs = deepcopy(rerecoOriginalArgs)
reqArgs['AgentJobInfo'] = {
"agent_a": {
"num_jobs": 2,
"status": "done"
}
}
reqInfo = RequestInfo(reqArgs)
self.assertItemsEqual(reqArgs, reqInfo.data)
# test `andFilterCheck` method
self.assertFalse(reqInfo.andFilterCheck(dict(RequestType="BAD_TYPE")))
self.assertTrue(reqInfo.andFilterCheck(dict(RequestType="ReReco")))
self.assertFalse(reqInfo.andFilterCheck(dict(IncludeParents="False")))
self.assertTrue(reqInfo.andFilterCheck(dict(IncludeParents="True")))
self.assertTrue(reqInfo.andFilterCheck(dict(IncludeParents=True)))
self.assertFalse(reqInfo.andFilterCheck(dict(AgentJobInfo="CLEANED")))
# test `isWorkflowCleaned` method
self.assertFalse(reqInfo.isWorkflowCleaned())
# finally, test the `get` method
self.assertIsNone(reqInfo.get(prop="NonExistent", default=None))
self.assertIsNone(reqInfo.get(prop="NonExistent"))
self.assertEqual(
reqInfo.get(prop="NonExistent", default="diff_default"),
"diff_default")
self.assertEqual(reqInfo.get(prop="Memory"), 234)
def filterData(filterDict, maskList):
reqData = DataCache.getlatestJobData()
for _, reqInfo in reqData.iteritems():
reqData = RequestInfo(reqInfo)
if reqData.andFilterCheck(filterDict):
for prop in maskList:
result = reqData.get(prop, [])
if isinstance(result, list):
for value in result:
yield value
elif result is not None and result != "":
yield result
def testRequestInfoStepChain(self):
"""
Test the RequestInfo class with a StepChain workflow.
"""
# test overwrite of original values
reqArgs = deepcopy(stepChainOriginalArgs)
reqArgs['AgentJobInfo'] = {
"agent_a": {
"num_jobs": 2,
"status": "done"
}
}
from pprint import pprint
pprint(reqArgs)
reqInfo = RequestInfo(reqArgs)
self.assertItemsEqual(reqArgs, reqInfo.data)
# test `andFilterCheck` method
self.assertFalse(reqInfo.andFilterCheck(dict(RequestType="BAD_TYPE")))
self.assertTrue(reqInfo.andFilterCheck(dict(RequestType="StepChain")))
self.assertFalse(reqInfo.andFilterCheck(dict(IncludeParents="False")))
self.assertFalse(reqInfo.andFilterCheck(dict(IncludeParents="True")))
self.assertFalse(reqInfo.andFilterCheck(dict(IncludeParents=True)))
self.assertTrue(reqInfo.andFilterCheck(dict(ProcessingVersion=3)))
self.assertTrue(reqInfo.andFilterCheck(dict(GlobalTag="PHYS18")))
self.assertFalse(reqInfo.andFilterCheck(dict(AgentJobInfo="CLEANED")))
# test `isWorkflowCleaned` method
self.assertFalse(reqInfo.isWorkflowCleaned())
# finally, test the `get` method
self.assertIsNone(reqInfo.get(prop="NonExistent", default=None))
self.assertIsNone(reqInfo.get(prop="NonExistent"))
self.assertEqual(
reqInfo.get(prop="NonExistent", default="diff_default"),
"diff_default")
# Memory is not inside any Step dict, that's why a plain integer...
self.assertEqual(reqInfo.get(prop="Memory"), 1234)
self.assertEqual(reqInfo.get(prop="RequestName"), "test_stepchain")
self.assertEqual(reqInfo.get(prop="AcquisitionEra"), ["ACQERA"])
self.assertEqual(reqInfo.get(prop="LumisPerJob"), [10])
# NOTE: inner step properties have precedence over the top level ones.
self.assertItemsEqual(reqInfo.get(prop="ScramArch"),
["slc6_amd64_gcc493", "slc7_amd64_gcc630"])
def filterDataByRequest(filterDict, maskList=None):
reqData = DataCache.getlatestJobData()
if maskList is not None:
if isinstance(maskList, basestring):
maskList = [maskList]
if "RequestName" not in maskList:
maskList.append("RequestName")
for _, reqDict in reqData.iteritems():
reqInfo = RequestInfo(reqDict)
if reqInfo.andFilterCheck(filterDict):
if maskList is None:
yield reqDict
else:
resultItem = {}
for prop in maskList:
resultItem[prop] = reqInfo.get(prop, None)
yield resultItem
