def testDatasetRunWhitelist(self):
"""
Test NumberOfRuns splitting with run white list
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151]
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(4, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(217, unit['NumberOfLumis'])
self.assertEqual(8, unit['NumberOfFiles'])
self.assertEqual(83444, unit['NumberOfEvents'])
def testDatasetLumiMask(self):
"""
Test NumberOfRuns splitting type with lumi mask
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
dqmHarvArgs["LumiList"] = {
"181358": [[71, 80], [95, 110]],
"181151": [[1, 20]]
}
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(2, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(46, unit['NumberOfLumis'])
self.assertEqual(4, unit['NumberOfFiles'])
self.assertEqual(12342, unit['NumberOfEvents'])
def testDatasetPolicy(self):
"""
Test ordinary NumberOfRuns splitting
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(
["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(47, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(list(unit['Inputs']), [inputDataset])
self.assertEqual(4855, unit['NumberOfLumis'])
self.assertEqual(72, unit['NumberOfFiles'])
self.assertEqual(743201, unit['NumberOfEvents'])
def testInvalidSpecs(self):
"""Specs with no work"""
dqmHarvArgs = getRequestArgs()
# no dataset
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('NoInputDatasetTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
getFirstTask(DQMHarvWorkload).data.input.dataset = None
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Dataset(), DQMHarvWorkload, task)
# invalid dataset name
DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidInputDatasetTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
getFirstTask(DQMHarvWorkload).data.input.dataset.name = '/MinimumBias/FAKE-Filter-v1/RECO'
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(DBSReaderError, Dataset(), DQMHarvWorkload, task)
# invalid run whitelist
DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidRunNumberTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
DQMHarvWorkload.setRunWhitelist([666]) # not in this dataset
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Dataset(), DQMHarvWorkload, task)
def testDatasetSingleJob(self):
"""
Test NumberOfRuns splitting type with very large SliceSize
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
dqmHarvArgs["DQMHarvestUnit"] = 'multiRun'
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(1, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(4855, unit['NumberOfLumis'])
self.assertEqual(72, unit['NumberOfFiles'])
self.assertEqual(743201, unit['NumberOfEvents'])
def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"DQMConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.getProcessingString(), "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertFalse(testWorkload.getPrepID(), "PrepId does not match")
self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_7_3_1_patch1'])
# test workload attributes
self.assertEqual(testWorkload.processingString, "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertEqual(sorted(testWorkload.lumiList.keys()), ['139788', '139790', '144011'])
self.assertEqual(sorted(testWorkload.lumiList.values()),
[[[5, 10], [15, 20], [25, 30]], [[25, 75],
[125, 175], [275, 325]], [[50, 100], [110, 125]]])
self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
self.assertEqual(sorted(task.inputRunWhitelist()),
[138923, 138924, 138934, 138937, 139788, 139789,
139790, 144011, 144083, 144084, 144086])
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
return
def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"DQMConfigCacheID": self.injectDQMHarvestConfig(),
"LumiList": {"251643": [[1, 15], [50,70]], "251721": [[50,100], [110,120]]}
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016B")
self.assertEqual(testWorkload.getProcessingString(), "23Sep2016")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016B-23Sep2016-0022")
self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20'])
# test workload attributes
self.assertEqual(testWorkload.processingString, "23Sep2016")
self.assertEqual(testWorkload.acquisitionEra, "Run2016B")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertEqual(sorted(testWorkload.lumiList.keys()), ['251643', '251721'])
self.assertEqual(sorted(testWorkload.lumiList.values()), [[[1,15], [50,70]], [[50,100], [110,120]]])
self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
self.assertFalse(task.inputRunWhitelist())
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
return
def testDataDirectiveFromQueue(self):
"""Test data directive from queue"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
splitArgs = DQMHarvWorkload.startPolicyParameters()
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(RuntimeError, Dataset(**splitArgs), DQMHarvWorkload, task, {NOT_EXIST_DATASET: []})
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = []
expWfTasks = [
'/TestWorkload/EndOfRunDQMHarvest',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect'
]
expFsets = [
'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive'
]
subMaps = [
(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect',
'MinFileBased', 'LogCollect'),
(1,
'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')
]
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"EndOfRunDQMHarvest",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments.update({"ConfigCacheID": self.injectDQMHarvestConfig()})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
with self.assertRaises(WMSpecFactoryException):
factory.factoryWorkloadConstruction("TestBadWorkload",
testArguments)
return
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments.update({
"ConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
with self.assertRaises(WMSpecFactoryException):
factory.factoryWorkloadConstruction("TestBadWorkload", testArguments)
return
def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments.update({
"DQMConfigCacheID": self.injectDQMHarvestConfig(),
"LumiList": {"251643": [[1, 15], [50, 70]], "251721": [[50, 100], [110, 120]]}
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016F")
self.assertEqual(testWorkload.getProcessingString(), "23Sep2016")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016F-v1-NoBPTX-23Sep2016-0001")
self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20'])
self.assertEqual(sorted(testWorkload.getLumiList().keys()), ['251643', '251721'])
self.assertEqual(sorted(testWorkload.getLumiList().values()), [[[1, 15], [50, 70]], [[50, 100], [110, 120]]])
self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
self.assertFalse(task.inputRunWhitelist())
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
return
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"ConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
self.assertRaises(WMSpecFactoryException, factory.validateSchema,
testArguments)
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = []
expWfTasks = ['/TestWorkload/EndOfRunDQMHarvest',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect']
expFsets = ['TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive']
subMaps = [(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'),
(1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')]
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def testDatasetPolicy(self):
"""
Test ordinary NumberOfRuns splitting
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(47, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(4855, unit['NumberOfLumis'])
self.assertEqual(72, unit['NumberOfFiles'])
self.assertEqual(743201, unit['NumberOfEvents'])
def testDatasetRunWhitelist(self):
"""
Test NumberOfRuns splitting with run white list
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151]
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(4, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(217, unit['NumberOfLumis'])
self.assertEqual(8, unit['NumberOfFiles'])
self.assertEqual(83444, unit['NumberOfEvents'])
def testDatasetLumiMask(self):
"""
Test NumberOfRuns splitting type with lumi mask
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
dqmHarvArgs["LumiList"] = {"181358": [[71, 80], [95, 110]], "181151": [[1, 20]]}
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(2, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(46, unit['NumberOfLumis'])
self.assertEqual(4, unit['NumberOfFiles'])
self.assertEqual(12342, unit['NumberOfEvents'])
def testDatasetSingleJob(self):
"""
Test NumberOfRuns splitting type with very large SliceSize
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
dqmHarvArgs["DQMHarvestUnit"] = 'multiRun'
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(1, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(4855, unit['NumberOfLumis'])
self.assertEqual(72, unit['NumberOfFiles'])
self.assertEqual(743201, unit['NumberOfEvents'])
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"ConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
self.assertRaises(WMSpecFactoryException, factory.validateSchema, testArguments)
return
def getRequestArgs():
""" Returns default values defined in the spec workload """
return DQMHarvestWorkloadFactory.getTestArguments()
def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"DQMConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(),
"CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.getProcessingString(),
"GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertFalse(testWorkload.getPrepID(), "PrepId does not match")
self.assertEqual(testWorkload.getCMSSWVersions(),
['CMSSW_7_3_1_patch1'])
# test workload attributes
self.assertEqual(testWorkload.processingString,
"GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertEqual(sorted(testWorkload.lumiList.keys()),
['139788', '139790', '144011'])
self.assertEqual(
sorted(testWorkload.lumiList.values()),
[[[5, 10], [15, 20], [25, 30]], [[25, 75], [125, 175], [275, 325]],
[[50, 100], [110, 125]]])
self.assertEqual(testWorkload.data.policies.start.policyName,
"DatasetBlock")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(
sorted(tasks),
['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(),
"/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest",
"Wrong job splitting algo")
self.assertFalse(task.inputLocationFlag(), "Wrong input location flag")
self.assertEqual(sorted(task.inputRunWhitelist()), [
138923, 138924, 138934, 138937, 139788, 139789, 139790, 144011,
144083, 144084, 144086
])
self.assertEqual(sorted(task.listAllStepNames()),
['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
#print "ALAN inspect %s" % inspect.getmembers(allTasks)
#for property, value in vars(allTasks).iteritems():
# print property, ": ", value
return
