def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"DQMConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.getProcessingString(), "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertFalse(testWorkload.getPrepID(), "PrepId does not match")
self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_7_3_1_patch1'])
# test workload attributes
self.assertEqual(testWorkload.processingString, "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertEqual(sorted(testWorkload.lumiList.keys()), ['139788', '139790', '144011'])
self.assertEqual(sorted(testWorkload.lumiList.values()),
[[[5, 10], [15, 20], [25, 30]], [[25, 75],
[125, 175], [275, 325]], [[50, 100], [110, 125]]])
self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
self.assertEqual(sorted(task.inputRunWhitelist()),
[138923, 138924, 138934, 138937, 139788, 139789,
139790, 144011, 144083, 144084, 144086])
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
return
def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"DQMConfigCacheID": self.injectDQMHarvestConfig(),
"LumiList": {"251643": [[1, 15], [50,70]], "251721": [[50,100], [110,120]]}
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016B")
self.assertEqual(testWorkload.getProcessingString(), "23Sep2016")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016B-23Sep2016-0022")
self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20'])
# test workload attributes
self.assertEqual(testWorkload.processingString, "23Sep2016")
self.assertEqual(testWorkload.acquisitionEra, "Run2016B")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertEqual(sorted(testWorkload.lumiList.keys()), ['251643', '251721'])
self.assertEqual(sorted(testWorkload.lumiList.values()), [[[1,15], [50,70]], [[50,100], [110,120]]])
self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
self.assertFalse(task.inputRunWhitelist())
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = []
expWfTasks = [
'/TestWorkload/EndOfRunDQMHarvest',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect'
]
expFsets = [
'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive'
]
subMaps = [
(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect',
'MinFileBased', 'LogCollect'),
(1,
'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')
]
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"EndOfRunDQMHarvest",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments.update({"ConfigCacheID": self.injectDQMHarvestConfig()})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
with self.assertRaises(WMSpecFactoryException):
factory.factoryWorkloadConstruction("TestBadWorkload",
testArguments)
return
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments.update({
"ConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
self.assertRaises(WMSpecFactoryException, factory.validateSchema, testArguments)
return
def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments.update({
"DQMConfigCacheID": self.injectDQMHarvestConfig(),
"LumiList": {"251643": [[1, 15], [50, 70]], "251721": [[50, 100], [110, 120]]}
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016F")
self.assertEqual(testWorkload.getProcessingString(), "23Sep2016")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016F-v1-NoBPTX-23Sep2016-0001")
self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20'])
self.assertEqual(sorted(testWorkload.getLumiList().keys()), ['251643', '251721'])
self.assertEqual(sorted(testWorkload.getLumiList().values()), [[[1, 15], [50, 70]], [[50, 100], [110, 120]]])
self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
self.assertFalse(task.inputRunWhitelist())
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
return
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"ConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
self.assertRaises(WMSpecFactoryException, factory.validateSchema, testArguments)
return
def testDQMHarvestFailed(self):
"""
Build a DQMHarvest workload without a DQM config doc
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"ConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("DQMConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
self.assertRaises(WMSpecFactoryException, factory.validateSchema,
testArguments)
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = []
expWfTasks = ['/TestWorkload/EndOfRunDQMHarvest',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect']
expFsets = ['TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive']
subMaps = [(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'),
(1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')]
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def getRequestArgs():
""" Returns default values defined in the spec workload """
return DQMHarvestWorkloadFactory.getTestArguments()
def testDQMHarvest(self):
"""
Build a DQMHarvest workload
"""
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "dqmharvest_t",
"DQMConfigCacheID": self.injectDQMHarvestConfig()
})
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(),
"CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.getProcessingString(),
"GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertFalse(testWorkload.getPrepID(), "PrepId does not match")
self.assertEqual(testWorkload.getCMSSWVersions(),
['CMSSW_7_3_1_patch1'])
# test workload attributes
self.assertEqual(testWorkload.processingString,
"GR_R_73_V0A_TEST_RelVal_jetHT2012c")
self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertEqual(sorted(testWorkload.lumiList.keys()),
['139788', '139790', '144011'])
self.assertEqual(
sorted(testWorkload.lumiList.values()),
[[[5, 10], [15, 20], [25, 30]], [[25, 75], [125, 175], [275, 325]],
[[50, 100], [110, 125]]])
self.assertEqual(testWorkload.data.policies.start.policyName,
"DatasetBlock")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(
sorted(tasks),
['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "EndOfRunDQMHarvest")
self.assertEqual(task.getPathName(),
"/TestWorkload/EndOfRunDQMHarvest")
self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
self.assertEqual(task.jobSplittingAlgorithm(), "Harvest",
"Wrong job splitting algo")
self.assertFalse(task.inputLocationFlag(), "Wrong input location flag")
self.assertEqual(sorted(task.inputRunWhitelist()), [
138923, 138924, 138934, 138937, 139788, 139789, 139790, 144011,
144083, 144084, 144086
])
self.assertEqual(sorted(task.listAllStepNames()),
['cmsRun1', 'logArch1', 'upload1'])
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")
#print "ALAN inspect %s" % inspect.getmembers(allTasks)
#for property, value in vars(allTasks).iteritems():
# print property, ": ", value
return