def testDatasetRunWhitelist(self): """ Test NumberOfRuns splitting with run white list """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151] factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(4, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(217, unit['NumberOfLumis']) self.assertEqual(8, unit['NumberOfFiles']) self.assertEqual(83444, unit['NumberOfEvents'])
def testDatasetLumiMask(self): """ Test NumberOfRuns splitting type with lumi mask """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) dqmHarvArgs["LumiList"] = { "181358": [[71, 80], [95, 110]], "181151": [[1, 20]] } factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(2, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(46, unit['NumberOfLumis']) self.assertEqual(4, unit['NumberOfFiles']) self.assertEqual(12342, unit['NumberOfEvents'])
def testDatasetPolicy(self): """ Test ordinary NumberOfRuns splitting """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist( ["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(47, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(list(unit['Inputs']), [inputDataset]) self.assertEqual(4855, unit['NumberOfLumis']) self.assertEqual(72, unit['NumberOfFiles']) self.assertEqual(743201, unit['NumberOfEvents'])
def testInvalidSpecs(self): """Specs with no work""" dqmHarvArgs = getRequestArgs() # no dataset dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('NoInputDatasetTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) getFirstTask(DQMHarvWorkload).data.input.dataset = None for task in DQMHarvWorkload.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Dataset(), DQMHarvWorkload, task) # invalid dataset name DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidInputDatasetTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) getFirstTask(DQMHarvWorkload).data.input.dataset.name = '/MinimumBias/FAKE-Filter-v1/RECO' for task in DQMHarvWorkload.taskIterator(): self.assertRaises(DBSReaderError, Dataset(), DQMHarvWorkload, task) # invalid run whitelist DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidRunNumberTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) DQMHarvWorkload.setRunWhitelist([666]) # not in this dataset for task in DQMHarvWorkload.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Dataset(), DQMHarvWorkload, task)
def testDatasetSingleJob(self): """ Test NumberOfRuns splitting type with very large SliceSize """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) dqmHarvArgs["DQMHarvestUnit"] = 'multiRun' factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(1, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(4855, unit['NumberOfLumis']) self.assertEqual(72, unit['NumberOfFiles']) self.assertEqual(743201, unit['NumberOfEvents'])
def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "DQMConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.getProcessingString(), "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertFalse(testWorkload.getPrepID(), "PrepId does not match") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_7_3_1_patch1']) # test workload attributes self.assertEqual(testWorkload.processingString, "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.processingVersion, 1) self.assertEqual(sorted(testWorkload.lumiList.keys()), ['139788', '139790', '144011']) self.assertEqual(sorted(testWorkload.lumiList.values()), [[[5, 10], [15, 20], [25, 30]], [[25, 75], [125, 175], [275, 325]], [[50, 100], [110, 125]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag") self.assertEqual(sorted(task.inputRunWhitelist()), [138923, 138924, 138934, 138937, 139788, 139789, 139790, 144011, 144083, 144084, 144086]) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") return
def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "DQMConfigCacheID": self.injectDQMHarvestConfig(), "LumiList": {"251643": [[1, 15], [50,70]], "251721": [[50,100], [110,120]]} }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016B") self.assertEqual(testWorkload.getProcessingString(), "23Sep2016") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016B-23Sep2016-0022") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20']) # test workload attributes self.assertEqual(testWorkload.processingString, "23Sep2016") self.assertEqual(testWorkload.acquisitionEra, "Run2016B") self.assertEqual(testWorkload.processingVersion, 1) self.assertEqual(sorted(testWorkload.lumiList.keys()), ['251643', '251721']) self.assertEqual(sorted(testWorkload.lumiList.values()), [[[1,15], [50,70]], [[50,100], [110,120]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag") self.assertFalse(task.inputRunWhitelist()) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") return
def testDataDirectiveFromQueue(self): """Test data directive from queue""" dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() for task in DQMHarvWorkload.taskIterator(): self.assertRaises(RuntimeError, Dataset(**splitArgs), DQMHarvWorkload, task, {NOT_EXIST_DATASET: []})
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = [] expWfTasks = [ '/TestWorkload/EndOfRunDQMHarvest', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect' ] expFsets = [ 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive' ] subMaps = [ (2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'), (1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting') ] testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig() testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
def testDQMHarvestFailed(self): """ Build a DQMHarvest workload without a DQM config doc """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments.update({"ConfigCacheID": self.injectDQMHarvestConfig()}) testArguments.pop("DQMConfigCacheID", None) factory = DQMHarvestWorkloadFactory() with self.assertRaises(WMSpecFactoryException): factory.factoryWorkloadConstruction("TestBadWorkload", testArguments) return
def testDQMHarvestFailed(self): """ Build a DQMHarvest workload without a DQM config doc """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments.update({ "ConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("DQMConfigCacheID", None) factory = DQMHarvestWorkloadFactory() with self.assertRaises(WMSpecFactoryException): factory.factoryWorkloadConstruction("TestBadWorkload", testArguments) return
def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments.update({ "DQMConfigCacheID": self.injectDQMHarvestConfig(), "LumiList": {"251643": [[1, 15], [50, 70]], "251721": [[50, 100], [110, 120]]} }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016F") self.assertEqual(testWorkload.getProcessingString(), "23Sep2016") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016F-v1-NoBPTX-23Sep2016-0001") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20']) self.assertEqual(sorted(testWorkload.getLumiList().keys()), ['251643', '251721']) self.assertEqual(sorted(testWorkload.getLumiList().values()), [[[1, 15], [50, 70]], [[50, 100], [110, 120]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag") self.assertFalse(task.inputRunWhitelist()) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") return
def testDQMHarvestFailed(self): """ Build a DQMHarvest workload without a DQM config doc """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "ConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("DQMConfigCacheID", None) factory = DQMHarvestWorkloadFactory() self.assertRaises(WMSpecFactoryException, factory.validateSchema, testArguments) return
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = [] expWfTasks = ['/TestWorkload/EndOfRunDQMHarvest', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect'] expFsets = ['TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive'] subMaps = [(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'), (1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')] testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig() testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
def testDatasetPolicy(self): """ Test ordinary NumberOfRuns splitting """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(47, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(4855, unit['NumberOfLumis']) self.assertEqual(72, unit['NumberOfFiles']) self.assertEqual(743201, unit['NumberOfEvents'])
def testDatasetRunWhitelist(self): """ Test NumberOfRuns splitting with run white list """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151] factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(4, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(217, unit['NumberOfLumis']) self.assertEqual(8, unit['NumberOfFiles']) self.assertEqual(83444, unit['NumberOfEvents'])
def testDatasetLumiMask(self): """ Test NumberOfRuns splitting type with lumi mask """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) dqmHarvArgs["LumiList"] = {"181358": [[71, 80], [95, 110]], "181151": [[1, 20]]} factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(2, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(46, unit['NumberOfLumis']) self.assertEqual(4, unit['NumberOfFiles']) self.assertEqual(12342, unit['NumberOfEvents'])
def testDatasetSingleJob(self): """ Test NumberOfRuns splitting type with very large SliceSize """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) dqmHarvArgs["DQMHarvestUnit"] = 'multiRun' factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(1, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(4855, unit['NumberOfLumis']) self.assertEqual(72, unit['NumberOfFiles']) self.assertEqual(743201, unit['NumberOfEvents'])
def getRequestArgs(): """ Returns default values defined in the spec workload """ return DQMHarvestWorkloadFactory.getTestArguments()
def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "DQMConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.getProcessingString(), "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertFalse(testWorkload.getPrepID(), "PrepId does not match") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_7_3_1_patch1']) # test workload attributes self.assertEqual(testWorkload.processingString, "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.processingVersion, 1) self.assertEqual(sorted(testWorkload.lumiList.keys()), ['139788', '139790', '144011']) self.assertEqual( sorted(testWorkload.lumiList.values()), [[[5, 10], [15, 20], [25, 30]], [[25, 75], [125, 175], [275, 325]], [[50, 100], [110, 125]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "DatasetBlock") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual( sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.inputLocationFlag(), "Wrong input location flag") self.assertEqual(sorted(task.inputRunWhitelist()), [ 138923, 138924, 138934, 138937, 139788, 139789, 139790, 144011, 144083, 144084, 144086 ]) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") #print "ALAN inspect %s" % inspect.getmembers(allTasks) #for property, value in vars(allTasks).iteritems(): # print property, ": ", value return