def testDataDirectiveFromQueue(self): """Test data directive from queue""" dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist( ["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() for task in DQMHarvWorkload.taskIterator(): self.assertRaises(RuntimeError, Dataset(**splitArgs), DQMHarvWorkload, task, {NOT_EXIST_DATASET: []})
def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments.update({ "DQMConfigCacheID": self.injectDQMHarvestConfig(), "LumiList": {"251643": [[1, 15], [50, 70]], "251721": [[50, 100], [110, 120]]} }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016F") self.assertEqual(testWorkload.getProcessingString(), "23Sep2016") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016F-v1-NoBPTX-23Sep2016-0001") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20']) self.assertEqual(sorted(testWorkload.getLumiList().keys()), ['251643', '251721']) self.assertEqual(sorted(testWorkload.getLumiList().values()), [[[1, 15], [50, 70]], [[50, 100], [110, 120]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag") self.assertFalse(task.inputRunWhitelist()) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") return
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = [] expWfTasks = ['/TestWorkload/EndOfRunDQMHarvest', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect'] expFsets = ['TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive'] subMaps = [(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'), (1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')] testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig() testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) print("Tasks producing output:\n%s" % pformat(testWorkload.listOutputProducingTasks())) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") print("List of workflows:\n%s" % pformat(workflows)) self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() print("List of filesets:\n%s" % pformat(filesets)) self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) print("List of subscriptions:\n%s" % pformat(subscriptions)) self.assertItemsEqual(subscriptions, subMaps)
def testDQMHarvestFailed(self): """ Build a DQMHarvest workload without a DQM config doc """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "ConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("DQMConfigCacheID", None) factory = DQMHarvestWorkloadFactory() self.assertRaises(WMSpecFactoryException, factory.validateSchema, testArguments) return
def testDatasetPolicy(self): """ Test ordinary NumberOfRuns splitting """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(47, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(4855, unit['NumberOfLumis']) self.assertEqual(72, unit['NumberOfFiles']) self.assertEqual(743201, unit['NumberOfEvents'])
def testDatasetLumiMask(self): """ Test NumberOfRuns splitting type with lumi mask """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) dqmHarvArgs["LumiList"] = {"181358": [[71, 80], [95, 110]], "181151": [[1, 20]]} factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(2, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(46, unit['NumberOfLumis']) self.assertEqual(4, unit['NumberOfFiles']) self.assertEqual(12342, unit['NumberOfEvents'])
def testInvalidSpecs(self): """Specs with no work""" dqmHarvArgs = getRequestArgs() # no dataset dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'NoInputDatasetTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist( ["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) getFirstTask(DQMHarvWorkload).data.input.dataset = None for task in DQMHarvWorkload.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Dataset(), DQMHarvWorkload, task) # invalid dataset name DQMHarvWorkload = factory.factoryWorkloadConstruction( 'InvalidInputDatasetTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist( ["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) getFirstTask( DQMHarvWorkload ).data.input.dataset.name = '/MinimumBias/FAKE-Filter-v1/RECO' for task in DQMHarvWorkload.taskIterator(): self.assertRaises(DBSReaderError, Dataset(), DQMHarvWorkload, task) # invalid run whitelist DQMHarvWorkload = factory.factoryWorkloadConstruction( 'InvalidRunNumberTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist( ["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) DQMHarvWorkload.setRunWhitelist([666]) # not in this dataset for task in DQMHarvWorkload.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Dataset(), DQMHarvWorkload, task)
def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "DQMConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.getProcessingString(), "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertFalse(testWorkload.getPrepID(), "PrepId does not match") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_7_3_1_patch1']) # test workload attributes self.assertEqual(testWorkload.processingString, "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.processingVersion, 1) self.assertEqual(sorted(testWorkload.lumiList.keys()), ['139788', '139790', '144011']) self.assertEqual( sorted(testWorkload.lumiList.values()), [[[5, 10], [15, 20], [25, 30]], [[25, 75], [125, 175], [275, 325]], [[50, 100], [110, 125]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "DatasetBlock") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual( sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.inputLocationFlag(), "Wrong input location flag") self.assertEqual(sorted(task.inputRunWhitelist()), [ 138923, 138924, 138934, 138937, 139788, 139789, 139790, 144011, 144083, 144084, 144086 ]) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") #print "ALAN inspect %s" % inspect.getmembers(allTasks) #for property, value in vars(allTasks).iteritems(): # print property, ": ", value return