class RESTBaseUnitTest(unittest.TestCase): def setUp(self): # default set self.schemaModules = [] self.initialize() if self.schemaModules: self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() # logLevel = logging.SQLDEBUG self.testInit.setDatabaseConnection(self.config.getDBUrl()) self.testInit.setSchema(customModules = self.schemaModules, useDefault = False) self.rt = Root(self.config) self.rt.start(blocking=False) def tearDown(self): self.rt.stop() if self.schemaModules: self.testInit.clearDatabase() self.config = None def initialize(self): """ i.e. self.config = DefaultConfig('WMCore.WebTools.RESTModel') self.config.setDBUrl("sqlite://") self.schemaModules = ["WMCore.ThreadPool", WMCore.WMBS"] """ message = "initialize method has to be implemented, self.restModel, self.schemaModules needs to be set" raise NotImplementedError, message
class PromptSkimTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("promptskim_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("promptskim_t") return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() return @attr("integration") def testPromptSkim(self): """ _testPromptSkim_ Verify that PromptSkim workflows can be created. Note that this requires a system that has all of the cms software installed on it. """ dataProcArguments = getTestArguments() dataProcArguments["CouchUrl"] = os.environ["COUCHURL"] dataProcArguments["CouchDBName"] = "promptskim_t" testWorkload = promptSkimWorkload("TestWorkload", dataProcArguments) return
class WorkloadSummary_t(unittest.TestCase): def setUp(self): """bootstrap tests""" self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() #self.testInit.setDatabaseConnection() #self.testInit.setSchema(customModules = ["WMCore.WMBS"], # useDefault = False) self.testInit.setupCouch("wmcore-workloadsummary", "WorkloadSummary") self.workload1 = newWorkload("WorkloadSummaryTest1") self.workload2 = newWorkload("WorkloadSummaryTest2") def tearDown(self): self.testInit.tearDownCouch() self.testInit.clearDatabase() pass def testA(self): """ register workloads in the couchapp and pulling back information from the views """ populateWorkload(self.workload1, "evansde77") populateWorkload(self.workload2, "drsm79") summary1 = self.workload1.generateWorkloadSummary() summary2 = self.workload2.generateWorkloadSummary() summ1 = WorkloadSummary(self.workload1.name(), self.testInit.couchUrl, self.testInit.couchDbName, self.workload1) summ2 = WorkloadSummary(self.workload2.name(), self.testInit.couchUrl, self.testInit.couchDbName, self.workload2) summ1.create() summ2.create() summ1.addACDCCollection(makeUUID()) for t in self.workload1.listAllTaskPathNames(): fakeDoc = makeUUID() summ1.addACDCFileset(t, fakeDoc)
class MonteCarloTest(EmulatedUnitTestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ super(MonteCarloTest, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch(TEST_DB_NAME, "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.generateWorkDir() couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase(TEST_DB_NAME) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName") self.listFilesets = self.daoFactory(classname="Fileset.List") self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow") return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() super(MonteCarloTest, self).tearDown() return def injectMonteCarloConfig(self): """ _injectMonteCarlo_ Create a bogus config cache document for the montecarlo generation and inject it into couch. Return the ID of the document. """ newConfig = Document() newConfig["info"] = None newConfig["config"] = None newConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f" newConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7" newConfig["owner"] = {"group": "cmsdataops", "user": "******"} newConfig["pset_tweak_details"] = {"process": {"outputModules_": ["OutputA", "OutputB"], "OutputA": {"dataset": {"filterName": "OutputAFilter", "dataTier": "RECO"}}, "OutputB": {"dataset": {"filterName": "OutputBFilter", "dataTier": "USER"}}}} result = self.configDatabase.commitOne(newConfig) return result[0]["id"] def _commonMonteCarloTest(self): """ Retrieve the workload from WMBS and test all its properties. """ goldenOutputMods = {"OutputA": "RECO", "OutputB": "USER"} prodWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Production") prodWorkflow.load() self.assertEqual(len(prodWorkflow.outputMap.keys()), 3, "Error: Wrong number of WF outputs.") for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier mergedOutput = prodWorkflow.outputMap[fset][0]["merged_output_fileset"] unmergedOutput = prodWorkflow.outputMap[fset][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Production/unmerged-%s" % (goldenOutputMod + tier), "Error: Unmerged output fileset is wrong.") logArchOutput = prodWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = prodWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Production/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Production/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") for goldenOutputMod, tier in goldenOutputMods.items(): mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Production/ProductionMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") from pprint import pformat print(pformat(mergeWorkflow.outputMap)) mergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-Production-SomeBlock") topLevelFileset.loadData() prodSubscription = Subscription(fileset=topLevelFileset, workflow=prodWorkflow) prodSubscription.loadData() self.assertEqual(prodSubscription["type"], "Production", "Error: Wrong subscription type.") self.assertEqual(prodSubscription["split_algo"], "EventBased", "Error: Wrong split algo.") for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier unmergedOutput = Fileset(name="/TestWorkload/Production/unmerged-%s" % fset) unmergedOutput.loadData() mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Production/ProductionMerge%s" % goldenOutputMod) mergeWorkflow.load() mergeSubscription = Subscription(fileset=unmergedOutput, workflow=mergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algo: %s" % mergeSubscription["split_algo"]) for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier unmerged = Fileset(name="/TestWorkload/Production/unmerged-%s" % fset) unmerged.loadData() cleanupWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Production/ProductionCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset=unmerged, workflow=cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong split algo.") procLogCollect = Fileset(name="/TestWorkload/Production/unmerged-logArchive") procLogCollect.loadData() procLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Production/LogCollect") procLogCollectWorkflow.load() logCollectSub = Subscription(fileset=procLogCollect, workflow=procLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") for goldenOutputMod in goldenOutputMods: mergeLogCollect = Fileset( name="/TestWorkload/Production/ProductionMerge%s/merged-logArchive" % goldenOutputMod) mergeLogCollect.loadData() mergeLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Production/ProductionMerge%s/Production%sMergeLogCollect" % ( goldenOutputMod, goldenOutputMod)) mergeLogCollectWorkflow.load() logCollectSub = Subscription(fileset=mergeLogCollect, workflow=mergeLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") def testMonteCarlo(self): """ _testMonteCarlo_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return def testMonteCarloExtension(self): """ _testMonteCarloExtension_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstLumi"] = 10001 defaultArguments["EventsPerJob"] = 100 defaultArguments["FirstEvent"] = 10001 # defaultArguments["FirstEvent"] = 10001 initial_lfn_counter = 100 # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath('/TestWorkload/Production/ProductionMerge%s' % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return def testMCWithPileup(self): """ _testMCWithPileup_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. The input configuration includes pileup input files. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() # Add pileup inputs defaultArguments["MCPileup"] = COSMICS_PU defaultArguments["DataPileup"] = DATA_PU defaultArguments["DeterministicPileup"] = True factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') cmsRunStep = productionTask.getStep("cmsRun1").getTypeHelper() pileupData = cmsRunStep.getPileup() self.assertEqual(pileupData.data.dataset, [DATA_PU]) self.assertEqual(pileupData.mc.dataset, [COSMICS_PU]) splitting = productionTask.jobSplittingParameters() self.assertTrue(splitting["deterministicPileup"]) return def testMCWithLHE(self): """ _testMCWithLHE_ Create a MonteCarlo workflow with a variation on the type of work done, this refers to the previous LHEStepZero where the input can be .lhe files and there is more than one lumi per job. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["LheInputFiles"] = "True" defaultArguments["EventsPerJob"] = 200 defaultArguments["EventsPerLumi"] = 50 factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') splitting = productionTask.jobSplittingParameters() self.assertEqual(splitting["events_per_job"], 200) self.assertEqual(splitting["events_per_lumi"], 50) self.assertEqual(splitting["lheInputFiles"], True) self.assertFalse(splitting["deterministicPileup"]) return def testMemCoresSettings(self): """ _testMemCoresSettings_ Make sure the multicore and memory setings are properly propagated to all tasks and steps. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) # test default values taskObj = testWorkload.getTask('Production') for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], 2300.0) # now test case where args are provided defaultArguments["Multicore"] = 6 defaultArguments["Memory"] = 4600.0 defaultArguments["EventStreams"] = 3 testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) taskObj = testWorkload.getTask('Production') for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) if step == 'cmsRun1': self.assertEqual(stepHelper.getNumberOfCores(), defaultArguments["Multicore"]) self.assertEqual(stepHelper.getNumberOfStreams(), defaultArguments["EventStreams"]) else: self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], defaultArguments["Memory"]) return def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = ['/TestWorkload/Production', '/TestWorkload/Production/ProductionMergeOutputB', '/TestWorkload/Production/ProductionMergeOutputA'] expWfTasks = ['/TestWorkload/Production', '/TestWorkload/Production/LogCollect', '/TestWorkload/Production/ProductionCleanupUnmergedOutputA', '/TestWorkload/Production/ProductionCleanupUnmergedOutputB', '/TestWorkload/Production/ProductionMergeOutputA', '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect', '/TestWorkload/Production/ProductionMergeOutputB', '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect'] expFsets = ['FILESET_DEFINED_DURING_RUNTIME', '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputA/merged-MergedRECO', '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputB/merged-MergedUSER', '/TestWorkload/Production/unmerged-logArchive', '/TestWorkload/Production/unmerged-OutputARECO'] subMaps = ['FILESET_DEFINED_DURING_RUNTIME', (6, '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect', 'MinFileBased', 'LogCollect'), (3, '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect', 'MinFileBased', 'LogCollect'), (8, '/TestWorkload/Production/unmerged-logArchive', '/TestWorkload/Production/LogCollect', 'MinFileBased', 'LogCollect'), (7, '/TestWorkload/Production/unmerged-OutputARECO', '/TestWorkload/Production/ProductionCleanupUnmergedOutputA', 'SiblingProcessingBased', 'Cleanup'), (5, '/TestWorkload/Production/unmerged-OutputARECO', '/TestWorkload/Production/ProductionMergeOutputA', 'ParentlessMergeBySize', 'Merge'), (4, '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionCleanupUnmergedOutputB', 'SiblingProcessingBased', 'Cleanup'), (2, '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionMergeOutputB', 'ParentlessMergeBySize', 'Merge')] testArguments = MonteCarloWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = TEST_DB_NAME testArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) myMask = Mask(FirstRun=1, FirstLumi=1, FirstEvent=1, LastRun=1, LastLumi=10, LastEvent=1000) testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask, cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # same function as in WMBSHelper, otherwise we cannot know which fileset name is maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)]) topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest() expFsets[0] = topFilesetName # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subMaps[0] = (1, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production') subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps) ### create another top level subscription myMask = Mask(FirstRun=1, FirstLumi=11, FirstEvent=1001, LastRun=1, LastLumi=20, LastEvent=2000) testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask, cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # same function as in WMBSHelper, otherwise we cannot know which fileset name is maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)]) topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest() expFsets.append(topFilesetName) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subMaps.append((9, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production')) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
class ReDigiTest(EmulatedUnitTestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ super(ReDigiTest, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("redigi_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) self.testInit.generateWorkDir() couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("redigi_t") return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() EmulatorHelper.resetEmulators() super(ReDigiTest, self).tearDown() return def testDependentReDigi(self): """ _testDependentReDigi_ Verfiy that a dependent ReDigi workflow that keeps stages out RAW data is created and installed into WMBS correctly. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput" defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput" factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) topLevelFileset = Fileset(name = "TestWorkload-StepOneProc-SomeBlock") topLevelFileset.loadData() stepOneUnmergedRAWFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-RAWDEBUGoutput") stepOneUnmergedRAWFileset.loadData() stepOneMergedRAWFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/merged-Merged") stepOneMergedRAWFileset.loadData() stepOneLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-logArchive") stepOneLogArchiveFileset.loadData() stepOneMergeLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/merged-logArchive") stepOneMergeLogArchiveFileset.loadData() stepTwoUnmergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/unmerged-DQMoutput") stepTwoUnmergedDQMFileset.loadData() stepTwoUnmergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/unmerged-RECODEBUGoutput") stepTwoUnmergedRECOFileset.loadData() stepTwoMergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeDQMoutput/merged-Merged") stepTwoMergedDQMFileset.loadData() stepTwoMergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/merged-Merged") stepTwoMergedRECOFileset.loadData() stepTwoLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/unmerged-logArchive") stepTwoLogArchiveFileset.loadData() stepTwoMergeDQMLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeDQMoutput/merged-logArchive") stepTwoMergeDQMLogArchiveFileset.loadData() stepTwoMergeRECOLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/merged-logArchive") stepTwoMergeRECOLogArchiveFileset.loadData() stepThreeUnmergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/unmerged-aodOutputModule") stepThreeUnmergedAODFileset.loadData() stepThreeMergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcMergeaodOutputModule/merged-Merged") stepThreeMergedAODFileset.loadData() stepThreeLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/unmerged-logArchive") stepThreeLogArchiveFileset.loadData() stepThreeMergeLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcMergeaodOutputModule/merged-logArchive") stepThreeMergeLogArchiveFileset.loadData() stepOneWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc") stepOneWorkflow.load() self.assertEqual(stepOneWorkflow.wfType, 'reprocessing') self.assertTrue("logArchive" in stepOneWorkflow.outputMap.keys(), "Error: Step one missing output module.") self.assertTrue("RAWDEBUGoutput" in stepOneWorkflow.outputMap.keys(), "Error: Step one missing output module.") self.assertEqual(stepOneWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepOneLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepOneLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneWorkflow.outputMap["RAWDEBUGoutput"][0]["merged_output_fileset"].id, stepOneMergedRAWFileset.id, "Error: RAWDEBUG output fileset is wrong.") self.assertEqual(stepOneWorkflow.outputMap["RAWDEBUGoutput"][0]["output_fileset"].id, stepOneUnmergedRAWFileset.id, "Error: RAWDEBUG output fileset is wrong.") for outputMod in stepOneWorkflow.outputMap.keys(): self.assertTrue(len(stepOneWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepOneSub = Subscription(workflow = stepOneWorkflow, fileset = topLevelFileset) stepOneSub.loadData() self.assertEqual(stepOneSub["type"], "Processing", "Error: Step one sub has wrong type.") stepOneCleanupWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedRAWDEBUGoutput") stepOneCleanupWorkflow.load() self.assertEqual(len(stepOneCleanupWorkflow.outputMap.keys()), 0, "Error: Cleanup should have no output.") stepOneCleanupSub = Subscription(workflow = stepOneCleanupWorkflow, fileset = stepOneUnmergedRAWFileset) stepOneCleanupSub.loadData() self.assertEqual(stepOneCleanupSub["type"], "Cleanup", "Error: Step one sub has wrong type.") stepOneLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/LogCollect") stepOneLogCollectWorkflow.load() self.assertEqual(len(stepOneLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect should have no output.") stepOneLogCollectSub = Subscription(workflow = stepOneLogCollectWorkflow, fileset = stepOneLogArchiveFileset) stepOneLogCollectSub.loadData() self.assertEqual(stepOneLogCollectSub["type"], "LogCollect", "Error: Step one sub has wrong type.") stepOneMergeWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput") stepOneMergeWorkflow.load() self.assertTrue("Merged" in stepOneMergeWorkflow.outputMap.keys(), "Error: Step one merge missing output module.") self.assertTrue("logArchive" in stepOneMergeWorkflow.outputMap.keys(), "Error: Step one merge missing output module.") self.assertEqual(stepOneMergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepOneMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneMergeWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepOneMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneMergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepOneMergedRAWFileset.id, "Error: RAWDEBUG merge output fileset is wrong.") self.assertEqual(stepOneMergeWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepOneMergedRAWFileset.id, "Error: RAWDEBUG merge output fileset is wrong.") for outputMod in stepOneMergeWorkflow.outputMap.keys(): self.assertTrue(len(stepOneMergeWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepOneMergeSub = Subscription(workflow = stepOneMergeWorkflow, fileset = stepOneUnmergedRAWFileset) stepOneMergeSub.loadData() self.assertEqual(stepOneMergeSub["type"], "Merge", "Error: Step one sub has wrong type.") stepTwoWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc") stepTwoWorkflow.load() self.assertTrue("RECODEBUGoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertTrue("DQMoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["output_fileset"].id, stepTwoUnmergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["output_fileset"].id, stepTwoUnmergedDQMFileset.id, "Error: DQM output fileset is wrong.") stepTwoSub = Subscription(workflow = stepTwoWorkflow, fileset = stepOneMergedRAWFileset) stepTwoSub.loadData() self.assertEqual(stepTwoSub["type"], "Processing", "Error: Step two sub has wrong type.") for outputMod in stepTwoWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoCleanupDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcCleanupUnmergedDQMoutput") stepTwoCleanupDQMWorkflow.load() self.assertEqual(len(stepTwoCleanupDQMWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupDQMSub = Subscription(workflow = stepTwoCleanupDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoCleanupDQMSub.loadData() self.assertEqual(stepTwoCleanupDQMSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoCleanupRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcCleanupUnmergedRECODEBUGoutput") stepTwoCleanupRECOWorkflow.load() self.assertEqual(len(stepTwoCleanupRECOWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupRECOSub = Subscription(workflow = stepTwoCleanupRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoCleanupRECOSub.loadData() self.assertEqual(stepTwoCleanupRECOSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcLogCollect") stepTwoLogCollectWorkflow.load() self.assertEqual(len(stepTwoLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect shouldn't have any output.") stepTwoLogCollectSub = Subscription(workflow = stepTwoLogCollectWorkflow, fileset = stepTwoLogArchiveFileset) stepTwoLogCollectSub.loadData() self.assertEqual(stepTwoLogCollectSub["type"], "LogCollect", "Error: Step two sub has wrong type.") stepTwoMergeRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput") stepTwoMergeRECOWorkflow.load() self.assertTrue("Merged" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") stepTwoMergeRECOSub = Subscription(workflow = stepTwoMergeRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoMergeRECOSub.loadData() self.assertEqual(stepTwoMergeRECOSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeRECOWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeRECOWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoMergeDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeDQMoutput") stepTwoMergeDQMWorkflow.load() self.assertTrue("Merged" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") stepTwoMergeDQMSub = Subscription(workflow = stepTwoMergeDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoMergeDQMSub.loadData() self.assertEqual(stepTwoMergeDQMSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeDQMWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeDQMWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepThreeWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc") stepThreeWorkflow.load() self.assertTrue("aodOutputModule" in stepThreeWorkflow.outputMap.keys(), "Error: Step three missing output module.") self.assertTrue("logArchive" in stepThreeWorkflow.outputMap.keys(), "Error: Step three missing output module.") self.assertEqual(stepThreeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepThreeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepThreeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeWorkflow.outputMap["aodOutputModule"][0]["merged_output_fileset"].id, stepThreeMergedAODFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepThreeWorkflow.outputMap["aodOutputModule"][0]["output_fileset"].id, stepThreeUnmergedAODFileset.id, "Error: RECODEBUG output fileset is wrong.") stepThreeSub = Subscription(workflow = stepThreeWorkflow, fileset = stepTwoMergedRECOFileset) stepThreeSub.loadData() self.assertEqual(stepThreeSub["type"], "Processing", "Error: Step three sub has wrong type.") for outputMod in stepThreeWorkflow.outputMap.keys(): self.assertTrue(len(stepThreeWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepThreeCleanupWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcCleanupUnmergedaodOutputModule") stepThreeCleanupWorkflow.load() self.assertEqual(len(stepThreeCleanupWorkflow.outputMap.keys()), 0, "Error: Cleanup should have no output.") stepThreeCleanupSub = Subscription(workflow = stepThreeCleanupWorkflow, fileset = stepThreeUnmergedAODFileset) stepThreeCleanupSub.loadData() self.assertEqual(stepThreeCleanupSub["type"], "Cleanup", "Error: Step three sub has wrong type.") stepThreeLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcLogCollect") stepThreeLogCollectWorkflow.load() self.assertEqual(len(stepThreeLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect should have no output.") stepThreeLogCollectSub = Subscription(workflow = stepThreeLogCollectWorkflow, fileset = stepThreeLogArchiveFileset) stepThreeLogCollectSub.loadData() self.assertEqual(stepThreeLogCollectSub["type"], "LogCollect", "Error: Step three sub has wrong type.") stepThreeMergeWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcMergeaodOutputModule") stepThreeMergeWorkflow.load() self.assertTrue("Merged" in stepThreeMergeWorkflow.outputMap.keys(), "Error: Step three merge missing output module.") self.assertTrue("logArchive" in stepThreeMergeWorkflow.outputMap.keys(), "Error: Step three merge missing output module.") self.assertEqual(stepThreeMergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepThreeMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeMergeWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepThreeMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeMergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepThreeMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") self.assertEqual(stepThreeMergeWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepThreeMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") stepThreeMergeSub = Subscription(workflow = stepThreeMergeWorkflow, fileset = stepThreeUnmergedAODFileset) stepThreeMergeSub.loadData() self.assertEqual(stepThreeMergeSub["type"], "Merge", "Error: Step three sub has wrong type.") for outputMod in stepThreeMergeWorkflow.outputMap.keys(): self.assertTrue(len(stepThreeMergeWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") return def verifyDiscardRAW(self): """ _verifyDiscardRAW_ Verify that a workflow that discards the RAW was installed into WMBS correctly. """ topLevelFileset = Fileset(name = "TestWorkload-StepOneProc-SomeBlock") topLevelFileset.loadData() stepTwoUnmergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-DQMoutput") stepTwoUnmergedDQMFileset.loadData() stepTwoUnmergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-RECODEBUGoutput") stepTwoUnmergedRECOFileset.loadData() stepTwoMergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeDQMoutput/merged-Merged") stepTwoMergedDQMFileset.loadData() stepTwoMergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRECODEBUGoutput/merged-Merged") stepTwoMergedRECOFileset.loadData() stepTwoLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-logArchive") stepTwoLogArchiveFileset.loadData() stepTwoMergeDQMLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeDQMoutput/merged-logArchive") stepTwoMergeDQMLogArchiveFileset.loadData() stepTwoMergeRECOLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRECODEBUGoutput/merged-logArchive") stepTwoMergeRECOLogArchiveFileset.loadData() stepTwoWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc") stepTwoWorkflow.load() self.assertTrue("RECODEBUGoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertTrue("DQMoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["output_fileset"].id, stepTwoUnmergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["output_fileset"].id, stepTwoUnmergedDQMFileset.id, "Error: DQM output fileset is wrong.") stepTwoSub = Subscription(workflow = stepTwoWorkflow, fileset = topLevelFileset) stepTwoSub.loadData() self.assertEqual(stepTwoSub["type"], "Processing", "Error: Step two sub has wrong type.") for outputMod in stepTwoWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoCleanupDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedDQMoutput") stepTwoCleanupDQMWorkflow.load() self.assertEqual(len(stepTwoCleanupDQMWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupDQMSub = Subscription(workflow = stepTwoCleanupDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoCleanupDQMSub.loadData() self.assertEqual(stepTwoCleanupDQMSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoCleanupRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedRECODEBUGoutput") stepTwoCleanupRECOWorkflow.load() self.assertEqual(len(stepTwoCleanupRECOWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupRECOSub = Subscription(workflow = stepTwoCleanupRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoCleanupRECOSub.loadData() self.assertEqual(stepTwoCleanupRECOSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/LogCollect") stepTwoLogCollectWorkflow.load() self.assertEqual(len(stepTwoLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect shouldn't have any output.") stepTwoLogCollectSub = Subscription(workflow = stepTwoLogCollectWorkflow, fileset = stepTwoLogArchiveFileset) stepTwoLogCollectSub.loadData() self.assertEqual(stepTwoLogCollectSub["type"], "LogCollect", "Error: Step two sub has wrong type.") stepTwoMergeRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRECODEBUGoutput") stepTwoMergeRECOWorkflow.load() self.assertTrue("Merged" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") stepTwoMergeRECOSub = Subscription(workflow = stepTwoMergeRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoMergeRECOSub.loadData() self.assertEqual(stepTwoMergeRECOSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeRECOWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeRECOWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoMergeDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeDQMoutput") stepTwoMergeDQMWorkflow.load() self.assertTrue("Merged" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") stepTwoMergeDQMSub = Subscription(workflow = stepTwoMergeDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoMergeDQMSub.loadData() self.assertEqual(stepTwoMergeDQMSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeDQMWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeDQMWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") return def verifyKeepAOD(self): """ _verifyKeepAOD_ Verify that a workflow that only produces AOD in a single step was installed correctly into WMBS. """ topLevelFileset = Fileset(name = "TestWorkload-StepOneProc-SomeBlock") topLevelFileset.loadData() stepTwoUnmergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-aodOutputModule") stepTwoUnmergedAODFileset.loadData() stepTwoMergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeaodOutputModule/merged-Merged") stepTwoMergedAODFileset.loadData() stepTwoLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-logArchive") stepTwoLogArchiveFileset.loadData() stepTwoMergeAODLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeaodOutputModule/merged-logArchive") stepTwoMergeAODLogArchiveFileset.loadData() stepTwoWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc") stepTwoWorkflow.load() self.assertTrue("aodOutputModule" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["aodOutputModule"][0]["merged_output_fileset"].id, stepTwoMergedAODFileset.id, "Error: AOD output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["aodOutputModule"][0]["output_fileset"].id, stepTwoUnmergedAODFileset.id, "Error: AOD output fileset is wrong.") stepTwoSub = Subscription(workflow = stepTwoWorkflow, fileset = topLevelFileset) stepTwoSub.loadData() self.assertEqual(stepTwoSub["type"], "Processing", "Error: Step two sub has wrong type.") for outputMod in stepTwoWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoCleanupAODWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedaodOutputModule") stepTwoCleanupAODWorkflow.load() self.assertEqual(len(stepTwoCleanupAODWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupAODSub = Subscription(workflow = stepTwoCleanupAODWorkflow, fileset = stepTwoUnmergedAODFileset) stepTwoCleanupAODSub.loadData() self.assertEqual(stepTwoCleanupAODSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/LogCollect") stepTwoLogCollectWorkflow.load() self.assertEqual(len(stepTwoLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect shouldn't have any output.") stepTwoLogCollectSub = Subscription(workflow = stepTwoLogCollectWorkflow, fileset = stepTwoLogArchiveFileset) stepTwoLogCollectSub.loadData() self.assertEqual(stepTwoLogCollectSub["type"], "LogCollect", "Error: Step two sub has wrong type.") stepTwoMergeAODWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeaodOutputModule") stepTwoMergeAODWorkflow.load() self.assertTrue("Merged" in stepTwoMergeAODWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeAODWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeAODLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeAODLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") stepTwoMergeAODSub = Subscription(workflow = stepTwoMergeAODWorkflow, fileset = stepTwoUnmergedAODFileset) stepTwoMergeAODSub.loadData() self.assertEqual(stepTwoMergeAODSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeAODWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeAODWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") return def testChainedReDigi(self): """ _testChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW data can be created and installed into WMBS correctly. This will only verify the step one/step two information in WMBS as the step three information is the same as the dependent workflow. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput" defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput" defaultArguments["MCPileup"] = PILEUP_DATASET defaultArguments["KeepStepOneOutput"] = False factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) # Verify that pileup is configured for both of the cmsRun steps in the # top level task. topLevelTask = testWorkload.getTopLevelTask()[0] cmsRun1Helper = topLevelTask.getStepHelper("cmsRun1") cmsRun2Helper = topLevelTask.getStepHelper("cmsRun2") cmsRun1PileupConfig = cmsRun1Helper.getPileup() cmsRun2PileupConfig = cmsRun2Helper.getPileup() self.assertTrue(cmsRun1PileupConfig.mc.dataset, "/some/cosmics/dataset") self.assertTrue(cmsRun2PileupConfig.mc.dataset, "/some/cosmics/dataset") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return def testThreeStepChainedReDigi(self): """ _testThreeStepChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW and RECO data can be created and installed into WMBS correctly. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False defaultArguments["KeepStepTwoOutput"] = False defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput" defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput" factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) self.assertTrue(len(testWorkload.getTopLevelTask()) == 1, "Error: Wrong number of top level tasks.") topLevelTask = testWorkload.getTopLevelTask()[0] topLevelStep = topLevelTask.steps() cmsRun2Step = topLevelStep.getStep("cmsRun2").getTypeHelper() self.assertTrue(len(cmsRun2Step.listOutputModules()) == 2, "Error: Wrong number of output modules in cmsRun2.") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyKeepAOD() return def testCombinedReDigiRecoConfig(self): """ _testCombinedReDigiRecoConfig_ Verify that a ReDigi workflow that uses a single step one config installs into WMBS correctly. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase, combinedStepOne = True) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[2] defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput" factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return def testSingleStepReDigi(self): """ _testSingleStepReDigi_ Verify that a single step ReDigi workflow can be created and installed correctly into WMBS. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[2] factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyKeepAOD() return
class ReportTest(unittest.TestCase): """ _ReportTest_ Unit tests for the Report class. """ def setUp(self): """ _setUp_ Figure out the location of the XML report produced by CMSSW. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase = True) self.testInit.setupCouch("report_t/fwjrs", "FWJRDump") self.xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") self.badxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml") self.skippedFilesxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml") self.skippedAllFilesxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml") self.fallbackXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml") self.testDir = self.testInit.generateWorkDir() return def tearDown(self): """ _tearDown_ Cleanup the databases. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def verifyInputData(self, report): """ _verifyInputData_ Verify that the input file in the Report class matches the input file in the XML generated by CMSSW. """ inputFiles = report.getInputFilesFromStep("cmsRun1") assert len(inputFiles) == 1, \ "Error: Wrong number of input files." assert inputFiles[0]["lfn"] == "/store/data/BeamCommissioning09/MinimumBias/RAW/v1/000/122/023/142F3F42-C5D6-DE11-945D-000423D94494.root", \ "Error: Wrong LFN on input file." assert inputFiles[0]["pfn"] == "dcap://cmsdca.fnal.gov:24137/pnfs/fnal.gov/usr/cms/WAX/11/store/data/BeamCommissioning09/MinimumBias/RAW/v1/000/122/023/142F3F42-C5D6-DE11-945D-000423D94494.root", \ "Error: Wrong PFN on input file." inputRun = list(inputFiles[0]["runs"]) assert len(inputRun) == 1, \ "Error: Wrong number of runs in input." assert inputRun[0].run == 122023, \ "Error: Wrong run number on input file." assert len(inputRun[0].lumis) == 1, \ "Error: Wrong number of lumis in input file." assert 215 in inputRun[0].lumis, \ "Error: Input file is missing lumis." assert inputFiles[0]["events"] == 2, \ "Error: Wrong number of events in input file." assert inputFiles[0]["size"] == 0, \ "Error: Wrong size in input file." assert inputFiles[0]["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \ "Error: Catalog on input file is wrong." assert inputFiles[0]["guid"] == "142F3F42-C5D6-DE11-945D-000423D94494", \ "Error: GUID of input file is wrong." return def verifyRecoOutput(self, report): """ _verifyRecoOutput_ Verify that all the metadata in the RECO output module is correct. """ outputFiles = report.getFilesFromOutputModule("cmsRun1", "outputRECORECO") assert len(outputFiles) == 1, \ "Error: Wrong number of output files." assert outputFiles[0]["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root", \ "Error: Wrong LFN on output file: %s" % outputFiles[0]["lfn"] assert outputFiles[0]["pfn"] == "outputRECORECO.root", \ "Error: Wrong PFN on output file." outputRun = list(outputFiles[0]["runs"]) assert len(outputRun) == 1, \ "Error: Wrong number of runs in output." assert outputRun[0].run == 122023, \ "Error: Wrong run number on output file." assert len(outputRun[0].lumis) == 1, \ "Error: Wrong number of lumis in output file." assert 215 in outputRun[0].lumis, \ "Error: Output file is missing lumis." assert outputFiles[0]["events"] == 2, \ "Error: Wrong number of events in output file." assert outputFiles[0]["size"] == 0, \ "Error: Wrong size in output file." assert len(outputFiles[0]["input"]) == 1, \ "Error: Wrong number of input files." assert outputFiles[0]["input"][0] == "/store/data/BeamCommissioning09/MinimumBias/RAW/v1/000/122/023/142F3F42-C5D6-DE11-945D-000423D94494.root", \ "Error: LFN of input file is wrong." assert len(outputFiles[0]["checksums"]) == 0, \ "Error: There should be no checksums in output file." assert outputFiles[0]["catalog"] == "", \ "Error: Catalog on output file is wrong." assert outputFiles[0]["guid"] == "7E3359C8-222E-DF11-B2B0-001731230E47", \ "Error: GUID of output file is wrong: %s" % outputFiles[0]["guid"] assert outputFiles[0]["module_label"] == "outputRECORECO", \ "Error: Module label of output file is wrong." assert outputFiles[0]["branch_hash"] == "cf37adeb60b427f4ccd0e21b5771146b", \ "Error: Branch has on output file is wrong." return def verifyAlcaOutput(self, report): """ _verifyAlcaOutput_ Verify that all of the meta data in the ALCARECO output module is correct. """ outputFiles = report.getFilesFromOutputModule("cmsRun1", "outputALCARECORECO") assert len(outputFiles) == 1, \ "Error: Wrong number of output files." assert outputFiles[0]["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/ALCARECO/rereco_GR09_R_34X_V5_All_v1/0000/B8F849C9-222E-DF11-B2B0-001731230E47.root", \ "Error: Wrong LFN on output file: %s" % outputFiles[0]["lfn"] assert outputFiles[0]["pfn"] == "outputALCARECORECO.root", \ "Error: Wrong PFN on output file." outputRun = list(outputFiles[0]["runs"]) assert len(outputRun) == 1, \ "Error: Wrong number of runs in output." assert outputRun[0].run == 122023, \ "Error: Wrong run number on output file." assert len(outputRun[0].lumis) == 1, \ "Error: Wrong number of lumis in output file." assert 215 in outputRun[0].lumis, \ "Error: Output file is missing lumis." assert outputFiles[0]["events"] == 2, \ "Error: Wrong number of events in output file." assert outputFiles[0]["size"] == 0, \ "Error: Wrong size in output file." assert len(outputFiles[0]["input"]) == 1, \ "Error: Wrong number of input files." assert outputFiles[0]["input"][0] == "/store/data/BeamCommissioning09/MinimumBias/RAW/v1/000/122/023/142F3F42-C5D6-DE11-945D-000423D94494.root", \ "Error: LFN of input file is wrong." assert len(outputFiles[0]["checksums"]) == 0, \ "Error: There should be no checksums in output file." assert outputFiles[0]["catalog"] == "", \ "Error: Catalog on output file is wrong." assert outputFiles[0]["guid"] == "B8F849C9-222E-DF11-B2B0-001731230E47", \ "Error: GUID of output file is wrong: %s" % outputFiles[0]["guid"] assert outputFiles[0]["module_label"] == "outputALCARECORECO", \ "Error: Module label of output file is wrong." assert outputFiles[0]["branch_hash"] == "cf37adeb60b427f4ccd0e21b5771146b", \ "Error: Branch has on output file is wrong." return def testXMLParsing(self): """ _testParsing_ Verify that the parsing of a CMSSW XML report works correctly. """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) self.verifyInputData(myReport) self.verifyRecoOutput(myReport) self.verifyAlcaOutput(myReport) return def testBadXMLParsing(self): """ _testBadXMLParsing_ Verify that the parsing of a CMSSW XML report works correctly even if the XML is malformed. This should raise a FwkJobReportException, which in CMSSW will be caught """ myReport = Report("cmsRun1") from WMCore.FwkJobReport.Report import FwkJobReportException self.assertRaises(FwkJobReportException, myReport.parse, self.badxmlPath) self.assertEqual(myReport.getStepErrors("cmsRun1")['error0'].type, 'BadFWJRXML') self.assertEqual(myReport.getStepErrors("cmsRun1")['error0'].exitCode, 50115) return def testErrorReporting(self): """ _testErrorReporting_ Verify that errors are correctly transfered from the XML report to the python report. """ cmsException = \ """cms::Exception caught in cmsRun ---- EventProcessorFailure BEGIN EventProcessingStopped ---- ScheduleExecutionFailure BEGIN ProcessingStopped ---- NoRecord BEGIN No "CastorDbRecord" record found in the EventSetup. Please add an ESSource or ESProducer that delivers such a record. cms::Exception going through module CastorRawToDigi/castorDigis run: 121849 lumi: 1 event: 23 ---- NoRecord END Exception going through path raw2digi_step ---- ScheduleExecutionFailure END an exception occurred during current event processing cms::Exception caught in EventProcessor and rethrown ---- EventProcessorFailure END""" xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWFailReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) assert hasattr(myReport.data.cmsRun1, "errors"), \ "Error: Error section missing." assert getattr(myReport.data.cmsRun1.errors, "errorCount") == 1, \ "Error: Error count is wrong." assert hasattr(myReport.data.cmsRun1.errors, "error0"), \ "Error: Error0 section is missing." assert myReport.data.cmsRun1.errors.error0.type == "CMSException", \ "Error: Wrong error type." assert myReport.data.cmsRun1.errors.error0.exitCode == "8001", \ "Error: Wrong exit code." assert myReport.data.cmsRun1.errors.error0.details == cmsException, \ "Error: Error details are wrong:\n|%s|\n|%s|" % (myReport.data.cmsRun1.errors.error0.details, cmsException) # Test getStepErrors self.assertEqual(myReport.getStepErrors("cmsRun1")['error0'].type, "CMSException") return def testMultipleInputs(self): """ _testMultipleInputs_ Verify that parsing XML reports with multiple inputs works correctly. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWMultipleInput.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) assert hasattr(myReport.data.cmsRun1.input, "source"), \ "Error: Report missing input source." inputFiles = myReport.getInputFilesFromStep("cmsRun1") assert len(inputFiles) == 2, \ "Error: Wrong number of input files." for inputFile in inputFiles: assert inputFile["input_type"] == "primaryFiles", \ "Error: Wrong input type." assert inputFile["module_label"] == "source", \ "Error: Module label is wrong" assert inputFile["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \ "Error: Catalog is wrong." assert inputFile["events"] == 2, \ "Error: Wrong number of events." assert inputFile["input_source_class"] == "PoolSource", \ "Error: Wrong input source class." if inputFile["guid"] == "F0875ECD-3347-DF11-9FE0-003048678A80": assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \ "Error: Input LFN is wrong." assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \ "Error: Input PFN is wrong." assert len(inputFile["runs"]) == 1, \ "Error: Wrong number of runs." assert list(inputFile["runs"])[0].run == 124216, \ "Error: Wrong run number." assert 1 in list(inputFile["runs"])[0], \ "Error: Wrong lumi sections in input file." else: assert inputFile["guid"] == "626D74CE-3347-DF11-9363-0030486790C0", \ "Error: Wrong guid." assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \ "Error: Input LFN is wrong." assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \ "Error: Input PFN is wrong." assert len(inputFile["runs"]) == 1, \ "Error: Wrong number of runs." assert list(inputFile["runs"])[0].run == 124216, \ "Error: Wrong run number." assert 2 in list(inputFile["runs"])[0], \ "Error: Wrong lumi sections in input file." return def testJSONEncoding(self): """ _testJSONEncoding_ Verify that turning the FWJR into a JSON object works correctly. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) jsonReport = myReport.__to_json__(None) assert "task" in jsonReport.keys(), \ "Error: Task name missing from report." assert len(jsonReport["steps"].keys()) == 1, \ "Error: Wrong number of steps in report." assert "cmsRun1" in jsonReport["steps"].keys(), \ "Error: Step missing from json report." cmsRunStep = jsonReport["steps"]["cmsRun1"] jsonReportSections = ["status", "errors", "logs", "parameters", "site", "analysis", "cleanup", "input", "output", "start"] for jsonReportSection in jsonReportSections: assert jsonReportSection in cmsRunStep.keys(), \ "Error: missing section: %s" % jsonReportSection return def testTimeSetting(self): """ _testTimeSetting_ Can we set the times correctly? """ stepName = "cmsRun1" timeDiff = 0.01 myReport = Report(stepName) localTime = time.time() myReport.setStepStartTime(stepName) myReport.setStepStopTime(stepName) repTime = myReport.getTimes(stepName) self.assertTrue(repTime["startTime"] - localTime < timeDiff) self.assertTrue(repTime["stopTime"] - localTime < timeDiff) myReport = Report("cmsRun1") myReport.addStep("cmsRun2") myReport.addStep("cmsRun3") step = myReport.retrieveStep("cmsRun1") step.startTime = 1 step.stopTime = 8 step = myReport.retrieveStep("cmsRun2") step.startTime = 2 step.stopTime = 9 step = myReport.retrieveStep("cmsRun3") step.startTime = 3 step.stopTime = 10 self.assertEqual(myReport.getFirstStartLastStop()['stopTime'], 10) self.assertEqual(myReport.getFirstStartLastStop()['startTime'], 1) return def testTaskJobID(self): """ _testTaskJobID_ Test the basic task and jobID functions """ report = Report('fake') self.assertEqual(report.getTaskName(), None) self.assertEqual(report.getJobID(), None) report.setTaskName('silly') report.setJobID(100) self.assertEqual(report.getTaskName(), 'silly') self.assertEqual(report.getJobID(), 100) return def test_PerformanceReport(self): """ _PerformanceReport_ Test the performance report part of the job report """ report = Report("cmsRun1") report.setStepVSize(stepName = "cmsRun1", min = 100, max = 800, average = 244) report.setStepRSS(stepName = "cmsRun1", min = 100, max = 800, average = 244) report.setStepPCPU(stepName = "cmsRun1", min = 100, max = 800, average = 244) report.setStepPMEM(stepName = "cmsRun1", min = 100, max = 800, average = 244) perf = report.retrieveStep("cmsRun1").performance for section in perf.dictionary_().values(): d = section.dictionary_() self.assertEqual(d['min'], 100) self.assertEqual(d['max'], 800) self.assertEqual(d['average'], 244) return def testPerformanceSummary(self): """ _testPerformanceSummary_ Test whether or not we can pull performance information out of a Timing/SimpleMemoryCheck jobReport """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) # Do a brief check of the three sections perf = myReport.data.cmsRun1.performance self.assertEqual(perf.memory.PeakValueRss, '492.293') self.assertEqual(perf.cpu.TotalJobCPU, '9.16361') self.assertEqual(perf.storage.writeTotalMB, 5.22226) self.assertEqual(perf.storage.writeTotalSecs, 60317.4) self.assertEqual(perf.storage.readPercentageOps, 0.98585512216030857) return def testPerformanceJSON(self): """ _testPerformanceJSON_ Verify that the performance section of the report is correctly converted to JSON. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) perfSection = myReport.__to_json__(thunker = None)["steps"]["cmsRun1"]["performance"] self.assertTrue(perfSection.has_key("storage"), "Error: Storage section is missing.") self.assertTrue(perfSection.has_key("memory"), "Error: Memory section is missing.") self.assertTrue(perfSection.has_key("cpu"), "Error: CPU section is missing.") self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105", "Error: AvgEventCPU is wrong.") self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703", "Error: TotalJobTime is wrong.") self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166, "Error: readTotalMB is wrong.") self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653, "Error: readMaxMSec is wrong") self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293", "Error: PeakValueRss is wrong.") self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281", "Error: PeakValueVsize is wrong.") return def testExitCode(self): """ _testExitCode_ Test and see if we can get an exit code out of a report Note: Errors without a return code return 99999 """ report = Report("cmsRun1") self.assertEqual(report.getExitCode(), 0) report.addError(stepName = "cmsRun1", exitCode = None, errorType = "test", errorDetails = "test") self.assertEqual(report.getExitCode(), 99999) self.assertEqual(report.getStepExitCode(stepName = "cmsRun1"), 99999) report.addError(stepName = "cmsRun1", exitCode = '12345', errorType = "test", errorDetails = "test") self.assertEqual(report.getExitCode(), 12345) self.assertEqual(report.getStepExitCode(stepName = "cmsRun1"), 12345) def testProperties(self): """ _testProperties_ Test data fields for the properties information for DBS """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) name = "ThisIsASillyString" myReport.setValidStatus(name) myReport.setGlobalTag(name) myReport.setAcquisitionProcessing(acquisitionEra = 'NULL', processingVer = name) myReport.setInputDataset(inputPath = '/lame/path') for f in myReport.getAllFilesFromStep("cmsRun1"): self.assertEqual(f['globalTag'], name) self.assertEqual(f['validStatus'], name) self.assertEqual(f['processingVer'], name) self.assertEqual(f['acquisitionEra'], 'NULL') self.assertEqual(f['inputPath'], '/lame/path') return def testOutputFiles(self): """ _testOutputFiles_ Test some basic manipulation of output files """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) files = myReport.getAllFilesFromStep(step = "cmsRun1") f1 = files[0] f2 = files[1] self.assertEqual(f1['outputModule'], 'outputRECORECO') self.assertEqual(f1['pfn'], 'outputRECORECO.root') self.assertEqual(f2['outputModule'], 'outputALCARECORECO') self.assertEqual(f2['pfn'], 'outputALCARECORECO.root') for f in files: self.assertEqual(f['events'], 2) self.assertEqual(f['configURL'], None) self.assertEqual(f['merged'], False) self.assertEqual(f['validStatus'], None) self.assertEqual(f['first_event'], 0) return def testGetAdlerChecksum(self): """ _testGetAdlerChecksum_ Test the function that sees if all files have an adler checksum. For some reason, our default XML report doesn't have checksums Therefore it should fail. """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) myReport.checkForAdlerChecksum(stepName = "cmsRun1") self.assertFalse(myReport.stepSuccessful(stepName = "cmsRun1")) self.assertEqual(myReport.getExitCode(), 60451) # Now see what happens if the adler32 is set to None myReport2 = Report("cmsRun1") myReport2.parse(self.xmlPath) fRefs = myReport2.getAllFileRefsFromStep(step = "cmsRun1") for fRef in fRefs: fRef.checksums = {'adler32': None} myReport2.checkForAdlerChecksum(stepName = "cmsRun1") self.assertFalse(myReport2.stepSuccessful(stepName = "cmsRun1")) self.assertEqual(myReport2.getExitCode(), 60451) myReport3 = Report("cmsRun1") myReport3.parse(self.xmlPath) fRefs = myReport3.getAllFileRefsFromStep(step = "cmsRun1") for fRef in fRefs: fRef.checksums = {'adler32': 100} myReport3.checkForAdlerChecksum(stepName = "cmsRun1") self.assertTrue(myReport3.getExitCode() != 60451) return def testCheckLumiInformation(self): """ _testCheckLumiInformation_ Test the function that checks if all files have run lumi information """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) myReport.checkForRunLumiInformation(stepName = "cmsRun1") self.assertNotEqual(myReport.getExitCode(), 60452) # Remove the lumi information on purpose myReport2 = Report("cmsRun1") myReport2.parse(self.xmlPath) fRefs = myReport2.getAllFileRefsFromStep(step = "cmsRun1") for fRef in fRefs: fRef.runs = ConfigSection() myReport2.checkForRunLumiInformation(stepName = "cmsRun1") self.assertFalse(myReport2.stepSuccessful(stepName = "cmsRun1")) self.assertEqual(myReport2.getExitCode(), 60452) return def testTaskSuccessful(self): """ _testTaskSuccessful_ Test whether or not the report marks the task successful """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) # First, the report should fail self.assertFalse(myReport.taskSuccessful()) # Second, if we ignore cmsRun, the task # should succeed self.assertTrue(myReport.taskSuccessful(ignoreString = 'cmsRun')) return def testMultiCoreReport(self): """ _testMultiCoreReport_ Verify that multicore reports can be json encoded and uploaded to couch. """ couchdb = CouchServer(os.environ["COUCHURL"]) fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs") self.mcPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/MulticoreReport.pkl") myReport = Report() myReport.unpersist(self.mcPath) fwjrDocument = {"_id": "303-0", "jobid": 303, "retrycount": 0, "fwjr": myReport.__to_json__(None), "type": "fwjr"} fwjrdatabase.queue(fwjrDocument, timestamp = True) fwjrdatabase.commit() return def testStripReport(self): """ _testStripReport_ Test whether or not we can strip input file information from a FWJR and create a smaller object. """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) path1 = os.path.join(self.testDir, 'testReport1.pkl') path2 = os.path.join(self.testDir, 'testReport2.pkl') myReport.save(path1) info = BasicAlgos.getFileInfo(filename = path1) self.assertEqual(info['Size'], 7101) inputFiles = myReport.getAllInputFiles() self.assertEqual(len(inputFiles), 1) myReport.stripInputFiles() self.assertEqual(len(myReport.getAllInputFiles()), 0) myReport.save(path2) info = BasicAlgos.getFileInfo(filename = path2) self.assertEqual(info['Size'], 6210) return def testDuplicatStep(self): """ _testDuplicateStep_ If the same step is added twice, it should act as a replacement, and raise an appropriate message """ baseReport = Report("cmsRun1") baseReport.parse(self.xmlPath) modReport = Report("cmsRun1") modReport.parse(self.xmlPath) setattr(modReport.data.cmsRun1, 'testVar', 'test01') report = Report() report.setStep(stepName = 'cmsRun1', stepSection = baseReport.retrieveStep('cmsRun1')) report.setStep(stepName = 'cmsRun1', stepSection = modReport.retrieveStep('cmsRun1')) self.assertEqual(report.listSteps(), ['cmsRun1']) self.assertEqual(report.data.cmsRun1.testVar, 'test01') return def testDeleteOutputModule(self): """ _testDeleteOutputModule_ If asked delete an output module, if it doesn't exist then do nothing """ originalReport = Report("cmsRun1") originalReport.parse(self.xmlPath) self.assertTrue(originalReport.getOutputModule("cmsRun1", "outputALCARECORECO"), "Error: Report XML doesn't have the module for the test, invalid test") originalOutputModules = len(originalReport.retrieveStep("cmsRun1").outputModules) originalReport.deleteOutputModuleForStep("cmsRun1", "outputALCARECORECO") self.assertFalse(originalReport.getOutputModule("cmsRun1", "outputALCARECORECO"), "Error: The output module persists after deletion") self.assertEqual(len(originalReport.retrieveStep("cmsRun1").outputModules), originalOutputModules - 1, "Error: The number of output modules is incorrect after deletion") def testSkippedFiles(self): """ _testSkippedFiles_ Test that skipped files are translated from FWJR into report """ # Check a report where some files were skipped but not all originalReport = Report("cmsRun1") originalReport.parse(self.skippedFilesxmlPath) self.assertEqual(originalReport.getAllSkippedFiles(), ['/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root']) # For negative control, check a good report with no skipped files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) self.assertEqual(goodReport.getAllSkippedFiles(), []) # Check a report where all files were skipped badReport = Report("cmsRun1") badReport.parse(self.skippedAllFilesxmlPath) self.assertEqual(sorted(badReport.getAllSkippedFiles()), ['/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root', '/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A622-E211-A851-0019B9F581C.root']) return def testSkippedFilesJSON(self): """ _testSkippedFilesJSON_ Test that skipped files are translated properly into JSON """ # Check a report where some files were skipped but not all originalReport = Report("cmsRun1") originalReport.parse(self.skippedFilesxmlPath) originalJSON = originalReport.__to_json__(None) self.assertEqual(len(originalJSON['skippedFiles']), 1) # For negative control, check a good report with no skipped files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) goodJSON = goodReport.__to_json__(None) self.assertEqual(goodJSON['skippedFiles'], []) # Check a report where all files were skipped badReport = Report("cmsRun1") badReport.parse(self.skippedAllFilesxmlPath) badJSON = badReport.__to_json__(None) self.assertEqual(len(badJSON['skippedFiles']), 2) return def testFallbackFiles(self): """ _testFallback_ Test that fallback files end up in the report """ # For negative control, check a good report with no fallback files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) self.assertEqual(goodReport.getAllFallbackFiles(), []) # Check a report where the file was a fallback badReport = Report("cmsRun1") badReport.parse(self.fallbackXmlPath) self.assertEqual(sorted(badReport.getAllFallbackFiles()), ['/store/data/Run2012D/SingleElectron/AOD/PromptReco-v1/000/207/279/D43A5B72-1831-E211-895D-001D09F24763.root']) return def testFallbackFilesJSON(self): """ _testFallbackFilesJSON_ Test that fallback attempt files are translated properly into JSON """ # For negative control, check a good report with no skipped files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) goodJSON = goodReport.__to_json__(None) self.assertEqual(goodJSON['fallbackFiles'], []) # Check a report where all files were skipped badReport = Report("cmsRun1") badReport.parse(self.fallbackXmlPath) badJSON = badReport.__to_json__(None) self.assertEqual(len(badJSON['fallbackFiles']), 1) return def testOutputCheck(self): """ _testOutputCheck_ Check that we can identify bad reports with no output files """ badReport = Report("cmsRun1") badReport.parse(self.skippedAllFilesxmlPath) badReport.checkForOutputFiles("cmsRun1") self.assertFalse(badReport.stepSuccessful(stepName = "cmsRun1")) self.assertEqual(badReport.getExitCode(), 60450) return
class RetryManagerTest(unittest.TestCase): """ TestCase for TestRetryManager module """ def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) self.testInit.setupCouch("retry_manager_t/jobs", "JobDump") self.testInit.setupCouch("retry_manager_t/fwjrs", "FWJRDump") self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs") self.setJobTime = self.daofactory(classname = "Jobs.SetStateTime") self.increaseRetry = self.daofactory(classname = "Jobs.IncrementRetry") self.testDir = self.testInit.generateWorkDir() self.configFile = EmulatorSetup.setupWMAgentConfig() self.nJobs = 10 return def tearDown(self): """ Database deletion """ self.testInit.clearDatabase() self.testInit.delWorkDir() self.testInit.tearDownCouch() EmulatorSetup.deleteConfig(self.configFile) return def getConfig(self): """ _getConfig_ """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("RetryManager") config.RetryManager.logLevel = 'DEBUG' config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager' config.RetryManager.pollInterval = 10 # These are the cooloff times for the RetryManager, the times it waits # Before attempting resubmission config.RetryManager.section_("DefaultRetryAlgo") config.RetryManager.DefaultRetryAlgo.section_("default") config.RetryManager.DefaultRetryAlgo.default.coolOffTime = {'create': 120, 'submit': 120, 'job': 120} # Path to plugin directory config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns' config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE() config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components') # ErrorHandler # Not essential, but useful for ProcessingAlgo config.component_("ErrorHandler") config.ErrorHandler.maxRetries = 5 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "retry_manager_t" return config def createTestJobGroup(self, nJobs, subType = "Processing", retryOnce = False): """ _createTestJobGroup_ Creates a group of several jobs """ testWorkflow = Workflow(spec = "spec.xml", owner = "Simon", name = makeUUID(), task="Test") testWorkflow.create() testWMBSFileset = Fileset(name = "TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow, type = subType) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileA.create() testFileB.create() for i in range(0, nJobs): testJob = Job(name = makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['cache_dir'] = os.path.join(self.testDir, testJob['name']) os.mkdir(testJob['cache_dir']) testJobGroup.add(testJob) testJobGroup.commit() if retryOnce: self.increaseRetry.execute(testJobGroup.jobs) return testJobGroup def testA_Create(self): """ WMComponent_t.RetryManager_t.RetryManager_t:testCreate() Mimics creation of component and test jobs failed in create stage. """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'createfailed', 'new') changer.propagate(testJobGroup.jobs, 'createcooloff', 'createfailed') idList = self.getJobs.execute(state = 'CreateCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'CreateCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'CreateCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) return def testB_Submit(self): """ WMComponent_t.RetryManager_t.RetryManager_t:testSubmit() Mimics creation of component and test jobs failed in create stage. """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) return def testC_Job(self): """ WMComponent_t.RetryManager_t.RetryManager_t:testJob() Mimics creation of component and test jobs failed in create stage. """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') idList = self.getJobs.execute(state = 'JobCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) return def testD_SquaredAlgo(self): """ _testSquaredAlgo_ Test the squared algorithm to make sure it loads and works """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing' : 'SquaredAlgo'} config.RetryManager.section_("SquaredAlgo") config.RetryManager.SquaredAlgo.section_("Processing") config.RetryManager.SquaredAlgo.Processing.coolOffTime = {'create': 10, 'submit': 10, 'job': 10} changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') changer.propagate(testJobGroup.jobs, 'created', 'submitcooloff') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 12) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) def testE_ExponentialAlgo(self): """ _testExponentialAlgo_ Test the exponential algorithm to make sure it loads and works """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing' : 'ExponentialAlgo'} config.RetryManager.section_("ExponentialAlgo") config.RetryManager.ExponentialAlgo.section_("Processing") config.RetryManager.ExponentialAlgo.Processing.coolOffTime = {'create': 10, 'submit': 10, 'job': 10} changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') changer.propagate(testJobGroup.jobs, 'created', 'submitcooloff') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 12) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) def testF_LinearAlgo(self): """ _testLinearAlgo_ Test the linear algorithm to make sure it loads and works """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing' : 'LinearAlgo'} config.RetryManager.section_("LinearAlgo") config.RetryManager.LinearAlgo.section_("Processing") config.RetryManager.LinearAlgo.Processing.coolOffTime = {'create': 10, 'submit': 10, 'job': 10} changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') changer.propagate(testJobGroup.jobs, 'created', 'submitcooloff') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 12) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) return def testG_ProcessingAlgo(self): """ _ProcessingAlgo_ Test for the ProcessingAlgo Prototype """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing' : 'ProcessingAlgo'} config.RetryManager.section_("ProcessingAlgo") config.RetryManager.ProcessingAlgo.section_("default") config.RetryManager.ProcessingAlgo.default.coolOffTime = {'create': 10, 'submit': 10, 'job': 10} changer = ChangeState(config) fwjrPath = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t", "fwjrs/badBackfillJobReport.pkl") report = Report() report.load(fwjrPath) for job in testJobGroup.jobs: job['fwjr'] = report job['retry_count'] = 0 report.save(os.path.join(job['cache_dir'], "Report.%i.pkl" % job['retry_count'])) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') testRetryManager = RetryManagerPoller(config) testRetryManager.algorithm() idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup.jobs: j = Job(id = job['id']) j.load() self.assertEqual(j['retry_count'], 1) report.save(os.path.join(j['cache_dir'], "Report.%i.pkl" % j['retry_count'])) config.RetryManager.ProcessingAlgo.default.OneMoreErrorCodes = [8020] testRetryManager2 = RetryManagerPoller(config) testRetryManager2.algorithm() idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: j = Job(id = job['id']) j.load() self.assertEqual(j['retry_count'], 5) # Now test timeout testJobGroup2 = self.createTestJobGroup(nJobs = self.nJobs) # Cycle jobs for job in testJobGroup2.jobs: job['fwjr'] = report job['retry_count'] = 0 report.save(os.path.join(job['cache_dir'], "Report.%i.pkl" % job['retry_count'])) changer.propagate(testJobGroup2.jobs, 'created', 'new') changer.propagate(testJobGroup2.jobs, 'executing', 'created') changer.propagate(testJobGroup2.jobs, 'complete', 'executing') changer.propagate(testJobGroup2.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup2.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup2.jobs: j = Job(id = job['id']) j.load() self.assertEqual(j['retry_count'], 0) config.RetryManager.ProcessingAlgo.default.OneMoreErrorCodes = [] config.RetryManager.ProcessingAlgo.default.MaxRunTime = 1 testRetryManager3 = RetryManagerPoller(config) testRetryManager3.algorithm() idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs * 2) for job in testJobGroup2.jobs: j = Job(id = job['id']) j.load() self.assertEqual(j['retry_count'], 5) return def testH_PauseAlgo(self): """ _testH_PauseAlgo_ Test the pause algorithm, note that given pauseCount = n, the job will run first n + 1 times before being paused. After that it will be paused each n times """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing' : 'PauseAlgo'} config.RetryManager.section_("PauseAlgo") config.RetryManager.PauseAlgo.section_("Processing") config.RetryManager.PauseAlgo.Processing.coolOffTime = {'create': 20, 'submit': 20, 'job': 20} config.RetryManager.PauseAlgo.Processing.pauseCount = 2 changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) # Making sure that jobs are not created ahead of time for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 15) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCoolOff') self.assertEqual(len(idList), self.nJobs) # Giving time so they can be retried for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 25) # Make sure that the plugin allowed them to go back to created state testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'created') self.assertEqual(len(idList), self.nJobs) # Fail them out again changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') # Make sure that no change happens before timeout for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 75) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCoolOff') self.assertEqual(len(idList), self.nJobs) # Giving time so they can be paused for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 85) # Make sure that the plugin pauses them testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'jobpaused') self.assertEqual(len(idList), self.nJobs) # Emulating ops retrying the job changer.propagate(testJobGroup.jobs, 'created', 'jobpaused') # Making sure it did the right thing idList = self.getJobs.execute(state = 'created') self.assertEqual(len(idList), self.nJobs) # Fail them out again changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 175) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCoolOff') self.assertEqual(len(idList), self.nJobs) # Giving time so they can be retried for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 185) # Make sure that the plugin allowed them to go back to created state testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'created') self.assertEqual(len(idList), self.nJobs) # Fail them out again changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 315) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'jobcooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 325) # Make sure that the plugin allowed them to go back to created state testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'jobpaused') self.assertEqual(len(idList), self.nJobs) return def testI_MultipleJobTypes(self): """ _testI_MultipleJobTypes_ Check that we can configure different retry algorithms for different job types, including a default for nonspecified types. Also check that two job types can share the same retry algorithm but with different parameters """ #Let's create 4 job groups processingJobGroup = self.createTestJobGroup(nJobs = 10, retryOnce = True) productionJobGroup = self.createTestJobGroup(nJobs = 15, subType = "Production", retryOnce = True) mergeJobGroup = self.createTestJobGroup(nJobs = 20, subType = "Merge", retryOnce = True) skimJobGroup = self.createTestJobGroup(nJobs = 5, subType = "Skim", retryOnce = True) #Set an adequate config #Processing jobs get the PauseAlgo with pauseCount 4 #Production jobs get the ExponentialAlgo #Merge jobs get the PauseAlgo but with pauseCount 2 which is the default #Skim jobs are not configured, so they get the default SquaredAlgo config = self.getConfig() config.RetryManager.plugins = {'Processing' : 'PauseAlgo', 'Production' : 'ExponentialAlgo', 'Merge' : 'PauseAlgo', 'default' : 'SquaredAlgo'} config.RetryManager.section_("PauseAlgo") config.RetryManager.PauseAlgo.section_("Processing") config.RetryManager.PauseAlgo.Processing.coolOffTime = {'create': 30, 'submit': 30, 'job': 30} config.RetryManager.PauseAlgo.Processing.pauseCount = 4 config.RetryManager.PauseAlgo.section_("default") config.RetryManager.PauseAlgo.default.coolOffTime = {'create': 60, 'submit': 60, 'job': 60} config.RetryManager.PauseAlgo.default.pauseCount = 2 config.RetryManager.section_("ExponentialAlgo") config.RetryManager.ExponentialAlgo.section_("Production") config.RetryManager.ExponentialAlgo.Production.coolOffTime = {'create': 30, 'submit': 30, 'job': 30} config.RetryManager.ExponentialAlgo.section_("default") config.RetryManager.ExponentialAlgo.default.coolOffTime = {'create': 60, 'submit': 60, 'job': 60} config.RetryManager.section_("SquaredAlgo") config.RetryManager.SquaredAlgo.section_("Skim") config.RetryManager.SquaredAlgo.Skim.coolOffTime = {'create': 30, 'submit': 30, 'job': 30} config.RetryManager.SquaredAlgo.section_("default") config.RetryManager.SquaredAlgo.default.coolOffTime = {'create': 60, 'submit': 60, 'job': 60} #Start the state changer and RetryManager changer = ChangeState(config) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) #Create the jobs for the first time changer.propagate(processingJobGroup.jobs, 'created', 'new') # Let's start with the processing jobs and the pauseAlgo for count in range(1,5): #Fail the jobs changer.propagate(processingJobGroup.jobs, 'executing', 'created') changer.propagate(processingJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(processingJobGroup.jobs, 'jobcooloff', 'jobfailed') #Check that the cooloff time is strictly enforced #First a job time just below the cooloff time for job in processingJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 30*pow(count,2) + 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCoolOff') self.assertEqual(len(idList), len(processingJobGroup.jobs), "Jobs went into cooloff without the proper timing") #Now above the cooloff time for job in processingJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 30*pow(count,2) - 5) testRetryManager.algorithm(None) #Make sure the jobs get created again or go to paused if count < 4: idList = self.getJobs.execute(state = 'created') else: idList = self.getJobs.execute(state = 'jobpaused') self.assertEqual(len(idList), len(processingJobGroup.jobs), "Jobs didn't change state correctly") #Unpause them so they don't interfere with subsequent tests changer.propagate(processingJobGroup.jobs, 'created', 'jobpaused') changer.propagate(processingJobGroup.jobs, 'executing', 'created') #Now the production jobs and the exponential algo changer.propagate(productionJobGroup.jobs, 'created', 'new') for count in range(1,3): changer.propagate(productionJobGroup.jobs, 'executing', 'created') changer.propagate(productionJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(productionJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in productionJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - pow(30,count) + 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCoolOff') self.assertEqual(len(idList), len(productionJobGroup.jobs), "Jobs went into cooloff without the proper timing") for job in productionJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - pow(30,count) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'created') self.assertEqual(len(idList), len(productionJobGroup.jobs), "Jobs didn't change state correctly") #Send them to executing changer.propagate(productionJobGroup.jobs, 'executing', 'created') #Now the merge jobs and the paused algo with different parameters changer.propagate(mergeJobGroup.jobs, 'created', 'new') for count in range(1,3): changer.propagate(mergeJobGroup.jobs, 'executing', 'created') changer.propagate(mergeJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(mergeJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in mergeJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 30*pow(count,2) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCoolOff') self.assertEqual(len(idList), len(mergeJobGroup.jobs), "Jobs went into cooloff without the proper timing") for job in mergeJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 60*pow(count,2) - 5) testRetryManager.algorithm(None) if count < 2: idList = self.getJobs.execute(state = 'created') else: idList = self.getJobs.execute(state = 'jobpaused') self.assertEqual(len(idList), len(mergeJobGroup.jobs), "Jobs didn't change state correctly") #Send them to executing changer.propagate(mergeJobGroup.jobs, 'created', 'jobpaused') changer.propagate(mergeJobGroup.jobs, 'executing', 'created') #Now the skim jobs and the squared algo changer.propagate(skimJobGroup.jobs, 'created', 'new') for count in range(1,3): changer.propagate(skimJobGroup.jobs, 'executing', 'created') changer.propagate(skimJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(skimJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in skimJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 30*pow(count,2) + 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'JobCoolOff') self.assertEqual(len(idList), len(skimJobGroup.jobs), "Jobs went into cooloff without the proper timing") for job in skimJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 30*pow(count,2) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'created') self.assertEqual(len(idList), len(skimJobGroup.jobs), "Jobs didn't change state correctly") def testY_MultipleIterations(self): """ _MultipleIterations_ Paranoia based check to see if I'm saving class instances correctly """ testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'submitfailed', 'Created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs) # Make a new jobGroup for a second run testJobGroup = self.createTestJobGroup(nJobs = self.nJobs) # Set job state changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') # Set them to go off for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 200) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'Created') self.assertEqual(len(idList), self.nJobs * 2) return def testZ_Profile(self): """ _Profile_ Do a basic profiling of the algo """ return import cProfile, pstats nJobs = 1000 testJobGroup = self.createTestJobGroup(nJobs = nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'createfailed', 'new') changer.propagate(testJobGroup.jobs, 'createcooloff', 'createfailed') idList = self.getJobs.execute(state = 'CreateCooloff') self.assertEqual(len(idList), nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state = 'CreateCooloff') self.assertEqual(len(idList), nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID = job["id"], stateTime = int(time.time()) - 150) startTime = time.time() #cProfile.runctx("testRetryManager.algorithm()", globals(), locals(), filename = "profStats.stat") testRetryManager.algorithm(None) stopTime = time.time() idList = self.getJobs.execute(state = 'CreateCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state = 'New') self.assertEqual(len(idList), nJobs) print("Took %f seconds to run polling algo" % (stopTime - startTime)) p = pstats.Stats('profStats.stat') p.sort_stats('cumulative') p.print_stats(0.2) return
class PromptRecoTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("promptreco_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("promptreco_t") self.testDir = self.testInit.generateWorkDir() return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def setupPromptSkimConfigObject(self): """ _setupPromptSkimConfigObject_ Creates a custom config object for testing of the skim functionality """ self.promptSkim = ConfigSection(name="Tier1Skim") self.promptSkim.SkimName = "TestSkim1" self.promptSkim.DataTier = "RECO" self.promptSkim.TwoFileRead = False self.promptSkim.ProcessingVersion = "PromptSkim-v1" self.promptSkim.ConfigURL = "http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi/CMSSW/Configuration/DataOps/python/prescaleskimmer.py?revision=1.1" def testPromptReco(self): """ _testPromptReco_ Create a Prompt Reconstruction workflow and verify it installs into WMBS correctly. """ testArguments = getTestArguments() testWorkload = promptrecoWorkload("TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Reco", "SomeBlock", cachepath = self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) recoWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco") recoWorkflow.load() self.assertEqual(len(recoWorkflow.outputMap.keys()), len(testArguments["WriteTiers"]) + 1, "Error: Wrong number of WF outputs in the Reco WF.") goldenOutputMods = ["write_RECO", "write_ALCARECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_ALCARECO": self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = recoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = recoWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") alcaSkimWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim") alcaSkimWorkflow.load() self.assertEqual(len(alcaSkimWorkflow.outputMap.keys()), len(testArguments["AlcaSkims"]) + 1, "Error: Wrong number of WF outputs in the AlcaSkim WF.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") dqmWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged") dqmWorkflow.load() logArchOutput = dqmWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = dqmWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-Reco-SomeBlock") topLevelFileset.loadData() recoSubscription = Subscription(fileset = topLevelFileset, workflow = recoWorkflow) recoSubscription.loadData() self.assertEqual(recoSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(recoSubscription["split_algo"], "EventBased", "Error: Wrong split algorithm. %s" % recoSubscription["split_algo"]) alcaRecoFileset = Fileset(name = "/TestWorkload/Reco/unmerged-write_ALCARECO") alcaRecoFileset.loadData() alcaSkimSubscription = Subscription(fileset = alcaRecoFileset, workflow = alcaSkimWorkflow) alcaSkimSubscription.loadData() self.assertEqual(alcaSkimSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(alcaSkimSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % alcaSkimSubscription["split_algo"]) mergedDQMFileset = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_DQM/merged-Merged") mergedDQMFileset.loadData() dqmSubscription = Subscription(fileset = mergedDQMFileset, workflow = dqmWorkflow) dqmSubscription.loadData() self.assertEqual(dqmSubscription["type"], "Harvesting", "Error: Wrong subscription type.") self.assertEqual(dqmSubscription["split_algo"], "Harvest", "Error: Wrong split algo.") unmergedOutputs = ["write_RECO", "write_AOD", "write_DQM"] for unmergedOutput in unmergedOutputs: unmergedDataTier = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % unmergedOutput) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedDataTier, workflow = dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = [] for alcaProd in testArguments["AlcaSkims"]: unmergedOutputs.append("ALCARECOStream%s" % alcaProd) for unmergedOutput in unmergedOutputs: unmergedAlcaSkim = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % unmergedOutput) unmergedAlcaSkim.loadData() alcaSkimMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % unmergedOutput) alcaSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedAlcaSkim, workflow = alcaSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM", "write_ALCARECO"] for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmerged%s" %goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") recoLogCollect = Fileset(name = "/TestWorkload/Reco/unmerged-logArchive") recoLogCollect.loadData() recoLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/LogCollect") recoLogCollectWorkflow.load() logCollectSub = Subscription(fileset = recoLogCollect, workflow = recoLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive") alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect") alcaSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: recoMergeLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod) recoMergeLogCollect.loadData() recoMergeLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s/Reco%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) recoMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = recoMergeLogCollect, workflow = recoMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod) alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/AlcaSkim%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) alcaSkimLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") dqmHarvestLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive") dqmHarvestLogCollect.loadData() dqmHarvestLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect") dqmHarvestLogCollectWorkflow.load() logCollectSub = Subscription(fileset = dqmHarvestLogCollect, workflow = dqmHarvestLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") return @attr("integration") def testPromptRecoWithSkims(self): """ _testT1PromptRecoWithSkim_ Create a T1 Prompt Reconstruction workflow with PromptSkims and verify it installs into WMBS correctly. """ self.setupPromptSkimConfigObject() testArguments = getTestArguments() testArguments["PromptSkims"] = [self.promptSkim] testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = "promptreco_t" testArguments["EnvPath"] = os.environ.get("EnvPath", None) testArguments["BinPath"] = os.environ.get("BinPath", None) testWorkload = promptrecoWorkload("TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Reco", "SomeBlock", cachepath = self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) recoWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco") recoWorkflow.load() self.assertEqual(len(recoWorkflow.outputMap.keys()), len(testArguments["WriteTiers"]) + 1, "Error: Wrong number of WF outputs in the Reco WF.") goldenOutputMods = ["write_RECO", "write_ALCARECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_ALCARECO": self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = recoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = recoWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") alcaSkimWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim") alcaSkimWorkflow.load() self.assertEqual(len(alcaSkimWorkflow.outputMap.keys()), len(testArguments["AlcaSkims"]) + 1, "Error: Wrong number of WF outputs in the AlcaSkim WF.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") promptSkimWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1") promptSkimWorkflow.load() self.assertEqual(len(promptSkimWorkflow.outputMap.keys()), 6, "Error: Wrong number of WF outputs.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: mergedOutput = promptSkimWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = promptSkimWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = promptSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = promptSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-Reco-SomeBlock") topLevelFileset.loadData() recoSubscription = Subscription(fileset = topLevelFileset, workflow = recoWorkflow) recoSubscription.loadData() self.assertEqual(recoSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(recoSubscription["split_algo"], "EventBased", "Error: Wrong split algorithm. %s" % recoSubscription["split_algo"]) alcaRecoFileset = Fileset(name = "/TestWorkload/Reco/unmerged-write_ALCARECO") alcaRecoFileset.loadData() alcaSkimSubscription = Subscription(fileset = alcaRecoFileset, workflow = alcaSkimWorkflow) alcaSkimSubscription.loadData() self.assertEqual(alcaSkimSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(alcaSkimSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % alcaSkimSubscription["split_algo"]) mergedRecoFileset = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/merged-Merged") mergedRecoFileset.loadData() promptSkimSubscription = Subscription(fileset = mergedRecoFileset, workflow = promptSkimWorkflow) promptSkimSubscription.loadData() self.assertEqual(promptSkimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(promptSkimSubscription["split_algo"], "FileBased", "Error: Wrong split algorithm. %s" % promptSkimSubscription["split_algo"]) unmergedOutputs = ["write_RECO", "write_AOD", "write_DQM"] for unmergedOutput in unmergedOutputs: unmergedDataTier = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % unmergedOutput) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedDataTier, workflow = dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = [] for alcaProd in testArguments["AlcaSkims"]: unmergedOutputs.append("ALCARECOStream%s" % alcaProd) for unmergedOutput in unmergedOutputs: unmergedAlcaSkim = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % unmergedOutput) unmergedAlcaSkim.loadData() alcaSkimMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % unmergedOutput) alcaSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedAlcaSkim, workflow = alcaSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for unmergedOutput in unmergedOutputs: unmergedPromptSkim = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-%s" % unmergedOutput) unmergedPromptSkim.loadData() promptSkimMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s" % unmergedOutput) promptSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedPromptSkim, workflow = promptSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM", "write_ALCARECO"] for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmerged%s" %goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-%s" % unmergedOutput) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1CleanupUnmerged%s" % unmergedOutput) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong split algorithm. %s" % cleanupSubscription["split_algo"]) recoLogCollect = Fileset(name = "/TestWorkload/Reco/unmerged-logArchive") recoLogCollect.loadData() recoLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/LogCollect") recoLogCollectWorkflow.load() logCollectSub = Subscription(fileset = recoLogCollect, workflow = recoLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive") alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect") alcaSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") promptSkimLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-logArchive") promptSkimLogCollect.loadData() promptSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1LogCollect") promptSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset = promptSkimLogCollect, workflow = promptSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: recoMergeLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod) recoMergeLogCollect.loadData() recoMergeLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s/Reco%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) recoMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = recoMergeLogCollect, workflow = recoMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod) alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/AlcaSkim%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) alcaSkimLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: promptSkimMergeLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-logArchive" % goldenOutputMod) promptSkimMergeLogCollect.loadData() promptSkimMergeLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/TestSkim1%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) promptSkimMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = promptSkimMergeLogCollect, workflow = promptSkimMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") return
class RetryManagerTest(EmulatedUnitTestCase): """ TestCase for TestRetryManager module """ def setUp(self): """ setup for test. """ super(RetryManagerTest, self).setUp() myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.setupCouch("retry_manager_t/jobs", "JobDump") self.testInit.setupCouch("retry_manager_t/fwjrs", "FWJRDump") self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daofactory(classname="Jobs.GetAllJobs") self.setJobTime = self.daofactory(classname="Jobs.SetStateTime") self.increaseRetry = self.daofactory(classname="Jobs.IncrementRetry") self.testDir = self.testInit.generateWorkDir() self.configFile = EmulatorSetup.setupWMAgentConfig() self.nJobs = 10 return def tearDown(self): """ Database deletion """ self.testInit.clearDatabase() self.testInit.delWorkDir() self.testInit.tearDownCouch() EmulatorSetup.deleteConfig(self.configFile) return def getConfig(self): """ _getConfig_ """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("RetryManager") config.RetryManager.logLevel = 'DEBUG' config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager' config.RetryManager.pollInterval = 10 # These are the cooloff times for the RetryManager, the times it waits # Before attempting resubmission config.RetryManager.section_("DefaultRetryAlgo") config.RetryManager.DefaultRetryAlgo.section_("default") config.RetryManager.DefaultRetryAlgo.default.coolOffTime = { 'create': 120, 'submit': 120, 'job': 120 } # Path to plugin directory config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns' config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE() config.RetryManager.componentDir = os.path.join( os.getcwd(), 'Components') # ErrorHandler # Not essential, but useful for ProcessingAlgo config.component_("ErrorHandler") config.ErrorHandler.maxRetries = 5 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "retry_manager_t" return config def createTestJobGroup(self, nJobs, subType="Processing", retryOnce=False): """ _createTestJobGroup_ Creates a group of several jobs """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name=makeUUID(), task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow, type=subType) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileB.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileA.create() testFileB.create() for i in range(0, nJobs): testJob = Job(name=makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['cache_dir'] = os.path.join(self.testDir, testJob['name']) os.mkdir(testJob['cache_dir']) testJobGroup.add(testJob) testJobGroup.commit() if retryOnce: self.increaseRetry.execute(testJobGroup.jobs) return testJobGroup def testA_Create(self): """ WMComponent_t.RetryManager_t.RetryManager_t:testCreate() Mimics creation of component and test jobs failed in create stage. """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'createfailed', 'new') changer.propagate(testJobGroup.jobs, 'createcooloff', 'createfailed') idList = self.getJobs.execute(state='CreateCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='CreateCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='CreateCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) return def testB_Submit(self): """ WMComponent_t.RetryManager_t.RetryManager_t:testSubmit() Mimics creation of component and test jobs failed in create stage. """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) return def testC_Job(self): """ WMComponent_t.RetryManager_t.RetryManager_t:testJob() Mimics creation of component and test jobs failed in create stage. """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') idList = self.getJobs.execute(state='JobCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) return def testD_SquaredAlgo(self): """ _testSquaredAlgo_ Test the squared algorithm to make sure it loads and works """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing': 'SquaredAlgo'} config.RetryManager.section_("SquaredAlgo") config.RetryManager.SquaredAlgo.section_("Processing") config.RetryManager.SquaredAlgo.Processing.coolOffTime = { 'create': 10, 'submit': 10, 'job': 10 } changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') changer.propagate(testJobGroup.jobs, 'created', 'submitcooloff') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 12) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) def testE_ExponentialAlgo(self): """ _testExponentialAlgo_ Test the exponential algorithm to make sure it loads and works """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing': 'ExponentialAlgo'} config.RetryManager.section_("ExponentialAlgo") config.RetryManager.ExponentialAlgo.section_("Processing") config.RetryManager.ExponentialAlgo.Processing.coolOffTime = { 'create': 10, 'submit': 10, 'job': 10 } changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') changer.propagate(testJobGroup.jobs, 'created', 'submitcooloff') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 12) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) def testF_LinearAlgo(self): """ _testLinearAlgo_ Test the linear algorithm to make sure it loads and works """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing': 'LinearAlgo'} config.RetryManager.section_("LinearAlgo") config.RetryManager.LinearAlgo.section_("Processing") config.RetryManager.LinearAlgo.Processing.coolOffTime = { 'create': 10, 'submit': 10, 'job': 10 } changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') changer.propagate(testJobGroup.jobs, 'created', 'submitcooloff') changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 12) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) return def testG_ProcessingAlgo(self): """ _ProcessingAlgo_ Test for the ProcessingAlgo Prototype """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing': 'ProcessingAlgo'} config.RetryManager.section_("ProcessingAlgo") config.RetryManager.ProcessingAlgo.section_("default") config.RetryManager.ProcessingAlgo.default.coolOffTime = { 'create': 10, 'submit': 10, 'job': 10 } changer = ChangeState(config) fwjrPath = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t", "fwjrs/badBackfillJobReport.pkl") report = Report() report.load(fwjrPath) for job in testJobGroup.jobs: job['fwjr'] = report job['retry_count'] = 0 report.save( os.path.join(job['cache_dir'], "Report.%i.pkl" % job['retry_count'])) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') testRetryManager = RetryManagerPoller(config) testRetryManager.algorithm() idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup.jobs: j = Job(id=job['id']) j.load() self.assertEqual(j['retry_count'], 1) report.save( os.path.join(j['cache_dir'], "Report.%i.pkl" % j['retry_count'])) config.RetryManager.ProcessingAlgo.default.OneMoreErrorCodes = [8020] testRetryManager2 = RetryManagerPoller(config) testRetryManager2.algorithm() idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: j = Job(id=job['id']) j.load() self.assertEqual(j['retry_count'], 5) # Now test timeout testJobGroup2 = self.createTestJobGroup(nJobs=self.nJobs) # Cycle jobs for job in testJobGroup2.jobs: job['fwjr'] = report job['retry_count'] = 0 report.save( os.path.join(job['cache_dir'], "Report.%i.pkl" % job['retry_count'])) changer.propagate(testJobGroup2.jobs, 'created', 'new') changer.propagate(testJobGroup2.jobs, 'executing', 'created') changer.propagate(testJobGroup2.jobs, 'complete', 'executing') changer.propagate(testJobGroup2.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup2.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup2.jobs: j = Job(id=job['id']) j.load() self.assertEqual(j['retry_count'], 0) config.RetryManager.ProcessingAlgo.default.OneMoreErrorCodes = [] config.RetryManager.ProcessingAlgo.default.MaxRunTime = 1 testRetryManager3 = RetryManagerPoller(config) testRetryManager3.algorithm() idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs * 2) for job in testJobGroup2.jobs: j = Job(id=job['id']) j.load() self.assertEqual(j['retry_count'], 5) return def testH_PauseAlgo(self): """ _testH_PauseAlgo_ Test the pause algorithm, note that given pauseCount = n, the job will run first n + 1 times before being paused. After that it will be paused each n times """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() config.RetryManager.plugins = {'Processing': 'PauseAlgo'} config.RetryManager.section_("PauseAlgo") config.RetryManager.PauseAlgo.section_("Processing") config.RetryManager.PauseAlgo.Processing.coolOffTime = { 'create': 20, 'submit': 20, 'job': 20 } config.RetryManager.PauseAlgo.Processing.pauseCount = 2 changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) # Making sure that jobs are not created ahead of time for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 15) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCoolOff') self.assertEqual(len(idList), self.nJobs) # Giving time so they can be retried for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 25) # Make sure that the plugin allowed them to go back to created state testRetryManager.algorithm(None) idList = self.getJobs.execute(state='created') self.assertEqual(len(idList), self.nJobs) # Fail them out again changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') # Make sure that no change happens before timeout for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 75) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCoolOff') self.assertEqual(len(idList), self.nJobs) # Giving time so they can be paused for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 85) # Make sure that the plugin pauses them testRetryManager.algorithm(None) idList = self.getJobs.execute(state='jobpaused') self.assertEqual(len(idList), self.nJobs) # Emulating ops retrying the job changer.propagate(testJobGroup.jobs, 'created', 'jobpaused') # Making sure it did the right thing idList = self.getJobs.execute(state='created') self.assertEqual(len(idList), self.nJobs) # Fail them out again changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 175) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCoolOff') self.assertEqual(len(idList), self.nJobs) # Giving time so they can be retried for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 185) # Make sure that the plugin allowed them to go back to created state testRetryManager.algorithm(None) idList = self.getJobs.execute(state='created') self.assertEqual(len(idList), self.nJobs) # Fail them out again changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 315) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='jobcooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 325) # Make sure that the plugin allowed them to go back to created state testRetryManager.algorithm(None) idList = self.getJobs.execute(state='jobpaused') self.assertEqual(len(idList), self.nJobs) return def testI_MultipleJobTypes(self): """ _testI_MultipleJobTypes_ Check that we can configure different retry algorithms for different job types, including a default for nonspecified types. Also check that two job types can share the same retry algorithm but with different parameters """ # Let's create 4 job groups processingJobGroup = self.createTestJobGroup(nJobs=10, retryOnce=True) productionJobGroup = self.createTestJobGroup(nJobs=15, subType="Production", retryOnce=True) mergeJobGroup = self.createTestJobGroup(nJobs=20, subType="Merge", retryOnce=True) skimJobGroup = self.createTestJobGroup(nJobs=5, subType="Skim", retryOnce=True) # Set an adequate config # Processing jobs get the PauseAlgo with pauseCount 4 # Production jobs get the ExponentialAlgo # Merge jobs get the PauseAlgo but with pauseCount 2 which is the default # Skim jobs are not configured, so they get the default SquaredAlgo config = self.getConfig() config.RetryManager.plugins = { 'Processing': 'PauseAlgo', 'Production': 'ExponentialAlgo', 'Merge': 'PauseAlgo', 'default': 'SquaredAlgo' } config.RetryManager.section_("PauseAlgo") config.RetryManager.PauseAlgo.section_("Processing") config.RetryManager.PauseAlgo.Processing.coolOffTime = { 'create': 30, 'submit': 30, 'job': 30 } config.RetryManager.PauseAlgo.Processing.pauseCount = 4 config.RetryManager.PauseAlgo.section_("default") config.RetryManager.PauseAlgo.default.coolOffTime = { 'create': 60, 'submit': 60, 'job': 60 } config.RetryManager.PauseAlgo.default.pauseCount = 2 config.RetryManager.section_("ExponentialAlgo") config.RetryManager.ExponentialAlgo.section_("Production") config.RetryManager.ExponentialAlgo.Production.coolOffTime = { 'create': 30, 'submit': 30, 'job': 30 } config.RetryManager.ExponentialAlgo.section_("default") config.RetryManager.ExponentialAlgo.default.coolOffTime = { 'create': 60, 'submit': 60, 'job': 60 } config.RetryManager.section_("SquaredAlgo") config.RetryManager.SquaredAlgo.section_("Skim") config.RetryManager.SquaredAlgo.Skim.coolOffTime = { 'create': 30, 'submit': 30, 'job': 30 } config.RetryManager.SquaredAlgo.section_("default") config.RetryManager.SquaredAlgo.default.coolOffTime = { 'create': 60, 'submit': 60, 'job': 60 } # Start the state changer and RetryManager changer = ChangeState(config) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) # Create the jobs for the first time changer.propagate(processingJobGroup.jobs, 'created', 'new') # Let's start with the processing jobs and the pauseAlgo for count in range(1, 5): # Fail the jobs changer.propagate(processingJobGroup.jobs, 'executing', 'created') changer.propagate(processingJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(processingJobGroup.jobs, 'jobcooloff', 'jobfailed') # Check that the cooloff time is strictly enforced # First a job time just below the cooloff time for job in processingJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 30 * pow(count, 2) + 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCoolOff') self.assertEqual( len(idList), len(processingJobGroup.jobs), "Jobs went into cooloff without the proper timing") # Now above the cooloff time for job in processingJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 30 * pow(count, 2) - 5) testRetryManager.algorithm(None) # Make sure the jobs get created again or go to paused if count < 4: idList = self.getJobs.execute(state='created') else: idList = self.getJobs.execute(state='jobpaused') self.assertEqual(len(idList), len(processingJobGroup.jobs), "Jobs didn't change state correctly") # Unpause them so they don't interfere with subsequent tests changer.propagate(processingJobGroup.jobs, 'created', 'jobpaused') changer.propagate(processingJobGroup.jobs, 'executing', 'created') # Now the production jobs and the exponential algo changer.propagate(productionJobGroup.jobs, 'created', 'new') for count in range(1, 3): changer.propagate(productionJobGroup.jobs, 'executing', 'created') changer.propagate(productionJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(productionJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in productionJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - pow(30, count) + 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCoolOff') self.assertEqual( len(idList), len(productionJobGroup.jobs), "Jobs went into cooloff without the proper timing") for job in productionJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - pow(30, count) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='created') self.assertEqual(len(idList), len(productionJobGroup.jobs), "Jobs didn't change state correctly") # Send them to executing changer.propagate(productionJobGroup.jobs, 'executing', 'created') # Now the merge jobs and the paused algo with different parameters changer.propagate(mergeJobGroup.jobs, 'created', 'new') for count in range(1, 3): changer.propagate(mergeJobGroup.jobs, 'executing', 'created') changer.propagate(mergeJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(mergeJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in mergeJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 30 * pow(count, 2) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCoolOff') self.assertEqual( len(idList), len(mergeJobGroup.jobs), "Jobs went into cooloff without the proper timing") for job in mergeJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 60 * pow(count, 2) - 5) testRetryManager.algorithm(None) if count < 2: idList = self.getJobs.execute(state='created') else: idList = self.getJobs.execute(state='jobpaused') self.assertEqual(len(idList), len(mergeJobGroup.jobs), "Jobs didn't change state correctly") # Send them to executing changer.propagate(mergeJobGroup.jobs, 'created', 'jobpaused') changer.propagate(mergeJobGroup.jobs, 'executing', 'created') # Now the skim jobs and the squared algo changer.propagate(skimJobGroup.jobs, 'created', 'new') for count in range(1, 3): changer.propagate(skimJobGroup.jobs, 'executing', 'created') changer.propagate(skimJobGroup.jobs, 'jobfailed', 'executing') changer.propagate(skimJobGroup.jobs, 'jobcooloff', 'jobfailed') for job in skimJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 30 * pow(count, 2) + 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='JobCoolOff') self.assertEqual( len(idList), len(skimJobGroup.jobs), "Jobs went into cooloff without the proper timing") for job in skimJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 30 * pow(count, 2) - 5) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='created') self.assertEqual(len(idList), len(skimJobGroup.jobs), "Jobs didn't change state correctly") def testY_MultipleIterations(self): """ _MultipleIterations_ Paranoia based check to see if I'm saving class instances correctly """ testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'submitfailed', 'Created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), self.nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 150) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs) # Make a new jobGroup for a second run testJobGroup = self.createTestJobGroup(nJobs=self.nJobs) # Set job state changer.propagate(testJobGroup.jobs, 'submitfailed', 'created') changer.propagate(testJobGroup.jobs, 'submitcooloff', 'submitfailed') # Set them to go off for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 200) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='SubmitCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='Created') self.assertEqual(len(idList), self.nJobs * 2) return @attr('integration') def testZ_Profile(self): """ _Profile_ Do a basic profiling of the algo """ import pstats nJobs = 1000 testJobGroup = self.createTestJobGroup(nJobs=nJobs) config = self.getConfig() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'createfailed', 'new') changer.propagate(testJobGroup.jobs, 'createcooloff', 'createfailed') idList = self.getJobs.execute(state='CreateCooloff') self.assertEqual(len(idList), nJobs) testRetryManager = RetryManagerPoller(config) testRetryManager.setup(None) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 50) testRetryManager.algorithm(None) idList = self.getJobs.execute(state='CreateCooloff') self.assertEqual(len(idList), nJobs) for job in testJobGroup.jobs: self.setJobTime.execute(jobID=job["id"], stateTime=int(time.time()) - 150) startTime = time.time() # cProfile.runctx("testRetryManager.algorithm()", globals(), locals(), filename = "profStats.stat") testRetryManager.algorithm(None) stopTime = time.time() idList = self.getJobs.execute(state='CreateCooloff') self.assertEqual(len(idList), 0) idList = self.getJobs.execute(state='New') self.assertEqual(len(idList), nJobs) print("Took %f seconds to run polling algo" % (stopTime - startTime)) p = pstats.Stats('profStats.stat') p.sort_stats('cumulative') p.print_stats(0.2) return
class MonteCarloFromGENTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("mclhe_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("mclhe_t") return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.clearDatabase() return def injectConfig(self): """ _injectConfig_ Create a bogus config cache document and inject it into couch. Return the ID of the document. """ newConfig = Document() newConfig["info"] = None newConfig["config"] = None newConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f" newConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7" newConfig["owner"] = {"group": "cmsdataops", "user": "******"} newConfig["pset_tweak_details"] ={"process": {"outputModules_": ["outputRECORECO", "outputALCARECOALCARECO"], "outputRECORECO": {"dataset": {"filterName": "FilterRECO", "dataTier": "RECO"}}, "outputALCARECOALCARECO": {"dataset": {"filterName": "FilterALCARECO", "dataTier": "ALCARECO"}}}} result = self.configDatabase.commitOne(newConfig) return result[0]["id"] def testMonteCarloFromGEN(self): """ _testMonteCarloFromGEN_ Create a MonteCarloFromGEN workflow and verify it installs into WMBS correctly. """ arguments = getTestArguments() arguments["ConfigCacheID"] = self.injectConfig() arguments["CouchDBName"] = "mclhe_t" testWorkload = monteCarloFromGENWorkload("TestWorkload", arguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DMWM") testWMBSHelper = WMBSHelper(testWorkload, "MonteCarloFromGEN", "SomeBlock") testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) procWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN") procWorkflow.load() self.assertEqual(len(procWorkflow.outputMap.keys()), 3, "Error: Wrong number of WF outputs.") self.assertEqual(procWorkflow.wfType, 'lheproduction') goldenOutputMods = ["outputRECORECO", "outputALCARECOALCARECO"] for goldenOutputMod in goldenOutputMods: mergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/MonteCarloFromGEN/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = procWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = procWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/MonteCarloFromGEN/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/MonteCarloFromGEN/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-MonteCarloFromGEN-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset = topLevelFileset, workflow = procWorkflow) procSubscription.loadData() self.assertEqual(procSubscription["type"], "Production", "Error: Wrong subscription type: %s" % procSubscription["type"]) self.assertEqual(procSubscription["split_algo"], "LumiBased", "Error: Wrong split algo.") unmergedReco = Fileset(name = "/TestWorkload/MonteCarloFromGEN/unmerged-outputRECORECO") unmergedReco.loadData() recoMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMergeoutputRECORECO") recoMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedReco, workflow = recoMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algo: %s" % mergeSubscription["split_algo"]) unmergedAlca = Fileset(name = "/TestWorkload/MonteCarloFromGEN/unmerged-outputALCARECOALCARECO") unmergedAlca.loadData() alcaMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMergeoutputALCARECOALCARECO") alcaMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedAlca, workflow = alcaMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algo: %s" % mergeSubscription["split_algo"]) for procOutput in ["outputRECORECO", "outputALCARECOALCARECO"]: unmerged = Fileset(name = "/TestWorkload/MonteCarloFromGEN/unmerged-%s" % procOutput) unmerged.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENCleanupUnmerged%s" % procOutput) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmerged, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong split algo.") procLogCollect = Fileset(name = "/TestWorkload/MonteCarloFromGEN/unmerged-logArchive") procLogCollect.loadData() procLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN/LogCollect") procLogCollectWorkflow.load() logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") procLogCollect = Fileset(name = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMergeoutputRECORECO/merged-logArchive") procLogCollect.loadData() procLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMergeoutputRECORECO/MonteCarloFromGENoutputRECORECOMergeLogCollect") procLogCollectWorkflow.load() logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") procLogCollect = Fileset(name = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMergeoutputALCARECOALCARECO/merged-logArchive") procLogCollect.loadData() procLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/MonteCarloFromGEN/MonteCarloFromGENMergeoutputALCARECOALCARECO/MonteCarloFromGENoutputALCARECOALCARECOMergeLogCollect") procLogCollectWorkflow.load() logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") return
class TaskArchiverTest(EmulatedUnitTestCase): """ TestCase for TestTaskArchiver module """ _setup_done = False _teardown = False _maxMessage = 10 OWNERDN = os.environ[ 'OWNERDN'] if 'OWNERDN' in os.environ else "Generic/OWNERDN" def setUp(self): """ setup for test. """ super(TaskArchiverTest, self).setUp() myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema( customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"], useDefault=False) self.databaseName = "taskarchiver_t_0" self.testInit.setupCouch("%s/workloadsummary" % self.databaseName, "WorkloadSummary") self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump") self.testInit.setupCouch("wmagent_summary_t", "WMStats") self.testInit.setupCouch("wmagent_summary_central_t", "WMStats") self.testInit.setupCouch("stat_summary_t", "SummaryStats") reqmgrdb = "reqmgrdb_t" self.testInit.setupCouch(reqmgrdb, "ReqMgr") reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb) self.requestWriter = RequestDBWriter(reqDBURL) self.requestWriter.defaultStale = {} self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daofactory(classname="Jobs.GetAllJobs") self.inject = self.daofactory( classname="Workflow.MarkInjectedWorkflows") self.testDir = self.testInit.generateWorkDir() os.makedirs(os.path.join(self.testDir, 'specDir')) self.nJobs = 10 self.campaignName = 'aCampaign' return def tearDown(self): """ Database deletion """ self.testInit.clearDatabase(modules=["WMCore.WMBS"]) self.testInit.delWorkDir() self.testInit.tearDownCouch() return def getConfig(self): """ _createConfig_ General config file """ config = self.testInit.getConfiguration() # self.testInit.generateWorkDir(config) config.section_("General") config.General.workDir = "." config.General.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2" config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv52.fnal.gov:5984") config.JobStateMachine.couchDBName = self.databaseName config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' config.JobStateMachine.summaryStatsDBName = 'stat_summary_t' config.component_("JobCreator") config.JobCreator.jobCacheDir = os.path.join(self.testDir, 'testDir') config.component_("TaskArchiver") config.TaskArchiver.componentDir = self.testDir config.TaskArchiver.WorkQueueParams = { 'CacheDir': config.JobCreator.jobCacheDir } config.TaskArchiver.pollInterval = 60 config.TaskArchiver.logLevel = 'INFO' config.TaskArchiver.timeOut = 0 config.TaskArchiver.histogramKeys = [ 'AvgEventTime', 'writeTotalMB', 'jobTime' ] config.TaskArchiver.histogramBins = 5 config.TaskArchiver.histogramLimit = 5 config.TaskArchiver.perfPrimaryDatasets = [ 'SingleMu', 'MuHad', 'MinimumBias' ] config.TaskArchiver.perfDashBoardMinLumi = 50 config.TaskArchiver.perfDashBoardMaxLumi = 9000 config.TaskArchiver.dqmUrl = 'https://cmsweb.cern.ch/dqm/dev/' config.TaskArchiver.dashBoardUrl = 'http://dashboard43.cern.ch/dashboard/request.py/putluminositydata' config.TaskArchiver.workloadSummaryCouchDBName = "%s/workloadsummary" % self.databaseName config.TaskArchiver.localWMStatsURL = "%s/%s" % ( config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) config.TaskArchiver.workloadSummaryCouchURL = config.JobStateMachine.couchurl config.TaskArchiver.requireCouch = True config.component_("AnalyticsDataCollector") config.AnalyticsDataCollector.centralRequestDBURL = '%s/reqmgrdb_t' % config.JobStateMachine.couchurl config.AnalyticsDataCollector.RequestCouchApp = "ReqMgr" config.section_("ACDC") config.ACDC.couchurl = config.JobStateMachine.couchurl config.ACDC.database = config.JobStateMachine.couchDBName # Make the jobCacheDir os.mkdir(config.JobCreator.jobCacheDir) # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" config.section_("Agent") config.Agent.serverDN = "/we/bypass/myproxy/logon" return config def createWorkload(self, workloadName): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload(workloadName) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.setCampaign(self.campaignName) workload.save(workloadName) return workload def createTestJobGroup(self, config, name="TestWorkthrough", filesetName="TestFileset", specLocation="spec.xml", error=False, task="/TestWorkload/ReReco", jobType="Processing"): """ Creates a group of several jobs """ testWorkflow = Workflow(spec=specLocation, owner=self.OWNERDN, name=name, task=task, owner_vogroup="", owner_vorole="") testWorkflow.create() self.inject.execute(names=[name], injected=True) testWMBSFileset = Fileset(name=filesetName) testWMBSFileset.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) outputWMBSFileset = Fileset(name='%sOutput' % filesetName) outputWMBSFileset.create() testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10) testFileC.addRun(Run(10, *[12312])) testFileC.setLocation('malpaquet') testFileC.create() outputWMBSFileset.addFile(testFileC) outputWMBSFileset.commit() outputWMBSFileset.markOpen(0) testWorkflow.addOutput('output', outputWMBSFileset) testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow, type=jobType) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() for i in range(0, self.nJobs): testJob = Job(name=makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() changer = ChangeState(config) report1 = Report() report2 = Report() if error: path1 = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl") path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') else: path1 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'mergeReport1.pkl') path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') report1.load(filename=path1) report2.load(filename=path2) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') for i in range(self.nJobs): if i < self.nJobs // 2: testJobGroup.jobs[i]['fwjr'] = report1 else: testJobGroup.jobs[i]['fwjr'] = report2 changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'retrydone', 'jobfailed') changer.propagate(testJobGroup.jobs, 'exhausted', 'retrydone') changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted') testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup def createGiantJobSet(self, name, config, nSubs=10, nJobs=10, nFiles=1, spec="spec.xml"): """ Creates a massive set of jobs """ jobList = [] for i in range(0, nSubs): # Make a bunch of subscriptions localName = '%s-%i' % (name, i) testWorkflow = Workflow(spec=spec, owner=self.OWNERDN, name=localName, task="Test", owner_vogroup="", owner_vorole="") testWorkflow.create() testWMBSFileset = Fileset(name=localName) testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() filesToComplete = [] for j in range(0, nJobs): # Create jobs for each subscription testFileA = File(lfn="%s-%i-lfnA" % (localName, j), size=1024, events=10) testFileA.addRun( Run( 10, *[ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ])) testFileA.setLocation('malpaquet') testFileA.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.commit() filesToComplete.append(testFileA) testJob = Job(name='%s-%i' % (localName, j)) testJob.addFile(testFileA) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJobGroup.add(testJob) jobList.append(testJob) for k in range(0, nFiles): # Create output files testFile = File(lfn="%s-%i-output" % (localName, k), size=1024, events=10) testFile.addRun(Run(10, *[12312])) testFile.setLocation('malpaquet') testFile.create() testJobGroup.output.addFile(testFile) testJobGroup.output.commit() testJobGroup.commit() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'success', 'complete') changer.propagate(testJobGroup.jobs, 'cleanout', 'success') testWMBSFileset.markOpen(0) testSubscription.completeFiles(filesToComplete) return jobList def getPerformanceFromDQM(self, dqmUrl, dataset, run): # Make function to fetch this from DQM. Returning Null or False if it fails getUrl = "%sjsonfairy/archive/%s%s/DQM/TimerService/event_byluminosity" % ( dqmUrl, run, dataset) # Assert if the URL is assembled as expected if run == 207214: self.assertEqual( 'https://cmsweb.cern.ch/dqm/dev/jsonfairy/archive/207214/MinimumBias/Commissioning10-v4/DQM/DQM/TimerService/event_byluminosity', getUrl) # let's suppose it works.. testResponseFile = open( os.path.join(getTestBase(), 'WMComponent_t/TaskArchiver_t/DQMGUIResponse.json'), 'r') response = testResponseFile.read() testResponseFile.close() responseJSON = json.loads(response) return responseJSON def filterInterestingPerfPoints(self, responseJSON, minLumi, maxLumi): worthPoints = {} points = responseJSON["hist"]["bins"]["content"] for i in range(responseJSON["hist"]["xaxis"]["first"]["id"], responseJSON["hist"]["xaxis"]["last"]["id"]): # is the point worth it? if yes add to interesting points dictionary. # 1 - non 0 # 2 - between minimum and maximum expected luminosity # FIXME : 3 - population in dashboard for the bin interval < 100 # Those should come from the config : if points[i] == 0: continue binSize = responseJSON["hist"]["xaxis"]["last"][ "value"] // responseJSON["hist"]["xaxis"]["last"]["id"] # Fetching the important values instLuminosity = i * binSize timePerEvent = points[i] if instLuminosity > minLumi and instLuminosity < maxLumi: worthPoints[instLuminosity] = timePerEvent return worthPoints def publishPerformanceDashBoard(self, dashBoardUrl, PD, release, worthPoints): dashboardPayload = [] for instLuminosity in worthPoints: timePerEvent = int(worthPoints[instLuminosity]) dashboardPayload.append({ "primaryDataset": PD, "release": release, "integratedLuminosity": instLuminosity, "timePerEvent": timePerEvent }) data = "{\"data\":%s}" % str(dashboardPayload).replace("\'", "\"") # let's suppose it works.. testDashBoardPayloadFile = open( os.path.join(getTestBase(), 'WMComponent_t/TaskArchiver_t/DashBoardPayload.json'), 'r') testDashBoardPayload = testDashBoardPayloadFile.read() testDashBoardPayloadFile.close() self.assertEqual(data, testDashBoardPayload) return True def populateWorkflowWithCompleteStatus(self, name="TestWorkload"): schema = generate_reqmgr_schema(1) schema[0]["RequestName"] = name self.requestWriter.insertGenericRequest(schema[0]) result = self.requestWriter.updateRequestStatus(name, "completed") return result def testA_BasicFunctionTest(self): """ _BasicFunctionTest_ Tests the components, by seeing if they can process a simple set of closeouts """ myThread = threading.currentThread() config = self.getConfig() workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl') workload = self.createWorkload(workloadName=workloadPath) testJobGroup = self.createTestJobGroup(config=config, name=workload.name(), specLocation=workloadPath, error=False) # Create second workload testJobGroup2 = self.createTestJobGroup( config=config, name=workload.name(), filesetName="TestFileset_2", specLocation=workloadPath, task="/TestWorkload/ReReco/LogCollect", jobType="LogCollect") cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "ReReco") os.makedirs(cachePath) self.assertTrue(os.path.exists(cachePath)) cachePath2 = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "LogCollect") os.makedirs(cachePath2) self.assertTrue(os.path.exists(cachePath2)) result = myThread.dbi.processData( "SELECT * FROM wmbs_subscription")[0].fetchall() self.assertEqual(len(result), 2) workflowName = "TestWorkload" dbname = config.TaskArchiver.workloadSummaryCouchDBName couchdb = CouchServer(config.JobStateMachine.couchurl) workdatabase = couchdb.connectDatabase(dbname) jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName) fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName) jobs = jobdb.loadView("JobDump", "jobsByWorkflowName", options={ "startkey": [workflowName], "endkey": [workflowName, {}] })['rows'] fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName", options={ "startkey": [workflowName], "endkey": [workflowName, {}] })['rows'] self.assertEqual(len(jobs), 2 * self.nJobs) from WMCore.WMBS.CreateWMBSBase import CreateWMBSBase create = CreateWMBSBase() tables = [] for x in create.requiredTables: tables.append(x[2:]) self.populateWorkflowWithCompleteStatus() testTaskArchiver = TaskArchiverPoller(config=config) testTaskArchiver.algorithm() cleanCouch = CleanCouchPoller(config=config) cleanCouch.setup() cleanCouch.algorithm() result = myThread.dbi.processData( "SELECT * FROM wmbs_job")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData( "SELECT * FROM wmbs_subscription")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData( "SELECT * FROM wmbs_jobgroup")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData( "SELECT * FROM wmbs_fileset")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData( "SELECT * FROM wmbs_file_details")[0].fetchall() self.assertEqual(len(result), 0) # Make sure we deleted the directory self.assertFalse(os.path.exists(cachePath)) self.assertFalse( os.path.exists( os.path.join(self.testDir, 'workloadTest/TestWorkload'))) testWMBSFileset = Fileset(id=1) self.assertEqual(testWMBSFileset.exists(), False) workloadSummary = workdatabase.document(id="TestWorkload") # Check ACDC self.assertEqual(workloadSummary['ACDCServer'], sanitizeURL(config.ACDC.couchurl)['url']) # Check the output self.assertEqual(list(workloadSummary['output']), ['/Electron/MorePenguins-v0/RECO']) self.assertEqual( sorted(workloadSummary['output']['/Electron/MorePenguins-v0/RECO'] ['tasks']), ['/TestWorkload/ReReco', '/TestWorkload/ReReco/LogCollect']) # Check performance # Check histograms self.assertAlmostEqual( workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1'] ['AvgEventTime']['histogram'][0]['average'], 0.89405199999999996, places=2) self.assertEqual( workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1'] ['AvgEventTime']['histogram'][0]['nEvents'], 10) # Check standard performance self.assertAlmostEqual( workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1'] ['TotalJobCPU']['average'], 17.786300000000001, places=2) self.assertAlmostEqual( workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1'] ['TotalJobCPU']['stdDev'], 0.0, places=2) # Check worstOffenders self.assertEqual( workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1'] ['AvgEventTime']['worstOffenders'], [{ 'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 1 }, { 'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 1 }, { 'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 2 }]) # Check retryData self.assertEqual(workloadSummary['retryData']['/TestWorkload/ReReco'], {'1': 10}) logCollectPFN = 'srm://srm-cms.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/cms/store/logs/prod/2012/11/WMAgent/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8-AlcaSkimLogCollect-1-logs.tar' self.assertEqual(workloadSummary['logArchives'], { '/TestWorkload/ReReco/LogCollect': [logCollectPFN for _ in range(10)] }) # LogCollect task is made out of identical FWJRs # assert that it is identical for x in workloadSummary['performance'][ '/TestWorkload/ReReco/LogCollect']['cmsRun1']: if x in config.TaskArchiver.histogramKeys: continue for y in ['average', 'stdDev']: self.assertAlmostEqual( workloadSummary['performance'] ['/TestWorkload/ReReco/LogCollect']['cmsRun1'][x][y], workloadSummary['performance']['/TestWorkload/ReReco'] ['cmsRun1'][x][y], places=2) return def testB_testErrors(self): """ _testErrors_ Test with a failed FWJR """ config = self.getConfig() workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl') workload = self.createWorkload(workloadName=workloadPath) testJobGroup = self.createTestJobGroup(config=config, name=workload.name(), specLocation=workloadPath, error=True) # Create second workload testJobGroup2 = self.createTestJobGroup( config=config, name=workload.name(), filesetName="TestFileset_2", specLocation=workloadPath, task="/TestWorkload/ReReco/LogCollect", jobType="LogCollect") cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "ReReco") os.makedirs(cachePath) self.assertTrue(os.path.exists(cachePath)) couchdb = CouchServer(config.JobStateMachine.couchurl) jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName) fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName) jobdb.loadView("JobDump", "jobsByWorkflowName", options={ "startkey": [workload.name()], "endkey": [workload.name(), {}] })['rows'] fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName", options={ "startkey": [workload.name()], "endkey": [workload.name(), {}] })['rows'] self.populateWorkflowWithCompleteStatus() testTaskArchiver = TaskArchiverPoller(config=config) testTaskArchiver.algorithm() cleanCouch = CleanCouchPoller(config=config) cleanCouch.setup() cleanCouch.algorithm() dbname = getattr(config.JobStateMachine, "couchDBName") workdatabase = couchdb.connectDatabase("%s/workloadsummary" % dbname) workloadSummary = workdatabase.document(id=workload.name()) self.assertEqual( workloadSummary['errors']['/TestWorkload/ReReco']['failureTime'], 500) self.assertTrue('99999' in workloadSummary['errors'] ['/TestWorkload/ReReco']['cmsRun1']) failedRunInfo = workloadSummary['errors']['/TestWorkload/ReReco'][ 'cmsRun1']['99999']['runs'] self.assertEqual( failedRunInfo, {'10': [[12312, 12312]]}, "Wrong lumi information in the summary for failed jobs") # Check the failures by site histograms self.assertEqual( workloadSummary['histograms']['workflowLevel']['failuresBySite'] ['data']['T1_IT_CNAF']['Failed Jobs'], 10) self.assertEqual( workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco'] ['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['99999'], 10) self.assertEqual( workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco'] ['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['8020'], 10) self.assertEqual( workloadSummary['histograms']['workflowLevel']['failuresBySite'] ['average']['Failed Jobs'], 10) self.assertEqual( workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco'] ['cmsRun1']['errorsBySite']['average']['99999'], 10) self.assertEqual( workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco'] ['cmsRun1']['errorsBySite']['average']['8020'], 10) self.assertEqual( workloadSummary['histograms']['workflowLevel']['failuresBySite'] ['stdDev']['Failed Jobs'], 0) self.assertEqual( workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco'] ['cmsRun1']['errorsBySite']['stdDev']['99999'], 0) self.assertEqual( workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco'] ['cmsRun1']['errorsBySite']['stdDev']['8020'], 0) return @attr("integration") def testC_Profile(self): """ _Profile_ DON'T RUN THIS! """ import cProfile import pstats name = makeUUID() config = self.getConfig() jobList = self.createGiantJobSet(name=name, config=config, nSubs=10, nJobs=1000, nFiles=10) cleanCouch = CleanCouchPoller(config=config) cleanCouch.setup() cProfile.runctx("cleanCouch.algorithm()", globals(), locals(), filename="testStats.stat") p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats() return @attr("integration") def testD_Timing(self): """ _Timing_ This is to see how fast things go. """ myThread = threading.currentThread() name = makeUUID() config = self.getConfig() jobList = self.createGiantJobSet(name=name, config=config, nSubs=10, nJobs=1000, nFiles=10) testTaskArchiver = TaskArchiverPoller(config=config) startTime = time.time() testTaskArchiver.algorithm() stopTime = time.time() result = myThread.dbi.processData( "SELECT * FROM wmbs_job")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData( "SELECT * FROM wmbs_subscription")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData( "SELECT * FROM wmbs_jobgroup")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData( "SELECT * FROM wmbs_file_details")[0].fetchall() self.assertEqual(len(result), 0) testWMBSFileset = Fileset(id=1) self.assertEqual(testWMBSFileset.exists(), False) logging.info("TaskArchiver took %f seconds", (stopTime - startTime)) def testDQMRecoPerformanceToDashBoard(self): myThread = threading.currentThread() listRunsWorkflow = self.dbsDaoFactory(classname="ListRunsWorkflow") # Didn't like to have done that, but the test doesn't provide all info I need in the system, so faking it: myThread.dbi.processData( """insert into dbsbuffer_workflow(id, name) values (1, 'TestWorkload')""", transaction=False) myThread.dbi.processData( """insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (1, '/store/t/e/s/t.test', 1, 1)""", transaction=False) myThread.dbi.processData( """insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (2, '/store/t/e/s/t.test2', 1, 1)""", transaction=False) myThread.dbi.processData( """insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207214, 100, 1)""", transaction=False) myThread.dbi.processData( """insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207215, 200, 2)""", transaction=False) config = self.getConfig() dqmUrl = getattr(config.TaskArchiver, "dqmUrl") perfDashBoardMinLumi = getattr(config.TaskArchiver, "perfDashBoardMinLumi") perfDashBoardMaxLumi = getattr(config.TaskArchiver, "perfDashBoardMaxLumi") dashBoardUrl = getattr(config.TaskArchiver, "dashBoardUrl") workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl') workload = self.createWorkload(workloadName=workloadPath) testJobGroup = self.createTestJobGroup(config=config, name=workload.name(), specLocation=workloadPath, error=True) testJobGroup2 = self.createTestJobGroup( config=config, name=workload.name(), filesetName="TestFileset_2", specLocation=workloadPath, task="/TestWorkload/ReReco/LogCollect", jobType="LogCollect") # Adding request type as ReReco, real ReqMgr requests have it workload.data.request.section_("schema") workload.data.request.schema.RequestType = "ReReco" workload.data.request.schema.CMSSWVersion = 'test_compops_CMSSW_5_3_6_patch1' workload.getTask('ReReco').addInputDataset(name='/a/b/c', primary='a', processed='b', tier='c') interestingPDs = getattr(config.TaskArchiver, "perfPrimaryDatasets") interestingDatasets = [] # Are the datasets from this request interesting? Do they have DQM output? One might ask afterwards if they have harvest for dataset in workload.listOutputDatasets(): (nothing, PD, procDataSet, dataTier) = dataset.split('/') if PD in interestingPDs and dataTier == "DQM": interestingDatasets.append(dataset) # We should have found 1 interesting dataset self.assertAlmostEqual(len(interestingDatasets), 1) if len(interestingDatasets) == 0: return # Request will be only interesting for performance if it's a ReReco or PromptReco (isReReco, isPromptReco) = (False, False) if getattr(workload.data.request.schema, "RequestType", None) == 'ReReco': isReReco = True # Yes, few people like magic strings, but have a look at : # https://github.com/dmwm/T0/blob/master/src/python/T0/RunConfig/RunConfigAPI.py#L718 # Might be safe enough # FIXME: in TaskArchiver, add a test to make sure that the dataset makes sense (procDataset ~= /a/ERA-PromptReco-vVERSON/DQM) if re.search('PromptReco', workload.name()): isPromptReco = True if not (isReReco or isPromptReco): return self.assertTrue(isReReco) self.assertFalse(isPromptReco) # We are not interested if it's not a PromptReco or a ReReco if not (isReReco or isPromptReco): return if isReReco: release = getattr(workload.data.request.schema, "CMSSWVersion") if not release: logging.info("no release for %s, bailing out", workload.name()) else: release = getattr( workload.tasks.Reco.steps.cmsRun1.application.setup, "cmsswVersion") if not release: logging.info("no release for %s, bailing out", workload.name()) self.assertEqual(release, "test_compops_CMSSW_5_3_6_patch1") # If all is true, get the run numbers processed by this worklfow runList = listRunsWorkflow.execute(workflow=workload.name()) self.assertEqual([207214, 207215], runList) # GO to DQM GUI, get what you want # https://cmsweb.cern.ch/dqm/offline/jsonfairy/archive/211313/PAMuon/HIRun2013-PromptReco-v1/DQM/DQM/TimerService/event for dataset in interestingDatasets: (nothing, PD, procDataSet, dataTier) = dataset.split('/') worthPoints = {} for run in runList: responseJSON = self.getPerformanceFromDQM(dqmUrl, dataset, run) worthPoints.update( self.filterInterestingPerfPoints(responseJSON, perfDashBoardMinLumi, perfDashBoardMaxLumi)) # Publish dataset performance to DashBoard. if not self.publishPerformanceDashBoard(dashBoardUrl, PD, release, worthPoints): logging.info( "something went wrong when publishing dataset %s to DashBoard", dataset) return
class PromptRecoTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("promptreco_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("promptreco_t") self.testDir = self.testInit.generateWorkDir() return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def setupPromptSkimConfigObject(self): """ _setupPromptSkimConfigObject_ Creates a custom config object for testing of the skim functionality """ self.promptSkim = ConfigSection(name="Tier1Skim") self.promptSkim.SkimName = "TestSkim1" self.promptSkim.DataTier = "RECO" self.promptSkim.TwoFileRead = False self.promptSkim.ProcessingVersion = "PromptSkim-v1" self.promptSkim.ConfigURL = "http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi/CMSSW/Configuration/DataOps/python/prescaleskimmer.py?revision=1.1" def testPromptReco(self): """ _testPromptReco_ Create a Prompt Reconstruction workflow and verify it installs into WMBS correctly. """ testArguments = PromptRecoWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["EnableHarvesting"] = True factory = PromptRecoWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Reco", "SomeBlock", cachepath = self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) recoWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco") recoWorkflow.load() self.assertEqual(len(recoWorkflow.outputMap.keys()), len(testArguments["WriteTiers"]) + 1, "Error: Wrong number of WF outputs in the Reco WF.") goldenOutputMods = ["write_RECO", "write_ALCARECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_ALCARECO": self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = recoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = recoWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") alcaSkimWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim") alcaSkimWorkflow.load() self.assertEqual(len(alcaSkimWorkflow.outputMap.keys()), len(testArguments["AlcaSkims"]) + 1, "Error: Wrong number of WF outputs in the AlcaSkim WF.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") dqmWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged") dqmWorkflow.load() logArchOutput = dqmWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = dqmWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-Reco-SomeBlock") topLevelFileset.loadData() recoSubscription = Subscription(fileset = topLevelFileset, workflow = recoWorkflow) recoSubscription.loadData() self.assertEqual(recoSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(recoSubscription["split_algo"], "EventBased", "Error: Wrong split algorithm. %s" % recoSubscription["split_algo"]) alcaRecoFileset = Fileset(name = "/TestWorkload/Reco/unmerged-write_ALCARECO") alcaRecoFileset.loadData() alcaSkimSubscription = Subscription(fileset = alcaRecoFileset, workflow = alcaSkimWorkflow) alcaSkimSubscription.loadData() self.assertEqual(alcaSkimSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(alcaSkimSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % alcaSkimSubscription["split_algo"]) mergedDQMFileset = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_DQM/merged-Merged") mergedDQMFileset.loadData() dqmSubscription = Subscription(fileset = mergedDQMFileset, workflow = dqmWorkflow) dqmSubscription.loadData() self.assertEqual(dqmSubscription["type"], "Harvesting", "Error: Wrong subscription type.") self.assertEqual(dqmSubscription["split_algo"], "Harvest", "Error: Wrong split algo.") unmergedOutputs = ["write_RECO", "write_AOD", "write_DQM"] for unmergedOutput in unmergedOutputs: unmergedDataTier = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % unmergedOutput) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedDataTier, workflow = dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = [] for alcaProd in testArguments["AlcaSkims"]: unmergedOutputs.append("ALCARECOStream%s" % alcaProd) for unmergedOutput in unmergedOutputs: unmergedAlcaSkim = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % unmergedOutput) unmergedAlcaSkim.loadData() alcaSkimMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % unmergedOutput) alcaSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedAlcaSkim, workflow = alcaSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM", "write_ALCARECO"] for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmerged%s" %goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") recoLogCollect = Fileset(name = "/TestWorkload/Reco/unmerged-logArchive") recoLogCollect.loadData() recoLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/LogCollect") recoLogCollectWorkflow.load() logCollectSub = Subscription(fileset = recoLogCollect, workflow = recoLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive") alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect") alcaSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: recoMergeLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod) recoMergeLogCollect.loadData() recoMergeLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s/Reco%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) recoMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = recoMergeLogCollect, workflow = recoMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod) alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/AlcaSkim%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) alcaSkimLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") dqmHarvestLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive") dqmHarvestLogCollect.loadData() dqmHarvestLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect") dqmHarvestLogCollectWorkflow.load() logCollectSub = Subscription(fileset = dqmHarvestLogCollect, workflow = dqmHarvestLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") return @attr("integration") def testPromptRecoWithSkims(self): """ _testT1PromptRecoWithSkim_ Create a T1 Prompt Reconstruction workflow with PromptSkims and verify it installs into WMBS correctly. """ self.setupPromptSkimConfigObject() testArguments = PromptRecoWorkloadFactory.getTestArguments() testArguments["PromptSkims"] = [self.promptSkim] testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = "promptreco_t" testArguments["EnvPath"] = os.environ.get("EnvPath", None) testArguments["BinPath"] = os.environ.get("BinPath", None) factory = PromptRecoWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Reco", "SomeBlock", cachepath = self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) recoWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco") recoWorkflow.load() self.assertEqual(len(recoWorkflow.outputMap.keys()), len(testArguments["WriteTiers"]) + 1, "Error: Wrong number of WF outputs in the Reco WF.") goldenOutputMods = ["write_RECO", "write_ALCARECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = recoWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_ALCARECO": self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = recoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = recoWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") alcaSkimWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim") alcaSkimWorkflow.load() self.assertEqual(len(alcaSkimWorkflow.outputMap.keys()), len(testArguments["AlcaSkims"]) + 1, "Error: Wrong number of WF outputs in the AlcaSkim WF.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = alcaSkimWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") promptSkimWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1") promptSkimWorkflow.load() self.assertEqual(len(promptSkimWorkflow.outputMap.keys()), 6, "Error: Wrong number of WF outputs.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: mergedOutput = promptSkimWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = promptSkimWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = promptSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = promptSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-Reco-SomeBlock") topLevelFileset.loadData() recoSubscription = Subscription(fileset = topLevelFileset, workflow = recoWorkflow) recoSubscription.loadData() self.assertEqual(recoSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(recoSubscription["split_algo"], "EventBased", "Error: Wrong split algorithm. %s" % recoSubscription["split_algo"]) alcaRecoFileset = Fileset(name = "/TestWorkload/Reco/unmerged-write_ALCARECO") alcaRecoFileset.loadData() alcaSkimSubscription = Subscription(fileset = alcaRecoFileset, workflow = alcaSkimWorkflow) alcaSkimSubscription.loadData() self.assertEqual(alcaSkimSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(alcaSkimSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % alcaSkimSubscription["split_algo"]) mergedRecoFileset = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/merged-Merged") mergedRecoFileset.loadData() promptSkimSubscription = Subscription(fileset = mergedRecoFileset, workflow = promptSkimWorkflow) promptSkimSubscription.loadData() self.assertEqual(promptSkimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(promptSkimSubscription["split_algo"], "FileBased", "Error: Wrong split algorithm. %s" % promptSkimSubscription["split_algo"]) unmergedOutputs = ["write_RECO", "write_AOD", "write_DQM"] for unmergedOutput in unmergedOutputs: unmergedDataTier = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % unmergedOutput) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedDataTier, workflow = dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = [] for alcaProd in testArguments["AlcaSkims"]: unmergedOutputs.append("ALCARECOStream%s" % alcaProd) for unmergedOutput in unmergedOutputs: unmergedAlcaSkim = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % unmergedOutput) unmergedAlcaSkim.loadData() alcaSkimMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % unmergedOutput) alcaSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedAlcaSkim, workflow = alcaSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for unmergedOutput in unmergedOutputs: unmergedPromptSkim = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-%s" % unmergedOutput) unmergedPromptSkim.loadData() promptSkimMergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s" % unmergedOutput) promptSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedPromptSkim, workflow = promptSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM", "write_ALCARECO"] for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-%s" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmerged%s" %goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-%s" % unmergedOutput) unmergedFileset.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1CleanupUnmerged%s" % unmergedOutput) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmergedFileset, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong split algorithm. %s" % cleanupSubscription["split_algo"]) recoLogCollect = Fileset(name = "/TestWorkload/Reco/unmerged-logArchive") recoLogCollect.loadData() recoLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/LogCollect") recoLogCollectWorkflow.load() logCollectSub = Subscription(fileset = recoLogCollect, workflow = recoLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive") alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect") alcaSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") promptSkimLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/unmerged-logArchive") promptSkimLogCollect.loadData() promptSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1LogCollect") promptSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset = promptSkimLogCollect, workflow = promptSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: recoMergeLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod) recoMergeLogCollect.loadData() recoMergeLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMerge%s/Reco%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) recoMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = recoMergeLogCollect, workflow = recoMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: alcaSkimLogCollect = Fileset(name = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod) alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/AlcaSkim%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) alcaSkimLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = alcaSkimLogCollect, workflow = alcaSkimLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["fakeSkimOut1", "fakeSkimOut2", "fakeSkimOut3", "fakeSkimOut4", "fakeSkimOut5"] for goldenOutputMod in goldenOutputMods: promptSkimMergeLogCollect = Fileset(name = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/merged-logArchive" % goldenOutputMod) promptSkimMergeLogCollect.loadData() promptSkimMergeLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Reco/RecoMergewrite_RECO/TestSkim1/TestSkim1Merge%s/TestSkim1%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) promptSkimMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset = promptSkimMergeLogCollect, workflow = promptSkimMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") return
class JobTrackerTest(unittest.TestCase): """ TestCase for TestJobTracker module """ _maxMessage = 10 def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"]) self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl" ], useDefault=False) self.testInit.setupCouch("jobtracker_t/jobs", "JobDump") self.testInit.setupCouch("jobtracker_t/fwjrs", "FWJRDump") self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs") #Create sites in resourceControl resourceControl = ResourceControl() resourceControl.insertSite(siteName='malpaquet', pnn='se.malpaquet', ceName='malpaquet', plugin="CondorPlugin") resourceControl.insertThreshold(siteName = 'malpaquet', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="malpaquet", pnn="malpaquet", ceName="malpaquet", plugin="CondorPlugin") # Create user newuser = self.daoFactory(classname="Users.New") newuser.execute(dn="jchurchill") # We actually need the user name self.user = getpass.getuser() self.testDir = self.testInit.generateWorkDir() self.configFile = EmulatorSetup.setupWMAgentConfig() def tearDown(self): """ Database deletion """ self.testInit.clearDatabase(modules=[ "WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl" ]) self.testInit.delWorkDir() self.testInit.tearDownCouch() EmulatorSetup.deleteConfig(self.configFile) return def getConfig(self): """ _getConfig_ Build a basic JobTracker config """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("Agent") config.Agent.agentName = 'testAgent' config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') #config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh') config.JobSubmitter.submitScript = os.path.join( WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # BossAir config.component_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "jobtracker_t" return config def createTestJobs(self, nJobs, cacheDir): """ _createTestJobs_ Create several jobs """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() # Create a file testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileA.create() baseName = makeUUID() # Now create a job for i in range(nJobs): testJob = Job(name='%s-%i' % (baseName, i)) testJob.addFile(testFileA) testJob['location'] = 'malpaquet' testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob.create(testJobGroup) testJob.save() testJobGroup.add(testJob) testJobGroup.commit() # Set test job caches for job in testJobGroup.jobs: job.setCache(cacheDir) return testJobGroup @attr('integration') def testA_CondorTest(self): """ _CondorTest_ Because I don't want this test to be submitter dependent: Create a dummy condor job. Submit a dummy condor job. Track it. Kill it. Exit """ myThread = threading.currentThread() # This has to be run with an empty queue nRunning = getCondorRunningJobs(self.user) self.assertEqual( nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) nJobs = 10 jobCE = 'cmsosgce.fnal.gov/jobmanager-condor' # Create directories cacheDir = os.path.join(self.testDir, 'CacheDir') submitDir = os.path.join(self.testDir, 'SubmitDir') if not os.path.isdir(cacheDir): os.makedirs(cacheDir) if not os.path.isdir(submitDir): os.makedirs(submitDir) # Get config config = self.getConfig() # Get jobGroup testJobGroup = self.createTestJobs(nJobs=nJobs, cacheDir=cacheDir) # Propogate jobs changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') result = self.getJobs.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nJobs) jobTracker = JobTrackerPoller(config) jobTracker.setup() # First iteration # There are no jobs in the tracker, # The tracker should register the jobs as missing # This should tell it that they've finished # So the tracker should send them onwards jobTracker.algorithm() result = self.getJobs.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nJobs) result = self.getJobs.execute(state='complete', jobType="Processing") self.assertEqual(len(result), 0) # Second iteration # Reset the jobs # This time submit them to the queue # The jobs should remain in holding changer.propagate(testJobGroup.jobs, 'executing', 'created') result = self.getJobs.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nJobs) # Create a submit script createSubmitScript(submitDir) jobPackage = os.path.join(self.testDir, 'JobPackage.pkl') f = open(jobPackage, 'w') f.write(' ') f.close() sandbox = os.path.join(self.testDir, 'sandbox.box') f = open(sandbox, 'w') f.write(' ') f.close() for job in testJobGroup.jobs: job['plugin'] = 'CondorPlugin' job['userdn'] = 'jchurchill' job['custom'] = {'location': 'malpaquet'} job['cache_dir'] = self.testDir job['sandbox'] = sandbox job['packageDir'] = self.testDir info = {} info['packageDir'] = self.testDir info['index'] = 0 info['sandbox'] = sandbox jobTracker.bossAir.submit(jobs=testJobGroup.jobs, info=info) time.sleep(1) # All jobs should be running nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, nJobs) # Run the algorithm. After this # all jobs should still be running jobTracker.algorithm() # Are jobs in the right state? result = self.getJobs.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nJobs) result = self.getJobs.execute(state='Complete', jobType="Processing") self.assertEqual(len(result), 0) # Are jobs still in the condor_q nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, nJobs) # Then we're done jobTracker.bossAir.kill(jobs=testJobGroup.jobs) # No jobs should be left nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, 0) jobTracker.algorithm() # Are jobs in the right state? result = self.getJobs.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), 0) result = self.getJobs.execute(state='Complete', jobType="Processing") self.assertEqual(len(result), nJobs) # This is optional if you want to look at what # files were actually created during running #if os.path.isdir('testDir'): # shutil.rmtree('testDir') #shutil.copytree('%s' %self.testDir, os.path.join(os.getcwd(), 'testDir')) return @attr('integration') def testB_ReallyLongTest(self): """ _ReallyLongTest_ Run a really long test using the condor plugin """ return # This has to be run with an empty queue nRunning = getCondorRunningJobs(self.user) self.assertEqual( nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) myThread = threading.currentThread() # This has to be run with an empty queue nRunning = getCondorRunningJobs(self.user) self.assertEqual( nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) nJobs = 500 jobCE = 'cmsosgce.fnal.gov/jobmanager-condor' # Create directories cacheDir = os.path.join(self.testDir, 'CacheDir') submitDir = os.path.join(self.testDir, 'SubmitDir') if not os.path.isdir(cacheDir): os.makedirs(cacheDir) if not os.path.isdir(submitDir): os.makedirs(submitDir) # Get config config = self.getConfig() # Get jobGroup testJobGroup = self.createTestJobs(nJobs=nJobs, cacheDir=cacheDir) # Propogate jobs changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') jobTracker = JobTrackerPoller(config) jobTracker.setup() # Now create some jobs for job in testJobGroup.jobs[:(nJobs / 2)]: jdl = createJDL(id=job['id'], directory=submitDir, jobCE=jobCE) jdlFile = os.path.join(submitDir, 'condorJDL_%i.jdl' % (job['id'])) handle = open(jdlFile, 'w') handle.writelines(jdl) handle.close() command = ["condor_submit", jdlFile] pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) pipe.communicate() startTime = time.time() cProfile.runctx("jobTracker.algorithm()", globals(), locals(), filename="testStats.stat") #jobTracker.algorithm() stopTime = time.time() # Are jobs in the right state? result = self.getJobs.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nJobs / 2) result = self.getJobs.execute(state='Complete', jobType="Processing") self.assertEqual(len(result), nJobs / 2) # Then we're done killList = [x['id'] for x in testJobGroup.jobs] jobTracker.killJobs(jobList=killList) # No jobs should be left nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, 0) print("Process took %f seconds to process %i classAds" % ((stopTime - startTime), nJobs / 2)) p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats() def testAlerts(self): """ Tests only alerts triggered from JobTrackerPoller. """ config = self.getConfig() jobTracker = JobTrackerPoller(config) jobTracker.sendAlert(6, msg="test message")
class DataCollectionService_t(unittest.TestCase): def setUp(self): """bootstrap tests""" self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.setupCouch("wmcore-acdc-datacollectionsvc", "GroupUser", "ACDC") return def tearDown(self): self.testInit.tearDownCouch() self.testInit.clearDatabase() return def testChunking(self): """ _testChunking_ Insert a workload and files that have several distinct sets of locations. Verify that the chunks are created correctly and that they only groups files that have the same set of locations. Also verify that the chunks are pulled out of ACDC correctly. """ dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc") def getJob(): job = Job() job["task"] = "/ACDCTest/reco" job["workflow"] = "ACDCTest" job["location"] = "cmssrm.fnal.gov" job["owner"] = "cmsdataops" job["group"] = "cmsdataops" return job testFileA = File(lfn=makeUUID(), size=1024, events=1024) testFileA.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileA.addRun(Run(1, 1, 2)) testFileB = File(lfn=makeUUID(), size=1024, events=1024) testFileB.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileB.addRun(Run(1, 3, 4)) testFileC = File(lfn=makeUUID(), size=1024, events=1024) testFileC.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileC.addRun(Run(1, 5, 6)) testJobA = getJob() testJobA.addFile(testFileA) testJobA.addFile(testFileB) testJobA.addFile(testFileC) testFileD = File(lfn=makeUUID(), size=1024, events=1024) testFileD.setLocation(["cmssrm.fnal.gov"]) testFileD.addRun(Run(2, 1, 2)) testFileE = File(lfn=makeUUID(), size=1024, events=1024) testFileE.setLocation(["cmssrm.fnal.gov"]) testFileE.addRun(Run(2, 3, 4)) testJobB = getJob() testJobB.addFile(testFileD) testJobB.addFile(testFileE) testFileF = File(lfn=makeUUID(), size=1024, events=1024, parents=set(["/some/parent/F"])) testFileF.setLocation( ["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"]) testFileF.addRun(Run(3, 1, 2)) testFileG = File(lfn=makeUUID(), size=1024, events=1024, parents=set(["/some/parent/G"])) testFileG.setLocation( ["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"]) testFileG.addRun(Run(3, 3, 4)) testFileH = File(lfn=makeUUID(), size=1024, events=1024, parents=set(["/some/parent/H"])) testFileH.setLocation( ["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"]) testFileH.addRun(Run(3, 5, 6)) testJobC = getJob() testJobC.addFile(testFileF) testJobC.addFile(testFileG) testJobC.addFile(testFileH) testFileI = File(lfn=makeUUID(), size=1024, events=1024, merged=True) testFileI.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileI.addRun(Run(4, 1, 2)) testFileJ = File(lfn=makeUUID(), size=1024, events=1024, merged=True) testFileJ.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileJ.addRun(Run(4, 3, 4)) testFileK = File(lfn=makeUUID(), size=1024, events=1024, merged=True) testFileK.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileK.addRun(Run(4, 5, 6)) testJobD = getJob() testJobD.addFile(testFileI) testJobD.addFile(testFileJ) testJobD.addFile(testFileK) dcs.failedJobs([testJobA, testJobB, testJobC, testJobD]) chunks = dcs.chunkFileset("ACDCTest", "/ACDCTest/reco", chunkSize=5) self.assertEqual( len(chunks), 4, "Error: There should be four chunks: %s" % len(chunks)) goldenMetaData = { 1: { "lumis": 2, "locations": ["castor.cern.ch", "cmssrm.fnal.gov"], "events": 1024 }, 2: { "lumis": 4, "locations": ["cmssrm.fnal.gov"], "events": 2048 }, 3: { "lumis": 6, "locations": ["castor.cern.ch", "cmssrm.fnal.gov", "srm.ral.uk"], "events": 3072 }, 5: { "lumis": 10, "locations": ["castor.cern.ch", "cmssrm.fnal.gov"], "events": 5120 } } testFiles = [ testFileA, testFileB, testFileC, testFileI, testFileJ, testFileK ] lastFile = testFileA for testFile in testFiles: if lastFile["lfn"] < testFile["lfn"]: lastFile = testFile testFiles.remove(lastFile) goldenFiles = { 1: [lastFile], 2: [testFileD, testFileE], 3: [testFileF, testFileG, testFileH], 5: testFiles } for chunk in chunks: chunkMetaData = dcs.getChunkInfo("ACDCTest", "/ACDCTest/reco", chunk["offset"], chunk["files"]) self.assertEqual(chunkMetaData["files"], chunk["files"], "Error: Metadata doesn't match.") self.assertEqual(chunkMetaData["lumis"], chunk["lumis"], "Error: Metadata doesn't match.") self.assertEqual(chunkMetaData["events"], chunk["events"], "Error: Metadata doesn't match.") self.assertEqual(chunkMetaData["locations"], chunk["locations"], "Error: Metadata doesn't match.") self.assertTrue(chunk["files"] in goldenMetaData.keys(), "Error: Extra chunk found.") self.assertEqual(chunk["lumis"], goldenMetaData[chunk["files"]]["lumis"], "Error: Lumis in chunk is wrong.") self.assertEqual(chunk["locations"], goldenMetaData[chunk["files"]]["locations"], "Error: Locations in chunk is wrong.") self.assertEqual(chunk["events"], goldenMetaData[chunk["files"]]["events"], "Error: Events in chunk is wrong.") del goldenMetaData[chunk["files"]] chunkFiles = dcs.getChunkFiles("ACDCTest", "/ACDCTest/reco", chunk["offset"], chunk["files"]) self.assertTrue(chunk["files"] in goldenFiles.keys(), "Error: Extra chunk found.") goldenChunkFiles = goldenFiles[chunk["files"]] self.assertEqual(len(chunkFiles), len(goldenChunkFiles)) for chunkFile in chunkFiles: foundFile = None for goldenChunkFile in goldenChunkFiles: if chunkFile["lfn"] == goldenChunkFile["lfn"]: foundFile = goldenChunkFile break self.assertTrue( foundFile != None, "Error: Missing chunk file: %s, %s" % (chunkFiles, goldenChunkFiles)) self.assertEqual(foundFile["parents"], chunkFile["parents"], "Error: File parents should match.") self.assertEqual(foundFile["merged"], chunkFile["merged"], "Error: File merged status should match.") self.assertEqual(foundFile["locations"], chunkFile["locations"], "Error: File locations should match.") self.assertEqual( foundFile["events"], chunkFile["events"], "Error: File locations should match: %s" % chunk["files"]) self.assertEqual(foundFile["size"], chunkFile["size"], "Error: File locations should match.") self.assertEqual(len(foundFile["runs"]), len(chunkFile["runs"]), "Error: Wrong number of runs.") for run in foundFile["runs"]: runMatch = False for chunkRun in chunkFile["runs"]: if chunkRun.run == run.run and chunkRun.lumis == run.lumis: runMatch = True break self.assertTrue(runMatch, "Error: Run information is wrong.") del goldenFiles[chunk["files"]] return def testGetLumiWhitelist(self): """ _testGetLumiWhitelist_ Verify that the ACDC whitelist generation code works correctly. We'll add jobs with the following lumi info: # Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12] # Run 2, lumis [5, 6, 7], [10, 11, 12], [15] # Run 3, lumis [20] And should get out a whitelist that looks like this: {"1": [[1, 4], [6, 7], [9, 9], [11, 12]], "2": [[5, 7], [10, 12], [15, 15]], "3": [[20, 20]]} """ dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc") def getJob(): job = Job() job["task"] = "/ACDCTest/reco" job["workflow"] = "ACDCTest" job["location"] = "cmssrm.fnal.gov" job["owner"] = "cmsdataops" job["group"] = "cmsdataops" return job testFileA = File(lfn=makeUUID(), size=1024, events=1024) testFileA.addRun(Run(1, 1, 2)) testFileB = File(lfn=makeUUID(), size=1024, events=1024) testFileB.addRun(Run(1, 3)) testJobA = getJob() testJobA.addFile(testFileA) testJobA.addFile(testFileB) testFileC = File(lfn=makeUUID(), size=1024, events=1024) testFileC.addRun(Run(1, 4, 6)) testJobB = getJob() testJobB.addFile(testFileC) testFileD = File(lfn=makeUUID(), size=1024, events=1024) testFileD.addRun(Run(1, 7)) testJobC = getJob() testJobC.addFile(testFileD) testFileE = File(lfn=makeUUID(), size=1024, events=1024) testFileE.addRun(Run(1, 11, 12)) testJobD = getJob() testJobD.addFile(testFileE) testFileF = File(lfn=makeUUID(), size=1024, events=1024) testFileF.addRun(Run(2, 5, 6, 7)) testJobE = getJob() testJobE.addFile(testFileF) testFileG = File(lfn=makeUUID(), size=1024, events=1024) testFileG.addRun(Run(2, 10, 11, 12)) testJobF = getJob() testJobF.addFile(testFileG) testFileH = File(lfn=makeUUID(), size=1024, events=1024) testFileH.addRun(Run(2, 15)) testJobG = getJob() testJobG.addFile(testFileH) testFileI = File(lfn=makeUUID(), size=1024, events=1024) testFileI.addRun(Run(3, 20)) testJobH = getJob() testJobH.addFile(testFileI) testFileJ = File(lfn=makeUUID(), size=1024, events=1024) testFileJ.addRun(Run(1, 9)) testJobI = getJob() testJobI.addFile(testFileJ) dcs.failedJobs([ testJobA, testJobB, testJobC, testJobD, testJobE, testJobF, testJobG, testJobH, testJobI ]) whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco") self.assertEqual(len(whiteList.keys()), 3, "Error: There should be 3 runs.") self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]], "Error: Whitelist for run 1 is wrong.") self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]], "Error: Whitelist for run 2 is wrong.") self.assertEqual(whiteList["3"], [[20, 20]], "Error: Whitelist for run 3 is wrong.") return
class JobSubmitterTest(unittest.TestCase): """ _JobSubmitterTest_ Test class for the JobSubmitterPoller """ def setUp(self): """ _setUp_ Standard setup: Now with 100% more couch """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database" ]) self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump") self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump") self.testInit.setupCouch("wmagent_summary_t", "WMStats") myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.baDaoFactory = DAOFactory(package="WMCore.BossAir", logger=myThread.logger, dbinterface=myThread.dbi) self.testDir = self.testInit.generateWorkDir() # Set heartbeat self.componentName = 'JobSubmitter' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ _tearDown_ Standard tearDown """ self.testInit.clearDatabase() self.testInit.delWorkDir() self.testInit.tearDownCouch() EmulatorSetup.deleteConfig(self.configFile) return def setResourceThresholds(self, site, **options): """ _setResourceThresholds_ Utility to set resource thresholds """ if not options: options = { 'state': 'Normal', 'runningSlots': 10, 'pendingSlots': 5, 'tasks': ['Processing', 'Merge'], 'Processing': { 'pendingSlots': 5, 'runningSlots': 10 }, 'Merge': { 'pendingSlots': 2, 'runningSlots': 5 } } resourceControl = ResourceControl() resourceControl.insertSite(siteName=site, pnn='se.%s' % (site), ceName=site, plugin="MockPlugin", pendingSlots=options['pendingSlots'], runningSlots=options['runningSlots'], cmsName=site) for task in options['tasks']: resourceControl.insertThreshold( siteName=site, taskType=task, maxSlots=options[task]['runningSlots'], pendingSlots=options[task]['pendingSlots']) if options.get('state'): resourceControl.changeSiteState(site, options.get('state')) return def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site, taskType='Processing', name=None): """ _createJobGroups_ Creates a series of jobGroups for submissions """ jobGroupList = [] if name is None: name = makeUUID() testWorkflow = Workflow(spec=workloadSpec, owner="tapas", name=name, task="basicWorkload/Production") testWorkflow.create() # Create subscriptions for _ in range(nSubs): name = makeUUID() # Create Fileset, Subscription, jobGroup testFileset = Fileset(name=name) testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type=taskType, split_algo="FileBased") testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() # Create jobs self.makeNJobs(name=name, task=task, nJobs=nJobs, jobGroup=testJobGroup, fileset=testFileset, sub=testSubscription.exists(), site=site) testFileset.commit() testJobGroup.commit() jobGroupList.append(testJobGroup) return jobGroupList def makeNJobs(self, name, task, nJobs, jobGroup, fileset, sub, site): """ _makeNJobs_ Make and return a WMBS Job and File This handles all those damn add-ons """ # Set the CacheDir cacheDir = os.path.join(self.testDir, 'CacheDir') for n in range(nJobs): # First make a file #site = self.sites[0] testFile = File(lfn="/singleLfn/%s/%s" % (name, n), size=1024, events=10) fileset.addFile(testFile) fileset.commit() location = None if isinstance(site, list): if len(site) > 0: location = site[0] else: location = site index = 0 for f in fileset.files: index += 1 testJob = Job(name='%s-%i' % (name, index)) testJob.addFile(f) testJob["location"] = location testJob["possiblePSN"] = set(site) if isinstance( site, list) else set([site]) testJob['task'] = task.getPathName() testJob['sandbox'] = task.data.input.sandbox testJob['spec'] = os.path.join(self.testDir, 'basicWorkload.pcl') testJob['mask']['FirstEvent'] = 101 testJob['priority'] = 101 testJob['numberOfCores'] = 1 jobCache = os.path.join(cacheDir, 'Sub_%i' % (sub), 'Job_%i' % (index)) os.makedirs(jobCache) testJob.create(jobGroup) testJob['cache_dir'] = jobCache testJob.save() jobGroup.add(testJob) output = open(os.path.join(jobCache, 'job.pkl'), 'w') pickle.dump(testJob, output) output.close() return testJob, testFile def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] #Here Test the CondorPlugin instead of MockPlugin #config.BossAir.pluginNames = ['CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config def createTestWorkload(self, workloadName='Tier1ReReco'): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload(workloadName) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload def testA_BasicTest(self): """ Use the MockPlugin to create a simple test Check to see that all the jobs were "submitted", don't care about thresholds """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 2 nJobs = 20 site = "T2_US_UCSD" self.setResourceThresholds(site, pendingSlots=50, runningSlots=100, tasks=['Processing', 'Merge'], Processing={ 'pendingSlots': 50, 'runningSlots': 100 }, Merge={ 'pendingSlots': 50, 'runningSlots': 100 }) jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Do pre-submit check getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), nSubs * nJobs) jobSubmitter = JobSubmitterPoller(config=config) jobSubmitter.algorithm() # Check that jobs are in the right state result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nSubs * nJobs) # Check assigned locations getLocationAction = self.daoFactory(classname="Jobs.GetLocation") for jobId in result: loc = getLocationAction.execute(jobid=jobId) self.assertEqual(loc, [['T2_US_UCSD']]) # Run another cycle, it shouldn't submit anything. There isn't anything to submit jobSubmitter.algorithm() result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nSubs * nJobs) nSubs = 1 nJobs = 10 # Submit another 10 jobs jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, taskType="Merge") for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Check that the jobs are available for submission and run another cycle result = getJobsAction.execute(state='Created', jobType="Merge") self.assertEqual(len(result), nSubs * nJobs) jobSubmitter.algorithm() #Check that the last 10 jobs were submitted as well. result = getJobsAction.execute(state='Created', jobType="Merge") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Executing', jobType="Merge") self.assertEqual(len(result), nSubs * nJobs) return def testB_thresholdTest(self): """ _testB_thresholdTest_ Check that the threshold management is working, this requires checks on pending/running jobs globally at a site and per task/site """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 5 nJobs = 10 site = "T1_US_FNAL" self.setResourceThresholds(site, pendingSlots=50, runningSlots=200, tasks=['Processing', 'Merge'], Processing={ 'pendingSlots': 45, 'runningSlots': -1 }, Merge={ 'pendingSlots': 10, 'runningSlots': 20, 'priority': 5 }) # Always initialize the submitter after setting the sites, flaky! jobSubmitter = JobSubmitterPoller(config=config) jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Do pre-submit check getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), nSubs * nJobs) jobSubmitter.algorithm() # Check that jobs are in the right state, # here we are limited by the pending threshold for the Processing task (45) result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 5) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), 45) # Check assigned locations getLocationAction = self.daoFactory(classname="Jobs.GetLocation") for jobId in result: loc = getLocationAction.execute(jobid=jobId) self.assertEqual(loc, [['T1_US_FNAL']]) # Run another cycle, it shouldn't submit anything. Jobs are still in pending jobSubmitter.algorithm() result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 5) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), 45) # Now put 10 Merge jobs, only 5 can be submitted, there we hit the global pending threshold for the site nSubs = 1 nJobs = 10 jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, taskType='Merge') for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() result = getJobsAction.execute(state='Created', jobType="Merge") self.assertEqual(len(result), 5) result = getJobsAction.execute(state='Executing', jobType="Merge") self.assertEqual(len(result), 5) result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 5) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), 45) # Now let's test running thresholds # The scenario will be setup as follows: Move all current jobs as running # Create 300 Processing jobs and 300 merge jobs # Run 5 polling cycles, moving all pending jobs to running in between # Result is, merge is left at 25 running 0 pending and processing is left at 215 running 0 pending # Processing has 135 jobs in queue and Merge 285 # This tests all threshold dynamics including the prioritization of merge over processing nSubs = 1 nJobs = 300 jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site) jobGroupList.extend( self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, taskType='Merge')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') getRunJobID = self.baDaoFactory(classname="LoadByWMBSID") setRunJobStatus = self.baDaoFactory(classname="SetStatus") for _ in range(5): result = getJobsAction.execute(state='Executing') binds = [] for jobId in result: binds.append({'id': jobId, 'retry_count': 0}) runJobIds = getRunJobID.execute(binds) setRunJobStatus.execute([x['id'] for x in runJobIds], 'Running') jobSubmitter.algorithm() result = getJobsAction.execute(state='Executing', jobType='Processing') self.assertEqual(len(result), 215) result = getJobsAction.execute(state='Created', jobType='Processing') self.assertEqual(len(result), 135) result = getJobsAction.execute(state='Executing', jobType='Merge') self.assertEqual(len(result), 25) result = getJobsAction.execute(state='Created', jobType='Merge') self.assertEqual(len(result), 285) return def testC_prioritization(self): """ _testC_prioritization_ Check that jobs are prioritized by job type and by oldest workflow """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 1 nJobs = 10 site = "T1_US_FNAL" self.setResourceThresholds(site, pendingSlots=10, runningSlots=-1, tasks=['Processing', 'Merge'], Processing={ 'pendingSlots': 50, 'runningSlots': -1 }, Merge={ 'pendingSlots': 10, 'runningSlots': -1, 'priority': 5 }) # Always initialize the submitter after setting the sites, flaky! jobSubmitter = JobSubmitterPoller(config=config) jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, name='OldestWorkflow') jobGroupList.extend( self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, taskType='Merge')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() # Merge goes first getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Merge") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Executing', jobType="Merge") self.assertEqual(len(result), 10) result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 10) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), 0) # Create a newer workflow processing, and after some new jobs for an old workflow jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, name='NewestWorkflow') jobGroupList.extend( self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, name='OldestWorkflow')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Move pending jobs to running getRunJobID = self.baDaoFactory(classname="LoadByWMBSID") setRunJobStatus = self.baDaoFactory(classname="SetStatus") for idx in range(2): result = getJobsAction.execute(state='Executing') binds = [] for jobId in result: binds.append({'id': jobId, 'retry_count': 0}) runJobIds = getRunJobID.execute(binds) setRunJobStatus.execute([x['id'] for x in runJobIds], 'Running') # Run again on created workflows jobSubmitter.algorithm() result = getJobsAction.execute(state='Created', jobType="Merge") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Executing', jobType="Merge") self.assertEqual(len(result), 10) result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 30 - (idx + 1) * 10) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), (idx + 1) * 10) # Check that older workflow goes first even with newer jobs getWorkflowAction = self.daoFactory( classname="Jobs.GetWorkflowTask") workflows = getWorkflowAction.execute(result) for workflow in workflows: self.assertEqual(workflow['name'], 'OldestWorkflow') return def testD_SubmitFailed(self): """ _testD_SubmitFailed_ Check if jobs without a possible site to run at go to SubmitFailed """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 2 nJobs = 10 jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), site=[], workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName)) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter = JobSubmitterPoller(config=config) jobSubmitter.algorithm() # Jobs should go to submit failed getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='SubmitFailed', jobType="Processing") self.assertEqual(len(result), nSubs * nJobs) return def testE_SiteModesTest(self): """ _testE_SiteModesTest_ Test the behavior of the submitter in response to the different states of the sites """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 1 nJobs = 20 sites = [ 'T2_US_Florida', 'T2_TW_Taiwan', 'T3_CO_Uniandes', 'T1_US_FNAL' ] for site in sites: self.setResourceThresholds(site, pendingSlots=10, runningSlots=-1, tasks=['Processing', 'Merge'], Processing={ 'pendingSlots': 10, 'runningSlots': -1 }, Merge={ 'pendingSlots': 10, 'runningSlots': -1, 'priority': 5 }) myResourceControl = ResourceControl(config) myResourceControl.changeSiteState('T2_US_Florida', 'Draining') # First test that we prefer Normal over drain, and T1 over T2/T3 jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, site=[x for x in sites], task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName)) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter = JobSubmitterPoller(config=config) # Actually run it jobSubmitter.algorithm() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nSubs * nJobs) # All jobs should be at either FNAL, Taiwan or Uniandes. It's a random selection # Check assigned locations getLocationAction = self.daoFactory(classname="Jobs.GetLocation") locationDict = getLocationAction.execute([{ 'jobid': x } for x in result]) for entry in locationDict: loc = entry['site_name'] self.assertNotEqual(loc, 'T2_US_Florida') # Now set everything to down, check we don't submit anything for site in sites: myResourceControl.changeSiteState(site, 'Down') jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, site=[x for x in sites], task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName)) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() # Nothing is submitted despite the empty slots at Uniandes and Florida result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nSubs * nJobs) # Now set everything to Drain and create Merge jobs. Those should be submitted for site in sites: myResourceControl.changeSiteState(site, 'Draining') nSubsMerge = 1 nJobsMerge = 5 jobGroupList = self.createJobGroups(nSubs=nSubsMerge, nJobs=nJobsMerge, site=[x for x in sites], task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), taskType='Merge') for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() result = getJobsAction.execute(state='Executing', jobType='Merge') self.assertEqual(len(result), nSubsMerge * nJobsMerge) # Now set everything to Aborted, and create Merge jobs. Those should fail # since the can only run at one place for site in sites: myResourceControl.changeSiteState(site, 'Aborted') nSubsMerge = 1 nJobsMerge = 5 jobGroupList = self.createJobGroups(nSubs=nSubsMerge, nJobs=nJobsMerge, site=[x for x in sites], task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), taskType='Merge') for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() result = getJobsAction.execute(state='SubmitFailed', jobType='Merge') self.assertEqual(len(result), nSubsMerge * nJobsMerge) result = getJobsAction.execute(state='Executing', jobType='Processing') self.assertEqual(len(result), nSubs * nJobs) return @attr('integration') def testF_PollerProfileTest(self): """ _testF_PollerProfileTest_ Submit a lot of jobs and test how long it takes for them to actually be submitted """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 100 nJobs = 100 site = "T1_US_FNAL" self.setResourceThresholds(site, pendingSlots=20000, runningSlots=-1, tasks=['Processing', 'Merge'], Processing={ 'pendingSlots': 10000, 'runningSlots': -1 }, Merge={ 'pendingSlots': 10000, 'runningSlots': -1, 'priority': 5 }) # Always initialize the submitter after setting the sites, flaky! JobSubmitterPoller(config=config) jobGroupList = self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site) jobGroupList.extend( self.createJobGroups(nSubs=nSubs, nJobs=nJobs, task=workload.getTask("ReReco"), workloadSpec=os.path.join( self.testDir, 'workloadTest', workloadName), site=site, taskType='Merge')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Actually run it startTime = time.time() cProfile.runctx("JobSubmitterPoller(config=config).algorithm()", globals(), locals(), filename="testStats.stat") stopTime = time.time() print("Job took %f seconds to complete" % (stopTime - startTime)) p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats() return
class WMBSHelperTest(EmulatedUnitTestCase): def setUp(self): """ _setUp_ """ super(WMBSHelperTest, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump") self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump") self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache") os.environ["COUCHDB"] = "wmbshelper_t" self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir", "WMCore.ResourceControl" ], useDefault=False) self.workDir = self.testInit.generateWorkDir() self.wmspec = self.createWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = DBSReader(self.inputDataset.dbsurl) self.rucioAcct = "wmcore_transferor" self.rucio = Rucio(self.rucioAcct) self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=threading.currentThread().logger, dbinterface=threading.currentThread().dbi) self.configFile = EmulatorSetup.setupWMAgentConfig() self.config = loadConfigurationFile(self.configFile) self.config.component_("JobSubmitter") self.config.JobSubmitter.submitDir = self.workDir self.config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.clearDatabase() self.testInit.tearDownCouch() self.testInit.delWorkDir() EmulatorSetup.deleteConfig(self.configFile) super(WMBSHelperTest, self).tearDown() return def setupForKillTest(self, baAPI=None): """ _setupForKillTest_ Inject a workflow into WMBS that has a processing task, a merge task and a cleanup task. Inject files into the various tasks at various processing states (acquired, complete, available...). Also create jobs for each subscription in various states. """ myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) dummyLocationAction = daoFactory(classname="Locations.New") changeStateAction = daoFactory(classname="Jobs.ChangeState") resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', pnn='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertThreshold(siteName='site1', taskType='Processing', \ maxSlots=10000, pendingSlots=10000) userDN = 'someDN' userAction = daoFactory(classname="Users.New") userAction.execute(dn=userDN, group_name='DEFAULT', role_name='DEFAULT') inputFileset = Fileset("input") inputFileset.create() inputFileA = File("lfnA", locations="goodse.cern.ch") inputFileB = File("lfnB", locations="goodse.cern.ch") inputFileC = File("lfnC", locations="goodse.cern.ch") inputFileA.create() inputFileB.create() inputFileC.create() inputFileset.addFile(inputFileA) inputFileset.addFile(inputFileB) inputFileset.addFile(inputFileC) inputFileset.commit() unmergedOutputFileset = Fileset("unmerged") unmergedOutputFileset.create() unmergedFileA = File("ulfnA", locations="goodse.cern.ch") unmergedFileB = File("ulfnB", locations="goodse.cern.ch") unmergedFileC = File("ulfnC", locations="goodse.cern.ch") unmergedFileA.create() unmergedFileB.create() unmergedFileC.create() unmergedOutputFileset.addFile(unmergedFileA) unmergedOutputFileset.addFile(unmergedFileB) unmergedOutputFileset.addFile(unmergedFileC) unmergedOutputFileset.commit() mainProcWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="Proc") mainProcWorkflow.create() mainProcMergeWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="ProcMerge") mainProcMergeWorkflow.create() mainCleanupWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="Cleanup") mainCleanupWorkflow.create() self.mainProcSub = Subscription(fileset=inputFileset, workflow=mainProcWorkflow, type="Processing") self.mainProcSub.create() self.mainProcSub.acquireFiles(inputFileA) self.mainProcSub.completeFiles(inputFileB) procJobGroup = JobGroup(subscription=self.mainProcSub) procJobGroup.create() self.procJobA = Job(name="ProcJobA") self.procJobA["state"] = "new" self.procJobA["location"] = "site1" self.procJobB = Job(name="ProcJobB") self.procJobB["state"] = "executing" self.procJobB["location"] = "site1" self.procJobC = Job(name="ProcJobC") self.procJobC["state"] = "complete" self.procJobC["location"] = "site1" self.procJobA.create(procJobGroup) self.procJobB.create(procJobGroup) self.procJobC.create(procJobGroup) self.mainMergeSub = Subscription(fileset=unmergedOutputFileset, workflow=mainProcMergeWorkflow, type="Merge") self.mainMergeSub.create() self.mainMergeSub.acquireFiles(unmergedFileA) self.mainMergeSub.failFiles(unmergedFileB) mergeJobGroup = JobGroup(subscription=self.mainMergeSub) mergeJobGroup.create() self.mergeJobA = Job(name="MergeJobA") self.mergeJobA["state"] = "exhausted" self.mergeJobA["location"] = "site1" self.mergeJobB = Job(name="MergeJobB") self.mergeJobB["state"] = "cleanout" self.mergeJobB["location"] = "site1" self.mergeJobC = Job(name="MergeJobC") self.mergeJobC["state"] = "new" self.mergeJobC["location"] = "site1" self.mergeJobA.create(mergeJobGroup) self.mergeJobB.create(mergeJobGroup) self.mergeJobC.create(mergeJobGroup) self.mainCleanupSub = Subscription(fileset=unmergedOutputFileset, workflow=mainCleanupWorkflow, type="Cleanup") self.mainCleanupSub.create() self.mainCleanupSub.acquireFiles(unmergedFileA) self.mainCleanupSub.completeFiles(unmergedFileB) cleanupJobGroup = JobGroup(subscription=self.mainCleanupSub) cleanupJobGroup.create() self.cleanupJobA = Job(name="CleanupJobA") self.cleanupJobA["state"] = "new" self.cleanupJobA["location"] = "site1" self.cleanupJobB = Job(name="CleanupJobB") self.cleanupJobB["state"] = "executing" self.cleanupJobB["location"] = "site1" self.cleanupJobC = Job(name="CleanupJobC") self.cleanupJobC["state"] = "complete" self.cleanupJobC["location"] = "site1" self.cleanupJobA.create(cleanupJobGroup) self.cleanupJobB.create(cleanupJobGroup) self.cleanupJobC.create(cleanupJobGroup) jobList = [ self.procJobA, self.procJobB, self.procJobC, self.mergeJobA, self.mergeJobB, self.mergeJobC, self.cleanupJobA, self.cleanupJobB, self.cleanupJobC ] changeStateAction.execute(jobList) if baAPI: for job in jobList: job['plugin'] = 'TestPlugin' job['userdn'] = userDN job['usergroup'] = 'DEFAULT' job['userrole'] = 'DEFAULT' job['custom']['location'] = 'site1' baAPI.createNewJobs(wmbsJobs=jobList) # We'll create an unrelated workflow to verify that it isn't affected # by the killing code. bogusFileset = Fileset("dontkillme") bogusFileset.create() bogusFileA = File("bogus/lfnA", locations="goodse.cern.ch") bogusFileA.create() bogusFileset.addFile(bogusFileA) bogusFileset.commit() bogusWorkflow = Workflow(spec="spec2", owner="Steve", name="Bogus", task="Proc") bogusWorkflow.create() self.bogusSub = Subscription(fileset=bogusFileset, workflow=bogusWorkflow, type="Processing") self.bogusSub.create() self.bogusSub.acquireFiles(bogusFileA) return def verifyFileKillStatus(self): """ _verifyFileKillStatus_ Verify that all files were killed correctly. The status of files in Cleanup and LogCollect subscriptions isn't modified. Status of already completed and failed files is not modified. Also verify that the bogus subscription is untouched. """ failedFiles = self.mainProcSub.filesOfStatus("Failed") acquiredFiles = self.mainProcSub.filesOfStatus("Acquired") completedFiles = self.mainProcSub.filesOfStatus("Completed") availableFiles = self.mainProcSub.filesOfStatus("Available") bogusAcquiredFiles = self.bogusSub.filesOfStatus("Acquired") self.assertEqual(len(availableFiles), 0, \ "Error: There should be no available files.") self.assertEqual(len(acquiredFiles), 0, \ "Error: There should be no acquired files.") self.assertEqual(len(bogusAcquiredFiles), 1, \ "Error: There should be one acquired file.") self.assertEqual(len(completedFiles), 3, \ "Error: There should be only one completed file.") goldenLFNs = ["lfnA", "lfnB", "lfnC"] for completedFile in completedFiles: self.assertTrue(completedFile["lfn"] in goldenLFNs, \ "Error: Extra completed file.") goldenLFNs.remove(completedFile["lfn"]) self.assertEqual(len(failedFiles), 0, \ "Error: There should be no failed files.") self.assertEqual(len(goldenLFNs), 0, \ "Error: Missing LFN") failedFiles = self.mainMergeSub.filesOfStatus("Failed") acquiredFiles = self.mainMergeSub.filesOfStatus("Acquired") completedFiles = self.mainMergeSub.filesOfStatus("Completed") availableFiles = self.mainMergeSub.filesOfStatus("Available") self.assertEqual(len(acquiredFiles), 0, \ "Error: Merge subscription should have 0 acq files.") self.assertEqual(len(availableFiles), 0, \ "Error: Merge subscription should have 0 avail files.") self.assertEqual(len(failedFiles), 1, \ "Error: Merge subscription should have 1 failed files.") self.assertEqual( list(failedFiles)[0]["lfn"], "ulfnB", "Error: Wrong failed file.") self.assertEqual(len(completedFiles), 2, \ "Error: Merge subscription should have 2 compl files.") goldenLFNs = ["ulfnA", "ulfnC"] for completedFile in completedFiles: self.assertTrue(completedFile["lfn"] in goldenLFNs, \ "Error: Extra complete file.") goldenLFNs.remove(completedFile["lfn"]) self.assertEqual(len(goldenLFNs), 0, \ "Error: Missing LFN") failedFiles = self.mainCleanupSub.filesOfStatus("Failed") acquiredFiles = self.mainCleanupSub.filesOfStatus("Acquired") completedFiles = self.mainCleanupSub.filesOfStatus("Completed") availableFiles = self.mainCleanupSub.filesOfStatus("Available") self.assertEqual(len(failedFiles), 0, \ "Error: Cleanup subscription should have 0 fai files.") self.assertEqual(len(acquiredFiles), 1, \ "Error: There should be only one acquired file.") self.assertEqual(list(acquiredFiles)[0]["lfn"], "ulfnA", \ "Error: Wrong acquired LFN.") self.assertEqual(len(completedFiles), 1, \ "Error: There should be only one completed file.") self.assertEqual(list(completedFiles)[0]["lfn"], "ulfnB", \ "Error: Wrong completed LFN.") self.assertEqual(len(availableFiles), 1, \ "Error: There should be only one available file.") self.assertEqual(list(availableFiles)[0]["lfn"], "ulfnC", \ "Error: Wrong completed LFN.") return def verifyJobKillStatus(self): """ _verifyJobKillStatus_ Verify that jobs are killed correctly. Jobs belonging to Cleanup and LogCollect subscriptions are not killed. The status of jobs that have already finished running is not changed. """ self.procJobA.load() self.procJobB.load() self.procJobC.load() self.assertEqual(self.procJobA["state"], "killed", \ "Error: Proc job A should be killed.") self.assertEqual(self.procJobB["state"], "killed", \ "Error: Proc job B should be killed.") self.assertEqual(self.procJobC["state"], "complete", \ "Error: Proc job C should be complete.") self.mergeJobA.load() self.mergeJobB.load() self.mergeJobC.load() self.assertEqual(self.mergeJobA["state"], "exhausted", \ "Error: Merge job A should be exhausted.") self.assertEqual(self.mergeJobB["state"], "cleanout", \ "Error: Merge job B should be cleanout.") self.assertEqual(self.mergeJobC["state"], "killed", \ "Error: Merge job C should be killed.") self.cleanupJobA.load() self.cleanupJobB.load() self.cleanupJobC.load() self.assertEqual(self.cleanupJobA["state"], "new", \ "Error: Cleanup job A should be new.") self.assertEqual(self.cleanupJobB["state"], "executing", \ "Error: Cleanup job B should be executing.") self.assertEqual(self.cleanupJobC["state"], "complete", \ "Error: Cleanup job C should be complete.") return def createTestWMSpec(self): """ _createTestWMSpec_ Create a WMSpec that has a processing, merge, cleanup and skims tasks that can be used by the subscription creation test. """ testWorkload = WMWorkloadHelper(WMWorkload("TestWorkload")) testWorkload.setDashboardActivity("TestReReco") testWorkload.setSpecUrl("/path/to/workload") testWorkload.setOwnerDetails("sfoulkes", "DMWM", {'dn': 'MyDN'}) procTask = testWorkload.newTask("ProcessingTask") procTask.setTaskType("Processing") procTask.setSplittingAlgorithm("FileBased", files_per_job=1) procTaskCMSSW = procTask.makeStep("cmsRun1") procTaskCMSSW.setStepType("CMSSW") procTaskCMSSWHelper = procTaskCMSSW.getTypeHelper() procTask.setTaskType("Processing") procTask.setSiteWhitelist(["site1"]) procTask.setSiteBlacklist(["site2"]) procTask.applyTemplates() procTaskCMSSWHelper.addOutputModule("OutputA", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) mergeTask = procTask.addTask("MergeTask") mergeTask.setInputReference(procTaskCMSSW, outputModule="OutputA", dataTier='DataTierA') mergeTask.setTaskType("Merge") mergeTask.setSplittingAlgorithm("WMBSMergeBySize", min_merge_size=1, max_merge_size=2, max_merge_events=3) mergeTaskCMSSW = mergeTask.makeStep("cmsRun1") mergeTaskCMSSW.setStepType("CMSSW") mergeTaskCMSSWHelper = mergeTaskCMSSW.getTypeHelper() mergeTask.setTaskType("Merge") mergeTask.applyTemplates() mergeTaskCMSSWHelper.addOutputModule("Merged", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) cleanupTask = procTask.addTask("CleanupTask") cleanupTask.setInputReference(procTaskCMSSW, outputModule="OutputA", dataTier="DataTierA") cleanupTask.setTaskType("Merge") cleanupTask.setSplittingAlgorithm("SiblingProcessingBased", files_per_job=50) cleanupTaskCMSSW = cleanupTask.makeStep("cmsRun1") cleanupTaskCMSSW.setStepType("CMSSW") cleanupTask.setTaskType("Cleanup") cleanupTask.applyTemplates() skimTask = mergeTask.addTask("SkimTask") skimTask.setTaskType("Skim") skimTask.setInputReference(mergeTaskCMSSW, outputModule="Merged", dataTier="DataTierA") skimTask.setSplittingAlgorithm("FileBased", files_per_job=1, include_parents=True) skimTaskCMSSW = skimTask.makeStep("cmsRun1") skimTaskCMSSW.setStepType("CMSSW") skimTaskCMSSWHelper = skimTaskCMSSW.getTypeHelper() skimTask.setTaskType("Skim") skimTask.applyTemplates() skimTaskCMSSWHelper.addOutputModule("SkimOutputA", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) skimTaskCMSSWHelper.addOutputModule("SkimOutputB", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierB", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) return testWorkload def setupMCWMSpec(self): """Setup MC workflow""" self.wmspec = self.createMCWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = None # add sites that would normally be added by operator via resource_control locationDAO = self.daoFactory(classname="Locations.New") self.pnns = [] for site in ['T2_XX_SiteA', 'T2_XX_SiteB']: locationDAO.execute(siteName=site, pnn=site) self.pnns.append(site) def createWMSpec(self, name='ReRecoWorkload'): factory = ReRecoWorkloadFactory() rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) wmspec = factory.factoryWorkloadConstruction(name, rerecoArgs) wmspec.setSpecUrl("/path/to/workload") wmspec.setSubscriptionInformation(custodialSites=[], nonCustodialSites=[], autoApproveSites=[], priority="Low", custodialSubType="Move") return wmspec def createMCWMSpec(self, name='MonteCarloWorkload'): mcArgs = TaskChainWorkloadFactory.getTestArguments() mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"] mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"]) wmspec = taskChainWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents=10000) return wmspec def getDBS(self, wmspec): topLevelTask = getFirstTask(wmspec) inputDataset = topLevelTask.inputDataset() dbs = DBSReader(inputDataset.dbsurl) # dbsDict = {self.inputDataset.dbsurl : self.dbs} return dbs def createWMBSHelperWithTopTask(self, wmspec, block, mask=None, parentFlag=False, detail=False, commonLocation=None): topLevelTask = getFirstTask(wmspec) wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath=self.workDir, commonLocation=commonLocation) if block: blockName = block if parentFlag: block = self.dbs.getFileBlockWithParents(blockName) data = self.rucio.getReplicaInfoForBlocks(block=[blockName]) block['PhEDExNodeNames'] = data[0]["replica"] else: block = self.dbs.getFileBlock(blockName) data = self.rucio.getReplicaInfoForBlocks(block=[blockName]) block['PhEDExNodeNames'] = data[0]["replica"] sub, files = wmbs.createSubscriptionAndAddFiles(block=block) if detail: return wmbs, sub, files else: return wmbs def testKillWorkflow(self): """ _testKillWorkflow_ Verify that workflow killing works correctly. """ baAPI = BossAirAPI(config=self.config, insertStates=True) # Create nine jobs self.setupForKillTest(baAPI=baAPI) self.assertEqual(len(baAPI._listRunJobs()), 9) killWorkflow("Main", self.config, self.config) self.verifyFileKillStatus() self.verifyJobKillStatus() self.assertEqual(len(baAPI._listRunJobs()), 8) return def testCreateSubscription(self): """ _testCreateSubscription_ Verify that the subscription creation code works correctly. """ resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', pnn='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertSite(siteName='site2', pnn='goodse2.cern.ch', ceName='site2', plugin="TestPlugin") testWorkload = self.createTestWMSpec() testTopLevelTask = getFirstTask(testWorkload) testWMBSHelper = WMBSHelper(testWorkload, testTopLevelTask.name(), "SomeBlock", cachepath=self.workDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testTopLevelTask, testWMBSHelper.topLevelFileset) procWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/ProcessingTask") procWorkflow.load() self.assertEqual(procWorkflow.owner, "sfoulkes", "Error: Wrong owner: %s" % procWorkflow.owner) self.assertEqual(procWorkflow.group, "DMWM", "Error: Wrong group: %s" % procWorkflow.group) self.assertEqual(procWorkflow.wfType, "TestReReco", "Error: Wrong type.") self.assertEqual( procWorkflow.spec, os.path.join(self.workDir, procWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(procWorkflow.outputMap), 1, "Error: Wrong number of WF outputs.") mergedProcOutput = procWorkflow.outputMap["OutputADataTierA"][0][ "merged_output_fileset"] unmergedProcOutput = procWorkflow.outputMap["OutputADataTierA"][0][ "output_fileset"] mergedProcOutput.loadData() unmergedProcOutput.loadData() self.assertEqual( mergedProcOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-MergedDataTierA", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedProcOutput.name, "/TestWorkload/ProcessingTask/unmerged-OutputADataTierA", "Error: Unmerged output fileset is wrong.") mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/ProcessingTask/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap), 1, "Error: Wrong number of WF outputs.") cleanupWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/ProcessingTask/CleanupTask") cleanupWorkflow.load() self.assertEqual(cleanupWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( cleanupWorkflow.spec, os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(cleanupWorkflow.outputMap), 0, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["MergedDataTierA"][0][ "output_fileset"] unmergedMergeOutput.loadData() self.assertEqual( unmergedMergeOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-MergedDataTierA", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/ProcessingTask/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][0][ "merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][ 0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][0][ "merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][ 0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual( mergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual( unmergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Unmerged output fileset is wrong.") self.assertEqual( mergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-ProcessingTask-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset=topLevelFileset, workflow=procWorkflow) procSubscription.loadData() self.assertEqual(len(procSubscription.getWhiteBlackList()), 2, "Error: Wrong site white/black list for proc sub.") for site in procSubscription.getWhiteBlackList(): if site["site_name"] == "site1": self.assertEqual(site["valid"], 1, "Error: Site should be white listed.") else: self.assertEqual(site["valid"], 0, "Error: Site should be black listed.") self.assertEqual(procSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") mergeSubscription = Subscription(fileset=unmergedProcOutput, workflow=mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset=unmergedMergeOutput, workflow=skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return def testTruncatedWFInsertion(self): """ _testTruncatedWFInsertion_ """ resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', pnn='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertSite(siteName='site2', pnn='goodse2.cern.ch', ceName='site2', plugin="TestPlugin") testWorkload = self.createTestWMSpec() testTopLevelTask = getFirstTask(testWorkload) testWMBSHelper = WMBSHelper(testWorkload, testTopLevelTask.name(), "SomeBlock", cachepath=self.workDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testTopLevelTask, testWMBSHelper.topLevelFileset) testWorkload.truncate("ResubmitTestWorkload", "/TestWorkload/ProcessingTask/MergeTask", "someserver", "somedatabase") # create the subscription for multiple top task (MergeTask and CleanupTask for the same block) for task in testWorkload.getTopLevelTask(): testResubmitWMBSHelper = WMBSHelper(testWorkload, task.name(), "SomeBlock2", cachepath=self.workDir) testResubmitWMBSHelper.createTopLevelFileset() testResubmitWMBSHelper._createSubscriptionsInWMBS( task, testResubmitWMBSHelper.topLevelFileset) mergeWorkflow = Workflow(name="ResubmitTestWorkload", task="/ResubmitTestWorkload/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap), 1, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["MergedDataTierA"][0][ "output_fileset"] unmergedMergeOutput.loadData() self.assertEqual( unmergedMergeOutput.name, "/ResubmitTestWorkload/MergeTask/merged-MergedDataTierA", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow( name="ResubmitTestWorkload", task="/ResubmitTestWorkload/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][0][ "merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][ 0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][0][ "merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][ 0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual( mergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual( unmergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Unmerged output fileset is wrong.") self.assertEqual( mergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset( name="ResubmitTestWorkload-MergeTask-SomeBlock2") topLevelFileset.loadData() mergeSubscription = Subscription(fileset=topLevelFileset, workflow=mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset=unmergedMergeOutput, workflow=skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return def testReReco(self): """ReReco workflow""" # create workflow block = self.dataset + "#" + BLOCK1 wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 5) def testReRecoBlackRunRestriction(self): """ReReco workflow with Run restrictions""" block = self.dataset + "#" + BLOCK2 self.topLevelTask.setInputRunBlacklist( [181183]) # Set run blacklist to only run in the block wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 0) def testReRecoWhiteRunRestriction(self): block = self.dataset + "#" + BLOCK2 self.topLevelTask.setInputRunWhitelist( [181183]) # Set run whitelist to only run in the block wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 1) def testLumiMaskRestrictionsOK(self): block = self.dataset + "#" + BLOCK1 self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['181367'] self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['57,80'] wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 1) def testLumiMaskRestrictionsKO(self): block = self.dataset + "#" + BLOCK1 self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = [ '123454321' ] self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = [ '123,123' ] wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 0) def testDuplicateFileInsert(self): # using default wmspec block = self.dataset + "#" + BLOCK1 wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) wmbs.topLevelFileset.loadData() numOfFiles = len(wmbs.topLevelFileset.files) # check initially inserted files. dbsFiles = self.dbs.getFileBlock(block)['Files'] self.assertEqual(numOfFiles, len(dbsFiles)) firstFileset = wmbs.topLevelFileset wmbsDao = wmbs.daofactory(classname="Files.InFileset") numOfFiles = len(wmbsDao.execute(firstFileset.id)) self.assertEqual(numOfFiles, len(dbsFiles)) # use the new spec with same inputdataset block = self.dataset + "#" + BLOCK1 wmspec = self.createWMSpec("TestSpec1") dbs = self.getDBS(wmspec) wmbs = self.createWMBSHelperWithTopTask(wmspec, block) # check duplicate insert dbsFiles = dbs.getFileBlock(block) data = self.rucio.getReplicaInfoForBlocks(block=[block]) dbsFiles['PhEDExNodeNames'] = data[0]["replica"] numOfFiles = wmbs.addFiles(dbsFiles) self.assertEqual(numOfFiles, 0) secondFileset = wmbs.topLevelFileset wmbsDao = wmbs.daofactory(classname="Files.InFileset") numOfFiles = len(wmbsDao.execute(secondFileset.id)) self.assertEqual(numOfFiles, len(dbsFiles['Files'])) self.assertNotEqual(firstFileset.id, secondFileset.id) def testDuplicateSubscription(self): """Can't duplicate subscriptions""" siteWhitelist = ["T2_XX_SiteA", "T2_XX_SiteB"] # using default wmspec block = self.dataset + "#" + BLOCK1 wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) wmbs.topLevelFileset.loadData() numOfFiles = len(wmbs.topLevelFileset.files) filesetId = wmbs.topLevelFileset.id subId = wmbs.topLevelSubscription['id'] # check initially inserted files. dbsFiles = self.dbs.getFileBlock(block)['Files'] self.assertEqual(numOfFiles, len(dbsFiles)) # Not clear what's supposed to happen here, 2nd test is completely redundant dummyFirstFileset = wmbs.topLevelFileset self.assertEqual(numOfFiles, len(dbsFiles)) # reinsert subscription - shouldn't create anything new wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) wmbs.topLevelFileset.loadData() self.assertEqual(numOfFiles, len(wmbs.topLevelFileset.files)) self.assertEqual(filesetId, wmbs.topLevelFileset.id) self.assertEqual(subId, wmbs.topLevelSubscription['id']) # now do a montecarlo workflow self.setupMCWMSpec() mask = Mask(FirstRun=12, FirstLumi=1234, FirstEvent=12345, LastEvent=999995, LastLumi=12345, LastRun=12) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask, commonLocation=siteWhitelist) wmbs.topLevelFileset.loadData() numOfFiles = len(wmbs.topLevelFileset.files) filesetId = wmbs.topLevelFileset.id subId = wmbs.topLevelSubscription['id'] # check initially inserted files. # Not clear what's supposed to happen here, 2nd test is completely redundant numDbsFiles = 1 self.assertEqual(numOfFiles, numDbsFiles) dummyFirstFileset = wmbs.topLevelFileset self.assertEqual(numOfFiles, numDbsFiles) # reinsert subscription - shouldn't create anything new wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask, commonLocation=siteWhitelist) wmbs.topLevelFileset.loadData() self.assertEqual(numOfFiles, len(wmbs.topLevelFileset.files)) self.assertEqual(filesetId, wmbs.topLevelFileset.id) self.assertEqual(subId, wmbs.topLevelSubscription['id']) def testParentage(self): """ 1. check whether parent files are created in wmbs. 2. check parent files are associated to child. 3. When 2 specs with the same input data (one with parent processing, one without it) is inserted, if one without parent processing inserted first then the other with parent processing insert, it still needs to create parent files although child files are duplicate """ # Swap out the dataset for one that has parents task = next(self.wmspec.taskIterator()) oldDS = task.inputDataset( ) # Copy the old dataset, only will use DBS URL from it task.addInputDataset(name="/Cosmics/ComissioningHI-PromptReco-v1/RECO", primary='Cosmics', processed='ComissioningHI-PromptReco-v1', tier='RECO', dbsurl=oldDS.dbsurl) block = '/Cosmics/ComissioningHI-PromptReco-v1/RECO' + '#5b89ba9c-0dbf-11e1-9b6c-003048caaace' # File creation without parents wmbs, _, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block, parentFlag=False, detail=True) self.assertEqual(8, numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: self.assertEqual(len(child["parents"]), 0) # no parents per child # File creation with parents wmbs, _, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block, parentFlag=True, detail=True) self.assertEqual(8, numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: self.assertEqual(len(child["parents"]), 1) # one parent per child def testMCFakeFileInjection(self): """Inject fake Monte Carlo files into WMBS""" # This test is failing because the name of the couch DB is set to None # in BasicProductionWorkload.getProdArgs() but changing it to # "reqmgr_config_cache_t" from StdBase test arguments does not fix the # situation. testDuplicateSubscription probably has the same issue siteWhitelist = ["T2_XX_SiteA", "T2_XX_SiteB"] self.setupMCWMSpec() mask = Mask(FirstRun=12, FirstLumi=1234, FirstEvent=12345, LastEvent=999995, LastLumi=12345, LastRun=12) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask, commonLocation=siteWhitelist) subscription = wmbs.topLevelSubscription self.assertEqual(1, subscription.exists()) fileset = subscription['fileset'] self.assertEqual(1, fileset.exists()) fileset.loadData() # need to refresh from database self.assertEqual(len(fileset.files), 1) self.assertEqual(len(fileset.parents), 0) self.assertFalse(fileset.open) firstFile = list(fileset.files)[0] self.assertEqual(firstFile['events'], mask['LastEvent'] - mask['FirstEvent'] + 1) # inclusive range self.assertEqual(firstFile['merged'], False) # merged files get added to dbs self.assertEqual(len(firstFile['parents']), 0) # firstFile.loadData() self.assertEqual(sorted(firstFile['locations']), sorted(self.pnns)) self.assertEqual(len(firstFile.getParentLFNs()), 0) self.assertEqual(len(firstFile.getRuns()), 1) run = firstFile.getRuns()[0] self.assertEqual(run.run, mask['FirstRun']) self.assertEqual(run.lumis[0], mask['FirstLumi']) self.assertEqual(run.lumis[-1], mask['LastLumi']) self.assertEqual(len(run.lumis), mask['LastLumi'] - mask['FirstLumi'] + 1)
class PrivateMCTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("privatemc_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("privatemc_t") self.testDir = self.testInit.generateWorkDir() return def injectAnalysisConfig(self): """ Create a bogus config cache document for the analysis workflow and inject it into couch. Return the ID of the document. """ newConfig = Document() newConfig["info"] = None newConfig["config"] = None newConfig["pset_hash"] = "21cb400c6ad63c3a97fa93f8e8785127" newConfig["owner"] = {"group": "Analysis", "user": "******"} newConfig["pset_tweak_details"] ={"process": {"outputModules_": ["OutputA", "OutputB"], "OutputA": {"dataset": {"filterName": "OutputAFilter", "dataTier": "RECO"}}, "OutputB": {"dataset": {"filterName": "OutputBFilter", "dataTier": "USER"}}}} result = self.configDatabase.commitOne(newConfig) return result[0]["id"] def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def testPrivateMC(self): """ _testAnalysis_ """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "privatemc_t" defaultArguments["AnalysisConfigCacheDoc"] = self.injectAnalysisConfig() defaultArguments["ProcessingVersion"] = 1 processingFactory = PrivateMCWorkloadFactory() testWorkload = processingFactory("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DMWM") testWMBSHelper = WMBSHelper(testWorkload, "PrivateMC", "SomeBlock", cachepath = self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) procWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/PrivateMC") procWorkflow.load() self.assertEqual(len(procWorkflow.outputMap.keys()), 3, "Error: Wrong number of WF outputs: %s" % len(procWorkflow.outputMap.keys())) logArchOutput = procWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]#Actually Analysis does not have a merge task unmergedLogArchOutput = procWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/PrivateMC/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/PrivateMC/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") goldenOutputMods = ["OutputA", "OutputB"] for goldenOutputMod in goldenOutputMods: mergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/PrivateMC/unmerged-%s" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/PrivateMC/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-PrivateMC-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset = topLevelFileset, workflow = procWorkflow) procSubscription.loadData() self.assertEqual(procSubscription["type"], "PrivateMC", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "EventBased", "Error: Wrong split algo.") procLogCollect = Fileset(name = "/TestWorkload/PrivateMC/unmerged-logArchive") procLogCollect.loadData() procLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/PrivateMC/LogCollect") procLogCollectWorkflow.load() logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.")
class ReDigiTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("redigi_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("redigi_t") return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() return def injectReDigiConfigs(self, combinedStepOne = False): """ _injectReDigiConfigs_ Create bogus config cache documents for the various steps of the ReDigi workflow. Return the IDs of the documents. """ stepOneConfig = Document() stepOneConfig["info"] = None stepOneConfig["config"] = None stepOneConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f" stepOneConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7" stepOneConfig["owner"] = {"group": "cmsdataops", "user": "******"} if combinedStepOne: stepOneConfig["pset_tweak_details"] ={"process": {"outputModules_": ["RECODEBUGoutput", "DQMoutput"], "RECODEBUGoutput": {"dataset": {"filterName": "", "dataTier": "RECO-DEBUG-OUTPUT"}}, "DQMoutput": {"dataset": {"filterName": "", "dataTier": "DQM"}}}} else: stepOneConfig["pset_tweak_details"] ={"process": {"outputModules_": ["RAWDEBUGoutput"], "RAWDEBUGoutput": {"dataset": {"filterName": "", "dataTier": "RAW-DEBUG-OUTPUT"}}}} stepTwoConfig = Document() stepTwoConfig["info"] = None stepTwoConfig["config"] = None stepTwoConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f" stepTwoConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7" stepTwoConfig["owner"] = {"group": "cmsdataops", "user": "******"} stepTwoConfig["pset_tweak_details"] ={"process": {"outputModules_": ["RECODEBUGoutput", "DQMoutput"], "RECODEBUGoutput": {"dataset": {"filterName": "", "dataTier": "RECO-DEBUG-OUTPUT"}}, "DQMoutput": {"dataset": {"filterName": "", "dataTier": "DQM"}}}} stepThreeConfig = Document() stepThreeConfig["info"] = None stepThreeConfig["config"] = None stepThreeConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f" stepThreeConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7" stepThreeConfig["owner"] = {"group": "cmsdataops", "user": "******"} stepThreeConfig["pset_tweak_details"] ={"process": {"outputModules_": ["aodOutputModule"], "aodOutputModule": {"dataset": {"filterName": "", "dataTier": "AODSIM"}}}} stepOne = self.configDatabase.commitOne(stepOneConfig)[0]["id"] stepTwo = self.configDatabase.commitOne(stepTwoConfig)[0]["id"] stepThree = self.configDatabase.commitOne(stepThreeConfig)[0]["id"] return (stepOne, stepTwo, stepThree) def testDependentReDigi(self): """ _testDependentReDigi_ Verfiy that a dependent ReDigi workflow that keeps stages out RAW data is created and installed into WMBS correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() topLevelFileset = Fileset(name = "TestWorkload-StepOneProc-SomeBlock") topLevelFileset.loadData() stepOneUnmergedRAWFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-RAWDEBUGoutput") stepOneUnmergedRAWFileset.loadData() stepOneMergedRAWFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/merged-Merged") stepOneMergedRAWFileset.loadData() stepOneLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-logArchive") stepOneLogArchiveFileset.loadData() stepOneMergeLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/merged-logArchive") stepOneMergeLogArchiveFileset.loadData() stepTwoUnmergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/unmerged-DQMoutput") stepTwoUnmergedDQMFileset.loadData() stepTwoUnmergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/unmerged-RECODEBUGoutput") stepTwoUnmergedRECOFileset.loadData() stepTwoMergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeDQMoutput/merged-Merged") stepTwoMergedDQMFileset.loadData() stepTwoMergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/merged-Merged") stepTwoMergedRECOFileset.loadData() stepTwoLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/unmerged-logArchive") stepTwoLogArchiveFileset.loadData() stepTwoMergeDQMLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeDQMoutput/merged-logArchive") stepTwoMergeDQMLogArchiveFileset.loadData() stepTwoMergeRECOLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/merged-logArchive") stepTwoMergeRECOLogArchiveFileset.loadData() stepThreeUnmergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/unmerged-aodOutputModule") stepThreeUnmergedAODFileset.loadData() stepThreeMergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcMergeaodOutputModule/merged-Merged") stepThreeMergedAODFileset.loadData() stepThreeLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/unmerged-logArchive") stepThreeLogArchiveFileset.loadData() stepThreeMergeLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcMergeaodOutputModule/merged-logArchive") stepThreeMergeLogArchiveFileset.loadData() stepOneWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc") stepOneWorkflow.load() self.assertTrue("logArchive" in stepOneWorkflow.outputMap.keys(), "Error: Step one missing output module.") self.assertTrue("RAWDEBUGoutput" in stepOneWorkflow.outputMap.keys(), "Error: Step one missing output module.") self.assertEqual(stepOneWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepOneLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepOneLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneWorkflow.outputMap["RAWDEBUGoutput"][0]["merged_output_fileset"].id, stepOneMergedRAWFileset.id, "Error: RAWDEBUG output fileset is wrong.") self.assertEqual(stepOneWorkflow.outputMap["RAWDEBUGoutput"][0]["output_fileset"].id, stepOneUnmergedRAWFileset.id, "Error: RAWDEBUG output fileset is wrong.") for outputMod in stepOneWorkflow.outputMap.keys(): self.assertTrue(len(stepOneWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepOneSub = Subscription(workflow = stepOneWorkflow, fileset = topLevelFileset) stepOneSub.loadData() self.assertEqual(stepOneSub["type"], "Processing", "Error: Step one sub has wrong type.") stepOneCleanupWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedRAWDEBUGoutput") stepOneCleanupWorkflow.load() self.assertEqual(len(stepOneCleanupWorkflow.outputMap.keys()), 0, "Error: Cleanup should have no output.") stepOneCleanupSub = Subscription(workflow = stepOneCleanupWorkflow, fileset = stepOneUnmergedRAWFileset) stepOneCleanupSub.loadData() self.assertEqual(stepOneCleanupSub["type"], "Cleanup", "Error: Step one sub has wrong type.") stepOneLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/LogCollect") stepOneLogCollectWorkflow.load() self.assertEqual(len(stepOneLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect should have no output.") stepOneLogCollectSub = Subscription(workflow = stepOneLogCollectWorkflow, fileset = stepOneLogArchiveFileset) stepOneLogCollectSub.loadData() self.assertEqual(stepOneLogCollectSub["type"], "LogCollect", "Error: Step one sub has wrong type.") stepOneMergeWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput") stepOneMergeWorkflow.load() self.assertTrue("Merged" in stepOneMergeWorkflow.outputMap.keys(), "Error: Step one merge missing output module.") self.assertTrue("logArchive" in stepOneMergeWorkflow.outputMap.keys(), "Error: Step one merge missing output module.") self.assertEqual(stepOneMergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepOneMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneMergeWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepOneMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepOneMergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepOneMergedRAWFileset.id, "Error: RAWDEBUG merge output fileset is wrong.") self.assertEqual(stepOneMergeWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepOneMergedRAWFileset.id, "Error: RAWDEBUG merge output fileset is wrong.") for outputMod in stepOneMergeWorkflow.outputMap.keys(): self.assertTrue(len(stepOneMergeWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepOneMergeSub = Subscription(workflow = stepOneMergeWorkflow, fileset = stepOneUnmergedRAWFileset) stepOneMergeSub.loadData() self.assertEqual(stepOneMergeSub["type"], "Merge", "Error: Step one sub has wrong type.") stepTwoWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc") stepTwoWorkflow.load() self.assertTrue("RECODEBUGoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertTrue("DQMoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["output_fileset"].id, stepTwoUnmergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["output_fileset"].id, stepTwoUnmergedDQMFileset.id, "Error: DQM output fileset is wrong.") stepTwoSub = Subscription(workflow = stepTwoWorkflow, fileset = stepOneMergedRAWFileset) stepTwoSub.loadData() self.assertEqual(stepTwoSub["type"], "Processing", "Error: Step two sub has wrong type.") for outputMod in stepTwoWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoCleanupDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcCleanupUnmergedDQMoutput") stepTwoCleanupDQMWorkflow.load() self.assertEqual(len(stepTwoCleanupDQMWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupDQMSub = Subscription(workflow = stepTwoCleanupDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoCleanupDQMSub.loadData() self.assertEqual(stepTwoCleanupDQMSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoCleanupRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcCleanupUnmergedRECODEBUGoutput") stepTwoCleanupRECOWorkflow.load() self.assertEqual(len(stepTwoCleanupRECOWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupRECOSub = Subscription(workflow = stepTwoCleanupRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoCleanupRECOSub.loadData() self.assertEqual(stepTwoCleanupRECOSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcLogCollect") stepTwoLogCollectWorkflow.load() self.assertEqual(len(stepTwoLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect shouldn't have any output.") stepTwoLogCollectSub = Subscription(workflow = stepTwoLogCollectWorkflow, fileset = stepTwoLogArchiveFileset) stepTwoLogCollectSub.loadData() self.assertEqual(stepTwoLogCollectSub["type"], "LogCollect", "Error: Step two sub has wrong type.") stepTwoMergeRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput") stepTwoMergeRECOWorkflow.load() self.assertTrue("Merged" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") stepTwoMergeRECOSub = Subscription(workflow = stepTwoMergeRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoMergeRECOSub.loadData() self.assertEqual(stepTwoMergeRECOSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeRECOWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeRECOWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoMergeDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeDQMoutput") stepTwoMergeDQMWorkflow.load() self.assertTrue("Merged" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") stepTwoMergeDQMSub = Subscription(workflow = stepTwoMergeDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoMergeDQMSub.loadData() self.assertEqual(stepTwoMergeDQMSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeDQMWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeDQMWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepThreeWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc") stepThreeWorkflow.load() self.assertTrue("aodOutputModule" in stepThreeWorkflow.outputMap.keys(), "Error: Step three missing output module.") self.assertTrue("logArchive" in stepThreeWorkflow.outputMap.keys(), "Error: Step three missing output module.") self.assertEqual(stepThreeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepThreeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepThreeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeWorkflow.outputMap["aodOutputModule"][0]["merged_output_fileset"].id, stepThreeMergedAODFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepThreeWorkflow.outputMap["aodOutputModule"][0]["output_fileset"].id, stepThreeUnmergedAODFileset.id, "Error: RECODEBUG output fileset is wrong.") stepThreeSub = Subscription(workflow = stepThreeWorkflow, fileset = stepTwoMergedRECOFileset) stepThreeSub.loadData() self.assertEqual(stepThreeSub["type"], "Processing", "Error: Step three sub has wrong type.") for outputMod in stepThreeWorkflow.outputMap.keys(): self.assertTrue(len(stepThreeWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepThreeCleanupWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcCleanupUnmergedaodOutputModule") stepThreeCleanupWorkflow.load() self.assertEqual(len(stepThreeCleanupWorkflow.outputMap.keys()), 0, "Error: Cleanup should have no output.") stepThreeCleanupSub = Subscription(workflow = stepThreeCleanupWorkflow, fileset = stepThreeUnmergedAODFileset) stepThreeCleanupSub.loadData() self.assertEqual(stepThreeCleanupSub["type"], "Cleanup", "Error: Step three sub has wrong type.") stepThreeLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcLogCollect") stepThreeLogCollectWorkflow.load() self.assertEqual(len(stepThreeLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect should have no output.") stepThreeLogCollectSub = Subscription(workflow = stepThreeLogCollectWorkflow, fileset = stepThreeLogArchiveFileset) stepThreeLogCollectSub.loadData() self.assertEqual(stepThreeLogCollectSub["type"], "LogCollect", "Error: Step three sub has wrong type.") stepThreeMergeWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRAWDEBUGoutput/StepTwoProc/StepTwoProcMergeRECODEBUGoutput/StepThreeProc/StepThreeProcMergeaodOutputModule") stepThreeMergeWorkflow.load() self.assertTrue("Merged" in stepThreeMergeWorkflow.outputMap.keys(), "Error: Step three merge missing output module.") self.assertTrue("logArchive" in stepThreeMergeWorkflow.outputMap.keys(), "Error: Step three merge missing output module.") self.assertEqual(stepThreeMergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepThreeMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeMergeWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepThreeMergeLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepThreeMergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepThreeMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") self.assertEqual(stepThreeMergeWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepThreeMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") stepThreeMergeSub = Subscription(workflow = stepThreeMergeWorkflow, fileset = stepThreeUnmergedAODFileset) stepThreeMergeSub.loadData() self.assertEqual(stepThreeMergeSub["type"], "Merge", "Error: Step three sub has wrong type.") for outputMod in stepThreeMergeWorkflow.outputMap.keys(): self.assertTrue(len(stepThreeMergeWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") return def verifyDiscardRAW(self): """ _verifyDiscardRAW_ Verify that a workflow that discards the RAW was installed into WMBS correctly. """ topLevelFileset = Fileset(name = "TestWorkload-StepOneProc-SomeBlock") topLevelFileset.loadData() stepTwoUnmergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-DQMoutput") stepTwoUnmergedDQMFileset.loadData() stepTwoUnmergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-RECODEBUGoutput") stepTwoUnmergedRECOFileset.loadData() stepTwoMergedDQMFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeDQMoutput/merged-Merged") stepTwoMergedDQMFileset.loadData() stepTwoMergedRECOFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRECODEBUGoutput/merged-Merged") stepTwoMergedRECOFileset.loadData() stepTwoLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-logArchive") stepTwoLogArchiveFileset.loadData() stepTwoMergeDQMLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeDQMoutput/merged-logArchive") stepTwoMergeDQMLogArchiveFileset.loadData() stepTwoMergeRECOLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeRECODEBUGoutput/merged-logArchive") stepTwoMergeRECOLogArchiveFileset.loadData() stepTwoWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc") stepTwoWorkflow.load() self.assertTrue("RECODEBUGoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertTrue("DQMoutput" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["RECODEBUGoutput"][0]["output_fileset"].id, stepTwoUnmergedRECOFileset.id, "Error: RECODEBUG output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["DQMoutput"][0]["output_fileset"].id, stepTwoUnmergedDQMFileset.id, "Error: DQM output fileset is wrong.") stepTwoSub = Subscription(workflow = stepTwoWorkflow, fileset = topLevelFileset) stepTwoSub.loadData() self.assertEqual(stepTwoSub["type"], "Processing", "Error: Step two sub has wrong type.") for outputMod in stepTwoWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoCleanupDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedDQMoutput") stepTwoCleanupDQMWorkflow.load() self.assertEqual(len(stepTwoCleanupDQMWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupDQMSub = Subscription(workflow = stepTwoCleanupDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoCleanupDQMSub.loadData() self.assertEqual(stepTwoCleanupDQMSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoCleanupRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedRECODEBUGoutput") stepTwoCleanupRECOWorkflow.load() self.assertEqual(len(stepTwoCleanupRECOWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupRECOSub = Subscription(workflow = stepTwoCleanupRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoCleanupRECOSub.loadData() self.assertEqual(stepTwoCleanupRECOSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/LogCollect") stepTwoLogCollectWorkflow.load() self.assertEqual(len(stepTwoLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect shouldn't have any output.") stepTwoLogCollectSub = Subscription(workflow = stepTwoLogCollectWorkflow, fileset = stepTwoLogArchiveFileset) stepTwoLogCollectSub.loadData() self.assertEqual(stepTwoLogCollectSub["type"], "LogCollect", "Error: Step two sub has wrong type.") stepTwoMergeRECOWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeRECODEBUGoutput") stepTwoMergeRECOWorkflow.load() self.assertTrue("Merged" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeRECOWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeRECOLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") self.assertEqual(stepTwoMergeRECOWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedRECOFileset.id, "Error: RECODEBUG merge output fileset is wrong.") stepTwoMergeRECOSub = Subscription(workflow = stepTwoMergeRECOWorkflow, fileset = stepTwoUnmergedRECOFileset) stepTwoMergeRECOSub.loadData() self.assertEqual(stepTwoMergeRECOSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeRECOWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeRECOWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoMergeDQMWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeDQMoutput") stepTwoMergeDQMWorkflow.load() self.assertTrue("Merged" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeDQMWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeDQMLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") self.assertEqual(stepTwoMergeDQMWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedDQMFileset.id, "Error: DQM merge output fileset is wrong.") stepTwoMergeDQMSub = Subscription(workflow = stepTwoMergeDQMWorkflow, fileset = stepTwoUnmergedDQMFileset) stepTwoMergeDQMSub.loadData() self.assertEqual(stepTwoMergeDQMSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeDQMWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeDQMWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") return def verifyKeepAOD(self): """ _verifyKeepAOD_ Verify that a workflow that only produces AOD in a single step was installed correctly into WMBS. """ topLevelFileset = Fileset(name = "TestWorkload-StepOneProc-SomeBlock") topLevelFileset.loadData() stepTwoUnmergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-aodOutputModule") stepTwoUnmergedAODFileset.loadData() stepTwoMergedAODFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeaodOutputModule/merged-Merged") stepTwoMergedAODFileset.loadData() stepTwoLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/unmerged-logArchive") stepTwoLogArchiveFileset.loadData() stepTwoMergeAODLogArchiveFileset = Fileset(name = "/TestWorkload/StepOneProc/StepOneProcMergeaodOutputModule/merged-logArchive") stepTwoMergeAODLogArchiveFileset.loadData() stepTwoWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc") stepTwoWorkflow.load() self.assertTrue("aodOutputModule" in stepTwoWorkflow.outputMap.keys(), "Error: Step two missing output module.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["aodOutputModule"][0]["merged_output_fileset"].id, stepTwoMergedAODFileset.id, "Error: AOD output fileset is wrong.") self.assertEqual(stepTwoWorkflow.outputMap["aodOutputModule"][0]["output_fileset"].id, stepTwoUnmergedAODFileset.id, "Error: AOD output fileset is wrong.") stepTwoSub = Subscription(workflow = stepTwoWorkflow, fileset = topLevelFileset) stepTwoSub.loadData() self.assertEqual(stepTwoSub["type"], "Processing", "Error: Step two sub has wrong type.") for outputMod in stepTwoWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") stepTwoCleanupAODWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcCleanupUnmergedaodOutputModule") stepTwoCleanupAODWorkflow.load() self.assertEqual(len(stepTwoCleanupAODWorkflow.outputMap.keys()), 0, "Error: Cleanup shouldn't have any output.") stepTwoCleanupAODSub = Subscription(workflow = stepTwoCleanupAODWorkflow, fileset = stepTwoUnmergedAODFileset) stepTwoCleanupAODSub.loadData() self.assertEqual(stepTwoCleanupAODSub["type"], "Cleanup", "Error: Step two sub has wrong type.") stepTwoLogCollectWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/LogCollect") stepTwoLogCollectWorkflow.load() self.assertEqual(len(stepTwoLogCollectWorkflow.outputMap.keys()), 0, "Error: LogCollect shouldn't have any output.") stepTwoLogCollectSub = Subscription(workflow = stepTwoLogCollectWorkflow, fileset = stepTwoLogArchiveFileset) stepTwoLogCollectSub.loadData() self.assertEqual(stepTwoLogCollectSub["type"], "LogCollect", "Error: Step two sub has wrong type.") stepTwoMergeAODWorkflow = Workflow(spec = "somespec", name = "TestWorkload", task = "/TestWorkload/StepOneProc/StepOneProcMergeaodOutputModule") stepTwoMergeAODWorkflow.load() self.assertTrue("Merged" in stepTwoMergeAODWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertTrue("logArchive" in stepTwoMergeAODWorkflow.outputMap.keys(), "Error: Step two merge missing output module.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["logArchive"][0]["merged_output_fileset"].id, stepTwoMergeAODLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["logArchive"][0]["output_fileset"].id, stepTwoMergeAODLogArchiveFileset.id, "Error: logArchive fileset is wrong.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["Merged"][0]["merged_output_fileset"].id, stepTwoMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") self.assertEqual(stepTwoMergeAODWorkflow.outputMap["Merged"][0]["output_fileset"].id, stepTwoMergedAODFileset.id, "Error: AOD merge output fileset is wrong.") stepTwoMergeAODSub = Subscription(workflow = stepTwoMergeAODWorkflow, fileset = stepTwoUnmergedAODFileset) stepTwoMergeAODSub.loadData() self.assertEqual(stepTwoMergeAODSub["type"], "Merge", "Error: Step two sub has wrong type.") for outputMod in stepTwoMergeAODWorkflow.outputMap.keys(): self.assertTrue(len(stepTwoMergeAODWorkflow.outputMap[outputMod]) == 1, "Error: more than one destination for output mod.") return def testChainedReDigi(self): """ _testChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW data can be created and installed into WMBS correctly. This will only verify the step one/step two information in WMBS as the step three information is the same as the dependent workflow. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() self.verifyDiscardRAW() return def testThreeStepChainedReDigi(self): """ _testThreeStepChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW and RECO data can be created and installed into WMBS correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False defaultArguments["KeepStepTwoOutput"] = False testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() self.verifyKeepAOD() return def testCombinedReDigiRecoConfig(self): """ _testCombinedReDigiRecoConfig_ Verify that a ReDigi workflow that uses a single step one config installs into WMBS correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs(combinedStepOne = True) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[2] defaultArguments["StepThreeConfigCacheID"] = None defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput" testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() self.verifyDiscardRAW() return def testSingleStepReDigi(self): """ _testSingleStepReDigi_ Verify that a single step ReDigi workflow can be created and installed correctly into WMBS. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[2] defaultArguments["StepTwoConfigCacheID"] = None defaultArguments["StepThreeConfigCacheID"] = None testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() self.verifyKeepAOD() return
class MonteCarloTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("montecarlo_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) self.testInit.generateWorkDir() couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("rereco_t") EmulatorHelper.setEmulators(dbs = True) return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() EmulatorHelper.resetEmulators() return def injectMonteCarloConfig(self): """ _injectMonteCarlo_ Create a bogus config cache document for the montecarlo generation and inject it into couch. Return the ID of the document. """ newConfig = Document() newConfig["info"] = None newConfig["config"] = None newConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f" newConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7" newConfig["owner"] = {"group": "cmsdataops", "user": "******"} newConfig["pset_tweak_details"] ={"process": {"outputModules_": ["OutputA", "OutputB"], "OutputA": {"dataset": {"filterName": "OutputAFilter", "dataTier": "RECO"}}, "OutputB": {"dataset": {"filterName": "OutputBFilter", "dataTier": "USER"}}}} result = self.configDatabase.commitOne(newConfig) return result[0]["id"] def _commonMonteCarloTest(self): """ Retrieve the workload from WMBS and test all its properties. """ prodWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Production") prodWorkflow.load() self.assertEqual(len(prodWorkflow.outputMap.keys()), 3, "Error: Wrong number of WF outputs.") goldenOutputMods = ["OutputA", "OutputB"] for goldenOutputMod in goldenOutputMods: mergedOutput = prodWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = prodWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Production/unmerged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = prodWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = prodWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Production/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Production/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Production/ProductionMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-Merged" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-Merged" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Production/ProductionMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-Production-SomeBlock") topLevelFileset.loadData() prodSubscription = Subscription(fileset = topLevelFileset, workflow = prodWorkflow) prodSubscription.loadData() self.assertEqual(prodSubscription["type"], "Production", "Error: Wrong subscription type.") self.assertEqual(prodSubscription["split_algo"], "EventBased", "Error: Wrong split algo.") for outputName in ["OutputA", "OutputB"]: unmergedOutput = Fileset(name = "/TestWorkload/Production/unmerged-%s" % outputName) unmergedOutput.loadData() mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Production/ProductionMerge%s" % outputName) mergeWorkflow.load() mergeSubscription = Subscription(fileset = unmergedOutput, workflow = mergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algo: %s" % mergeSubscription["split_algo"]) for outputName in ["OutputA", "OutputB"]: unmerged = Fileset(name = "/TestWorkload/Production/unmerged-%s" % outputName) unmerged.loadData() cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Production/ProductionCleanupUnmerged%s" % outputName) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset = unmerged, workflow = cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong split algo.") procLogCollect = Fileset(name = "/TestWorkload/Production/unmerged-logArchive") procLogCollect.loadData() procLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Production/LogCollect") procLogCollectWorkflow.load() logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") for outputName in ["OutputA", "OutputB"]: mergeLogCollect = Fileset(name = "/TestWorkload/Production/ProductionMerge%s/merged-logArchive" % outputName) mergeLogCollect.loadData() mergeLogCollectWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/Production/ProductionMerge%s/Production%sMergeLogCollect" % (outputName, outputName)) mergeLogCollectWorkflow.load() logCollectSub = Subscription(fileset = mergeLogCollect, workflow = mergeLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") def testMonteCarlo(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig() testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return def testMonteCarloExtension(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first event and lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstEvent"] = 3571428573 defaultArguments["FirstLumi"] = 26042 defaultArguments["TimePerEvent"] = 15 defaultArguments["FilterEfficiency"] = 0.014 defaultArguments["TotalTime"] = 28800 initial_lfn_counter = 26042 # Same as the previous number of jobs + 1 which is the same value of the first lumi testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath('/TestWorkload/Production/ProductionMerge%s' % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return def testRelValMCWithPileup(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. The input configuration includes pileup input files. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig() # add pile up configuration defaultArguments["PileupConfig"] = {"mc": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"], "data": ["/some/minbias/dataset3"]} testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
class TaskArchiverTest(unittest.TestCase): """ TestCase for TestTaskArchiver module """ _setup_done = False _teardown = False _maxMessage = 10 OWNERDN = os.environ['OWNERDN'] if 'OWNERDN' in os.environ else "Generic/OWNERDN" def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase = True) self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMComponent.DBS3Buffer"], useDefault = False) self.databaseName = "taskarchiver_t_0" self.testInit.setupCouch("%s/workloadsummary" % self.databaseName, "WorkloadSummary") self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump") self.testInit.setupCouch("wmagent_summary_t", "WMStats") self.testInit.setupCouch("wmagent_summary_central_t", "WMStats") self.testInit.setupCouch("stat_summary_t", "SummaryStats") reqmgrdb = "reqmgrdb_t" self.testInit.setupCouch(reqmgrdb, "ReqMgr") reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb) self.requestWriter = RequestDBWriter(reqDBURL) self.requestWriter.defaultStale = {} self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs") self.inject = self.daofactory(classname = "Workflow.MarkInjectedWorkflows") self.testDir = self.testInit.generateWorkDir() os.makedirs(os.path.join(self.testDir, 'specDir')) self.nJobs = 10 self.campaignName = 'aCampaign' self.uploadPublishInfo = False self.uploadPublishDir = None return def tearDown(self): """ Database deletion """ myThread = threading.currentThread() self.testInit.clearDatabase(modules = ["WMCore.WMBS"]) self.testInit.delWorkDir() self.testInit.tearDownCouch() return def getConfig(self): """ _createConfig_ General config file """ config = self.testInit.getConfiguration() #self.testInit.generateWorkDir(config) config.section_("General") config.General.workDir = "." config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv52.fnal.gov:5984") config.JobStateMachine.couchDBName = self.databaseName config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' config.JobStateMachine.summaryStatsDBName = 'stat_summary_t' config.component_("JobCreator") config.JobCreator.jobCacheDir = os.path.join(self.testDir, 'testDir') config.component_("TaskArchiver") config.TaskArchiver.componentDir = self.testDir config.TaskArchiver.WorkQueueParams = {} config.TaskArchiver.pollInterval = 60 config.TaskArchiver.logLevel = 'INFO' config.TaskArchiver.timeOut = 0 config.TaskArchiver.histogramKeys = ['AvgEventTime', 'writeTotalMB', 'jobTime'] config.TaskArchiver.histogramBins = 5 config.TaskArchiver.histogramLimit = 5 config.TaskArchiver.perfPrimaryDatasets = ['SingleMu', 'MuHad', 'MinimumBias'] config.TaskArchiver.perfDashBoardMinLumi = 50 config.TaskArchiver.perfDashBoardMaxLumi = 9000 config.TaskArchiver.dqmUrl = 'https://cmsweb.cern.ch/dqm/dev/' config.TaskArchiver.dashBoardUrl = 'http://dashboard43.cern.ch/dashboard/request.py/putluminositydata' config.TaskArchiver.workloadSummaryCouchDBName = "%s/workloadsummary" % self.databaseName config.TaskArchiver.workloadSummaryCouchURL = config.JobStateMachine.couchurl config.TaskArchiver.requireCouch = True config.TaskArchiver.uploadPublishInfo = self.uploadPublishInfo config.TaskArchiver.uploadPublishDir = self.uploadPublishDir config.TaskArchiver.userFileCacheURL = os.getenv('UFCURL', 'http://cms-xen38.fnal.gov:7725/userfilecache/') config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2" config.TaskArchiver.ReqMgrServiceURL = "https://cmsweb-dev.cern.ch/reqmgr/rest" config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) config.component_("AnalyticsDataCollector") config.AnalyticsDataCollector.centralRequestDBURL = '%s/reqmgrdb_t' % config.JobStateMachine.couchurl config.AnalyticsDataCollector.RequestCouchApp = "ReqMgr" config.section_("ACDC") config.ACDC.couchurl = config.JobStateMachine.couchurl config.ACDC.database = config.JobStateMachine.couchDBName # Make the jobCacheDir os.mkdir(config.JobCreator.jobCacheDir) # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" config.section_("BossAir") config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh' config.BossAir.gliteConf = '/afs/cern.ch/cms/LCG/LCG-2/UI/conf/glite_wms_CERN.conf' config.BossAir.credentialDir = '/home/crab/ALL_SETUP/credentials/' config.BossAir.gLiteProcesses = 2 config.BossAir.gLitePrefixEnv = "/lib64/" config.BossAir.pluginNames = ["gLitePlugin"] config.BossAir.proxyDir = "/tmp/credentials" config.BossAir.manualProxyPath = os.environ['X509_USER_PROXY'] if 'X509_USER_PROXY' in os.environ else None config.section_("Agent") config.Agent.serverDN = "/we/bypass/myproxy/logon" return config def createWorkload(self, workloadName = 'Test', emulator = True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.setCampaign(self.campaignName) workload.save(workloadName) return workload def createTestJobGroup(self, config, name = "TestWorkthrough", filesetName = "TestFileset", specLocation = "spec.xml", error = False, task = "/TestWorkload/ReReco", type = "Processing"): """ Creates a group of several jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec = specLocation, owner = self.OWNERDN, name = name, task = task, owner_vogroup="", owner_vorole="") testWorkflow.create() self.inject.execute(names = [name], injected = True) testWMBSFileset = Fileset(name = filesetName) testWMBSFileset.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) outputWMBSFileset = Fileset(name = '%sOutput' % filesetName) outputWMBSFileset.create() testFileC = File(lfn = "/this/is/a/lfnC" , size = 1024, events = 10) testFileC.addRun(Run(10, *[12312])) testFileC.setLocation('malpaquet') testFileC.create() outputWMBSFileset.addFile(testFileC) outputWMBSFileset.commit() outputWMBSFileset.markOpen(0) testWorkflow.addOutput('output', outputWMBSFileset) testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow, type = type) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() for i in range(0,self.nJobs): testJob = Job(name = makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run = 10, lumis = [12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() changer = ChangeState(config) report1 = Report() report2 = Report() if error: path1 = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl") path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') else: path1 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'mergeReport1.pkl') path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') report1.load(filename = path1) report2.load(filename = path2) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') for i in range(self.nJobs): if i < self.nJobs/2: testJobGroup.jobs[i]['fwjr'] = report1 else: testJobGroup.jobs[i]['fwjr'] = report2 changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'retrydone', 'jobfailed') changer.propagate(testJobGroup.jobs, 'exhausted', 'retrydone') changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted') testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup def createGiantJobSet(self, name, config, nSubs = 10, nJobs = 10, nFiles = 1, spec = "spec.xml"): """ Creates a massive set of jobs """ jobList = [] for i in range(0, nSubs): # Make a bunch of subscriptions localName = '%s-%i' % (name, i) testWorkflow = Workflow(spec = spec, owner = self.OWNERDN, name = localName, task="Test", owner_vogroup="", owner_vorole="") testWorkflow.create() testWMBSFileset = Fileset(name = localName) testWMBSFileset.create() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() filesToComplete = [] for j in range(0, nJobs): # Create jobs for each subscription testFileA = File(lfn = "%s-%i-lfnA" % (localName, j) , size = 1024, events = 10) testFileA.addRun(Run(10, *[11,12,13,14,15,16,17,18,19,20, 21,22,23,24,25,26,27,28,29,30, 31,32,33,34,35,36,37,38,39,40])) testFileA.setLocation('malpaquet') testFileA.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.commit() filesToComplete.append(testFileA) testJob = Job(name = '%s-%i' % (localName, j)) testJob.addFile(testFileA) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJobGroup.add(testJob) jobList.append(testJob) for k in range(0, nFiles): # Create output files testFile = File(lfn = "%s-%i-output" % (localName, k) , size = 1024, events = 10) testFile.addRun(Run(10, *[12312])) testFile.setLocation('malpaquet') testFile.create() testJobGroup.output.addFile(testFile) testJobGroup.output.commit() testJobGroup.commit() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'success', 'complete') changer.propagate(testJobGroup.jobs, 'cleanout', 'success') testWMBSFileset.markOpen(0) testSubscription.completeFiles(filesToComplete) return jobList def getPerformanceFromDQM(self, dqmUrl, dataset, run): # Make function to fetch this from DQM. Returning Null or False if it fails getUrl = "%sjsonfairy/archive/%s%s/DQM/TimerService/event_byluminosity" % (dqmUrl, run, dataset) # Assert if the URL is assembled as expected if run == 207214: self.assertEqual('https://cmsweb.cern.ch/dqm/dev/jsonfairy/archive/207214/MinimumBias/Commissioning10-v4/DQM/DQM/TimerService/event_byluminosity', getUrl) # let's suppose it works.. testResponseFile = open(os.path.join(getTestBase(), 'WMComponent_t/TaskArchiver_t/DQMGUIResponse.json'), 'r') response = testResponseFile.read() testResponseFile.close() responseJSON = json.loads(response) return responseJSON def filterInterestingPerfPoints(self, responseJSON, minLumi, maxLumi): worthPoints = {} points = responseJSON["hist"]["bins"]["content"] for i in range(responseJSON["hist"]["xaxis"]["first"]["id"], responseJSON["hist"]["xaxis"]["last"]["id"]): # is the point worth it? if yes add to interesting points dictionary. # 1 - non 0 # 2 - between minimum and maximum expected luminosity # FIXME : 3 - population in dashboard for the bin interval < 100 # Those should come from the config : if points[i] == 0: continue binSize = responseJSON["hist"]["xaxis"]["last"]["value"]/responseJSON["hist"]["xaxis"]["last"]["id"] # Fetching the important values instLuminosity = i*binSize timePerEvent = points[i] if instLuminosity > minLumi and instLuminosity < maxLumi : worthPoints[instLuminosity] = timePerEvent return worthPoints def publishPerformanceDashBoard(self, dashBoardUrl, PD, release, worthPoints): dashboardPayload = [] for instLuminosity in worthPoints : timePerEvent = int(worthPoints[instLuminosity]) dashboardPayload.append({"primaryDataset" : PD, "release" : release, "integratedLuminosity" : instLuminosity, "timePerEvent" : timePerEvent}) data = "{\"data\":%s}" % str(dashboardPayload).replace("\'","\"") # let's suppose it works.. testDashBoardPayloadFile = open(os.path.join(getTestBase(), 'WMComponent_t/TaskArchiver_t/DashBoardPayload.json'), 'r') testDashBoardPayload = testDashBoardPayloadFile.read() testDashBoardPayloadFile.close() self.assertEqual(data, testDashBoardPayload) return True def populateWorkflowWithCompleteStatus(self, name ="TestWorkload"): schema = generate_reqmgr_schema(1) schema[0]["RequestName"] = name self.requestWriter.insertGenericRequest(schema[0]) result = self.requestWriter.updateRequestStatus(name, "completed") return result def testA_BasicFunctionTest(self): """ _BasicFunctionTest_ Tests the components, by seeing if they can process a simple set of closeouts """ myThread = threading.currentThread() config = self.getConfig() workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl') workload = self.createWorkload(workloadName = workloadPath) testJobGroup = self.createTestJobGroup(config = config, name = workload.name(), specLocation = workloadPath, error = False) # Create second workload testJobGroup2 = self.createTestJobGroup(config = config, name = workload.name(), filesetName = "TestFileset_2", specLocation = workloadPath, task = "/TestWorkload/ReReco/LogCollect", type = "LogCollect") cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "ReReco") os.makedirs(cachePath) self.assertTrue(os.path.exists(cachePath)) cachePath2 = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "LogCollect") os.makedirs(cachePath2) self.assertTrue(os.path.exists(cachePath2)) result = myThread.dbi.processData("SELECT * FROM wmbs_subscription")[0].fetchall() self.assertEqual(len(result), 2) workflowName = "TestWorkload" dbname = config.TaskArchiver.workloadSummaryCouchDBName couchdb = CouchServer(config.JobStateMachine.couchurl) workdatabase = couchdb.connectDatabase(dbname) jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName) fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName) jobs = jobdb.loadView("JobDump", "jobsByWorkflowName", options = {"startkey": [workflowName], "endkey": [workflowName, {}]})['rows'] fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName", options = {"startkey": [workflowName], "endkey": [workflowName, {}]})['rows'] self.assertEqual(len(jobs), 2*self.nJobs) from WMCore.WMBS.CreateWMBSBase import CreateWMBSBase create = CreateWMBSBase() tables = [] for x in create.requiredTables: tables.append(x[2:]) self.populateWorkflowWithCompleteStatus() testTaskArchiver = TaskArchiverPoller(config = config) testTaskArchiver.algorithm() cleanCouch = CleanCouchPoller(config = config) cleanCouch.setup() cleanCouch.algorithm() result = myThread.dbi.processData("SELECT * FROM wmbs_job")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData("SELECT * FROM wmbs_subscription")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData("SELECT * FROM wmbs_jobgroup")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData("SELECT * FROM wmbs_fileset")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData("SELECT * FROM wmbs_file_details")[0].fetchall() self.assertEqual(len(result), 0) # Make sure we deleted the directory self.assertFalse(os.path.exists(cachePath)) self.assertFalse(os.path.exists(os.path.join(self.testDir, 'workloadTest/TestWorkload'))) testWMBSFileset = Fileset(id = 1) self.assertEqual(testWMBSFileset.exists(), False) workloadSummary = workdatabase.document(id = "TestWorkload") # Check ACDC self.assertEqual(workloadSummary['ACDCServer'], sanitizeURL(config.ACDC.couchurl)['url']) # Check the output self.assertEqual(workloadSummary['output'].keys(), ['/Electron/MorePenguins-v0/RECO']) self.assertEqual(sorted(workloadSummary['output']['/Electron/MorePenguins-v0/RECO']['tasks']), ['/TestWorkload/ReReco', '/TestWorkload/ReReco/LogCollect']) # Check performance # Check histograms self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['AvgEventTime']['histogram'][0]['average'], 0.89405199999999996, places = 2) self.assertEqual(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['AvgEventTime']['histogram'][0]['nEvents'], 10) # Check standard performance self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['TotalJobCPU']['average'], 17.786300000000001, places = 2) self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['TotalJobCPU']['stdDev'], 0.0, places = 2) # Check worstOffenders self.assertEqual(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['AvgEventTime']['worstOffenders'], [{'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 1}, {'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 1}, {'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 2}]) # Check retryData self.assertEqual(workloadSummary['retryData']['/TestWorkload/ReReco'], {'1': 10}) logCollectPFN = 'srm://srm-cms.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/cms/store/logs/prod/2012/11/WMAgent/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8-AlcaSkimLogCollect-1-logs.tar' self.assertEqual(workloadSummary['logArchives'], {'/TestWorkload/ReReco/LogCollect' : [logCollectPFN for _ in range(10)]}) # LogCollect task is made out of identical FWJRs # assert that it is identical for x in workloadSummary['performance']['/TestWorkload/ReReco/LogCollect']['cmsRun1'].keys(): if x in config.TaskArchiver.histogramKeys: continue for y in ['average', 'stdDev']: self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco/LogCollect']['cmsRun1'][x][y], workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1'][x][y], places = 2) return def testB_testErrors(self): """ _testErrors_ Test with a failed FWJR """ myThread = threading.currentThread() config = self.getConfig() workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl') workload = self.createWorkload(workloadName = workloadPath) testJobGroup = self.createTestJobGroup(config = config, name = workload.name(), specLocation = workloadPath, error = True) # Create second workload testJobGroup2 = self.createTestJobGroup(config = config, name = workload.name(), filesetName = "TestFileset_2", specLocation = workloadPath, task = "/TestWorkload/ReReco/LogCollect", type = "LogCollect") cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "ReReco") os.makedirs(cachePath) self.assertTrue(os.path.exists(cachePath)) couchdb = CouchServer(config.JobStateMachine.couchurl) jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName) fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName) jobdb.loadView("JobDump", "jobsByWorkflowName", options = {"startkey": [workload.name()], "endkey": [workload.name(), {}]})['rows'] fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName", options = {"startkey": [workload.name()], "endkey": [workload.name(), {}]})['rows'] self.populateWorkflowWithCompleteStatus() testTaskArchiver = TaskArchiverPoller(config = config) testTaskArchiver.algorithm() cleanCouch = CleanCouchPoller(config = config) cleanCouch.setup() cleanCouch.algorithm() dbname = getattr(config.JobStateMachine, "couchDBName") workdatabase = couchdb.connectDatabase("%s/workloadsummary" % dbname) workloadSummary = workdatabase.document(id = workload.name()) self.assertEqual(workloadSummary['errors']['/TestWorkload/ReReco']['failureTime'], 500) self.assertTrue('99999' in workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1']) failedRunInfo = workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1']['99999']['runs'] self.assertEqual(failedRunInfo, {'10' : [[12312, 12312]]}, "Wrong lumi information in the summary for failed jobs") # Check the failures by site histograms self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['data']['T1_IT_CNAF']['Failed Jobs'], 10) self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['99999'], 10) self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['8020'], 10) self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['average']['Failed Jobs'], 10) self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['99999'], 10) self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['8020'], 10) self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['stdDev']['Failed Jobs'], 0) self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['99999'], 0) self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['8020'], 0) return def testC_Profile(self): """ _Profile_ DON'T RUN THIS! """ return import cProfile, pstats myThread = threading.currentThread() name = makeUUID() config = self.getConfig() jobList = self.createGiantJobSet(name = name, config = config, nSubs = 10, nJobs = 1000, nFiles = 10) cleanCouch = CleanCouchPoller(config = config) cleanCouch.setup() cProfile.runctx("cleanCouch.algorithm()", globals(), locals(), filename = "testStats.stat") p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats() return def testD_Timing(self): """ _Timing_ This is to see how fast things go. """ return myThread = threading.currentThread() name = makeUUID() config = self.getConfig() jobList = self.createGiantJobSet(name = name, config = config, nSubs = 10, nJobs = 1000, nFiles = 10) testTaskArchiver = TaskArchiverPoller(config = config) startTime = time.time() testTaskArchiver.algorithm() stopTime = time.time() result = myThread.dbi.processData("SELECT * FROM wmbs_job")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData("SELECT * FROM wmbs_subscription")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData("SELECT * FROM wmbs_jobgroup")[0].fetchall() self.assertEqual(len(result), 0) result = myThread.dbi.processData("SELECT * FROM wmbs_file_details")[0].fetchall() self.assertEqual(len(result), 0) testWMBSFileset = Fileset(id = 1) self.assertEqual(testWMBSFileset.exists(), False) logging.info("TaskArchiver took %f seconds" % (stopTime - startTime)) def testDQMRecoPerformanceToDashBoard(self): myThread = threading.currentThread() listRunsWorkflow = self.dbsDaoFactory(classname="ListRunsWorkflow") # Didn't like to have done that, but the test doesn't provide all info I need in the system, so faking it: myThread.dbi.processData("""insert into dbsbuffer_workflow(id, name) values (1, 'TestWorkload')""" , transaction = False) myThread.dbi.processData("""insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (1, '/store/t/e/s/t.test', 1, 1)""" , transaction = False) myThread.dbi.processData("""insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (2, '/store/t/e/s/t.test2', 1, 1)""" , transaction = False) myThread.dbi.processData("""insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207214, 100, 1)""" , transaction = False) myThread.dbi.processData("""insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207215, 200, 2)""" , transaction = False) config = self.getConfig() dqmUrl = getattr(config.TaskArchiver, "dqmUrl") perfDashBoardMinLumi = getattr(config.TaskArchiver, "perfDashBoardMinLumi") perfDashBoardMaxLumi = getattr(config.TaskArchiver, "perfDashBoardMaxLumi") dashBoardUrl = getattr(config.TaskArchiver, "dashBoardUrl") workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl') workload = self.createWorkload(workloadName = workloadPath) testJobGroup = self.createTestJobGroup(config = config, name = workload.name(), specLocation = workloadPath, error = True) testJobGroup2 = self.createTestJobGroup(config = config, name = workload.name(), filesetName = "TestFileset_2", specLocation = workloadPath, task = "/TestWorkload/ReReco/LogCollect", type = "LogCollect") # Adding request type as ReReco, real ReqMgr requests have it workload.data.request.section_("schema") workload.data.request.schema.RequestType = "ReReco" workload.data.request.schema.CMSSWVersion = 'test_compops_CMSSW_5_3_6_patch1' workload.getTask('ReReco').addInputDataset(primary='a',processed='b',tier='c') interestingPDs = getattr(config.TaskArchiver, "perfPrimaryDatasets") interestingDatasets = [] # Are the datasets from this request interesting? Do they have DQM output? One might ask afterwards if they have harvest for dataset in workload.listOutputDatasets(): (nothing, PD, procDataSet, dataTier) = dataset.split('/') if PD in interestingPDs and dataTier == "DQM": interestingDatasets.append(dataset) # We should have found 1 interesting dataset self.assertAlmostEquals(len(interestingDatasets), 1) if len(interestingDatasets) == 0 : return # Request will be only interesting for performance if it's a ReReco or PromptReco (isReReco, isPromptReco) = (False, False) if getattr(workload.data.request.schema, "RequestType", None) == 'ReReco': isReReco=True # Yes, few people like magic strings, but have a look at : # https://github.com/dmwm/T0/blob/master/src/python/T0/RunConfig/RunConfigAPI.py#L718 # Might be safe enough # FIXME: in TaskArchiver, add a test to make sure that the dataset makes sense (procDataset ~= /a/ERA-PromptReco-vVERSON/DQM) if re.search('PromptReco', workload.name()): isPromptReco = True if not (isReReco or isPromptReco): return self.assertTrue(isReReco) self.assertFalse(isPromptReco) # We are not interested if it's not a PromptReco or a ReReco if (isReReco or isPromptReco) == False: return if isReReco : release = getattr(workload.data.request.schema, "CMSSWVersion") if not release : logging.info("no release for %s, bailing out" % workload.name()) else : release = getattr(workload.tasks.Reco.steps.cmsRun1.application.setup, "cmsswVersion") if not release : logging.info("no release for %s, bailing out" % workload.name()) self.assertEqual(release, "test_compops_CMSSW_5_3_6_patch1") # If all is true, get the run numbers processed by this worklfow runList = listRunsWorkflow.execute(workflow = workload.name()) self.assertEqual([207214, 207215], runList) # GO to DQM GUI, get what you want # https://cmsweb.cern.ch/dqm/offline/jsonfairy/archive/211313/PAMuon/HIRun2013-PromptReco-v1/DQM/DQM/TimerService/event for dataset in interestingDatasets : (nothing, PD, procDataSet, dataTier) = dataset.split('/') worthPoints = {} for run in runList : responseJSON = self.getPerformanceFromDQM(dqmUrl, dataset, run) worthPoints.update(self.filterInterestingPerfPoints(responseJSON, perfDashBoardMinLumi, perfDashBoardMaxLumi)) # Publish dataset performance to DashBoard. if self.publishPerformanceDashBoard(dashBoardUrl, PD, release, worthPoints) == False: logging.info("something went wrong when publishing dataset %s to DashBoard" % dataset) return # Requires a running UserFileCache to succeed. https://cmsweb.cern.ch worked for me # The environment variable OWNERDN needs to be set. Used to retrieve an already delegated proxy and contact the ufc @attr('integration') def testPublishJSONCreate(self): """ Re-run testA_BasicFunctionTest with data in DBSBuffer Make sure files are generated """ # Set up uploading and write them elsewhere since the test deletes them. self.uploadPublishInfo = True self.uploadPublishDir = self.testDir # Insert some DBSFiles testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024, events = 20) testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024, events = 20) testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024, events = 20) testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildA.setDatasetPath("/Cosmics/USER-DATASET1-v1/USER") testFileChildB.setDatasetPath("/Cosmics/USER-DATASET1-v1/USER") testFileChildC.setDatasetPath("/Cosmics/USER-DATASET2-v1/USER") testFileChildA.create() testFileChildB.create() testFileChildC.create() testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChildA.addParents([testFile["lfn"]]) testFileChildB.addParents([testFile["lfn"]]) testFileChildC.addParents([testFile["lfn"]]) myThread = threading.currentThread() self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) self.insertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow") workflowID = self.insertWorkflow.execute(requestName='TestWorkload', taskPath='TestWorkload/Production', blockMaxCloseTime=100, blockMaxFiles=100, blockMaxEvents=100, blockMaxSize=100) myThread.dbi.processData("update dbsbuffer_file set workflow=1 where id < 4") # Run the test again self.testA_BasicFunctionTest() # Reset default values self.uploadPublishInfo = False self.uploadPublishDir = None # Make sure the files are there self.assertTrue(os.path.exists( os.path.join(self.testDir, 'TestWorkload_publish.json'))) self.assertTrue(os.path.getsize(os.path.join(self.testDir, 'TestWorkload_publish.json')) > 100) self.assertTrue(os.path.exists( os.path.join(self.testDir, 'TestWorkload_publish.tgz' ))) return
class Tier0PluginTest(unittest.TestCase): def setUp(self): """ _setUp_ Setup the test environment """ self.testInit = TestInit(__file__) self.testInit.setDatabaseConnection() self.testInit.setSchema(["WMCore.WMBS"]) self.requestCouchDB = 'wmstats_plugin_t' self.testInit.setupCouch(self.requestCouchDB, 'T0Request') self.testDir = self.testInit.generateWorkDir() reqDBURL = "%s/%s" % (os.environ['COUCHURL'], self.requestCouchDB) self.requestDBWriter = RequestDBWriter(reqDBURL, couchapp="T0Request") self.requestDBWriter._setNoStale() self.stateMap = {} self.orderedStates = [] self.plugin = None return def tearDown(self): """ _tearDown_ Clear databases and delete files """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def setupRepackWorkflow(self): """ _setupRepackWorkflow_ Populate WMBS with a repack-like workflow, every subscription must be unfinished at first """ workflowName = 'Repack_Run481516_StreamZ' mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW', 'RepackMergewrite_SingleNeutrino_RAW'] self.stateMap = {'Merge': [], 'Processing Done': []} self.orderedStates = ['Merge', 'Processing Done'] # Populate WMStats self.requestDBWriter.insertGenericRequest({'RequestName': workflowName}) self.requestDBWriter.updateRequestStatus(workflowName, 'Closed') # Create a wmspec in disk workload = newWorkload(workflowName) repackTask = workload.newTask('Repack') for task in mergeTasks: repackTask.addTask(task) repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW') specPath = os.path.join(self.testDir, 'Repack.pkl') workload.save(specPath) # Populate WMBS topFileset = Fileset(name='TestStreamerFileset') topFileset.create() options = {'spec': specPath, 'owner': 'ItsAMeMario', 'name': workflowName, 'wfType': 'tier0'} topLevelWorkflow = Workflow(task='/%s/Repack' % workflowName, **options) topLevelWorkflow.create() topLevelSub = Subscription(topFileset, topLevelWorkflow) topLevelSub.create() self.stateMap['Merge'].append(topFileset) for task in mergeTasks: mergeWorkflow = Workflow(task='/%s/Repack/%s' % (workflowName, task), **options) mergeWorkflow.create() unmergedFileset = Fileset(name='TestUnmergedFileset%s' % task) unmergedFileset.create() mergeSub = Subscription(unmergedFileset, mergeWorkflow) mergeSub.create() self.stateMap['Processing Done'].append(unmergedFileset) cleanupWorkflow = Workflow(task='/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW', **options) cleanupWorkflow.create() unmergedFileset = Fileset(name='TestUnmergedFilesetToCleanup') unmergedFileset.create() cleanupSub = Subscription(unmergedFileset, cleanupWorkflow) cleanupSub.create() return def setupExpressWorkflow(self): """ _setupExpressWorkflow_ Populate WMBS with a express-like workflow, every subscription must be unfinished at first """ workflowName = 'Express_Run481516_StreamZFast' secondLevelTasks = ['ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT', 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM', 'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT', 'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO'] alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd' dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged' self.stateMap = {'Merge': [], 'Harvesting': [], 'Processing Done': []} self.orderedStates = ['Merge', 'Harvesting', 'Processing Done'] # Populate WMStats self.requestDBWriter.insertGenericRequest({'RequestName': workflowName}) self.requestDBWriter.updateRequestStatus(workflowName, 'Closed') # Create a wmspec in disk workload = newWorkload(workflowName) expressTask = workload.newTask('Express') for task in secondLevelTasks: secondLevelTask = expressTask.addTask(task) if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO': secondLevelTask.addTask(alcaHarvestTask) elif task == 'ExpressMergewrite_StreamZFast_DQM': secondLevelTask.addTask(dqmHarvestTask) specPath = os.path.join(self.testDir, 'Express.pkl') workload.save(specPath) # Populate WMBS sharedFileset = Fileset(name='TestFileset') sharedFileset.create() sharedFileset.markOpen(False) options = {'spec': specPath, 'owner': 'ItsAMeMario', 'name': workflowName, 'wfType': 'tier0'} topLevelWorkflow = Workflow(task='/%s/Express' % workflowName, **options) topLevelWorkflow.create() topLevelSub = Subscription(sharedFileset, topLevelWorkflow) topLevelSub.create() self.stateMap['Merge'].append(topLevelSub) for task in [x for x in secondLevelTasks if not x.count('CleanupUnmerged')]: secondLevelWorkflow = Workflow(task='/%s/Express/%s' % (workflowName, task), **options) secondLevelWorkflow.create() mergeSub = Subscription(sharedFileset, secondLevelWorkflow) mergeSub.create() self.stateMap['Harvesting'].append(mergeSub) for (parent, child) in [('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask), ('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)]: harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' % (workflowName, parent, child), **options) harvestingWorkflow.create() harvestingSub = Subscription(sharedFileset, harvestingWorkflow) harvestingSub.create() self.stateMap['Processing Done'].append(harvestingSub) return def setupPromptRecoWorkflow(self): """ _setupPromptRecoWorkflow_ Populate WMBS with a real PromptReco workflow, every subscription must be unfinished at first """ # Populate disk and WMBS testArguments = PromptRecoWorkloadFactory.getTestArguments() workflowName = 'PromptReco_Run195360_Cosmics' factory = PromptRecoWorkloadFactory() testArguments["EnableHarvesting"] = True testArguments["CouchURL"] = os.environ["COUCHURL"] workload = factory.factoryWorkloadConstruction(workflowName, testArguments) wmbsHelper = WMBSHelper(workload, 'Reco', 'SomeBlock', cachepath=self.testDir) wmbsHelper.createTopLevelFileset() wmbsHelper._createSubscriptionsInWMBS(wmbsHelper.topLevelTask, wmbsHelper.topLevelFileset) self.stateMap = {'AlcaSkim': [], 'Merge': [], 'Harvesting': [], 'Processing Done': []} self.orderedStates = ['AlcaSkim', 'Merge', 'Harvesting', 'Processing Done'] # Populate WMStats self.requestDBWriter.insertGenericRequest({'RequestName': workflowName}) self.requestDBWriter.updateRequestStatus(workflowName, 'Closed') topLevelTask = '/%s/Reco' % workflowName alcaSkimTask = '%s/AlcaSkim' % topLevelTask mergeTasks = ['%s/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics', '%s/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T', '%s/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics', '%s/RecoMergewrite_AOD', '%s/RecoMergewrite_DQM', '%s/RecoMergewrite_RECO'] harvestingTask = '%s/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged' % topLevelTask self.stateMap['AlcaSkim'].append(wmbsHelper.topLevelSubscription) alcaSkimWorkflow = Workflow(name=workflowName, task=alcaSkimTask) alcaSkimWorkflow.load() alcarecoFileset = Fileset(name='/PromptReco_Run195360_Cosmics/Reco/unmerged-write_ALCARECOALCARECO') alcarecoFileset.load() alcaSkimSub = Subscription(alcarecoFileset, alcaSkimWorkflow) alcaSkimSub.load() self.stateMap['Merge'].append(alcaSkimSub) for task in mergeTasks: mergeTask = task % topLevelTask mergeWorkflow = Workflow(name=workflowName, task=mergeTask) mergeWorkflow.load() if 'AlcaSkim' in mergeTask: stream = mergeTask.split('/')[-1][13:] unmergedFileset = Fileset(name='%s/unmerged-%sALCARECO' % (alcaSkimTask, stream)) unmergedFileset.load() else: dataTier = mergeTask.split('/')[-1].split('_')[-1] unmergedFileset = Fileset(name='%s/unmerged-write_%s%s' % (topLevelTask, dataTier, dataTier)) unmergedFileset.load() mergeSub = Subscription(unmergedFileset, mergeWorkflow) mergeSub.load() self.stateMap['Harvesting'].append(mergeSub) harvestingWorkflow = Workflow(name=workflowName, task=harvestingTask) harvestingWorkflow.load() harvestingFileset = Fileset(name='/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-MergedDQM') harvestingFileset.load() harvestingSub = Subscription(harvestingFileset, harvestingWorkflow) harvestingSub.load() self.stateMap['Processing Done'].append(harvestingSub) return def verifyStateTransitions(self, transitionMethod='markFinished', transitionTrigger=True): """ _verifyStateTransitions_ Utility method which goes through the list of states in self.orderedStates and finishes the tasks that demand a state transition in each step. This according to the defined transition method and trigger. It verifies that the request document in WMStats is moving according to the transitions """ for idx in range(0, len(self.orderedStates) * 2): nextState = self.orderedStates[idx / 2] if (idx / 2) == 0: currentState = 'Closed' else: currentState = self.orderedStates[idx / 2 - 1] if idx % 2 == 0: for transitionObject in self.stateMap[nextState][:-1]: method = getattr(transitionObject, transitionMethod) method(transitionTrigger) self.plugin([], self.requestDBWriter, self.requestDBWriter) currentStateWorkflows = self.requestDBWriter.getRequestByStatus([currentState]) nextStateWorkflows = self.requestDBWriter.getRequestByStatus([nextState]) self.assertEqual(len(currentStateWorkflows), 1, 'Workflow moved incorrectly from %s' % currentState) self.assertEqual(len(nextStateWorkflows), 0, 'Workflow moved incorrectly to %s' % nextState) else: transitionObject = self.stateMap[nextState][-1] method = getattr(transitionObject, transitionMethod) method(transitionTrigger) self.plugin([], self.requestDBWriter, self.requestDBWriter) currentStateWorkflows = self.requestDBWriter.getRequestByStatus([currentState]) nextStateWorkflows = self.requestDBWriter.getRequestByStatus([nextState]) self.assertEqual(len(currentStateWorkflows), 0, 'Workflow did not move correctly from %s' % currentState) self.assertEqual(len(nextStateWorkflows), 1, 'Workflow did not move correctly to %s' % nextState) return def testA_RepackStates(self): """ _testA_RepackStates_ Setup an environment with a Repack workflow and traverse through the different states. Check that the transitions are sane. """ # Set the environment self.setupRepackWorkflow() self.plugin = Tier0Plugin() # Verify the transitions self.verifyStateTransitions('markOpen', False) return def testB_ExpressStates(self): """ _testB_ExpressStates_ Setup an environment with a Express workflow and traverse through the different states. Check that the transitions are sane. """ # Set the environment self.setupExpressWorkflow() self.plugin = Tier0Plugin() # Verify the transitions self.verifyStateTransitions() return def testC_PromptRecoStates(self): """ _testC_PromptRecoStates_ Setup an environment with a PromptReco workflow and traverse through the different states. Check that the transitions are sane. """ # Set the environment self.setupPromptRecoWorkflow() self.plugin = Tier0Plugin() # Verify the transitions self.verifyStateTransitions() return
class DQMHarvestTests(EmulatedUnitTestCase): """ _DQMHarvestTests_ Tests the DQMHarvest spec file """ def setUp(self): """ _setUp_ Initialize the database and couch. """ super(DQMHarvestTests, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("dqmharvest_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("dqmharvest_t") self.testInit.generateWorkDir() self.workload = None return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() super(DQMHarvestTests, self).tearDown() return def injectDQMHarvestConfig(self): """ _injectDQMHarvest_ Create a bogus config cache document for DQMHarvest and inject it into couch. Return the ID of the document. """ newConfig = Document() newConfig["info"] = None newConfig["config"] = None newConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e234f" newConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10876a7" newConfig["owner"] = {"group": "DATAOPS", "user": "******"} newConfig["pset_tweak_details"] = {"process": {"outputModules_": []}} result = self.configDatabase.commitOne(newConfig) return result[0]["id"] def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments.update({ "DQMConfigCacheID": self.injectDQMHarvestConfig(), "LumiList": { "251643": [[1, 15], [50, 70]], "251721": [[50, 100], [110, 120]] } }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016F") self.assertEqual(testWorkload.getProcessingString(), "23Sep2016") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertEqual( testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016F-v1-NoBPTX-23Sep2016-0001") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20']) self.assertEqual(sorted(testWorkload.getLumiList().keys()), ['251643', '251721']) self.assertEqual(sorted(testWorkload.getLumiList().values()), [[[1, 15], [50, 70]], [[50, 100], [110, 120]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual( sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag") self.assertFalse(task.inputRunWhitelist()) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") return def testDQMHarvestFailed(self): """ Build a DQMHarvest workload without a DQM config doc """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments.update({"ConfigCacheID": self.injectDQMHarvestConfig()}) testArguments.pop("DQMConfigCacheID", None) factory = DQMHarvestWorkloadFactory() with self.assertRaises(WMSpecFactoryException): factory.factoryWorkloadConstruction("TestBadWorkload", testArguments) return
class TaskChainTests(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("taskchain_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("taskchain_t") self.workload = None return def tearDown(self): """ _tearDown_ Clear out the database. """ del self.workload self.testInit.tearDownCouch() self.testInit.clearDatabase() return def testA(self): """ test creating workload with generator config """ generatorDoc = makeGeneratorConfig(self.configDatabase) processorDocs = makeProcessingConfigs(self.configDatabase) arguments = { "AcquisitionEra": "ReleaseValidation", "Requestor": "*****@*****.**", "CMSSWVersion": "CMSSW_3_5_8", "ScramArch": "slc5_ia32_gcc434", "ProcessingVersion": 1, "GlobalTag": "GR10_P_v4::All", "CouchURL": self.testInit.couchUrl, "CouchDBName": self.testInit.couchDbName, "SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"], "DashboardHost": "127.0.0.1", "DashboardPort": 8884, "TaskChain" : 5, "Task1" :{ "TaskName" : "GenSim", "ConfigCacheID" : generatorDoc, "SplittingAlgorithm" : "EventBased", "SplittingArguments" : {"events_per_job" : 250}, "RequestNumEvents" : 10000, "Seeding" : "Automatic", "PrimaryDataset" : "RelValTTBar", }, "Task2" : { "TaskName" : "DigiHLT", "InputTask" : "GenSim", "InputFromOutputModule" : "writeGENSIM", "ConfigCacheID" : processorDocs['DigiHLT'], "SplittingAlgorithm" : "FileBased", "SplittingArguments" : {"files_per_job" : 1 }, }, "Task3" : { "TaskName" : "Reco", "InputTask" : "DigiHLT", "InputFromOutputModule" : "writeRAWDIGI", "ConfigCacheID" : processorDocs['Reco'], "SplittingAlgorithm" : "FileBased", "SplittingArguments" : {"files_per_job" : 1 }, }, "Task4" : { "TaskName" : "ALCAReco", "InputTask" : "Reco", "InputFromOutputModule" : "writeALCA", "ConfigCacheID" : processorDocs['ALCAReco'], "SplittingAlgorithm" : "FileBased", "SplittingArguments" : {"files_per_job" : 1 }, }, "Task5" : { "TaskName" : "Skims", "InputTask" : "Reco", "InputFromOutputModule" : "writeRECO", "ConfigCacheID" : processorDocs['Skims'], "SplittingAlgorithm" : "FileBased", "SplittingArguments" : {"files_per_job" : 10 }, } } factory = TaskChainWorkloadFactory() try: self.workload = factory("PullingTheChain", arguments) except Exception, ex: msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex) self.fail(msg) self.workload.setSpecUrl("somespec") self.workload.setOwnerDetails("*****@*****.**", "DMWM") testWMBSHelper = WMBSHelper(self.workload, "GenSim", "SomeBlock") testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) firstTask = self.workload.getTaskByPath("/PullingTheChain/GenSim") self._checkTask(firstTask, arguments['Task1']) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT"), arguments['Task2']) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"), arguments['Task3']) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"), arguments['Task4']) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"), arguments['Task5'])
class DashboardReporterTest(unittest.TestCase): """ _DashboardReporterTest_ Test class for dashboardReporter """ def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also, create some dummy locations. """ myThread = threading.currentThread() self.sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN'] self.testInit = TestInit(__file__) self.testInit.setLogging(logLevel = logging.DEBUG) self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'], useDefault = False) self.testInit.setupCouch("dashboardreporter_t/jobs", "JobDump") self.testInit.setupCouch("dashboardreporter_t/fwjrs", "FWJRDump") resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName = site, seName = site, ceName = site) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 10000) self.testDir = self.testInit.generateWorkDir() self.alertsReceiver = None return def tearDown(self): """ _tearDown_ Rip things down. """ self.testInit.clearDatabase() self.testInit.delWorkDir() self.testInit.tearDownCouch() if self.alertsReceiver: self.alertsReceiver.shutdown() return def getConfig(self): """ _getConfig_ Creates a common config. """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "DashboardReporter" config.Agent.useHeartbeat = False config.section_("DashboardReporter") config.DashboardReporter.dashboardHost = "cmssrv52.fnal.gov" config.DashboardReporter.dashboardPort = 8884 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "dashboardreporter_t" # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" return config def createWorkload(self, workloadName = 'Test', emulator = True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") rereco = workload.getTask("ReReco") # Add RequestManager stuff workload.data.request.section_('schema') workload.data.request.schema.Requestor = 'nobody' workload.data.request.schema.Group = 'testers' taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadName) return workload def createTestJobGroup(self, nJobs = 10, retry_count = 0, workloadPath = 'test'): """ Creates a group of several jobs """ myThread = threading.currentThread() myThread.transaction.begin() testWorkflow = Workflow(spec = workloadPath, owner = "Simon", name = "wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name = "TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() testFile0 = File(lfn = "/this/is/a/parent", size = 1024, events = 10) testFile0.addRun(Run(10, *[12312])) testFile0.setLocation('malpaquet') testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12312])) testFileB.setLocation('malpaquet') testFile0.create() testFileA.create() testFileB.create() testFileA.addParent(lfn = "/this/is/a/parent") testFileB.addParent(lfn = "/this/is/a/parent") for i in range(0, nJobs): testJob = Job(name = makeUUID()) testJob['retry_count'] = retry_count testJob['retry_max'] = 10 testJob['group'] = 'BadGuys' testJob['user'] = '******' testJob['taskType'] = 'Merge' #testJob['fwjr'] = myReport testJobGroup.add(testJob) testJob.create(group = testJobGroup) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob.save() testJobGroup.commit() testSubscription.acquireFiles(files = [testFileA, testFileB]) testSubscription.save() myThread.transaction.commit() return testJobGroup def testA_testSubmit(self): """ _testSubmit_ Test whether we pick up submitted jobs """ #workload = self.createWorkload() jobGroup = self.createTestJobGroup() config = self.getConfig() xmlPath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) changer = ChangeState(config) for job in jobGroup.jobs: job['fwjr'] = myReport changer.propagate(jobGroup.jobs, "complete", "executing") changer.propagate(jobGroup.jobs, "success", "complete") dashboardReporter = DashboardReporterPoller(config = config) dashboardReporter.algorithm() # What the hell am I supposed to check? changer.propagate(jobGroup.jobs, 'jobfailed', 'executing') dashboardReporter.algorithm() return def testB_CheckExecutingJobsAndProfile(self): """ _CheckExecutingJobsAndProfile_ Pull up some executing jobs and profile them. """ return jobGroup = self.createTestJobGroup() config = self.getConfig() changer = ChangeState(config) changer.propagate(jobGroup.jobs, "executing", "created") dashboardReporter = DashboardReporterPoller(config = config) import cProfile, pstats cProfile.runctx("dashboardReporter.algorithm()", globals(), locals(), filename = "testStats.stat") p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats(.2) #dashboardReporter.algorithm() return def testC_DashboardReporterPollerAlertSending_algorithm(self): """ Cause exception (alert-worthy situation) in the algorithm() method. """ myThread = threading.currentThread() config = self.getConfig() handler, self.alertsReceiver = \ utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr) # emulate exception behaviour def raiseException(): raise Exception("My test exception.") dashboardReporter = DashboardReporterPoller(config = config) dashboardReporter.pollCouch = raiseException self.assertRaises(Exception, dashboardReporter.algorithm) # wait for the generated alert to arrive while len(handler.queue) == 0: time.sleep(0.3) print "%s waiting for alert to arrive ..." % inspect.stack()[0][3] self.alertsReceiver.shutdown() self.alertsReceiver = None # now check if the alert was properly sent self.assertEqual(len(handler.queue), 1) alert = handler.queue[0] self.assertEqual(alert["Source"], dashboardReporter.__class__.__name__) self.assertEqual(alert["Component"], "DashboardReporter") return
class RESTBaseUnitTestWithDBBackend(unittest.TestCase): def setUp(self, initRoot = True): """ unittest inherits this class should have set self.setConfig(config) config is WMCore.REST application config i.e. class ClildClass(RESTBaseUnitTestWithDBBackend): def setUp(self): self.setConfig(WMCore.ReqMgr.Config) # following setters are optional self.setCouchDBs([("reqmgr_workload_config", "ReqMgr")]) self.setSchemaModules(["WMCore.WMBS"]) """ if self.schemaModules or self.couchDBs: from WMQuality.TestInitCouchApp import TestInitCouchApp self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() # logLevel = logging.SQLDEBUG if self.schemaModules: self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schemaModules, useDefault = False) # Now pull the dbURL from the factory # I prefer this method because the factory has better error handling # Also because then you know everything is the same myThread = threading.currentThread() self.config.setDBUrl(myThread.dbFactory.dburl) if self.couchDBs: for (dbName, couchApp) in self.couchDBs: if couchApp: self.testInit.setupCouch(dbName, couchApp) else: self.testInit.setupCouch(dbName) logging.info("This is our config: %s" % self.config) self.initRoot = initRoot if initRoot: self.server = RESTMainTestServer(self.config, os.getcwd(), self._testMethodName) CherrypyTestInit.start(self.server) self.jsonSender = self.server.jsonSender # find the way to check the api with the permission self.test_authz_key = self.server.test_authz_key print("init root") def tearDown(self): if self.initRoot: CherrypyTestInit.stop(self.server) self.test_authz_key = None if self.schemaModules: self.testInit.clearDatabase() if self.couchDBs: self.testInit.tearDownCouch() self.config = None self.jsonSender = None return def setSchemaModules(self, schemaModules): """ This need to be set if backend db connection is needed ie. schemaModules = ["WMCore.WMBS","WMComponent.DBS3Buffer","WMCore.BossAir"] """ self.schemaModules = schemaModules or [] def setCouchDBs(self, couchDBs): """ This need to be set if counchdb connection is needed couchDBs = [("reqmgr_workload_config", "ReqMgr"), ] """ self.couchDBs = couchDBs or [] def setConfig(self, config): self.config = config
class DBSUploadTest(unittest.TestCase): """ TestCase for DBSUpload module Note: This fails if you use the in-memory syntax for sqlite i.e. (DATABASE = sqlite://) """ _maxMessage = 10 def setUp(self): """ _setUp_ setUp function for unittest """ # Set constants self.couchDB = "config_test" self.configURL = "RANDOM;;URL;;NAME" self.configString = "This is a random string" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.Agent.Database"], useDefault=False) self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache") myThread = threading.currentThread() self.bufferFactory = DAOFactory( package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi ) locationAction = self.bufferFactory(classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov") locationAction.execute(siteName="malpaquet") # Set heartbeat self.componentName = "JobSubmitter" self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() # Set up a config cache configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDB) configCache.createUserGroup(groupname="testGroup", username="******") self.testDir = self.testInit.generateWorkDir() psetPath = os.path.join(self.testDir, "PSet.txt") f = open(psetPath, "w") f.write(self.configString) f.close() configCache.addConfig(newConfig=psetPath, psetHash=None) configCache.save() self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB, configCache.getCouchID()) return def tearDown(self): """ _tearDown_ tearDown function for unittest """ self.testInit.clearDatabase(modules=["WMComponent.DBS3Buffer", "WMCore.Agent.Database"]) def createConfig(self): """ _createConfig_ This creates the actual config file used by the component """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "DBSUpload" config.Agent.useHeartbeat = False # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = "ERROR" config.DBSUpload.maxThreads = 1 config.DBSUpload.namespace = "WMComponent.DBSUpload.DBSUpload" config.DBSUpload.componentDir = os.path.join(os.getcwd(), "Components") config.DBSUpload.workerThreads = 4 config.section_("DBSInterface") config.DBSInterface.globalDBSUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet" config.DBSInterface.globalDBSVersion = "DBS_2_0_9" config.DBSInterface.DBSUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" config.DBSInterface.DBSVersion = "DBS_2_0_9" config.DBSInterface.DBSBlockMaxFiles = 10 config.DBSInterface.DBSBlockMaxSize = 9999999999 config.DBSInterface.DBSBlockMaxTime = 10000 config.DBSInterface.MaxFilesToCommit = 10 # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" # configure threshold of DBS upload queue size alert threshold # reference: trac ticket #1628 config.DBSUpload.alertUploadQueueSize = 2000 return config def getFiles(self, name, tier, nFiles=12, site="malpaquet"): """ Create some quick dummy test files """ files = [] for f in range(0, nFiles): testFile = DBSBufferFile(lfn="%s-%s-%i" % (name, site, f), size=1024, events=20, checksums={"cksum": 1}) testFile.setAlgorithm( appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL ) testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) testFile.addRun(Run(1, *[f])) testFile.setGlobalTag("aGlobalTag") testFile.create() testFile.setLocation(site) files.append(testFile) testFileChild = DBSBufferFile(lfn="%s-%s-child" % (name, site), size=1024, events=10, checksums={"cksum": 1}) testFileChild.setAlgorithm( appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL ) testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.setGlobalTag("aGlobalTag") testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x["lfn"] for x in files]) return files @attr("integration") def testA_basicUploadTest(self): """ _basicUploadTest_ Do everything simply once Create dataset, algo, files, blocks, upload them, mark as done, finish them, migrate them Also check the timeout """ myThread = threading.currentThread() config = self.createConfig() config.DBSInterface.DBSBlockMaxTime = 3 config.DBSUpload.pollInterval = 4 name = "ThisIsATest_%s" % (makeUUID()) tier = "RECO" nFiles = 12 files = self.getFiles(name=name, tier=tier, nFiles=nFiles) datasetPath = "/%s/%s/%s" % (name, name, tier) # Load components that are necessary to check status factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") dbinterface = factory.loadObject("UploadToDBS") dbsInterface = DBSInterface(config=config) localAPI = dbsInterface.getAPIRef() globeAPI = dbsInterface.getAPIRef(globalRef=True) # In the first round we should create blocks for the first dataset # The child dataset should not be handled until the parent is uploaded testDBSUpload = DBSUploadPoller(config=config) testDBSUpload.algorithm() # First, see if there are any blocks # One in DBS, one not in DBS result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall() self.assertEqual(len(result), 2) self.assertEqual(result, [("InGlobalDBS",), ("Open",)]) # Check to see if datasets and algos are in local DBS result = listAlgorithms(apiRef=localAPI, patternExe=name) self.assertEqual(len(result), 1) self.assertEqual(result[0]["ExecutableName"], name) result = listPrimaryDatasets(apiRef=localAPI, match=name) self.assertEqual(result, [name]) result = listProcessedDatasets(apiRef=localAPI, primary=name, dataTier="*") # Then check and see that the closed block made it into local DBS affectedBlocks = listBlocks(apiRef=localAPI, datasetPath=datasetPath) if affectedBlocks[0]["OpenForWriting"] == "0": self.assertEqual(affectedBlocks[1]["OpenForWriting"], "1") self.assertEqual(affectedBlocks[0]["NumberOfFiles"], 10) self.assertEqual(affectedBlocks[1]["NumberOfFiles"], 2) else: self.assertEqual(affectedBlocks[0]["OpenForWriting"], "1") self.assertEqual(affectedBlocks[1]["NumberOfFiles"], 10) self.assertEqual(affectedBlocks[0]["NumberOfFiles"], 2) # Check to make sure all the files are in local result = listDatasetFiles(apiRef=localAPI, datasetPath=datasetPath) fileLFNs = [x["lfn"] for x in files] for lfn in fileLFNs: self.assertTrue(lfn in result) # Make sure the child files aren't there flag = False try: listDatasetFiles(apiRef=localAPI, datasetPath="/%s/%s_2/%s" % (name, name, tier)) except Exception, ex: flag = True self.assertTrue(flag) # There should be one blocks in global # It should have ten files and be closed result = listBlocks(apiRef=globeAPI, datasetPath=datasetPath) self.assertEqual(len(result), 1) for block in result: self.assertEqual(block["OpenForWriting"], "0") self.assertTrue(block["NumberOfFiles"] in [2, 10]) # Okay, deep breath. First round done # In the second round, the second block of the parent fileset should transfer # Make sure that the timeout functions work time.sleep(10) testDBSUpload.algorithm() result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall() self.assertEqual(len(result), 2) self.assertEqual(result, [("InGlobalDBS",), ("InGlobalDBS",)]) # Check to make sure all the files are in global result = listDatasetFiles(apiRef=globeAPI, datasetPath=datasetPath) for lfn in fileLFNs: self.assertTrue(lfn in result) # Make sure the child files aren't there flag = False try: listDatasetFiles(apiRef=localAPI, datasetPath="/%s/%s_2/%s" % (name, name, tier)) except Exception, ex: flag = True
class JobSubmitterTest(unittest.TestCase): """ _JobSubmitterTest_ Test class for the JobSubmitterPoller """ def setUp(self): """ _setUp_ Standard setup: Now with 100% more couch """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"]) self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump") self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump") self.testInit.setupCouch("wmagent_summary_t", "WMStats") myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.baDaoFactory = DAOFactory(package = "WMCore.BossAir", logger = myThread.logger, dbinterface = myThread.dbi) self.testDir = self.testInit.generateWorkDir() # Set heartbeat self.componentName = 'JobSubmitter' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() return def tearDown(self): """ _tearDown_ Standard tearDown """ self.testInit.clearDatabase() self.testInit.delWorkDir() self.testInit.tearDownCouch() return def setResourceThresholds(self, site, **options): """ _setResourceThresholds_ Utility to set resource thresholds """ if not options: options = {'state' : 'Normal', 'runningSlots' : 10, 'pendingSlots' : 5, 'tasks' : ['Processing', 'Merge'], 'Processing' : {'pendingSlots' : 5, 'runningSlots' : 10}, 'Merge' : {'pendingSlots' : 2, 'runningSlots' : 5}} resourceControl = ResourceControl() resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), ceName = site, plugin = "MockPlugin", pendingSlots = options['pendingSlots'], runningSlots = options['runningSlots'], cmsName = site) for task in options['tasks']: resourceControl.insertThreshold(siteName = site, taskType = task, maxSlots = options[task]['runningSlots'], pendingSlots = options[task]['pendingSlots']) if options.get('state'): resourceControl.changeSiteState(site, options.get('state')) return def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site, bl = [], wl = [], taskType = 'Processing', name = None): """ _createJobGroups_ Creates a series of jobGroups for submissions """ jobGroupList = [] if name is None: name = makeUUID() testWorkflow = Workflow(spec = workloadSpec, owner = "mnorman", name = name, task = "basicWorkload/Production") testWorkflow.create() # Create subscriptions for _ in range(nSubs): name = makeUUID() # Create Fileset, Subscription, jobGroup testFileset = Fileset(name = name) testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = taskType, split_algo = "FileBased") testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() # Create jobs self.makeNJobs(name = name, task = task, nJobs = nJobs, jobGroup = testJobGroup, fileset = testFileset, sub = testSubscription.exists(), site = site, bl = bl, wl = wl) testFileset.commit() testJobGroup.commit() jobGroupList.append(testJobGroup) return jobGroupList def makeNJobs(self, name, task, nJobs, jobGroup, fileset, sub, site, bl = [], wl = []): """ _makeNJobs_ Make and return a WMBS Job and File This handles all those damn add-ons """ # Set the CacheDir cacheDir = os.path.join(self.testDir, 'CacheDir') for n in range(nJobs): # First make a file #site = self.sites[0] testFile = File(lfn = "/singleLfn/%s/%s" % (name, n), size = 1024, events = 10) if type(site) == list: for singleSite in site: testFile.setLocation(singleSite) else: testFile.setLocation(site) testFile.create() fileset.addFile(testFile) fileset.commit() index = 0 for f in fileset.files: index += 1 testJob = Job(name = '%s-%i' % (name, index)) testJob.addFile(f) testJob["location"] = f.getLocations()[0] testJob['task'] = task.getPathName() testJob['sandbox'] = task.data.input.sandbox testJob['spec'] = os.path.join(self.testDir, 'basicWorkload.pcl') testJob['mask']['FirstEvent'] = 101 testJob["siteBlacklist"] = bl testJob["siteWhitelist"] = wl testJob['priority'] = 101 jobCache = os.path.join(cacheDir, 'Sub_%i' % (sub), 'Job_%i' % (index)) os.makedirs(jobCache) testJob.create(jobGroup) testJob['cache_dir'] = jobCache testJob.save() jobGroup.add(testJob) output = open(os.path.join(jobCache, 'job.pkl'), 'w') pickle.dump(testJob, output) output.close() return testJob, testFile def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction'] config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config def createTestWorkload(self, workloadName = 'Tier1ReReco'): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload(workloadName) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload def testA_BasicTest(self): """ Use the MockPlugin to create a simple test Check to see that all the jobs were "submitted", don't care about thresholds """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 2 nJobs = 20 site = 'T2_US_UCSD' self.setResourceThresholds(site, pendingSlots = 50, runningSlots = 100, tasks = ['Processing', 'Merge'], Processing = {'pendingSlots' : 50, 'runningSlots' : 100}, Merge = {'pendingSlots' : 50, 'runningSlots' : 100}) jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % site) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Do pre-submit check getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) jobSubmitter = JobSubmitterPoller(config = config) jobSubmitter.algorithm() # Check that jobs are in the right state result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) # Check assigned locations getLocationAction = self.daoFactory(classname = "Jobs.GetLocation") for jobId in result: loc = getLocationAction.execute(jobid = jobId) self.assertEqual(loc, [['T2_US_UCSD']]) # Run another cycle, it shouldn't submit anything. There isn't anything to submit jobSubmitter.algorithm() result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) nSubs = 1 nJobs = 10 # Submit another 10 jobs jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % site, taskType = "Merge") for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Check that the jobs are available for submission and run another cycle result = getJobsAction.execute(state = 'Created', jobType = "Merge") self.assertEqual(len(result), nSubs * nJobs) jobSubmitter.algorithm() #Check that the last 10 jobs were submitted as well. result = getJobsAction.execute(state = 'Created', jobType = "Merge") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Executing', jobType = "Merge") self.assertEqual(len(result), nSubs * nJobs) return def testB_thresholdTest(self): """ _testB_thresholdTest_ Check that the threshold management is working, this requires checks on pending/running jobs globally at a site and per task/site """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 5 nJobs = 10 sites = ['T1_US_FNAL'] for site in sites: self.setResourceThresholds(site, pendingSlots = 50, runningSlots = 200, tasks = ['Processing', 'Merge'], Processing = {'pendingSlots' : 45, 'runningSlots' :-1}, Merge = {'pendingSlots' : 10, 'runningSlots' : 20, 'priority' : 5}) # Always initialize the submitter after setting the sites, flaky! jobSubmitter = JobSubmitterPoller(config = config) jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL') for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Do pre-submit check getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) jobSubmitter.algorithm() # Check that jobs are in the right state, # here we are limited by the pending threshold for the Processing task (45) result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 5) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), 45) # Check assigned locations getLocationAction = self.daoFactory(classname = "Jobs.GetLocation") for jobId in result: loc = getLocationAction.execute(jobid = jobId) self.assertEqual(loc, [['T1_US_FNAL']]) # Run another cycle, it shouldn't submit anything. Jobs are still in pending jobSubmitter.algorithm() result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 5) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), 45) # Now put 10 Merge jobs, only 5 can be submitted, there we hit the global pending threshold for the site nSubs = 1 nJobs = 10 jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL', taskType = 'Merge') for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() result = getJobsAction.execute(state = 'Created', jobType = "Merge") self.assertEqual(len(result), 5) result = getJobsAction.execute(state = 'Executing', jobType = "Merge") self.assertEqual(len(result), 5) result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 5) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), 45) # Now let's test running thresholds # The scenario will be setup as follows: Move all current jobs as running # Create 300 Processing jobs and 300 merge jobs # Run 5 polling cycles, moving all pending jobs to running in between # Result is, merge is left at 25 running 0 pending and processing is left at 215 running 0 pending # Processing has 135 jobs in queue and Merge 285 # This tests all threshold dynamics including the prioritization of merge over processing nSubs = 1 nJobs = 300 jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL') jobGroupList.extend(self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL', taskType = 'Merge')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') getRunJobID = self.baDaoFactory(classname = "LoadByWMBSID") setRunJobStatus = self.baDaoFactory(classname = "SetStatus") for _ in range(5): result = getJobsAction.execute(state = 'Executing') binds = [] for jobId in result: binds.append({'id' : jobId, 'retry_count' : 0}) runJobIds = getRunJobID.execute(binds) setRunJobStatus.execute([x['id'] for x in runJobIds], 'Running') jobSubmitter.algorithm() result = getJobsAction.execute(state = 'Executing', jobType = 'Processing') self.assertEqual(len(result), 215) result = getJobsAction.execute(state = 'Created', jobType = 'Processing') self.assertEqual(len(result), 135) result = getJobsAction.execute(state = 'Executing', jobType = 'Merge') self.assertEqual(len(result), 25) result = getJobsAction.execute(state = 'Created', jobType = 'Merge') self.assertEqual(len(result), 285) return def testC_prioritization(self): """ _testC_prioritization_ Check that jobs are prioritized by job type and by oldest workflow """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 1 nJobs = 10 sites = ['T1_US_FNAL'] for site in sites: self.setResourceThresholds(site, pendingSlots = 10, runningSlots = -1, tasks = ['Processing', 'Merge'], Processing = {'pendingSlots' : 50, 'runningSlots' :-1}, Merge = {'pendingSlots' : 10, 'runningSlots' :-1, 'priority' : 5}) # Always initialize the submitter after setting the sites, flaky! jobSubmitter = JobSubmitterPoller(config = config) jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL', name = 'OldestWorkflow') jobGroupList.extend(self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL', taskType = 'Merge')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() # Merge goes first getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Merge") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Executing', jobType = "Merge") self.assertEqual(len(result), 10) result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 10) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), 0) # Create a newer workflow processing, and after some new jobs for an old workflow jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL', name = 'NewestWorkflow') jobGroupList.extend(self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL', name = 'OldestWorkflow')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Move pending jobs to running getRunJobID = self.baDaoFactory(classname = "LoadByWMBSID") setRunJobStatus = self.baDaoFactory(classname = "SetStatus") for idx in range(2): result = getJobsAction.execute(state = 'Executing') binds = [] for jobId in result: binds.append({'id' : jobId, 'retry_count' : 0}) runJobIds = getRunJobID.execute(binds) setRunJobStatus.execute([x['id'] for x in runJobIds], 'Running') # Run again on created workflows jobSubmitter.algorithm() result = getJobsAction.execute(state = 'Created', jobType = "Merge") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Executing', jobType = "Merge") self.assertEqual(len(result), 10) result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 30 - (idx + 1) * 10) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), (idx + 1) * 10) # Check that older workflow goes first even with newer jobs getWorkflowAction = self.daoFactory(classname = "Jobs.GetWorkflowTask") workflows = getWorkflowAction.execute(result) for workflow in workflows: self.assertEqual(workflow['name'], 'OldestWorkflow') return def testD_WhiteListBlackList(self): """ _testD_WhiteListBlackList_ Test the whitelist/blacklist implementation Trust the jobCreator to get this in the job right """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 2 nJobs = 10 sites = ['T2_US_Florida', 'T2_TW_Taiwan', 'T2_CH_CERN', 'T3_CO_Uniandes'] for site in sites: self.setResourceThresholds(site, pendingSlots = 1000, runningSlots = -1, tasks = ['Processing', 'Merge'], Processing = {'pendingSlots' : 5000, 'runningSlots' :-1}, Merge = {'pendingSlots' : 1000, 'runningSlots' :-1, 'priority' : 5}) jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, site = 'se.%s' % sites[-1], task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), bl = sites[:-1]) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter = JobSubmitterPoller(config = config) # Actually run it jobSubmitter.algorithm() getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) # All jobs should be at T3_CO_Uniandes # Check assigned locations getLocationAction = self.daoFactory(classname = "Jobs.GetLocation") locationDict = getLocationAction.execute([{'jobid' : x} for x in result]) for entry in locationDict: loc = entry['site_name'] self.assertEqual(loc, 'T3_CO_Uniandes') # Run again and test the whiteList jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), site = 'se.%s' % 'T2_CH_CERN', workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), wl = ['T2_CH_CERN']) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Run it jobSubmitter.algorithm() # You'll have jobs from the previous run still in the database result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs * 2) # All jobs should be at CERN or Uniandes locationDict = getLocationAction.execute([{'jobid' : x} for x in result]) for entry in locationDict[nSubs * nJobs:]: loc = entry['site_name'] self.assertEqual(loc, 'T2_CH_CERN') # Run again with an invalid whitelist # After this point, the original two sets of jobs will be executing # The rest of the jobs should move to submitFailed jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), site = 'se.%s' % 'T2_CH_CERN', workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), wl = ['T2_US_Namibia']) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() # Jobs should be gone getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs * 2) result = getJobsAction.execute(state = 'SubmitFailed', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) # Run again with all sites blacklisted jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), site = ['se.%s' % x for x in sites], workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), bl = sites) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() # Jobs should go to submit failed getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs * 2) result = getJobsAction.execute(state = 'SubmitFailed', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs * 2) return def testE_SiteModesTest(self): """ _testE_SiteModesTest_ Test the behavior of the submitter in response to the different states of the sites """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 1 nJobs = 20 sites = ['T2_US_Florida', 'T2_TW_Taiwan', 'T3_CO_Uniandes', 'T1_US_FNAL'] for site in sites: self.setResourceThresholds(site, pendingSlots = 10, runningSlots = -1, tasks = ['Processing', 'Merge'], Processing = {'pendingSlots' : 10, 'runningSlots' :-1}, Merge = {'pendingSlots' : 10, 'runningSlots' :-1, 'priority' : 5}) myResourceControl = ResourceControl() myResourceControl.changeSiteState('T2_US_Florida', 'Draining') # First test that we prefer Normal over drain, and T1 over T2/T3 jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, site = ['se.%s' % x for x in sites], task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName)) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter = JobSubmitterPoller(config = config) # Actually run it jobSubmitter.algorithm() getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) # All jobs should be at either FNAL, Taiwan or Uniandes. It's a random selection # Check assigned locations getLocationAction = self.daoFactory(classname = "Jobs.GetLocation") locationDict = getLocationAction.execute([{'jobid' : x} for x in result]) for entry in locationDict: loc = entry['site_name'] self.assertNotEqual(loc, 'T2_US_Florida') # Now set everything to down, check we don't submit anything for site in sites: myResourceControl.changeSiteState(site, 'Down') jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, site = ['se.%s' % x for x in sites], task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName)) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() # Nothing is submitted despite the empty slots at Uniandes and Florida result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nJobs) # Now set everything to Aborted, and create Merge jobs. Those should fail # since the can only run at one place for site in sites: myResourceControl.changeSiteState(site, 'Aborted') nSubsMerge = 1 nJobsMerge = 5 jobGroupList = self.createJobGroups(nSubs = nSubsMerge, nJobs = nJobsMerge, site = ['se.%s' % x for x in sites], task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), taskType = 'Merge') for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') jobSubmitter.algorithm() result = getJobsAction.execute(state = 'SubmitFailed', jobType = 'Merge') self.assertEqual(len(result), nSubsMerge * nJobsMerge) result = getJobsAction.execute(state = 'Executing', jobType = 'Processing') self.assertEqual(len(result), nSubs * nJobs) return @attr('performance') def testF_PollerProfileTest(self): """ _testF_PollerProfileTest_ Submit a lot of jobs and test how long it takes for them to actually be submitted """ workloadName = "basicWorkload" workload = self.createTestWorkload() config = self.getConfig() changeState = ChangeState(config) nSubs = 100 nJobs = 100 sites = ['T1_US_FNAL'] for site in sites: self.setResourceThresholds(site, pendingSlots = 20000, runningSlots = -1, tasks = ['Processing', 'Merge'], Processing = {'pendingSlots' : 10000, 'runningSlots' :-1}, Merge = {'pendingSlots' : 10000, 'runningSlots' :-1, 'priority' : 5}) # Always initialize the submitter after setting the sites, flaky! jobSubmitter = JobSubmitterPoller(config = config) jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL') jobGroupList.extend(self.createJobGroups(nSubs = nSubs, nJobs = nJobs, task = workload.getTask("ReReco"), workloadSpec = os.path.join(self.testDir, 'workloadTest', workloadName), site = 'se.%s' % 'T1_US_FNAL', taskType = 'Merge')) for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') # Actually run it startTime = time.time() cProfile.runctx("jobSubmitter.algorithm()", globals(), locals(), filename = "testStats.stat") stopTime = time.time() print "Job took %f seconds to complete" % (stopTime - startTime) p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats() return
class RepackTests(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testDir = self.testInit.generateWorkDir() myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName") self.listFilesets = self.daoFactory(classname="Fileset.List") self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow") return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.clearDatabase() self.testInit.delWorkDir() return def testRepack(self): """ _testRepack_ Create a Repack workflow and verify it installs into WMBS correctly. """ testArguments = RepackWorkloadFactory.getTestArguments() testArguments.update(deepcopy(REQUEST)) factory = RepackWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Repack", cachepath=self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) repackWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Repack") repackWorkflow.load() self.assertEqual(len(repackWorkflow.outputMap.keys()), len(testArguments["Outputs"]) + 1, "Error: Wrong number of WF outputs in the Repack WF.") goldenOutputMods = {"write_PrimaryDataset1_RAW": "RAW", "write_PrimaryDataset2_RAW": "RAW"} for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier mergedOutput = repackWorkflow.outputMap[fset][0]["merged_output_fileset"] unmergedOutput = repackWorkflow.outputMap[fset][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_PrimaryDataset1_RAW": self.assertEqual(mergedOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Repack/unmerged-%s" % fset, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = repackWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = repackWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Repack/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Repack/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") for goldenOutputMod, tier in goldenOutputMods.items(): mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Repack/RepackMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 3, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-Repack") topLevelFileset.loadData() repackSubscription = Subscription(fileset=topLevelFileset, workflow=repackWorkflow) repackSubscription.loadData() self.assertEqual(repackSubscription["type"], "Repack", "Error: Wrong subscription type.") self.assertEqual(repackSubscription["split_algo"], "Repack", "Error: Wrong split algorithm. %s" % repackSubscription["split_algo"]) unmergedOutputs = {"write_PrimaryDataset1_RAW": "RAW", "write_PrimaryDataset2_RAW": "RAW"} for unmergedOutput, tier in unmergedOutputs.items(): fset = unmergedOutput + tier unmergedDataTier = Fileset(name="/TestWorkload/Repack/unmerged-%s" % fset) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Repack/RepackMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset=unmergedDataTier, workflow=dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "RepackMerge", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier unmergedFileset = Fileset(name="/TestWorkload/Repack/unmerged-%s" % fset) unmergedFileset.loadData() cleanupWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Repack/RepackCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") repackLogCollect = Fileset(name="/TestWorkload/Repack/unmerged-logArchive") repackLogCollect.loadData() repackLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Repack/LogCollect") repackLogCollectWorkflow.load() logCollectSub = Subscription(fileset=repackLogCollect, workflow=repackLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") for goldenOutputMod, tier in goldenOutputMods.items(): repackMergeLogCollect = Fileset( name="/TestWorkload/Repack/RepackMerge%s/merged-logArchive" % goldenOutputMod) repackMergeLogCollect.loadData() repackMergeLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Repack/RepackMerge%s/Repack%sMergeLogCollect" % ( goldenOutputMod, goldenOutputMod)) repackMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset=repackMergeLogCollect, workflow=repackMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") return def testMemCoresSettings(self): """ _testMemCoresSettings_ Make sure the multicore and memory setings are properly propagated to all tasks and steps. """ testArguments = RepackWorkloadFactory.getTestArguments() testArguments.update(deepcopy(REQUEST)) factory = RepackWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) # test default values taskPaths = ['/TestWorkload/Repack'] for task in taskPaths: taskObj = testWorkload.getTaskByPath(task) for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], 2300.0) # now test case where args are provided testArguments["Multicore"] = 6 testArguments["Memory"] = 4600.0 testArguments["EventStreams"] = 3 testArguments["Outputs"] = deepcopy(REQUEST['Outputs']) factory = RepackWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) for task in taskPaths: taskObj = testWorkload.getTaskByPath(task) for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) if task == '/TestWorkload/Repack' and step == 'cmsRun1': self.assertEqual(stepHelper.getNumberOfCores(), testArguments["Multicore"]) self.assertEqual(stepHelper.getNumberOfStreams(), testArguments["EventStreams"]) elif step in ('stageOut1', 'logArch1'): self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) else: self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should be single-core" % task) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"]) return def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = ['/TestWorkload/Repack', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW'] expWfTasks = ['/TestWorkload/Repack', '/TestWorkload/Repack/LogCollect', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset1_RAW', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset2_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/Repackwrite_PrimaryDataset1_RAWMergeLogCollect', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/Repackwrite_PrimaryDataset2_RAWMergeLogCollect'] expFsets = ['TestWorkload-Repack-StreamerFiles', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-MergedRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-MergedErrorRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-MergedRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-MergedErrorRAW', '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW', '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW', '/TestWorkload/Repack/unmerged-logArchive'] subMaps = [(3, '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/Repackwrite_PrimaryDataset1_RAWMergeLogCollect', 'MinFileBased', 'LogCollect'), (6, '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/Repackwrite_PrimaryDataset2_RAWMergeLogCollect', 'MinFileBased', 'LogCollect'), (8, '/TestWorkload/Repack/unmerged-logArchive', '/TestWorkload/Repack/LogCollect', 'MinFileBased', 'LogCollect'), (4, '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset1_RAW', 'SiblingProcessingBased', 'Cleanup'), (2, '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW', 'RepackMerge', 'Merge'), (7, '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset2_RAW', 'SiblingProcessingBased', 'Cleanup'), (5, '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW', 'RepackMerge', 'Merge'), (1, 'TestWorkload-Repack-StreamerFiles', '/TestWorkload/Repack', 'Repack', 'Repack')] testArguments = RepackWorkloadFactory.getTestArguments() testArguments.update(deepcopy(REQUEST)) factory = RepackWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "Repack", blockName='StreamerFiles', cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
class CouchappTest(unittest.TestCase): def setUp(self): myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) self.databaseName = "couchapp_t_0" self.testInit.setupCouch(self.databaseName, "WorkloadSummary") self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump") # Setup config for couch connections config = self.testInit.getConfiguration() config.section_("JobStateMachine") config.JobStateMachine.couchDBName = self.databaseName # Create couch server and connect to databases self.couchdb = CouchServer(config.JobStateMachine.couchurl) self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % config.JobStateMachine.couchDBName) self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % config.JobStateMachine.couchDBName) # Create changeState self.changeState = ChangeState(config) self.config = config # Create testDir self.testDir = self.testInit.generateWorkDir() return def tearDown(self): self.testInit.clearDatabase(modules = ["WMCore.WMBS"]) self.testInit.delWorkDir() #self.testInit.tearDownCouch() return def createWorkload(self, workloadName = 'Test', emulator = True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadName) return workload def createTestJobGroup(self, name = "TestWorkthrough", specLocation = "spec.xml", error = False, task = "/TestWorkload/ReReco", nJobs = 10): """ _createTestJobGroup_ Generate a test WMBS JobGroup with real FWJRs """ myThread = threading.currentThread() testWorkflow = Workflow(spec = specLocation, owner = "Simon", name = name, task = task) testWorkflow.create() testWMBSFileset = Fileset(name = name) testWMBSFileset.create() testFileA = File(lfn = makeUUID(), size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = makeUUID(), size = 1024, events = 10) testFileB.addRun(Run(10, *[12312])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() for i in range(0, nJobs): testJob = Job(name = makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run = 10, lumis = [12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() report = Report() if error: path = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl") else: path = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "PerformanceReport2.pkl") report.load(filename = path) self.changeState.propagate(testJobGroup.jobs, 'created', 'new') self.changeState.propagate(testJobGroup.jobs, 'executing', 'created') self.changeState.propagate(testJobGroup.jobs, 'complete', 'executing') for job in testJobGroup.jobs: job['fwjr'] = report self.changeState.propagate(testJobGroup.jobs, 'jobfailed', 'complete') self.changeState.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed') self.changeState.propagate(testJobGroup.jobs, 'cleanout', 'exhausted') testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup def testHighestJobID(self): """ _highestJobID_ This is a jobDump function that should tell us the highest jobID currently being stored in the couch DB. """ workloadPath = os.path.join(self.testDir, 'spec.pkl') workload = self.createWorkload(workloadName = workloadPath) testJobGroup = self.createTestJobGroup(name = workload.name(), specLocation = workloadPath, error = False, nJobs = 10) jobID = self.jobsdatabase.loadView("JobDump", "highestJobID")['rows'][0]['value'] self.assertEqual(jobID, 9) testJobGroup2 = self.createTestJobGroup(name = workload.name(), specLocation = workloadPath, error = False, nJobs = 10) jobID = self.jobsdatabase.loadView("JobDump", "highestJobID")['rows'][0]['value'] self.assertEqual(jobID, 19) return
class PileupFetcherTest(unittest.TestCase): def setUp(self): """ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("pileupfetcher_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("pileupfetcher_t") self.testDir = self.testInit.generateWorkDir() EmulatorHelper.setEmulators(dbs = True) def tearDown(self): """ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() EmulatorHelper.resetEmulators() def injectGenerationConfig(self): """ _injectGenerationConfig_ Inject a generation config for the MC workflow. """ config = Document() config["info"] = None config["config"] = None config["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e580f" config["pset_hash"] = "7c856ad35f9f544839d8525ca10259a7" config["owner"] = {"group": "cmsdataops", "user": "******"} config["pset_tweak_details"] = None config["pset_tweak_details"] = \ {"process": {"outputModules_": ["OutputA"], "OutputA": {"dataset": {"filterName": "OutputAFilter", "dataTier": "GEN-SIM-RAW"}}}} result = self.configDatabase.commitOne(config) return result[0]["id"] def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl): """ pileupDict is a Python dictionary containing particular pileup configuration information. Query DBS on given dataset contained now in both input defaultArguments as well as in the pileupDict and compare values. """ args = {} args["version"] = "DBS_2_0_9" args["mode"] = "GET" reader = DBSReader(dbsUrl, **args) inputArgs = defaultArguments["PileupConfig"] self.assertEqual(len(inputArgs), len(pileupDict), "Number of pileup types different.") for pileupType in inputArgs: m = ("pileup type '%s' not in PileupFetcher-produced pileup " "configuration: '%s'" % (pileupType, pileupDict)) self.assertTrue(pileupType in pileupDict, m) # now query DBS for compare actual results on files lists for each # pileup type and dataset and location (storage element names) # pileupDict is saved in the file and now comparing items of this # configuration with actual DBS results, the structure of pileupDict: # {"pileupTypeA": {"BlockA": {"FileList": [], "StorageElementNames": []}, # "BlockB": {"FileList": [], "StorageElementName": []}, ....} for pileupType, datasets in inputArgs.items(): # this is from the pileup configuration produced by PileupFetcher blockDict = pileupDict[pileupType] for dataset in datasets: dbsFileBlocks = reader.listFileBlocks(dataset = dataset) for dbsFileBlockName in dbsFileBlocks: fileList = [] # list of files in the block (dbsFile["LogicalFileName"]) storageElemNames = set() # list of StorageElementName # each DBS block has a list under 'StorageElementList', iterate over storageElements = reader.listFileBlockLocation(dbsFileBlockName) for storElem in storageElements: storageElemNames.add(storElem) # now get list of files in the block dbsFiles = reader.listFilesInBlock(dbsFileBlockName) for dbsFile in dbsFiles: fileList.append(dbsFile["LogicalFileName"]) # now compare the sets: m = ("StorageElementNames don't agree for pileup type '%s', " "dataset '%s' in configuration: '%s'" % (pileupType, dataset, pileupDict)) self.assertEqual(set(blockDict[dbsFileBlockName]["StorageElementNames"]), storageElemNames, m) m = ("FileList don't agree for pileup type '%s', dataset '%s' " " in configuration: '%s'" % (pileupType, dataset, pileupDict)) print fileList print blockDict[dbsFileBlockName]["FileList"] self.assertEqual(sorted(blockDict[dbsFileBlockName]["FileList"]), sorted(fileList)) def _queryPileUpConfigFile(self, defaultArguments, task, taskPath): """ Query and compare contents of the the pileup JSON configuration files. Iterate over tasks's steps as it happens in the PileupFetcher. """ for step in task.steps().nodeIterator(): helper = WMStep.WMStepHelper(step) # returns e.g. instance of CMSSWHelper if hasattr(helper.data, "pileup"): decoder = JSONDecoder() stepPath = "%s/%s" % (taskPath, helper.name()) pileupConfig = "%s/%s" % (stepPath, "pileupconf.json") try: f = open(pileupConfig, 'r') json = f.read() pileupDict = decoder.decode(json) f.close() except IOError: m = "Could not read pileup JSON configuration file: '%s'" % pileupConfig self.fail(m) self._queryAndCompareWithDBS(pileupDict, defaultArguments, helper.data.dbsUrl) def testPileupFetcherOnMC(self): pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments() pileupMcArgs["PileupConfig"] = {"cosmics": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"], "minbias": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]} pileupMcArgs["CouchURL"] = os.environ["COUCHURL"] pileupMcArgs["CouchDBName"] = "pileupfetcher_t" pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs) # Since this is test of the fetcher - The loading from WMBS isn't # really necessary because the fetching happens before the workflow # is inserted into WMBS: feed the workload instance directly into fetcher: fetcher = PileupFetcher() creator = SandboxCreator() pathBase = "%s/%s" % (self.testDir, testWorkload.name()) for topLevelTask in testWorkload.taskIterator(): for taskNode in topLevelTask.nodeIterator(): # this is how the call to PileupFetcher is happening # from the SandboxCreator test task = WMTask.WMTaskHelper(taskNode) taskPath = "%s/WMSandbox/%s" % (pathBase, task.name()) fetcher.setWorkingDirectory(taskPath) # create Sandbox for the fetcher ... creator._makePathonPackage(taskPath) fetcher(task) self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
class TestChangeState(unittest.TestCase): def setUp(self): """ _setUp_ """ self.transitions = Transitions() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("changestate_t/jobs", "JobDump") self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump") self.testInit.setupCouch("job_summary", "WMStats") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) couchurl = os.getenv("COUCHURL") self.couchServer = CouchServer(dburl = couchurl) self.config = self.testInit.getConfiguration() self.taskName = "/TestWorkflow/Task" return def tearDown(self): """ _tearDown_ Cleanup the databases. """ self.testInit.clearDatabase() self.testInit.tearDownCouch() return def testCheck(self): """ This is the test class for function Check from module ChangeState """ change = ChangeState(self.config, "changestate_t") # Run through all good state transitions and assert that they work for state in self.transitions.keys(): for dest in self.transitions[state]: change.check(dest, state) dummystates = ['dummy1', 'dummy2', 'dummy3', 'dummy4'] # Then run through some bad state transistions and assertRaises(AssertionError) for state in self.transitions.keys(): for dest in dummystates: self.assertRaises(AssertionError, change.check, dest, state) return def testRecordInCouch(self): """ _testRecordInCouch_ Verify that jobs, state transitions and fwjrs are recorded correctly. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048, locations = set(["somese.cern.ch"])) testFileB = File(lfn = "SomeLFNB", events = 1025, size = 2049, locations = set(["somese.cern.ch"])) testFileA.create() testFileB.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 2, \ "Error: Splitting should have created two jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" change.propagate([testJobA, testJobB], "new", "none") change.propagate([testJobA, testJobB], "created", "new") change.propagate([testJobA, testJobB], "executing", "created") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) for transition in testJobADoc["states"].itervalues(): self.assertTrue(type(transition["timestamp"]) in (int, long)) self.assertEqual(testJobADoc["jobid"] , testJobA["id"], "Error: ID parameter is incorrect.") assert testJobADoc["name"] == testJobA["name"], \ "Error: Name parameter is incorrect." assert testJobADoc["jobgroup"] == testJobA["jobgroup"], \ "Error: Jobgroup parameter is incorrect." assert testJobADoc["workflow"] == testJobA["workflow"], \ "Error: Workflow parameter is incorrect." assert testJobADoc["task"] == testJobA["task"], \ "Error: Task parameter is incorrect." assert testJobADoc["owner"] == testJobA["owner"], \ "Error: Owner parameter is incorrect." assert testJobADoc["mask"]["FirstEvent"] == testJobA["mask"]["FirstEvent"], \ "Error: First event in mask is incorrect." assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastEvent"], \ "Error: Last event in mask is incorrect." assert testJobADoc["mask"]["FirstLumi"] == testJobA["mask"]["FirstLumi"], \ "Error: First lumi in mask is incorrect." assert testJobADoc["mask"]["LastLumi"] == testJobA["mask"]["LastLumi"], \ "Error: First lumi in mask is incorrect." assert testJobADoc["mask"]["FirstRun"] == testJobA["mask"]["FirstRun"], \ "Error: First run in mask is incorrect." assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastRun"], \ "Error: First event in mask is incorrect." assert len(testJobADoc["inputfiles"]) == 1, \ "Error: Input files parameter is incorrect." testJobBDoc = change.jobsdatabase.document(testJobB["couch_record"]) assert testJobBDoc["jobid"] == testJobB["id"], \ "Error: ID parameter is incorrect." assert testJobBDoc["name"] == testJobB["name"], \ "Error: Name parameter is incorrect." assert testJobBDoc["jobgroup"] == testJobB["jobgroup"], \ "Error: Jobgroup parameter is incorrect." assert testJobBDoc["mask"]["FirstEvent"] == testJobB["mask"]["FirstEvent"], \ "Error: First event in mask is incorrect." assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastEvent"], \ "Error: Last event in mask is incorrect." assert testJobBDoc["mask"]["FirstLumi"] == testJobB["mask"]["FirstLumi"], \ "Error: First lumi in mask is incorrect." assert testJobBDoc["mask"]["LastLumi"] == testJobB["mask"]["LastLumi"], \ "Error: First lumi in mask is incorrect." assert testJobBDoc["mask"]["FirstRun"] == testJobB["mask"]["FirstRun"], \ "Error: First run in mask is incorrect." assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastRun"], \ "Error: First event in mask is incorrect." assert len(testJobBDoc["inputfiles"]) == 1, \ "Error: Input files parameter is incorrect." changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/jobs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 3, "Error: Wrong number of documents.") couchJobDoc = changeStateDB.document("1") assert couchJobDoc["name"] == testJobA["name"], \ "Error: Name is wrong" assert len(couchJobDoc["inputfiles"]) == 1, \ "Error: Wrong number of input files." result = changeStateDB.loadView("JobDump", "jobsByWorkflowName") self.assertEqual(len(result["rows"]), 2, "Error: Wrong number of rows.") for row in result["rows"]: couchJobDoc = changeStateDB.document(row["value"]["id"]) self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"], "Error: Rev is wrong.") return def testUpdateFailedDoc(self): """ _testUpdateFailedDoc_ Verify that the update function will work correctly and not throw a 500 error if the doc didn't make it into the database for some reason. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048, locations = set(["somese.cern.ch"])) testFileA.create() testFileset.addFile(testFileA) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobA["couch_record"] = str(testJobA["id"]) change.propagate([testJobA], "new", "none") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) self.assertTrue("states" in testJobADoc) self.assertTrue("1" in testJobADoc["states"]) return def testPersist(self): """ _testPersist_ This is the test class for function Propagate from module ChangeState """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() for i in range(4): newFile = File(lfn = "File%s" % i, locations = set(["somese.cern.ch"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 4, \ "Error: Splitting should have created four jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobC = jobGroup.jobs[2] testJobC["user"] = "******" testJobC["group"] = "DMWM" testJobC["taskType"] = "Processing" testJobD = jobGroup.jobs[3] testJobD["user"] = "******" testJobD["group"] = "DMWM" testJobD["taskType"] = "Processing" change.persist([testJobA, testJobB], "created", "new") change.persist([testJobC, testJobD], "new", "none") stateDAO = self.daoFactory(classname = "Jobs.GetState") jobAState = stateDAO.execute(id = testJobA["id"]) jobBState = stateDAO.execute(id = testJobB["id"]) jobCState = stateDAO.execute(id = testJobC["id"]) jobDState = stateDAO.execute(id = testJobD["id"]) assert jobAState == "created" and jobBState =="created" and \ jobCState == "new" and jobDState == "new", \ "Error: Jobs didn't change state correctly." return def testRetryCount(self): """ _testRetryCount_ Verify that the retry count is incremented when we move out of the submitcooloff or jobcooloff state. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() for i in range(4): newFile = File(lfn = "File%s" % i, locations = set(["somese.cern.ch"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 4, \ "Error: Splitting should have created four jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobC = jobGroup.jobs[2] testJobC["user"] = "******" testJobC["group"] = "DMWM" testJobC["taskType"] = "Processing" testJobD = jobGroup.jobs[3] testJobD["user"] = "******" testJobD["group"] = "DMWM" testJobD["taskType"] = "Processing" change.persist([testJobA], "created", "submitcooloff") change.persist([testJobB], "created", "jobcooloff") change.persist([testJobC, testJobD], "new", "none") testJobA.load() testJobB.load() testJobC.load() testJobD.load() assert testJobA["retry_count"] == 1, \ "Error: Retry count is wrong." assert testJobB["retry_count"] == 1, \ "Error: Retry count is wrong." assert testJobC["retry_count"] == 0, \ "Error: Retry count is wrong." assert testJobD["retry_count"] == 0, \ "Error: Retry count is wrong." return def testJobSerialization(self): """ _testJobSerialization_ Verify that serialization of a job works when adding a FWJR. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'executing', 'created') changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") result = changeStateDB.loadView("FWJRDump", "fwjrsByWorkflowName") self.assertEqual(len(result["rows"]), 1, "Error: Wrong number of rows.") for row in result["rows"]: couchJobDoc = changeStateDB.document(row["value"]["id"]) self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"], "Error: Rev is wrong.") for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/FWJRDump": fwjrDoc = changeStateDB.document(resultRow["id"]) break assert fwjrDoc["retrycount"] == 0, \ "Error: Retry count is wrong." assert len(fwjrDoc["fwjr"]["steps"].keys()) == 2, \ "Error: Wrong number of steps in FWJR." assert "cmsRun1" in fwjrDoc["fwjr"]["steps"].keys(), \ "Error: cmsRun1 step is missing from FWJR." assert "stageOut1" in fwjrDoc["fwjr"]["steps"].keys(), \ "Error: stageOut1 step is missing from FWJR." return def testDuplicateJobReports(self): """ _testDuplicateJobReports_ Verify that everything works correctly if a job report is added to the database more than once. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'executing', 'created') change.propagate([testJobA], 'executing', 'created') changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/FWJRDump": fwjrDoc = changeStateDB.document(resultRow["id"]) break return def testJobKilling(self): """ _testJobKilling_ Test that we can successfully set jobs to the killed state """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() for i in range(4): newFile = File(lfn = "File%s" % i, locations = set(["somese.cern.ch"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 4, \ "Error: Splitting should have created four jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobC = jobGroup.jobs[2] testJobC["user"] = "******" testJobC["group"] = "DMWM" testJobC["taskType"] = "Processing" testJobD = jobGroup.jobs[3] testJobD["user"] = "******" testJobD["group"] = "DMWM" testJobD["taskType"] = "Processing" change.persist([testJobA], "created", "new") change.persist([testJobB], "jobfailed", "executing") change.persist([testJobC, testJobD], "executing", "created") change.persist([testJobA], "killed", "created") change.persist([testJobB], "killed", "jobfailed") change.persist([testJobC, testJobD], "killed", "executing") for job in [testJobA, testJobB, testJobC, testJobD]: job.load() self.assertEqual(job['retry_count'], 99999) self.assertEqual(job['state'], 'killed') return def testFWJRInputFileTruncation(self): """ _testFWJRInputFileTruncation_ Test and see whether the ChangeState code can be used to automatically truncate the number of input files in a FWJR Code stolen from the serialization test """ self.config.JobStateMachine.maxFWJRInputFiles = 0 change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] self.assertEqual(len(jobGroup.jobs), 1, "Error: Splitting should have created one job.") testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'executing', 'created') changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") result = changeStateDB.loadView("FWJRDump", "fwjrsByWorkflowName") self.assertEqual(len(result["rows"]), 1, "Error: Wrong number of rows.") for row in result["rows"]: couchJobDoc = changeStateDB.document(row["value"]["id"]) self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"], "Error: Rev is wrong.") for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/FWJRDump": fwjrDoc = changeStateDB.document(resultRow["id"]) break self.assertEqual(fwjrDoc["fwjr"]["steps"]['cmsRun1']['input']['source'], []) return def testJobSummary(self): """ _testJobSummary_ verify that job summary for jobs with fwjr are correctly created and that status is updated when updatesummary flag is enabled """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Analysis" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) change.propagate([testJobA], 'executing', 'created') testJobA["fwjr"] = myReport change.propagate([testJobA], 'jobfailed', 'executing') changeStateDB = self.couchServer.connectDatabase(dbname = self.config.JobStateMachine.jobSummaryDBName) allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") fwjrDoc = {'state': None} for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/WMStats": fwjrDoc = changeStateDB.document(resultRow["id"]) break self.assertEqual(fwjrDoc['state'], 'jobfailed', "Error: summary doesn't have the expected job state") del testJobA["fwjr"] change.propagate([testJobA], 'jobcooloff', 'jobfailed', updatesummary = True) return def testIndexConflict(self): """ _testIndexConflict_ Verify that in case of conflict in the job index we discard the old document and replace with a new one """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "CompOps" testJobA["taskType"] = "Processing" myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'created', 'new') jobdatabase = self.couchServer.connectDatabase('changestate_t/jobs', False) fwjrdatabase = self.couchServer.connectDatabase('changestate_t/fwjrs', False) jobDoc = jobdatabase.document("1") fwjrDoc = fwjrdatabase.document("1-0") self.assertEqual(jobDoc["workflow"], "wf001", "Wrong workflow in couch job document") self.assertEqual(fwjrDoc["fwjr"]["task"], self.taskName, "Wrong task in fwjr couch document") testJobA.delete() myThread = threading.currentThread() myThread.dbi.processData("ALTER TABLE wmbs_job AUTO_INCREMENT = 1") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf002", task = "/TestWorkflow/Test2") testWorkflow.create() testFileset = Fileset(name = "TestFilesetB") testFileset.create() testFile = File(lfn = "SomeLFNB", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] testJobB = jobGroup.jobs[0] testJobB["user"] = "******" testJobB["group"] = "CompOps" testJobB["taskType"] = "Processing" testJobB["fwjr"] = myReport change.propagate([testJobB], 'created', 'new') jobDoc = jobdatabase.document("1") fwjrDoc = fwjrdatabase.document("1-0") self.assertEqual(jobDoc["workflow"], "wf002", "Job document was not overwritten") self.assertEqual(fwjrDoc["fwjr"]["task"], "/TestWorkflow/Test2", "FWJR document was not overwritten") return def testUpdateLocation(self): """ _testUpdateLocation_ Check that we can update the location of a job through the state machine. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") locationAction.execute("site2", seName = "somese2.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048, locations = set(["somese.cern.ch", "somese2.cern.ch"])) testFileB = File(lfn = "SomeLFNB", events = 1025, size = 2049, locations = set(["somese.cern.ch", "somese2.cern.ch"])) testFileA.create() testFileB.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 2, \ "Error: Splitting should have created two jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobA["site_cms_name"] = "site1" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobB["site_cms_name"] = "site2" change.propagate([testJobA, testJobB], "new", "none") change.propagate([testJobA, testJobB], "created", "new") change.propagate([testJobA, testJobB], "executing", "created") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) maxKey = max(testJobADoc["states"].keys()) transition = testJobADoc["states"][maxKey] self.assertEqual(transition["location"], "site1") testJobBDoc = change.jobsdatabase.document(testJobB["couch_record"]) maxKey = max(testJobBDoc["states"].keys()) transition = testJobBDoc["states"][maxKey] self.assertEqual(transition["location"], "site2") jobs = [{'jobid' : 1, 'location' : 'site2'}] change.recordLocationChange(jobs) testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) maxKey = max(testJobADoc["states"].keys()) transition = testJobADoc["states"][maxKey] self.assertEqual(transition["location"], "site2") listJobsDAO = self.daoFactory(classname = "Jobs.GetLocation") jobid = [{'jobid' : 1}, {'jobid' : 2}] jobsLocation = listJobsDAO.execute(jobid) for job in jobsLocation: self.assertEqual(job['site_name'], 'site2') return
class JobArchiverTest(EmulatedUnitTestCase): """ TestCase for TestJobArchiver module """ _maxMessage = 10 def setUp(self): """ setup for test. """ super(JobArchiverTest, self).setUp() myThread = threading.currentThread() super(JobArchiverTest, self).setUp() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() # self.tearDown() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump") self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump") self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daofactory(classname="Jobs.GetAllJobs") self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False) self.nJobs = 10 self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ Database deletion """ self.testInit.clearDatabase(modules=["WMCore.WMBS"]) self.testInit.tearDownCouch() self.testInit.delWorkDir() EmulatorSetup.deleteConfig(self.configFile) super(JobArchiverTest, self).tearDown() return def getConfig(self): """ _createConfig_ General config file """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.General.WorkDir = os.getenv("TESTDIR", os.getcwd()) # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984") config.JobStateMachine.couchDBName = "jobarchiver_t_0" config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' # config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = self.testDir config.JobArchiver.numberOfJobsToCluster = 1000 config.component_('WorkQueueManager') config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager" config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager" config.WorkQueueManager.level = 'LocalQueue' config.WorkQueueManager.logLevel = 'DEBUG' config.WorkQueueManager.couchurl = 'https://None' config.WorkQueueManager.dbname = 'whatever' config.WorkQueueManager.inboxDatabase = 'whatever2' config.WorkQueueManager.queueParams = {} config.WorkQueueManager.queueParams["ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue" return config def createTestJobGroup(self): """ Creates a group of several jobs """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileB.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() for _ in range(0, self.nJobs): testJob = Job(name=makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJobGroup.add(testJob) testJobGroup.commit() return testJobGroup def testBasicFunctionTest(self): """ _BasicFunctionTest_ Tests the components, by seeing if they can process a simple set of closeouts """ myThread = threading.currentThread() config = self.getConfig() testJobGroup = self.createTestJobGroup() changer = ChangeState(config) cacheDir = os.path.join(self.testDir, 'test') if not os.path.isdir(cacheDir): os.mkdir(cacheDir) # if os.path.isdir(config.JobArchiver.logDir): # shutil.rmtree(config.JobArchiver.logDir) for job in testJobGroup.jobs: myThread.transaction.begin() job["outcome"] = "success" job.save() myThread.transaction.commit() path = os.path.join(cacheDir, job['name']) os.makedirs(path) f = open('%s/%s.out' % (path, job['name']), 'w') f.write(job['name']) f.close() job.setCache(path) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'success', 'complete') testJobArchiver = JobArchiverPoller(config=config) testJobArchiver.algorithm() result = myThread.dbi.processData( "SELECT wmbs_job_state.name FROM wmbs_job_state INNER JOIN wmbs_job ON wmbs_job.state = wmbs_job_state.id")[ 0].fetchall() for val in result: self.assertEqual(val.values(), ['cleanout']) dirList = os.listdir(cacheDir) for job in testJobGroup.jobs: self.assertEqual(job["name"] in dirList, False) logPath = os.path.join(config.JobArchiver.componentDir, 'logDir', 'w', 'wf001', 'JobCluster_0') logList = os.listdir(logPath) for job in testJobGroup.jobs: self.assertEqual('Job_%i.tar.bz2' % (job['id']) in logList, True, 'Could not find transferred tarball for job %i' % (job['id'])) pipe = Popen(['tar', '-jxvf', os.path.join(logPath, 'Job_%i.tar.bz2' % (job['id']))], stdout=PIPE, stderr=PIPE, shell=False) pipe.wait() # filename = '%s/%s/%s.out' %(cacheDir[1:], job['name'], job['name']) filename = 'Job_%i/%s.out' % (job['id'], job['name']) self.assertEqual(os.path.isfile(filename), True, 'Could not find file %s' % (filename)) f = open(filename, 'r') fileContents = f.readlines() f.close() self.assertEqual(fileContents[0].find(job['name']) > -1, True) shutil.rmtree('Job_%i' % (job['id'])) if os.path.isfile('Job_%i.tar.bz2' % (job['id'])): os.remove('Job_%i.tar.bz2' % (job['id'])) return @attr('integration') def testSpeedTest(self): """ _SpeedTest_ Tests the components, as in sees if they load. Otherwise does nothing. """ return
class JobCreatorTest(unittest.TestCase): """ Test case for the JobCreator """ sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN'] def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also, create some dummy locations. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'], useDefault=False) self.couchdbname = "jobcreator_t" self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump") self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.daoFactory(classname="Locations.New") for site in self.sites: locationAction.execute(siteName=site, pnn=site) # Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName=site, pnn=site, ceName=site) resourceControl.insertThreshold(siteName=site, taskType='Processing', maxSlots=10000, pendingSlots=10000) self.resourceControl = resourceControl self._setup = True self._teardown = False self.testDir = self.testInit.generateWorkDir() self.cwd = os.getcwd() # Set heartbeat self.componentName = 'JobCreator' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.clearDatabase(modules=['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database']) self.testInit.delWorkDir() self._teardown = True self.testInit.tearDownCouch() EmulatorSetup.deleteConfig(self.configFile) return def createJobCollection(self, name, nSubs, nFiles, workflowURL='test'): """ _createJobCollection_ Create a collection of jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec=workflowURL, owner="mnorman", name=name, task="/TestWorkload/ReReco") testWorkflow.create() for sub in range(nSubs): nameStr = '%s-%i' % (name, sub) myThread.transaction.begin() testFileset = Fileset(name=nameStr) testFileset.create() for f in range(nFiles): # pick a random site site = random.choice(self.sites) testFile = File(lfn="/lfn/%s/%i" % (nameStr, f), size=1024, events=10) testFile.setLocation(site) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() myThread.transaction.commit() return def createWorkload(self, workloadName='Test', emulator=True, priority=1): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") rereco = workload.getTask("ReReco") seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001} rereco.addGenerator("PresetSeeder", **seederDict) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload def getConfig(self): """ _getConfig_ Creates a common config. """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = self.componentName # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' # The log level of the component. # config.JobCreator.logLevel = 'SQLDEBUG' config.JobCreator.logLevel = 'INFO' # maximum number of threads we want to deal # with messages per pool. config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromResourceControl = True config.JobCreator.pollInterval = 10 # config.JobCreator.jobCacheDir = self.testDir config.JobCreator.defaultJobType = 'processing' # Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 4 config.JobCreator.componentDir = self.testDir config.JobCreator.useWorkQueue = True config.JobCreator.WorkQueueParams = {'emulateDBSReader': True} # We now call the JobMaker from here config.component_('JobMaker') config.JobMaker.logLevel = 'INFO' config.JobMaker.namespace = 'WMCore.WMSpec.Makers.JobMaker' config.JobMaker.maxThreads = 1 config.JobMaker.makeJobsHandler = 'WMCore.WMSpec.Makers.Handlers.MakeJobs' # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = self.couchdbname return config def testVerySimpleTest(self): """ _VerySimpleTest_ Just test that everything works...more or less """ # return myThread = threading.currentThread() config = self.getConfig() name = makeUUID() nSubs = 5 nFiles = 10 workloadName = 'TestWorkload' dummyWorkload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) testJobCreator = JobCreatorPoller(config=config) # First, can we run once without everything crashing? testJobCreator.algorithm() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) # Count database objects result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) # Find the test directory testDirectory = os.path.join(self.testDir, 'jobCacheDir', 'TestWorkload', 'ReReco') # It should have at least one jobGroup self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory)) # But no more then twenty self.assertTrue(len(os.listdir(testDirectory)) <= 20) groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0') # First job should be in here listOfDirs = [] for tmpDirectory in os.listdir(testDirectory): listOfDirs.extend(os.listdir(os.path.join(testDirectory, tmpDirectory))) self.assertTrue('job_1' in listOfDirs) self.assertTrue('job_2' in listOfDirs) self.assertTrue('job_3' in listOfDirs) jobDir = os.listdir(groupDirectory)[0] jobFile = os.path.join(groupDirectory, jobDir, 'job.pkl') self.assertTrue(os.path.isfile(jobFile)) f = open(jobFile, 'r') job = pickle.load(f) f.close() self.assertEqual(job.baggage.PresetSeeder.generator.initialSeed, 1001) self.assertEqual(job.baggage.PresetSeeder.evtgenproducer.initialSeed, 1001) self.assertEqual(job['workflow'], name) self.assertEqual(len(job['input_files']), 1) self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2') return @attr('performance', 'integration') def testProfilePoller(self): """ Profile your performance You shouldn't be running this normally because it doesn't do anything """ myThread = threading.currentThread() name = makeUUID() nSubs = 5 nFiles = 1500 workloadName = 'TestWorkload' workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) config = self.getConfig() testJobCreator = JobCreatorPoller(config=config) cProfile.runctx("testJobCreator.algorithm()", globals(), locals(), filename="testStats.stat") getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") time.sleep(10) self.assertEqual(len(result), nSubs * nFiles) p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats(.2) return @attr('integration') def testProfileWorker(self): """ Profile where the work actually gets done You shouldn't be running this one either, since it doesn't test anything. """ myThread = threading.currentThread() name = makeUUID() nSubs = 5 nFiles = 500 workloadName = 'TestWorkload' workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) config = self.getConfig() configDict = {"couchURL": config.JobStateMachine.couchurl, "couchDBName": config.JobStateMachine.couchDBName, 'jobCacheDir': config.JobCreator.jobCacheDir, 'defaultJobType': config.JobCreator.defaultJobType} subs = [{"subscription": 1}, {"subscription": 2}, {"subscription": 3}, {"subscription": 4}, {"subscription": 5}] testJobCreator = JobCreatorPoller(**configDict) cProfile.runctx("testJobCreator.algorithm(parameters = input)", globals(), locals(), filename="workStats.stat") p = pstats.Stats('workStats.stat') p.sort_stats('cumulative') p.print_stats(.2) return @attr('integration') def testHugeTest(self): """ Don't run this one either """ myThread = threading.currentThread() config = self.getConfig() name = makeUUID() nSubs = 10 nFiles = 5000 workloadName = 'Tier1ReReco' dummyWorkload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) testJobCreator = JobCreatorPoller(config=config) # First, can we run once without everything crashing? startTime = time.time() testJobCreator.algorithm() stopTime = time.time() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) print("Job took %f seconds to run" % (stopTime - startTime)) # Count database objects result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) return def stuffWMBS(self, workflowURL, name): """ _stuffWMBS_ Insert some dummy jobs, jobgroups, filesets, files and subscriptions into WMBS to test job creation. Three completed job groups each containing several files are injected. Another incomplete job group is also injected. Also files are added to the "Mergeable" subscription as well as to the output fileset for their jobgroups. """ locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="s1", pnn="somese.cern.ch") mergeFileset = Fileset(name="mergeFileset") mergeFileset.create() bogusFileset = Fileset(name="bogusFileset") bogusFileset.create() mergeWorkflow = Workflow(spec=workflowURL, owner="mnorman", name=name, task="/TestWorkload/ReReco") mergeWorkflow.create() mergeSubscription = Subscription(fileset=mergeFileset, workflow=mergeWorkflow, split_algo="ParentlessMergeBySize") mergeSubscription.create() dummySubscription = Subscription(fileset=bogusFileset, workflow=mergeWorkflow, split_algo="ParentlessMergeBySize") file1 = File(lfn="file1", size=1024, events=1024, first_event=0, locations={"somese.cern.ch"}) file1.addRun(Run(1, *[45])) file1.create() file2 = File(lfn="file2", size=1024, events=1024, first_event=1024, locations={"somese.cern.ch"}) file2.addRun(Run(1, *[45])) file2.create() file3 = File(lfn="file3", size=1024, events=1024, first_event=2048, locations={"somese.cern.ch"}) file3.addRun(Run(1, *[45])) file3.create() file4 = File(lfn="file4", size=1024, events=1024, first_event=3072, locations={"somese.cern.ch"}) file4.addRun(Run(1, *[45])) file4.create() fileA = File(lfn="fileA", size=1024, events=1024, first_event=0, locations={"somese.cern.ch"}) fileA.addRun(Run(1, *[46])) fileA.create() fileB = File(lfn="fileB", size=1024, events=1024, first_event=1024, locations={"somese.cern.ch"}) fileB.addRun(Run(1, *[46])) fileB.create() fileC = File(lfn="fileC", size=1024, events=1024, first_event=2048, locations={"somese.cern.ch"}) fileC.addRun(Run(1, *[46])) fileC.create() fileI = File(lfn="fileI", size=1024, events=1024, first_event=0, locations={"somese.cern.ch"}) fileI.addRun(Run(2, *[46])) fileI.create() fileII = File(lfn="fileII", size=1024, events=1024, first_event=1024, locations={"somese.cern.ch"}) fileII.addRun(Run(2, *[46])) fileII.create() fileIII = File(lfn="fileIII", size=1024, events=1024, first_event=2048, locations={"somese.cern.ch"}) fileIII.addRun(Run(2, *[46])) fileIII.create() fileIV = File(lfn="fileIV", size=1024 * 1000000, events=1024, first_event=3072, locations={"somese.cern.ch"}) fileIV.addRun(Run(2, *[46])) fileIV.create() for fileObj in [file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII, fileIII, fileIV]: mergeFileset.addFile(fileObj) bogusFileset.addFile(fileObj) mergeFileset.commit() bogusFileset.commit() return def testTestNonProxySplitting(self): """ _TestNonProxySplitting_ Test and see if we can split things without a proxy. """ config = self.getConfig() config.JobCreator.workerThreads = 1 name = makeUUID() workloadName = 'TestWorkload' workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.stuffWMBS(workflowURL=workloadPath, name=name) testJobCreator = JobCreatorPoller(config=config) testJobCreator.algorithm() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 1) result = getJobsAction.execute(state='Created', jobType="Merge") self.assertEqual(len(result), 0) return
class WMBSHelperTest(unittest.TestCase): def setUp(self): """ _setUp_ """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump") self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump") os.environ["COUCHDB"] = "wmbshelper_t" self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMComponent.DBSBuffer.Database", "WMCore.BossAir", "WMCore.ResourceControl"], useDefault = False) self.workDir = self.testInit.generateWorkDir() self.wmspec = self.createWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = MockDBSReader(self.inputDataset.dbsurl) self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = threading.currentThread().logger, dbinterface = threading.currentThread().dbi) return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.clearDatabase() self.testInit.tearDownCouch() self.testInit.delWorkDir() return def setupForKillTest(self, baAPI = None): """ _setupForKillTest_ Inject a workflow into WMBS that has a processing task, a merge task and a cleanup task. Inject files into the various tasks at various processing states (acquired, complete, available...). Also create jobs for each subscription in various states. """ myThread = threading.currentThread() daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = daoFactory(classname = "Locations.New") changeStateAction = daoFactory(classname = "Jobs.ChangeState") resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'site1', seName = 'goodse.cern.ch', ceName = 'site1', plugin = "TestPlugin") resourceControl.insertThreshold(siteName = 'site1', taskType = 'Processing', \ maxSlots = 10000) inputFileset = Fileset("input") inputFileset.create() inputFileA = File("lfnA", locations = "goodse.cern.ch") inputFileB = File("lfnB", locations = "goodse.cern.ch") inputFileC = File("lfnC", locations = "goodse.cern.ch") inputFileA.create() inputFileB.create() inputFileC.create() inputFileset.addFile(inputFileA) inputFileset.addFile(inputFileB) inputFileset.addFile(inputFileC) inputFileset.commit() unmergedOutputFileset = Fileset("unmerged") unmergedOutputFileset.create() unmergedFileA = File("ulfnA", locations = "goodse.cern.ch") unmergedFileB = File("ulfnB", locations = "goodse.cern.ch") unmergedFileC = File("ulfnC", locations = "goodse.cern.ch") unmergedFileA.create() unmergedFileB.create() unmergedFileC.create() unmergedOutputFileset.addFile(unmergedFileA) unmergedOutputFileset.addFile(unmergedFileB) unmergedOutputFileset.addFile(unmergedFileC) unmergedOutputFileset.commit() mainProcWorkflow = Workflow(spec = "spec1", owner = "Steve", name = "Main", task = "Proc") mainProcWorkflow.create() mainProcMergeWorkflow = Workflow(spec = "spec1", owner = "Steve", name = "Main", task = "ProcMerge") mainProcMergeWorkflow.create() mainCleanupWorkflow = Workflow(spec = "spec1", owner = "Steve", name = "Main", task = "Cleanup") mainCleanupWorkflow.create() self.mainProcSub = Subscription(fileset = inputFileset, workflow = mainProcWorkflow, type = "Processing") self.mainProcSub.create() self.mainProcSub.acquireFiles(inputFileA) self.mainProcSub.completeFiles(inputFileB) procJobGroup = JobGroup(subscription = self.mainProcSub) procJobGroup.create() self.procJobA = Job(name = "ProcJobA") self.procJobA["state"] = "new" self.procJobA["location"] = "site1" self.procJobB = Job(name = "ProcJobB") self.procJobB["state"] = "executing" self.procJobB["location"] = "site1" self.procJobC = Job(name = "ProcJobC") self.procJobC["state"] = "complete" self.procJobC["location"] = "site1" self.procJobA.create(procJobGroup) self.procJobB.create(procJobGroup) self.procJobC.create(procJobGroup) self.mainMergeSub = Subscription(fileset = unmergedOutputFileset, workflow = mainProcMergeWorkflow, type = "Merge") self.mainMergeSub.create() self.mainMergeSub.acquireFiles(unmergedFileA) self.mainMergeSub.failFiles(unmergedFileB) mergeJobGroup = JobGroup(subscription = self.mainMergeSub) mergeJobGroup.create() self.mergeJobA = Job(name = "MergeJobA") self.mergeJobA["state"] = "exhausted" self.mergeJobA["location"] = "site1" self.mergeJobB = Job(name = "MergeJobB") self.mergeJobB["state"] = "cleanout" self.mergeJobB["location"] = "site1" self.mergeJobC = Job(name = "MergeJobC") self.mergeJobC["state"] = "new" self.mergeJobC["location"] = "site1" self.mergeJobA.create(mergeJobGroup) self.mergeJobB.create(mergeJobGroup) self.mergeJobC.create(mergeJobGroup) self.mainCleanupSub = Subscription(fileset = unmergedOutputFileset, workflow = mainCleanupWorkflow, type = "Cleanup") self.mainCleanupSub.create() self.mainCleanupSub.acquireFiles(unmergedFileA) self.mainCleanupSub.completeFiles(unmergedFileB) cleanupJobGroup = JobGroup(subscription = self.mainCleanupSub) cleanupJobGroup.create() self.cleanupJobA = Job(name = "CleanupJobA") self.cleanupJobA["state"] = "new" self.cleanupJobA["location"] = "site1" self.cleanupJobB = Job(name = "CleanupJobB") self.cleanupJobB["state"] = "executing" self.cleanupJobB["location"] = "site1" self.cleanupJobC = Job(name = "CleanupJobC") self.cleanupJobC["state"] = "complete" self.cleanupJobC["location"] = "site1" self.cleanupJobA.create(cleanupJobGroup) self.cleanupJobB.create(cleanupJobGroup) self.cleanupJobC.create(cleanupJobGroup) jobList = [self.procJobA, self.procJobB, self.procJobC, self.mergeJobA, self.mergeJobB, self.mergeJobC, self.cleanupJobA, self.cleanupJobB, self.cleanupJobC] changeStateAction.execute(jobList) if baAPI: for job in jobList: job['plugin'] = 'TestPlugin' job['userdn'] = 'Steve' job['custom']['location'] = 'site1' baAPI.createNewJobs(wmbsJobs = jobList) # We'll create an unrelated workflow to verify that it isn't affected # by the killing code. bogusFileset = Fileset("dontkillme") bogusFileset.create() bogusFileA = File("bogus/lfnA", locations = "goodse.cern.ch") bogusFileA.create() bogusFileset.addFile(bogusFileA) bogusFileset.commit() bogusWorkflow = Workflow(spec = "spec2", owner = "Steve", name = "Bogus", task = "Proc") bogusWorkflow.create() self.bogusSub = Subscription(fileset = bogusFileset, workflow = bogusWorkflow, type = "Processing") self.bogusSub.create() self.bogusSub.acquireFiles(bogusFileA) return def verifyFileKillStatus(self): """ _verifyFileKillStatus_ Verify that all files were killed correctly. The status of files in Cleanup and LogCollect subscriptions isn't modified. Status of already completed and failed files is not modified. Also verify that the bogus subscription is untouched. """ failedFiles = self.mainProcSub.filesOfStatus("Failed") acquiredFiles = self.mainProcSub.filesOfStatus("Acquired") completedFiles = self.mainProcSub.filesOfStatus("Completed") availableFiles = self.mainProcSub.filesOfStatus("Available") bogusAcquiredFiles = self.bogusSub.filesOfStatus("Acquired") self.assertEqual(len(availableFiles), 0, \ "Error: There should be no available files.") self.assertEqual(len(acquiredFiles), 0, \ "Error: There should be no acquired files.") self.assertEqual(len(bogusAcquiredFiles), 1, \ "Error: There should be one acquired file.") self.assertEqual(len(completedFiles), 3, \ "Error: There should be only one completed file.") goldenLFNs = ["lfnA", "lfnB", "lfnC"] for completedFile in completedFiles: self.assertTrue(completedFile["lfn"] in goldenLFNs, \ "Error: Extra completed file.") goldenLFNs.remove(completedFile["lfn"]) self.assertEqual(len(failedFiles), 0, \ "Error: There should be no failed files.") self.assertEqual(len(goldenLFNs), 0, \ "Error: Missing LFN") failedFiles = self.mainMergeSub.filesOfStatus("Failed") acquiredFiles = self.mainMergeSub.filesOfStatus("Acquired") completedFiles = self.mainMergeSub.filesOfStatus("Completed") availableFiles = self.mainMergeSub.filesOfStatus("Available") self.assertEqual(len(acquiredFiles), 0, \ "Error: Merge subscription should have 0 acq files.") self.assertEqual(len(availableFiles), 0, \ "Error: Merge subscription should have 0 avail files.") self.assertEqual(len(failedFiles), 1, \ "Error: Merge subscription should have 1 failed files.") self.assertEqual(list(failedFiles)[0]["lfn"], "ulfnB", "Error: Wrong failed file.") self.assertEqual(len(completedFiles), 2, \ "Error: Merge subscription should have 2 compl files.") goldenLFNs = ["ulfnA", "ulfnC"] for completedFile in completedFiles: self.assertTrue(completedFile["lfn"] in goldenLFNs, \ "Error: Extra complete file.") goldenLFNs.remove(completedFile["lfn"]) self.assertEqual(len(goldenLFNs), 0, \ "Error: Missing LFN") failedFiles = self.mainCleanupSub.filesOfStatus("Failed") acquiredFiles = self.mainCleanupSub.filesOfStatus("Acquired") completedFiles = self.mainCleanupSub.filesOfStatus("Completed") availableFiles = self.mainCleanupSub.filesOfStatus("Available") self.assertEqual(len(failedFiles), 0, \ "Error: Cleanup subscription should have 0 fai files.") self.assertEqual(len(acquiredFiles), 1, \ "Error: There should be only one acquired file.") self.assertEqual(list(acquiredFiles)[0]["lfn"], "ulfnA", \ "Error: Wrong acquired LFN.") self.assertEqual(len(completedFiles), 1, \ "Error: There should be only one completed file.") self.assertEqual(list(completedFiles)[0]["lfn"], "ulfnB", \ "Error: Wrong completed LFN.") self.assertEqual(len(availableFiles), 1, \ "Error: There should be only one available file.") self.assertEqual(list(availableFiles)[0]["lfn"], "ulfnC", \ "Error: Wrong completed LFN.") return def verifyJobKillStatus(self): """ _verifyJobKillStatus_ Verify that jobs are killed correctly. Jobs belonging to Cleanup and LogCollect subscriptions are not killed. The status of jobs that have already finished running is not changed. """ self.procJobA.load() self.procJobB.load() self.procJobC.load() self.assertEqual(self.procJobA["state"], "killed", \ "Error: Proc job A should be killed.") self.assertEqual(self.procJobB["state"], "killed", \ "Error: Proc job B should be killed.") self.assertEqual(self.procJobC["state"], "complete", \ "Error: Proc job C should be complete.") self.mergeJobA.load() self.mergeJobB.load() self.mergeJobC.load() self.assertEqual(self.mergeJobA["state"], "exhausted", \ "Error: Merge job A should be exhausted.") self.assertEqual(self.mergeJobB["state"], "cleanout", \ "Error: Merge job B should be cleanout.") self.assertEqual(self.mergeJobC["state"], "killed", \ "Error: Merge job C should be killed.") self.cleanupJobA.load() self.cleanupJobB.load() self.cleanupJobC.load() self.assertEqual(self.cleanupJobA["state"], "new", \ "Error: Cleanup job A should be new.") self.assertEqual(self.cleanupJobB["state"], "executing", \ "Error: Cleanup job B should be executing.") self.assertEqual(self.cleanupJobC["state"], "complete", \ "Error: Cleanup job C should be complete.") return def testKillWorkflow(self): """ _testKillWorkflow_ Verify that workflow killing works correctly. """ configFile = EmulatorSetup.setupWMAgentConfig() config = loadConfigurationFile(configFile) baAPI = BossAirAPI(config = config) # Create nine jobs self.setupForKillTest(baAPI = baAPI) self.assertEqual(len(baAPI._listRunJobs()), 9) killWorkflow("Main", config, config) self.verifyFileKillStatus() self.verifyJobKillStatus() self.assertEqual(len(baAPI._listRunJobs()), 8) EmulatorSetup.deleteConfig(configFile) return def createTestWMSpec(self): """ _createTestWMSpec_ Create a WMSpec that has a processing, merge, cleanup and skims tasks that can be used by the subscription creation test. """ testWorkload = WMWorkloadHelper(WMWorkload("TestWorkload")) testWorkload.setDashboardActivity("TestReReco") testWorkload.setSpecUrl("/path/to/workload") testWorkload.setOwnerDetails("sfoulkes", "DMWM", {'dn': 'MyDN'}) procTask = testWorkload.newTask("ProcessingTask") procTask.setTaskType("Processing") procTask.setSplittingAlgorithm("FileBased", files_per_job = 1) procTaskCMSSW = procTask.makeStep("cmsRun1") procTaskCMSSW.setStepType("CMSSW") procTaskCMSSWHelper = procTaskCMSSW.getTypeHelper() procTask.setTaskType("Processing") procTask.setSiteWhitelist(["site1"]) procTask.setSiteBlacklist(["site2"]) procTask.applyTemplates() procTaskCMSSWHelper.addOutputModule("OutputA", primaryDataset = "bogusPrimary", processedDataset = "bogusProcessed", dataTier = "DataTierA", lfnBase = "bogusUnmerged", mergedLFNBase = "bogusMerged", filterName = None) mergeTask = procTask.addTask("MergeTask") mergeTask.setInputReference(procTaskCMSSW, outputModule = "OutputA") mergeTask.setTaskType("Merge") mergeTask.setSplittingAlgorithm("WMBSMergeBySize", min_merge_size = 1, max_merge_size = 2, max_merge_events = 3) mergeTaskCMSSW = mergeTask.makeStep("cmsRun1") mergeTaskCMSSW.setStepType("CMSSW") mergeTaskCMSSWHelper = mergeTaskCMSSW.getTypeHelper() mergeTask.setTaskType("Merge") mergeTask.applyTemplates() mergeTaskCMSSWHelper.addOutputModule("Merged", primaryDataset = "bogusPrimary", processedDataset = "bogusProcessed", dataTier = "DataTierA", lfnBase = "bogusUnmerged", mergedLFNBase = "bogusMerged", filterName = None) cleanupTask = procTask.addTask("CleanupTask") cleanupTask.setInputReference(procTaskCMSSW, outputModule = "OutputA") cleanupTask.setTaskType("Merge") cleanupTask.setSplittingAlgorithm("SiblingProcessingBase", files_per_job = 50) cleanupTaskCMSSW = cleanupTask.makeStep("cmsRun1") cleanupTaskCMSSW.setStepType("CMSSW") cleanupTaskCMSSWHelper = cleanupTaskCMSSW.getTypeHelper() cleanupTask.setTaskType("Cleanup") cleanupTask.applyTemplates() skimTask = mergeTask.addTask("SkimTask") skimTask.setTaskType("Skim") skimTask.setInputReference(mergeTaskCMSSW, outputModule = "Merged") skimTask.setSplittingAlgorithm("FileBased", files_per_job = 1, include_parents = True) skimTaskCMSSW = skimTask.makeStep("cmsRun1") skimTaskCMSSW.setStepType("CMSSW") skimTaskCMSSWHelper = skimTaskCMSSW.getTypeHelper() skimTask.setTaskType("Skim") skimTask.applyTemplates() skimTaskCMSSWHelper.addOutputModule("SkimOutputA", primaryDataset = "bogusPrimary", processedDataset = "bogusProcessed", dataTier = "DataTierA", lfnBase = "bogusUnmerged", mergedLFNBase = "bogusMerged", filterName = None) skimTaskCMSSWHelper.addOutputModule("SkimOutputB", primaryDataset = "bogusPrimary", processedDataset = "bogusProcessed", dataTier = "DataTierA", lfnBase = "bogusUnmerged", mergedLFNBase = "bogusMerged", filterName = None) return testWorkload def testCreateSubscription(self): """ _testCreateSubscription_ Verify that the subscription creation code works correctly. """ resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'site1', seName = 'goodse.cern.ch', ceName = 'site1', plugin = "TestPlugin") resourceControl.insertSite(siteName = 'site2', seName = 'goodse2.cern.ch', ceName = 'site2', plugin = "TestPlugin") testWorkload = self.createTestWMSpec() testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock", cachepath = self.workDir) testWMBSHelper.createSubscription() procWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask") procWorkflow.load() self.assertEqual(procWorkflow.owner, "sfoulkes", "Error: Wrong owner: %s" % procWorkflow.owner) self.assertEqual(procWorkflow.group, "DMWM", "Error: Wrong group: %s" % procWorkflow.group) self.assertEqual(procWorkflow.wfType, "TestReReco", "Error: Wrong type.") self.assertEqual(procWorkflow.spec, os.path.join(self.workDir, procWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(procWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") mergedProcOutput = procWorkflow.outputMap["OutputA"][0]["merged_output_fileset"] unmergedProcOutput = procWorkflow.outputMap["OutputA"][0]["output_fileset"] mergedProcOutput.loadData() unmergedProcOutput.loadData() self.assertEqual(mergedProcOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-Merged", "Error: Merged output fileset is wrong.") self.assertEqual(unmergedProcOutput.name, "/TestWorkload/ProcessingTask/unmerged-OutputA", "Error: Unmerged output fileset is wrong.") mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask/CleanupTask") cleanupWorkflow.load() self.assertEqual(cleanupWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(cleanupWorkflow.spec, os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(cleanupWorkflow.outputMap.keys()), 0, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] unmergedMergeOutput.loadData() self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-Merged", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual(mergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual(unmergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Unmerged output fileset is wrong.") self.assertEqual(mergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Merged output fileset is wrong.") self.assertEqual(unmergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-ProcessingTask-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset = topLevelFileset, workflow = procWorkflow) procSubscription.loadData() self.assertEqual(len(procSubscription.getWhiteBlackList()), 2, "Error: Wrong site white/black list for proc sub.") for site in procSubscription.getWhiteBlackList(): if site["site_name"] == "site1": self.assertEqual(site["valid"], 1, "Error: Site should be white listed.") else: self.assertEqual(site["valid"], 0, "Error: Site should be black listed.") self.assertEqual(procSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") mergeSubscription = Subscription(fileset = unmergedProcOutput, workflow = mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset = unmergedMergeOutput, workflow = skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return def testTruncatedWFInsertion(self): """ _testTruncatedWFInsertion_ """ resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'site1', seName = 'goodse.cern.ch', ceName = 'site1', plugin = "TestPlugin") resourceControl.insertSite(siteName = 'site2', seName = 'goodse2.cern.ch', ceName = 'site2', plugin = "TestPlugin") testWorkload = self.createTestWMSpec() testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock", cachepath = self.workDir) testWMBSHelper.createSubscription() testWorkload.truncate("ResubmitTestWorkload", "/TestWorkload/ProcessingTask/MergeTask", "someserver", "somedatabase") testResubmitWMBSHelper = WMBSHelper(testWorkload, "SomeBlock2", cachepath = self.workDir) testResubmitWMBSHelper.createSubscription() mergeWorkflow = Workflow(name = "ResubmitTestWorkload", task = "/ResubmitTestWorkload/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") cleanupWorkflow = Workflow(name = "ResubmitTestWorkload", task = "/ResubmitTestWorkload/CleanupTask") cleanupWorkflow.load() self.assertEqual(cleanupWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(cleanupWorkflow.spec, os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(cleanupWorkflow.outputMap.keys()), 0, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] unmergedMergeOutput.loadData() self.assertEqual(unmergedMergeOutput.name, "/ResubmitTestWorkload/MergeTask/merged-Merged", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow(name = "ResubmitTestWorkload", task = "/ResubmitTestWorkload/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual(mergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual(unmergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Unmerged output fileset is wrong.") self.assertEqual(mergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Merged output fileset is wrong.") self.assertEqual(unmergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name = "ResubmitTestWorkload-MergeTask-SomeBlock2") topLevelFileset.loadData() mergeSubscription = Subscription(fileset = topLevelFileset, workflow = mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset = unmergedMergeOutput, workflow = skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return def setupMCWMSpec(self): """Setup MC workflow""" self.wmspec = self.createMCWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = None self.siteDB = fakeSiteDB() def createWMSpec(self, name = 'ReRecoWorkload'): wmspec = rerecoWorkload(name, rerecoArgs) wmspec.setSpecUrl("/path/to/workload") return wmspec def createMCWMSpec(self, name = 'MonteCarloWorkload'): wmspec = monteCarloWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents = 10000) return wmspec def getDBS(self, wmspec): topLevelTask = getFirstTask(wmspec) inputDataset = topLevelTask.inputDataset() dbs = MockDBSReader(inputDataset.dbsurl) #dbsDict = {self.inputDataset.dbsurl : self.dbs} return dbs def createWMBSHelperWithTopTask(self, wmspec, block, mask = None): topLevelTask = getFirstTask(wmspec) wmbs = WMBSHelper(wmspec, block, mask, cachepath = self.workDir) if block: block = self.dbs.getFileBlock(block)[block] wmbs.createSubscriptionAndAddFiles(block = block) return wmbs # def testProduction(self): # """Production workflow""" # pass def testReReco(self): """ReReco workflow""" # create workflow block = self.dataset + "#1" wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)) self.assertEqual(len(files), 1) def testReRecoBlackRunRestriction(self): """ReReco workflow with Run restrictions""" block = self.dataset + "#2" #add run blacklist self.topLevelTask.setInputRunBlacklist([1, 2, 3, 4]) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files']) self.assertEqual(len(files), 0) def testReRecoWhiteRunRestriction(self): block = self.dataset + "#2" # Run Whitelist self.topLevelTask.setInputRunWhitelist([2]) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files']) self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock()) def testDuplicateFileInsert(self): # using default wmspec block = self.dataset + "#1" wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) wmbs.topLevelFileset.loadData() numOfFiles = len(wmbs.topLevelFileset.files) # check initially inserted files. dbsFiles = self.dbs.getFileBlock(block)[block]['Files'] self.assertEqual(numOfFiles, len(dbsFiles)) firstFileset = wmbs.topLevelFileset wmbsDao = wmbs.daofactory(classname = "Files.InFileset") numOfFiles = len(wmbsDao.execute(firstFileset.id)) self.assertEqual(numOfFiles, len(dbsFiles)) # use the new spec with same inputdataset block = self.dataset + "#1" wmspec = self.createWMSpec("TestSpec1") dbs = self.getDBS(wmspec) wmbs = self.createWMBSHelperWithTopTask(wmspec, block) # check duplicate insert dbsFiles = dbs.getFileBlock(block)[block]['Files'] numOfFiles = wmbs.addFiles(dbs.getFileBlock(block)[block]) self.assertEqual(numOfFiles, 0) secondFileset = wmbs.topLevelFileset wmbsDao = wmbs.daofactory(classname = "Files.InFileset") numOfFiles = len(wmbsDao.execute(secondFileset.id)) self.assertEqual(numOfFiles, len(dbsFiles)) self.assertNotEqual(firstFileset.id, secondFileset.id) def testParentage(self): """ TODO: add the parentage test. 1. check whether parent files are created in wmbs. 2. check parent files are associated to child. 3. When 2 specs with the same input data (one with parent processing, one without it) is inserted, if one without parent processing inserted first then the other with parent processing insert, it still needs to create parent files although child files are duplicate """ pass def testMCFakeFileInjection(self): """Inject fake Monte Carlo files into WMBS""" self.setupMCWMSpec() mask = Mask(FirstRun = 12, FirstLumi = 1234, FirstEvent = 12345, LastEvent = 999995, LastLumi = 12345, LastRun = 12) # add sites that would normally be added by operator via resource_control locationDAO = self.daoFactory(classname = "Locations.New") ses = [] for site in ['T2_XX_SiteA', 'T2_XX_SiteB']: locationDAO.execute(siteName = site, seName = self.siteDB.cmsNametoSE(site)) ses.append(self.siteDB.cmsNametoSE(site)) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask) subscription = wmbs.topLevelSubscription self.assertEqual(1, subscription.exists()) fileset = subscription['fileset'] self.assertEqual(1, fileset.exists()) fileset.loadData() # need to refresh from database self.assertEqual(len(fileset.files), 1) self.assertEqual(len(fileset.parents), 0) self.assertFalse(fileset.open) file = list(fileset.files)[0] self.assertEqual(file['events'], mask['LastEvent'] - mask['FirstEvent'] + 1) # inclusive range self.assertEqual(file['merged'], False) # merged files get added to dbs self.assertEqual(len(file['parents']), 0) #file.loadData() self.assertEqual(sorted(file['locations']), sorted(ses)) self.assertEqual(len(file.getParentLFNs()), 0) self.assertEqual(len(file.getRuns()), 1) run = file.getRuns()[0] self.assertEqual(run.run, mask['FirstRun']) self.assertEqual(run.lumis[0], mask['FirstLumi']) self.assertEqual(run.lumis[-1], mask['LastLumi']) self.assertEqual(len(run.lumis), mask['LastLumi'] - mask['FirstLumi'] + 1)
class TaskChainTests(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("taskchain_t", "ConfigCache") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("taskchain_t") self.testInit.generateWorkDir() self.workload = None return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def testGeneratorWorkflow(self): """ _testGeneratorWorkflow_ Test creating a request with an initial generator task it mocks a request where there are 2 similar paths starting from the generator, each one with a different PrimaryDataset, CMSSW configuration and processed dataset. Dropping the RAW output as well. Also include an ignored output module to keep things interesting... """ generatorDoc = makeGeneratorConfig(self.configDatabase) processorDocs = makeProcessingConfigs(self.configDatabase) arguments = { "AcquisitionEra": "ReleaseValidation", "Requestor": "*****@*****.**", "CMSSWVersion": "CMSSW_3_5_8", "ScramArch": "slc5_ia32_gcc434", "ProcessingVersion": 1, "GlobalTag": "GR10_P_v4::All", "CouchURL": self.testInit.couchUrl, "CouchDBName": self.testInit.couchDbName, "SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"], "DashboardHost": "127.0.0.1", "DashboardPort": 8884, "TaskChain" : 6, "IgnoredOutputModules" : ["writeSkim2", "writeRAWDEBUGDIGI"], "Task1" :{ "TaskName" : "GenSim", "ConfigCacheID" : generatorDoc, "SplittingAlgorithm" : "EventBased", "SplittingArguments" : {"events_per_job" : 250}, "RequestNumEvents" : 10000, "Seeding" : "Automatic", "PrimaryDataset" : "RelValTTBar", }, "Task2" : { "TaskName" : "DigiHLT_new", "InputTask" : "GenSim", "InputFromOutputModule" : "writeGENSIM", "ConfigCacheID" : processorDocs['DigiHLT'], "SplittingAlgorithm" : "LumiBased", "SplittingArguments" : {"lumis_per_job" : 2 }, "CMSSWVersion" : "CMSSW_5_2_6", "GlobalTag" : "GR_39_P_V5:All", "PrimaryDataset" : "PURelValTTBar", "KeepOutput" : False }, "Task3" : { "TaskName" : "DigiHLT_ref", "InputTask" : "GenSim", "InputFromOutputModule" : "writeGENSIM", "ConfigCacheID" : processorDocs['DigiHLT'], "SplittingAlgorithm" : "EventBased", "SplittingArguments" : {"events_per_job" : 100 }, "CMSSWVersion" : "CMSSW_5_2_7", "GlobalTag" : "GR_40_P_V5:All", "AcquisitionEra" : "ReleaseValidationNewConditions", "ProcessingVersion" : 3, "ProcessingString" : "Test", "KeepOutput" : False }, "Task4" : { "TaskName" : "Reco", "InputTask" : "DigiHLT_new", "InputFromOutputModule" : "writeRAWDIGI", "ConfigCacheID" : processorDocs['Reco'], "SplittingAlgorithm" : "FileBased", "SplittingArguments" : {"files_per_job" : 1 }, "TransientOutputModules" : ["writeRECO"] }, "Task5" : { "TaskName" : "ALCAReco", "InputTask" : "DigiHLT_ref", "InputFromOutputModule" : "writeRAWDIGI", "ConfigCacheID" : processorDocs['ALCAReco'], "SplittingAlgorithm" : "LumiBased", "SplittingArguments" : {"lumis_per_job" : 8 }, }, "Task6" : { "TaskName" : "Skims", "InputTask" : "Reco", "InputFromOutputModule" : "writeRECO", "ConfigCacheID" : processorDocs['Skims'], "SplittingAlgorithm" : "LumiBased", "SplittingArguments" : {"lumis_per_job" : 10 }, } } factory = TaskChainWorkloadFactory() # Test a malformed task chain definition arguments['Task4']['TransientOutputModules'].append('writeAOD') self.assertRaises(WMSpecFactoryException, factory.validateSchema, arguments) arguments['Task4']['TransientOutputModules'].remove('writeAOD') try: factory.validateSchema(arguments) self.workload = factory("PullingTheChain", arguments) except Exception, ex: msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex) self.fail(msg) self.workload.setSpecUrl("somespec") self.workload.setOwnerDetails("*****@*****.**", "DMWM") testWMBSHelper = WMBSHelper(self.workload, "GenSim", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) firstTask = self.workload.getTaskByPath("/PullingTheChain/GenSim") self._checkTask(firstTask, arguments['Task1'], arguments) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new"), arguments['Task2'], arguments) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref"), arguments['Task3'], arguments) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco"), arguments['Task4'], arguments) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref/ALCAReco"), arguments['Task5'], arguments) self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco/Skims"), arguments['Task6'], arguments) # Verify the output datasets outputDatasets = self.workload.listOutputDatasets() self.assertEqual(len(outputDatasets), 11, "Number of output datasets doesn't match") self.assertTrue("/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM" in outputDatasets, "/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM not in output datasets") self.assertFalse("/RelValTTBar/ReleaseValidation-reco-v1/RECO" in outputDatasets, "/RelValTTBar/ReleaseValidation-reco-v1/RECO in output datasets") self.assertTrue("/RelValTTBar/ReleaseValidation-AOD-v1/AOD" in outputDatasets, "/RelValTTBar/ReleaseValidation-AOD-v1/AOD not in output datasets") self.assertTrue("/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets, "/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO not in output datasets") for i in range(1, 5): self.assertTrue("/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO" % i in outputDatasets, "/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets" % i) for i in range(1, 6): if i == 2: continue self.assertTrue("/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD" % i in outputDatasets, "/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets" % i) return
class JobUpdaterTest(unittest.TestCase): """ _JobUpdaterTest_ Test class for the JobUpdater """ def setUp(self): """ _setUp_ Set up test environment """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir"], useDefault = False) self.testInit.setupCouch('workqueue_t', 'WorkQueue') self.testInit.setupCouch('workqueue_inbox_t', 'WorkQueue') self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False) EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True, requestMgr = True) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = logging, dbinterface = myThread.dbi) self.listWorkflows = self.daoFactory(classname = "Workflow.ListForSubmitter") self.configFile = EmulatorSetup.setupWMAgentConfig() def tearDown(self): """ _tearDown_ Tear down the databases """ self.testInit.clearDatabase() self.testInit.tearDownCouch() self.testInit.delWorkDir() EmulatorHelper.resetEmulators() EmulatorSetup.deleteConfig(self.configFile) def getConfig(self): """ _getConfig_ Get a test configuration for the JobUpdater tests """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_('Agent') config.Agent.agentName = 'testAgent' config.section_('CoreDatabase') config.CoreDatabase.connectUrl = os.environ['DATABASE'] config.CoreDatabase.socket = os.getenv('DBSOCK') # JobTracker config.component_('JobUpdater') config.JobUpdater.reqMgrUrl = 'https://cmsweb-dev.cern.ch/reqmgr/reqMgr' # JobStateMachine config.section_('JobStateMachine') config.JobStateMachine.couchDBName = 'bogus' # BossAir config.section_('BossAir') config.BossAir.pluginNames = ['MockPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction'] config.BossAir.nCondorProcesses = 1 config.BossAir.section_('MockPlugin') config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobAccountant_t/fwjrs', 'MergedSkimSuccess.pkl') # WorkQueue config.component_('WorkQueueManager') config.WorkQueueManager.couchurl = os.environ['COUCHURL'] config.WorkQueueManager.dbname = 'workqueue_t' config.WorkQueueManager.inboxDatabase = 'workqueue_inbox_t' return config def stuffWMBS(self): """ _stuffWMBS_ Stuff WMBS with workflows """ workflow = Workflow(spec = 'spec.xml', name = 'ReRecoTest_v0Emulator', task = '/ReRecoTest_v0Emulator/Test', priority = 10) workflow.create() inputFileset = Fileset(name = 'TestFileset') inputFileset.create() subscription = Subscription(inputFileset, workflow) subscription.create() def test_BasicTest(self): """ _BasicTest_ Basic sanity check """ self.stuffWMBS() poller = JobUpdaterPoller(self.getConfig()) poller.reqmgr.getAssignment(self) result = self.listWorkflows.execute() self.assertEqual(len(result), 1) self.assertEqual(result[0]['priority'], 10) poller.algorithm() result = self.listWorkflows.execute() self.assertEqual(result[0]['priority'], 100)
class DQMHarvestTests(unittest.TestCase): """ _DQMHarvestTests_ Tests the DQMHarvest spec file """ def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("dqmharvest_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("dqmharvest_t") self.testInit.generateWorkDir() self.workload = None self.jsonTemplate = getTestFile('data/ReqMgr/requests/DMWM/DQMHarvesting.json') return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def injectDQMHarvestConfig(self): """ _injectDQMHarvest_ Create a bogus config cache document for DQMHarvest and inject it into couch. Return the ID of the document. """ newConfig = Document() newConfig["info"] = None newConfig["config"] = None newConfig["md5hash"] = "eb1c38cf50e14cf9fc31278a5c8e234f" newConfig["pset_hash"] = "7c856ad35f9f544839d8525ca10876a7" newConfig["owner"] = {"group": "DATAOPS", "user": "******"} newConfig["pset_tweak_details"] = {"process": {"outputModules_": []}} result = self.configDatabase.commitOne(newConfig) return result[0]["id"] def testDQMHarvest(self): """ Build a DQMHarvest workload """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "DQMConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "harvesting") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.getProcessingString(), "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertFalse(testWorkload.getPrepID(), "PrepId does not match") self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_7_3_1_patch1']) # test workload attributes self.assertEqual(testWorkload.processingString, "GR_R_73_V0A_TEST_RelVal_jetHT2012c") self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1") self.assertEqual(testWorkload.processingVersion, 1) self.assertEqual(sorted(testWorkload.lumiList.keys()), ['139788', '139790', '144011']) self.assertEqual(sorted(testWorkload.lumiList.values()), [[[5, 10], [15, 20], [25, 30]], [[25, 75], [125, 175], [275, 325]], [[50, 100], [110, 125]]]) self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 2) self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect']) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "EndOfRunDQMHarvest") self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest") self.assertEqual(task.taskType(), "Harvesting", "Wrong task type") self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo") self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag") self.assertEqual(sorted(task.inputRunWhitelist()), [138923, 138924, 138934, 138937, 139788, 139789, 139790, 144011, 144083, 144084, 144086]) self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1']) self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive") self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload") return def testDQMHarvestFailed(self): """ Build a DQMHarvest workload without a DQM config doc """ testArguments = DQMHarvestWorkloadFactory.getTestArguments() # Read in the request request = json.load(open(self.jsonTemplate)) testArguments.update(request['createRequest']) testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "dqmharvest_t", "ConfigCacheID": self.injectDQMHarvestConfig() }) testArguments.pop("DQMConfigCacheID", None) factory = DQMHarvestWorkloadFactory() self.assertRaises(WMSpecFactoryException, factory.validateSchema, testArguments) return
class JobTrackerTest(unittest.TestCase): """ TestCase for TestJobTracker module """ _maxMessage = 10 def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"]) self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"], useDefault = False) # self.testInit.setupCouch("jobtracker_t/jobs", "JobDump") # self.testInit.setupCouch("jobtracker_t/fwjrs", "FWJRDump") self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs") #Create sites in resourceControl resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'malpaquet', seName = 'se.malpaquet', ceName = 'malpaquet', plugin = "CondorPlugin") resourceControl.insertThreshold(siteName = 'malpaquet', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "malpaquet", seName = "malpaquet", ceName = "malpaquet", plugin = "CondorPlugin") # Create user newuser = self.daoFactory(classname = "Users.New") newuser.execute(dn = "jchurchill") # We actually need the user name self.user = getpass.getuser() self.testDir = self.testInit.generateWorkDir() def tearDown(self): """ Database deletion """ self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"]) self.testInit.delWorkDir() self.testInit.tearDownCouch() return def getConfig(self): """ _getConfig_ Build a basic JobTracker config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') #config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh') config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # BossAir config.component_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "jobtracker_t" return config def createTestJobs(self, nJobs, cacheDir): """ _createTestJobs_ Create several jobs """ testWorkflow = Workflow(spec = "spec.xml", owner = "Simon", name = "wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name = "TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow, type = "Processing", split_algo = "FileBased") testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() # Create a file testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileA.create() baseName = makeUUID() # Now create a job for i in range(nJobs): testJob = Job(name = '%s-%i' % (baseName, i)) testJob.addFile(testFileA) testJob['location'] = 'malpaquet' testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob.create(testJobGroup) testJob.save() testJobGroup.add(testJob) testJobGroup.commit() # Set test job caches for job in testJobGroup.jobs: job.setCache(cacheDir) return testJobGroup @attr('integration') def testA_CondorTest(self): """ _CondorTest_ Because I don't want this test to be submitter dependent: Create a dummy condor job. Submit a dummy condor job. Track it. Kill it. Exit """ myThread = threading.currentThread() # This has to be run with an empty queue nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) nJobs = 10 jobCE = 'cmsosgce.fnal.gov/jobmanager-condor' # Create directories cacheDir = os.path.join(self.testDir, 'CacheDir') submitDir = os.path.join(self.testDir, 'SubmitDir') if not os.path.isdir(cacheDir): os.makedirs(cacheDir) if not os.path.isdir(submitDir): os.makedirs(submitDir) # Get config config = self.getConfig() # Get jobGroup testJobGroup = self.createTestJobs(nJobs = nJobs, cacheDir = cacheDir) # Propogate jobs changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') result = self.getJobs.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nJobs) jobTracker = JobTrackerPoller(config) jobTracker.setup() # First iteration # There are no jobs in the tracker, # The tracker should register the jobs as missing # This should tell it that they've finished # So the tracker should send them onwards jobTracker.algorithm() result = self.getJobs.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nJobs) result = self.getJobs.execute(state = 'complete', jobType = "Processing") self.assertEqual(len(result), 0) # Second iteration # Reset the jobs # This time submit them to the queue # The jobs should remain in holding changer.propagate(testJobGroup.jobs, 'executing', 'created') result = self.getJobs.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nJobs) # Create a submit script createSubmitScript(submitDir) jobPackage = os.path.join(self.testDir, 'JobPackage.pkl') f = open(jobPackage, 'w') f.write(' ') f.close() sandbox = os.path.join(self.testDir, 'sandbox.box') f = open(sandbox, 'w') f.write(' ') f.close() for job in testJobGroup.jobs: job['plugin'] = 'CondorPlugin' job['userdn'] = 'jchurchill' job['custom'] = {'location': 'malpaquet'} job['cache_dir'] = self.testDir job['sandbox'] = sandbox job['packageDir'] = self.testDir info = {} info['packageDir'] = self.testDir info['index'] = 0 info['sandbox'] = sandbox jobTracker.bossAir.submit(jobs = testJobGroup.jobs, info = info) time.sleep(1) # All jobs should be running nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, nJobs) # Run the algorithm. After this # all jobs should still be running jobTracker.algorithm() # Are jobs in the right state? result = self.getJobs.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nJobs) result = self.getJobs.execute(state = 'Complete', jobType = "Processing") self.assertEqual(len(result), 0) # Are jobs still in the condor_q nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, nJobs) # Then we're done jobTracker.bossAir.kill(jobs = testJobGroup.jobs) # No jobs should be left nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, 0) jobTracker.algorithm() # Are jobs in the right state? result = self.getJobs.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), 0) result = self.getJobs.execute(state = 'Complete', jobType = "Processing") self.assertEqual(len(result), nJobs) # This is optional if you want to look at what # files were actually created during running #if os.path.isdir('testDir'): # shutil.rmtree('testDir') #shutil.copytree('%s' %self.testDir, os.path.join(os.getcwd(), 'testDir')) return @attr('integration') def testB_ReallyLongTest(self): """ _ReallyLongTest_ Run a really long test using the condor plugin """ return # This has to be run with an empty queue nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) myThread = threading.currentThread() # This has to be run with an empty queue nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) nJobs = 500 jobCE = 'cmsosgce.fnal.gov/jobmanager-condor' # Create directories cacheDir = os.path.join(self.testDir, 'CacheDir') submitDir = os.path.join(self.testDir, 'SubmitDir') if not os.path.isdir(cacheDir): os.makedirs(cacheDir) if not os.path.isdir(submitDir): os.makedirs(submitDir) # Get config config = self.getConfig() # Get jobGroup testJobGroup = self.createTestJobs(nJobs = nJobs, cacheDir = cacheDir) # Propogate jobs changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') jobTracker = JobTrackerPoller(config) jobTracker.setup() # Now create some jobs for job in testJobGroup.jobs[:(nJobs/2)]: jdl = createJDL(id = job['id'], directory = submitDir, jobCE = jobCE) jdlFile = os.path.join(submitDir, 'condorJDL_%i.jdl' % (job['id'])) handle = open(jdlFile, 'w') handle.writelines(jdl) handle.close() command = ["condor_submit", jdlFile] pipe = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = False) pipe.communicate() startTime = time.time() cProfile.runctx("jobTracker.algorithm()", globals(), locals(), filename = "testStats.stat") #jobTracker.algorithm() stopTime = time.time() # Are jobs in the right state? result = self.getJobs.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nJobs/2) result = self.getJobs.execute(state = 'Complete', jobType = "Processing") self.assertEqual(len(result), nJobs/2) # Then we're done killList = [x['id'] for x in testJobGroup.jobs] jobTracker.killJobs(jobList = killList) # No jobs should be left nRunning = getCondorRunningJobs(self.user) self.assertEqual(nRunning, 0) print ("Process took %f seconds to process %i classAds" %((stopTime - startTime), nJobs/2)) p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats() def testAlerts(self): """ Tests only alerts triggered from JobTrackerPoller. """ config = self.getConfig() jobTracker = JobTrackerPoller(config) jobTracker.sendAlert(6, msg = "test message")
class PromptRecoTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("promptreco_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("promptreco_t") self.testDir = self.testInit.generateWorkDir() myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName") self.listFilesets = self.daoFactory(classname="Fileset.List") self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow") return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.tearDownCouch() self.testInit.clearDatabase() self.testInit.delWorkDir() return def setupPromptSkimConfigObject(self): """ _setupPromptSkimConfigObject_ Creates a custom config object for testing of the skim functionality """ self.promptSkim = ConfigSection(name="Tier1Skim") self.promptSkim.SkimName = "TestSkim1" self.promptSkim.DataTier = "RECO" self.promptSkim.TwoFileRead = False self.promptSkim.ProcessingVersion = "PromptSkim-v1" self.promptSkim.ConfigURL = "http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi/CMSSW/Configuration/DataOps/python/prescaleskimmer.py?revision=1.1" # def testPromptReco(self): # """ # _testPromptReco_ # # Create a Prompt Reconstruction workflow # and verify it installs into WMBS correctly. # """ def testPromptRecoWithSkims(self): """ _testT1PromptRecoWithSkim_ Create a T1 Prompt Reconstruction workflow with PromptSkims and verify it installs into WMBS correctly. """ testArguments = PromptRecoWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["EnableHarvesting"] = True factory = PromptRecoWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Reco", "SomeBlock", cachepath=self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) recoWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco") recoWorkflow.load() self.assertEqual(len(recoWorkflow.outputMap.keys()), len(testArguments["WriteTiers"]) + 1, "Error: Wrong number of WF outputs in the Reco WF.") goldenOutputMods = {"write_RECO": "RECO", "write_ALCARECO": "ALCARECO", "write_AOD": "AOD", "write_DQM": "DQM"} for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier mergedOutput = recoWorkflow.outputMap[fset][0]["merged_output_fileset"] unmergedOutput = recoWorkflow.outputMap[fset][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_ALCARECO": self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/unmerged-%s" % fset, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = recoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = recoWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") alcaSkimWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim") alcaSkimWorkflow.load() self.assertEqual(len(alcaSkimWorkflow.outputMap.keys()), len(testArguments["AlcaSkims"]) + 1, "Error: Wrong number of WF outputs in the AlcaSkim WF.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: fset = goldenOutputMod + "ALCARECO" mergedOutput = alcaSkimWorkflow.outputMap[fset][0]["merged_output_fileset"] unmergedOutput = alcaSkimWorkflow.outputMap[fset][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") dqmWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged") dqmWorkflow.load() logArchOutput = dqmWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = dqmWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") goldenOutputMods = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"} for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["MergedALCARECO"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["MergedALCARECO"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-Reco-SomeBlock") topLevelFileset.loadData() recoSubscription = Subscription(fileset=topLevelFileset, workflow=recoWorkflow) recoSubscription.loadData() self.assertEqual(recoSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(recoSubscription["split_algo"], "EventAwareLumiBased", "Error: Wrong split algorithm. %s" % recoSubscription["split_algo"]) alcaRecoFileset = Fileset(name="/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO") alcaRecoFileset.loadData() alcaSkimSubscription = Subscription(fileset=alcaRecoFileset, workflow=alcaSkimWorkflow) alcaSkimSubscription.loadData() self.assertEqual(alcaSkimSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(alcaSkimSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % alcaSkimSubscription["split_algo"]) mergedDQMFileset = Fileset(name="/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM") mergedDQMFileset.loadData() dqmSubscription = Subscription(fileset=mergedDQMFileset, workflow=dqmWorkflow) dqmSubscription.loadData() self.assertEqual(dqmSubscription["type"], "Harvesting", "Error: Wrong subscription type.") self.assertEqual(dqmSubscription["split_algo"], "Harvest", "Error: Wrong split algo.") unmergedOutputs = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"} for unmergedOutput, tier in unmergedOutputs.items(): fset = unmergedOutput + tier unmergedDataTier = Fileset(name="/TestWorkload/Reco/unmerged-%s" % fset) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset=unmergedDataTier, workflow=dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = [] for alcaProd in testArguments["AlcaSkims"]: unmergedOutputs.append("ALCARECOStream%s" % alcaProd) for unmergedOutput in unmergedOutputs: unmergedAlcaSkim = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % unmergedOutput) unmergedAlcaSkim.loadData() alcaSkimMergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % unmergedOutput) alcaSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset=unmergedAlcaSkim, workflow=alcaSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) goldenOutputMods = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"} for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier unmergedFileset = Fileset(name="/TestWorkload/Reco/unmerged-%s" % fset) unmergedFileset.loadData() cleanupWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") recoLogCollect = Fileset(name="/TestWorkload/Reco/unmerged-logArchive") recoLogCollect.loadData() recoLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/LogCollect") recoLogCollectWorkflow.load() logCollectSub = Subscription(fileset=recoLogCollect, workflow=recoLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") alcaSkimLogCollect = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-logArchive") alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect") alcaSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset=alcaSkimLogCollect, workflow=alcaSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: recoMergeLogCollect = Fileset(name="/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod) recoMergeLogCollect.loadData() recoMergeLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMerge%s/Reco%sMergeLogCollect" % ( goldenOutputMod, goldenOutputMod)) recoMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset=recoMergeLogCollect, workflow=recoMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: alcaSkimLogCollect = Fileset( name="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod) alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/AlcaSkim%sMergeLogCollect" % ( goldenOutputMod, goldenOutputMod)) alcaSkimLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset=alcaSkimLogCollect, workflow=alcaSkimLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") dqmHarvestLogCollect = Fileset( name="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive") dqmHarvestLogCollect.loadData() dqmHarvestLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect") dqmHarvestLogCollectWorkflow.load() logCollectSub = Subscription(fileset=dqmHarvestLogCollect, workflow=dqmHarvestLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") return def testMemCoresSettings(self): """ _testMemCoresSettings_ Make sure the multicore and memory setings are properly propagated to all tasks and steps. """ testArguments = PromptRecoWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = "promptreco_t" testArguments["EnableHarvesting"] = True factory = PromptRecoWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) # test default values taskPaths = ['/TestWorkload/Reco', '/TestWorkload/Reco/AlcaSkim'] for task in taskPaths: taskObj = testWorkload.getTaskByPath(task) for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], 2300.0) # now test case where args are provided testArguments["Multicore"] = 6 testArguments["Memory"] = 4600.0 testArguments["EventStreams"] = 3 testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) for task in taskPaths: taskObj = testWorkload.getTaskByPath(task) for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) if task == '/TestWorkload/Reco' and step == 'cmsRun1': self.assertEqual(stepHelper.getNumberOfCores(), testArguments["Multicore"]) self.assertEqual(stepHelper.getNumberOfStreams(), testArguments["EventStreams"]) elif step in ('stageOut1', 'logArch1'): self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) else: self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should be single-core" % task) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"]) return def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = ['/TestWorkload/Reco', '/TestWorkload/Reco/AlcaSkim', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics', '/TestWorkload/Reco/RecoMergewrite_AOD', '/TestWorkload/Reco/RecoMergewrite_DQM', '/TestWorkload/Reco/RecoMergewrite_RECO'] expWfTasks = ['/TestWorkload/Reco', '/TestWorkload/Reco/AlcaSkim', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamHcalCalHOCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamMuAlGlobalCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamTkAlCosmics0T', '/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/AlcaSkimALCARECOStreamHcalCalHOCosmicsMergeLogCollect', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/AlcaSkimALCARECOStreamMuAlGlobalCosmicsMergeLogCollect', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/AlcaSkimALCARECOStreamTkAlCosmics0TMergeLogCollect', '/TestWorkload/Reco/LogCollect', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_ALCARECO', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_AOD', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_DQM', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_RECO', '/TestWorkload/Reco/RecoMergewrite_AOD', '/TestWorkload/Reco/RecoMergewrite_AOD/Recowrite_AODMergeLogCollect', '/TestWorkload/Reco/RecoMergewrite_DQM', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect', '/TestWorkload/Reco/RecoMergewrite_DQM/Recowrite_DQMMergeLogCollect', '/TestWorkload/Reco/RecoMergewrite_RECO', '/TestWorkload/Reco/RecoMergewrite_RECO/Recowrite_RECOMergeLogCollect'] expFsets = ['TestWorkload-Reco-/MinimumBias/ComissioningHI-v1/RAW', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-MergedALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO', '/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-MergedALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-MergedALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-logArchive', '/TestWorkload/Reco/RecoMergewrite_AOD/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_AOD/merged-MergedAOD', '/TestWorkload/Reco/unmerged-write_AODAOD', '/TestWorkload/Reco/unmerged-write_DQMDQM', '/TestWorkload/Reco/RecoMergewrite_DQM/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive', '/TestWorkload/Reco/RecoMergewrite_RECO/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_RECO/merged-MergedRECO', '/TestWorkload/Reco/unmerged-logArchive', '/TestWorkload/Reco/unmerged-write_RECORECO'] subMaps = [(4, '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/AlcaSkimALCARECOStreamHcalCalHOCosmicsMergeLogCollect', 'MinFileBased', 'LogCollect'), (10, '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/AlcaSkimALCARECOStreamMuAlGlobalCosmicsMergeLogCollect', 'MinFileBased', 'LogCollect'), (7, '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/AlcaSkimALCARECOStreamTkAlCosmics0TMergeLogCollect', 'MinFileBased', 'LogCollect'), (5, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamHcalCalHOCosmics', 'SiblingProcessingBased', 'Cleanup'), (3, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics', 'ParentlessMergeBySize', 'Merge'), (11, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamMuAlGlobalCosmics', 'SiblingProcessingBased', 'Cleanup'), (9, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics', 'ParentlessMergeBySize', 'Merge'), (8, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamTkAlCosmics0T', 'SiblingProcessingBased', 'Cleanup'), (6, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T', 'ParentlessMergeBySize', 'Merge'), (12, '/TestWorkload/Reco/AlcaSkim/unmerged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect', 'MinFileBased', 'LogCollect'), (15, '/TestWorkload/Reco/RecoMergewrite_AOD/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_AOD/Recowrite_AODMergeLogCollect', 'MinFileBased', 'LogCollect'), (20, '/TestWorkload/Reco/RecoMergewrite_DQM/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_DQM/Recowrite_DQMMergeLogCollect', 'MinFileBased', 'LogCollect'), (18, '/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged', 'Harvest', 'Harvesting'), (19, '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'), (23, '/TestWorkload/Reco/RecoMergewrite_RECO/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_RECO/Recowrite_RECOMergeLogCollect', 'MinFileBased', 'LogCollect'), (25, '/TestWorkload/Reco/unmerged-logArchive', '/TestWorkload/Reco/LogCollect', 'MinFileBased', 'LogCollect'), (2, '/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO', '/TestWorkload/Reco/AlcaSkim', 'ParentlessMergeBySize', 'Processing'), (13, '/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_ALCARECO', 'SiblingProcessingBased', 'Cleanup'), (16, '/TestWorkload/Reco/unmerged-write_AODAOD', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_AOD', 'SiblingProcessingBased', 'Cleanup'), (14, '/TestWorkload/Reco/unmerged-write_AODAOD', '/TestWorkload/Reco/RecoMergewrite_AOD', 'ParentlessMergeBySize', 'Merge'), (21, '/TestWorkload/Reco/unmerged-write_DQMDQM', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_DQM', 'SiblingProcessingBased', 'Cleanup'), (17, '/TestWorkload/Reco/unmerged-write_DQMDQM', '/TestWorkload/Reco/RecoMergewrite_DQM', 'ParentlessMergeBySize', 'Merge'), (24, '/TestWorkload/Reco/unmerged-write_RECORECO', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_RECO', 'SiblingProcessingBased', 'Cleanup'), (22, '/TestWorkload/Reco/unmerged-write_RECORECO', '/TestWorkload/Reco/RecoMergewrite_RECO', 'ParentlessMergeBySize', 'Merge'), (1, 'TestWorkload-Reco-/MinimumBias/ComissioningHI-v1/RAW', '/TestWorkload/Reco', 'EventAwareLumiBased', 'Processing')] testArguments = PromptRecoWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = "promptreco_t" testArguments["EnableHarvesting"] = True factory = PromptRecoWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "Reco", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
class TestChangeState(unittest.TestCase): def setUp(self): """ _setUp_ """ self.transitions = Transitions() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("changestate_t/jobs", "JobDump") self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump") self.testInit.setupCouch("job_summary", "WMStats") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) couchurl = os.getenv("COUCHURL") self.couchServer = CouchServer(dburl = couchurl) self.config = self.testInit.getConfiguration() self.taskName = "/TestWorkflow/Task" return def tearDown(self): """ _tearDown_ Cleanup the databases. """ self.testInit.clearDatabase() self.testInit.tearDownCouch() return def testCheck(self): """ This is the test class for function Check from module ChangeState """ change = ChangeState(self.config, "changestate_t") # Run through all good state transitions and assert that they work for state in self.transitions.keys(): for dest in self.transitions[state]: change.check(dest, state) dummystates = ['dummy1', 'dummy2', 'dummy3', 'dummy4'] # Then run through some bad state transistions and assertRaises(AssertionError) for state in self.transitions.keys(): for dest in dummystates: self.assertRaises(AssertionError, change.check, dest, state) return def testRecordInCouch(self): """ _testRecordInCouch_ Verify that jobs, state transitions and fwjrs are recorded correctly. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048, locations = set(["somese.cern.ch"])) testFileB = File(lfn = "SomeLFNB", events = 1025, size = 2049, locations = set(["somese.cern.ch"])) testFileA.create() testFileB.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 2, \ "Error: Splitting should have created two jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" change.propagate([testJobA, testJobB], "new", "none") change.propagate([testJobA, testJobB], "created", "new") change.propagate([testJobA, testJobB], "executing", "created") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) for transition in testJobADoc["states"].itervalues(): self.assertTrue(type(transition["timestamp"]) in (types.IntType, types.LongType)) assert testJobADoc["jobid"] == testJobA["id"], \ "Error: ID parameter is incorrect." assert testJobADoc["name"] == testJobA["name"], \ "Error: Name parameter is incorrect." assert testJobADoc["jobgroup"] == testJobA["jobgroup"], \ "Error: Jobgroup parameter is incorrect." assert testJobADoc["workflow"] == testJobA["workflow"], \ "Error: Workflow parameter is incorrect." assert testJobADoc["task"] == testJobA["task"], \ "Error: Task parameter is incorrect." assert testJobADoc["owner"] == testJobA["owner"], \ "Error: Owner parameter is incorrect." assert testJobADoc["mask"]["FirstEvent"] == testJobA["mask"]["FirstEvent"], \ "Error: First event in mask is incorrect." assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastEvent"], \ "Error: Last event in mask is incorrect." assert testJobADoc["mask"]["FirstLumi"] == testJobA["mask"]["FirstLumi"], \ "Error: First lumi in mask is incorrect." assert testJobADoc["mask"]["LastLumi"] == testJobA["mask"]["LastLumi"], \ "Error: First lumi in mask is incorrect." assert testJobADoc["mask"]["FirstRun"] == testJobA["mask"]["FirstRun"], \ "Error: First run in mask is incorrect." assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastRun"], \ "Error: First event in mask is incorrect." assert len(testJobADoc["inputfiles"]) == 1, \ "Error: Input files parameter is incorrect." testJobBDoc = change.jobsdatabase.document(testJobB["couch_record"]) assert testJobBDoc["jobid"] == testJobB["id"], \ "Error: ID parameter is incorrect." assert testJobBDoc["name"] == testJobB["name"], \ "Error: Name parameter is incorrect." assert testJobBDoc["jobgroup"] == testJobB["jobgroup"], \ "Error: Jobgroup parameter is incorrect." assert testJobBDoc["mask"]["FirstEvent"] == testJobB["mask"]["FirstEvent"], \ "Error: First event in mask is incorrect." assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastEvent"], \ "Error: Last event in mask is incorrect." assert testJobBDoc["mask"]["FirstLumi"] == testJobB["mask"]["FirstLumi"], \ "Error: First lumi in mask is incorrect." assert testJobBDoc["mask"]["LastLumi"] == testJobB["mask"]["LastLumi"], \ "Error: First lumi in mask is incorrect." assert testJobBDoc["mask"]["FirstRun"] == testJobB["mask"]["FirstRun"], \ "Error: First run in mask is incorrect." assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastRun"], \ "Error: First event in mask is incorrect." assert len(testJobBDoc["inputfiles"]) == 1, \ "Error: Input files parameter is incorrect." changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/jobs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 3, "Error: Wrong number of documents.") couchJobDoc = changeStateDB.document("1") assert couchJobDoc["name"] == testJobA["name"], \ "Error: Name is wrong" assert len(couchJobDoc["inputfiles"]) == 1, \ "Error: Wrong number of input files." result = changeStateDB.loadView("JobDump", "jobsByWorkflowName") self.assertEqual(len(result["rows"]), 2, "Error: Wrong number of rows.") for row in result["rows"]: couchJobDoc = changeStateDB.document(row["value"]["id"]) self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"], "Error: Rev is wrong.") return def testUpdateFailedDoc(self): """ _testUpdateFailedDoc_ Verify that the update function will work correctly and not throw a 500 error if the doc didn't make it into the database for some reason. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048, locations = set(["somese.cern.ch"])) testFileA.create() testFileset.addFile(testFileA) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobA["couch_record"] = str(testJobA["id"]) change.propagate([testJobA], "new", "none") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) self.assertTrue("states" in testJobADoc) self.assertTrue("1" in testJobADoc["states"]) return def testPersist(self): """ _testPersist_ This is the test class for function Propagate from module ChangeState """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() for i in range(4): newFile = File(lfn = "File%s" % i, locations = set(["somese.cern.ch"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 4, \ "Error: Splitting should have created four jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobC = jobGroup.jobs[2] testJobC["user"] = "******" testJobC["group"] = "DMWM" testJobC["taskType"] = "Processing" testJobD = jobGroup.jobs[3] testJobD["user"] = "******" testJobD["group"] = "DMWM" testJobD["taskType"] = "Processing" change.persist([testJobA, testJobB], "created", "new") change.persist([testJobC, testJobD], "new", "none") stateDAO = self.daoFactory(classname = "Jobs.GetState") jobAState = stateDAO.execute(id = testJobA["id"]) jobBState = stateDAO.execute(id = testJobB["id"]) jobCState = stateDAO.execute(id = testJobC["id"]) jobDState = stateDAO.execute(id = testJobD["id"]) assert jobAState == "created" and jobBState =="created" and \ jobCState == "new" and jobDState == "new", \ "Error: Jobs didn't change state correctly." return def testRetryCount(self): """ _testRetryCount_ Verify that the retry count is incremented when we move out of the submitcooloff or jobcooloff state. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() for i in range(4): newFile = File(lfn = "File%s" % i, locations = set(["somese.cern.ch"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 4, \ "Error: Splitting should have created four jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobC = jobGroup.jobs[2] testJobC["user"] = "******" testJobC["group"] = "DMWM" testJobC["taskType"] = "Processing" testJobD = jobGroup.jobs[3] testJobD["user"] = "******" testJobD["group"] = "DMWM" testJobD["taskType"] = "Processing" change.persist([testJobA], "created", "submitcooloff") change.persist([testJobB], "created", "jobcooloff") change.persist([testJobC, testJobD], "new", "none") testJobA.load() testJobB.load() testJobC.load() testJobD.load() assert testJobA["retry_count"] == 1, \ "Error: Retry count is wrong." assert testJobB["retry_count"] == 1, \ "Error: Retry count is wrong." assert testJobC["retry_count"] == 0, \ "Error: Retry count is wrong." assert testJobD["retry_count"] == 0, \ "Error: Retry count is wrong." return def testJobSerialization(self): """ _testJobSerialization_ Verify that serialization of a job works when adding a FWJR. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'executing', 'created') changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") result = changeStateDB.loadView("FWJRDump", "fwjrsByWorkflowName") self.assertEqual(len(result["rows"]), 1, "Error: Wrong number of rows.") for row in result["rows"]: couchJobDoc = changeStateDB.document(row["value"]["id"]) self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"], "Error: Rev is wrong.") for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/FWJRDump": fwjrDoc = changeStateDB.document(resultRow["id"]) break assert fwjrDoc["retrycount"] == 0, \ "Error: Retry count is wrong." assert len(fwjrDoc["fwjr"]["steps"].keys()) == 2, \ "Error: Wrong number of steps in FWJR." assert "cmsRun1" in fwjrDoc["fwjr"]["steps"].keys(), \ "Error: cmsRun1 step is missing from FWJR." assert "stageOut1" in fwjrDoc["fwjr"]["steps"].keys(), \ "Error: stageOut1 step is missing from FWJR." return def testDuplicateJobReports(self): """ _testDuplicateJobReports_ Verify that everything works correctly if a job report is added to the database more than once. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'executing', 'created') change.propagate([testJobA], 'executing', 'created') changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/FWJRDump": fwjrDoc = changeStateDB.document(resultRow["id"]) break return def testJobKilling(self): """ _testJobKilling_ Test that we can successfully set jobs to the killed state """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() for i in range(4): newFile = File(lfn = "File%s" % i, locations = set(["somese.cern.ch"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 4, \ "Error: Splitting should have created four jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobC = jobGroup.jobs[2] testJobC["user"] = "******" testJobC["group"] = "DMWM" testJobC["taskType"] = "Processing" testJobD = jobGroup.jobs[3] testJobD["user"] = "******" testJobD["group"] = "DMWM" testJobD["taskType"] = "Processing" change.persist([testJobA], "created", "new") change.persist([testJobB], "jobfailed", "executing") change.persist([testJobC, testJobD], "executing", "created") change.persist([testJobA], "killed", "created") change.persist([testJobB], "killed", "jobfailed") change.persist([testJobC, testJobD], "killed", "executing") for job in [testJobA, testJobB, testJobC, testJobD]: job.load() self.assertEqual(job['retry_count'], 99999) self.assertEqual(job['state'], 'killed') return def testFWJRInputFileTruncation(self): """ _testFWJRInputFileTruncation_ Test and see whether the ChangeState code can be used to automatically truncate the number of input files in a FWJR Code stolen from the serialization test """ self.config.JobStateMachine.maxFWJRInputFiles = 0 change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] self.assertEqual(len(jobGroup.jobs), 1, "Error: Splitting should have created one job.") testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'executing', 'created') changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") result = changeStateDB.loadView("FWJRDump", "fwjrsByWorkflowName") self.assertEqual(len(result["rows"]), 1, "Error: Wrong number of rows.") for row in result["rows"]: couchJobDoc = changeStateDB.document(row["value"]["id"]) self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"], "Error: Rev is wrong.") for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/FWJRDump": fwjrDoc = changeStateDB.document(resultRow["id"]) break self.assertEqual(fwjrDoc["fwjr"]["steps"]['cmsRun1']['input']['source'], []) return def testJobSummary(self): """ _testJobSummary_ verify that job summary for jobs with fwjr are correctly created and that status is updated when updatesummary flag is enabled """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Analysis" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) change.propagate([testJobA], 'executing', 'created') testJobA["fwjr"] = myReport change.propagate([testJobA], 'jobfailed', 'executing') changeStateDB = self.couchServer.connectDatabase(dbname = self.config.JobStateMachine.jobSummaryDBName) allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") fwjrDoc = {'state': None} for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/WMStats": fwjrDoc = changeStateDB.document(resultRow["id"]) break self.assertEqual(fwjrDoc['state'], 'jobfailed', "Error: summary doesn't have the expected job state") del testJobA["fwjr"] change.propagate([testJobA], 'jobcooloff', 'jobfailed', updatesummary = True) return def testIndexConflict(self): """ _testIndexConflict_ Verify that in case of conflict in the job index we discard the old document and replace with a new one """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "CompOps" testJobA["taskType"] = "Processing" myReport = Report() reportPath = os.path.join(getTestBase(), "WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'created', 'new') jobdatabase = self.couchServer.connectDatabase('changestate_t/jobs', False) fwjrdatabase = self.couchServer.connectDatabase('changestate_t/fwjrs', False) jobDoc = jobdatabase.document("1") fwjrDoc = fwjrdatabase.document("1-0") self.assertEqual(jobDoc["workflow"], "wf001", "Wrong workflow in couch job document") self.assertEqual(fwjrDoc["fwjr"]["task"], self.taskName, "Wrong task in fwjr couch document") testJobA.delete() myThread = threading.currentThread() myThread.dbi.processData("ALTER TABLE wmbs_job AUTO_INCREMENT = 1") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf002", task = "/TestWorkflow/Test2") testWorkflow.create() testFileset = Fileset(name = "TestFilesetB") testFileset.create() testFile = File(lfn = "SomeLFNB", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] testJobB = jobGroup.jobs[0] testJobB["user"] = "******" testJobB["group"] = "CompOps" testJobB["taskType"] = "Processing" testJobB["fwjr"] = myReport change.propagate([testJobB], 'created', 'new') jobDoc = jobdatabase.document("1") fwjrDoc = fwjrdatabase.document("1-0") self.assertEqual(jobDoc["workflow"], "wf002", "Job document was not overwritten") self.assertEqual(fwjrDoc["fwjr"]["task"], "/TestWorkflow/Test2", "FWJR document was not overwritten") return def testUpdateLocation(self): """ _testUpdateLocation_ Check that we can update the location of a job through the state machine. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") locationAction.execute("site2", seName = "somese2.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = self.taskName) testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048, locations = set(["somese.cern.ch", "somese2.cern.ch"])) testFileB = File(lfn = "SomeLFNB", events = 1025, size = 2049, locations = set(["somese.cern.ch", "somese2.cern.ch"])) testFileA.create() testFileB.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 2, \ "Error: Splitting should have created two jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobA["site_cms_name"] = "site1" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" testJobB["site_cms_name"] = "site2" change.propagate([testJobA, testJobB], "new", "none") change.propagate([testJobA, testJobB], "created", "new") change.propagate([testJobA, testJobB], "executing", "created") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) maxKey = max(testJobADoc["states"].keys()) transition = testJobADoc["states"][maxKey] self.assertEqual(transition["location"], "site1") testJobBDoc = change.jobsdatabase.document(testJobB["couch_record"]) maxKey = max(testJobBDoc["states"].keys()) transition = testJobBDoc["states"][maxKey] self.assertEqual(transition["location"], "site2") jobs = [{'jobid' : 1, 'location' : 'site2'}] change.recordLocationChange(jobs) testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) maxKey = max(testJobADoc["states"].keys()) transition = testJobADoc["states"][maxKey] self.assertEqual(transition["location"], "site2") listJobsDAO = self.daoFactory(classname = "Jobs.GetLocation") jobid = [{'jobid' : 1}, {'jobid' : 2}] jobsLocation = listJobsDAO.execute(jobid) for job in jobsLocation: self.assertEqual(job['site_name'], 'site2') return