コード例 #1
0
    def testDatasetRunWhitelist(self):
        """
        Test NumberOfRuns splitting with run white list
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(
            dqmHarvArgs["CouchDBName"])
        dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151]
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction(
            'DQMHarvestTest', dqmHarvArgs)
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(4, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(unit['Inputs'].keys(), [inputDataset])
                self.assertEqual(217, unit['NumberOfLumis'])
                self.assertEqual(8, unit['NumberOfFiles'])
                self.assertEqual(83444, unit['NumberOfEvents'])
コード例 #2
0
    def testDatasetLumiMask(self):
        """
        Test NumberOfRuns splitting type with lumi mask
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(
            dqmHarvArgs["CouchDBName"])
        dqmHarvArgs["LumiList"] = {
            "181358": [[71, 80], [95, 110]],
            "181151": [[1, 20]]
        }
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction(
            'DQMHarvestTest', dqmHarvArgs)
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(2, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(unit['Inputs'].keys(), [inputDataset])
                self.assertEqual(46, unit['NumberOfLumis'])
                self.assertEqual(4, unit['NumberOfFiles'])
                self.assertEqual(12342, unit['NumberOfEvents'])
コード例 #3
0
    def testDatasetPolicy(self):
        """
        Test ordinary NumberOfRuns splitting
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(
            dqmHarvArgs["CouchDBName"])
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction(
            'DQMHarvestTest', dqmHarvArgs)
        DQMHarvWorkload.setSiteWhitelist(
            ["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(47, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(list(unit['Inputs']), [inputDataset])
                self.assertEqual(4855, unit['NumberOfLumis'])
                self.assertEqual(72, unit['NumberOfFiles'])
                self.assertEqual(743201, unit['NumberOfEvents'])
コード例 #4
0
    def testInvalidSpecs(self):
        """Specs with no work"""
        dqmHarvArgs = getRequestArgs()
        # no dataset
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction('NoInputDatasetTest', dqmHarvArgs)
        DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
        getFirstTask(DQMHarvWorkload).data.input.dataset = None
        for task in DQMHarvWorkload.taskIterator():
            self.assertRaises(WorkQueueWMSpecError, Dataset(), DQMHarvWorkload, task)

        # invalid dataset name
        DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidInputDatasetTest', dqmHarvArgs)
        DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
        getFirstTask(DQMHarvWorkload).data.input.dataset.name = '/MinimumBias/FAKE-Filter-v1/RECO'
        for task in DQMHarvWorkload.taskIterator():
            self.assertRaises(DBSReaderError, Dataset(), DQMHarvWorkload, task)

        # invalid run whitelist
        DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidRunNumberTest', dqmHarvArgs)
        DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
        DQMHarvWorkload.setRunWhitelist([666])  # not in this dataset
        for task in DQMHarvWorkload.taskIterator():
            self.assertRaises(WorkQueueNoWorkError, Dataset(), DQMHarvWorkload, task)
コード例 #5
0
    def testDatasetSingleJob(self):
        """
        Test NumberOfRuns splitting type with very large SliceSize
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(
            dqmHarvArgs["CouchDBName"])
        dqmHarvArgs["DQMHarvestUnit"] = 'multiRun'
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction(
            'DQMHarvestTest', dqmHarvArgs)
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(1, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(unit['Inputs'].keys(), [inputDataset])
                self.assertEqual(4855, unit['NumberOfLumis'])
                self.assertEqual(72, unit['NumberOfFiles'])
                self.assertEqual(743201, unit['NumberOfEvents'])
コード例 #6
0
ファイル: DQMHarvest_t.py プロジェクト: huohuo21/WMCore
    def testDQMHarvest(self):
        """
        Build a DQMHarvest workload
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        # Read in the request
        request = json.load(open(self.jsonTemplate))
        testArguments.update(request['createRequest'])
        testArguments.update({
            "CouchURL": os.environ["COUCHURL"],
            "ConfigCacheUrl": os.environ["COUCHURL"],
            "CouchDBName": "dqmharvest_t",
            "DQMConfigCacheID": self.injectDQMHarvestConfig()
        })
        testArguments.pop("ConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
        self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
        self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_3_1_patch1")
        self.assertEqual(testWorkload.getProcessingString(), "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertFalse(testWorkload.getPrepID(), "PrepId does not match")
        self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_7_3_1_patch1'])

        # test workload attributes
        self.assertEqual(testWorkload.processingString, "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
        self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1")
        self.assertEqual(testWorkload.processingVersion, 1)
        self.assertEqual(sorted(testWorkload.lumiList.keys()), ['139788', '139790', '144011'])
        self.assertEqual(sorted(testWorkload.lumiList.values()),
                         [[[5, 10], [15, 20], [25, 30]], [[25, 75],
                                                          [125, 175], [275, 325]], [[50, 100], [110, 125]]])
        self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 2)
        self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "EndOfRunDQMHarvest")
        self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
        self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
        self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
        self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
        self.assertEqual(sorted(task.inputRunWhitelist()),
                         [138923, 138924, 138934, 138937, 139788, 139789,
                          139790, 144011, 144083, 144084, 144086])

        self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
        self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")

        return
コード例 #7
0
ファイル: DQMHarvest_t.py プロジェクト: dciangot/WMCore
    def testDQMHarvest(self):
        """
        Build a DQMHarvest workload
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        # Read in the request
        request = json.load(open(self.jsonTemplate))
        testArguments.update(request['createRequest'])
        testArguments.update({
            "CouchURL": os.environ["COUCHURL"],
            "ConfigCacheUrl": os.environ["COUCHURL"],
            "CouchDBName": "dqmharvest_t",
            "DQMConfigCacheID": self.injectDQMHarvestConfig(),
            "LumiList": {"251643": [[1, 15], [50,70]], "251721": [[50,100], [110,120]]}
        })
        testArguments.pop("ConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
        self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
        self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016B")
        self.assertEqual(testWorkload.getProcessingString(), "23Sep2016")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016B-23Sep2016-0022")
        self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20'])

        # test workload attributes
        self.assertEqual(testWorkload.processingString, "23Sep2016")
        self.assertEqual(testWorkload.acquisitionEra, "Run2016B")
        self.assertEqual(testWorkload.processingVersion, 1)
        self.assertEqual(sorted(testWorkload.lumiList.keys()), ['251643', '251721'])
        self.assertEqual(sorted(testWorkload.lumiList.values()), [[[1,15], [50,70]], [[50,100], [110,120]]])
        self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 2)
        self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "EndOfRunDQMHarvest")
        self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
        self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
        self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
        self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
        self.assertFalse(task.inputRunWhitelist())

        self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
        self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")

        return
コード例 #8
0
    def testDataDirectiveFromQueue(self):
        """Test data directive from queue"""
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
        DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
        splitArgs = DQMHarvWorkload.startPolicyParameters()

        for task in DQMHarvWorkload.taskIterator():
            self.assertRaises(RuntimeError, Dataset(**splitArgs), DQMHarvWorkload, task, {NOT_EXIST_DATASET: []})
コード例 #9
0
    def testFilesets(self):
        """
        Test workflow tasks, filesets and subscriptions creation
        """
        # expected tasks, filesets, subscriptions, etc
        expOutTasks = []
        expWfTasks = [
            '/TestWorkload/EndOfRunDQMHarvest',
            '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect'
        ]
        expFsets = [
            'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
            '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive'
        ]

        subMaps = [
            (2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
             '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect',
             'MinFileBased', 'LogCollect'),
            (1,
             'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
             '/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')
        ]

        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        testArguments.update(REQUEST)
        testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
        testArguments.pop("ConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "EndOfRunDQMHarvest",
                                    blockName=testArguments['InputDataset'],
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
                              expOutTasks)

        workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
        self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)

        # returns a tuple of id, name, open and last_update
        filesets = self.listFilesets.execute()
        self.assertItemsEqual([item[1] for item in filesets], expFsets)

        subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
                                                     returnTuple=True)
        self.assertItemsEqual(subscriptions, subMaps)
コード例 #10
0
    def testDQMHarvestFailed(self):
        """
        Build a DQMHarvest workload without a DQM config doc
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        testArguments.update(REQUEST)
        testArguments.update({"ConfigCacheID": self.injectDQMHarvestConfig()})
        testArguments.pop("DQMConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        with self.assertRaises(WMSpecFactoryException):
            factory.factoryWorkloadConstruction("TestBadWorkload",
                                                testArguments)
        return
コード例 #11
0
    def testDQMHarvestFailed(self):
        """
        Build a DQMHarvest workload without a DQM config doc
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        testArguments.update(REQUEST)
        testArguments.update({
            "ConfigCacheID": self.injectDQMHarvestConfig()
        })
        testArguments.pop("DQMConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        with self.assertRaises(WMSpecFactoryException):
            factory.factoryWorkloadConstruction("TestBadWorkload", testArguments)
        return
コード例 #12
0
ファイル: DQMHarvest_t.py プロジェクト: rishiloyola/WMCore
    def testDQMHarvest(self):
        """
        Build a DQMHarvest workload
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        testArguments.update(REQUEST)
        testArguments.update({
            "DQMConfigCacheID": self.injectDQMHarvestConfig(),
            "LumiList": {"251643": [[1, 15], [50, 70]], "251721": [[50, 100], [110, 120]]}
        })
        testArguments.pop("ConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
        self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
        self.assertEqual(testWorkload.getAcquisitionEra(), "Run2016F")
        self.assertEqual(testWorkload.getProcessingString(), "23Sep2016")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertEqual(testWorkload.getPrepID(), "TEST-Harvest-ReReco-Run2016F-v1-NoBPTX-23Sep2016-0001")
        self.assertEqual(testWorkload.getCMSSWVersions(), ['CMSSW_8_0_20'])
        self.assertEqual(sorted(testWorkload.getLumiList().keys()), ['251643', '251721'])
        self.assertEqual(sorted(testWorkload.getLumiList().values()), [[[1, 15], [50, 70]], [[50, 100], [110, 120]]])
        self.assertEqual(testWorkload.data.policies.start.policyName, "Dataset")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 2)
        self.assertEqual(sorted(tasks), ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "EndOfRunDQMHarvest")
        self.assertEqual(task.getPathName(), "/TestWorkload/EndOfRunDQMHarvest")
        self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
        self.assertEqual(task.jobSplittingAlgorithm(), "Harvest", "Wrong job splitting algo")
        self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag")
        self.assertFalse(task.inputRunWhitelist())

        self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'upload1'])
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
        self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")

        return
コード例 #13
0
ファイル: DQMHarvest_t.py プロジェクト: lucacopa/WMCore
    def testDQMHarvestFailed(self):
        """
        Build a DQMHarvest workload without a DQM config doc
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        # Read in the request
        request = json.load(open(self.jsonTemplate))
        testArguments.update(request['createRequest'])
        testArguments.update({
            "CouchURL": os.environ["COUCHURL"],
            "ConfigCacheUrl": os.environ["COUCHURL"],
            "CouchDBName": "dqmharvest_t",
            "ConfigCacheID": self.injectDQMHarvestConfig()
        })
        testArguments.pop("DQMConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        self.assertRaises(WMSpecFactoryException, factory.validateSchema,
                          testArguments)
        return
コード例 #14
0
    def testFilesets(self):
        """
        Test workflow tasks, filesets and subscriptions creation
        """
        # expected tasks, filesets, subscriptions, etc
        expOutTasks = []
        expWfTasks = ['/TestWorkload/EndOfRunDQMHarvest',
                      '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect']
        expFsets = ['TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
                    '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive']

        subMaps = [(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
                    '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'),
                   (1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
                    '/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')]

        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        testArguments.update(REQUEST)
        testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
        testArguments.pop("ConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest",
                                    blockName=testArguments['InputDataset'],
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)

        workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
        self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)

        # returns a tuple of id, name, open and last_update
        filesets = self.listFilesets.execute()
        self.assertItemsEqual([item[1] for item in filesets], expFsets)

        subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
        self.assertItemsEqual(subscriptions, subMaps)
コード例 #15
0
ファイル: Dataset_t.py プロジェクト: dciangot/WMCore
    def testDatasetPolicy(self):
        """
        Test ordinary NumberOfRuns splitting
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(47, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(unit['Inputs'].keys(), [inputDataset])
                self.assertEqual(4855, unit['NumberOfLumis'])
                self.assertEqual(72, unit['NumberOfFiles'])
                self.assertEqual(743201, unit['NumberOfEvents'])
コード例 #16
0
ファイル: Dataset_t.py プロジェクト: dciangot/WMCore
    def testDatasetRunWhitelist(self):
        """
        Test NumberOfRuns splitting with run white list
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
        dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151]
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(4, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(unit['Inputs'].keys(), [inputDataset])
                self.assertEqual(217, unit['NumberOfLumis'])
                self.assertEqual(8, unit['NumberOfFiles'])
                self.assertEqual(83444, unit['NumberOfEvents'])
コード例 #17
0
ファイル: Dataset_t.py プロジェクト: dciangot/WMCore
    def testDatasetLumiMask(self):
        """
        Test NumberOfRuns splitting type with lumi mask
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
        dqmHarvArgs["LumiList"] = {"181358": [[71, 80], [95, 110]], "181151": [[1, 20]]}
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(2, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(unit['Inputs'].keys(), [inputDataset])
                self.assertEqual(46, unit['NumberOfLumis'])
                self.assertEqual(4, unit['NumberOfFiles'])
                self.assertEqual(12342, unit['NumberOfEvents'])
コード例 #18
0
ファイル: Dataset_t.py プロジェクト: vkuznet/WMCore
    def testDatasetSingleJob(self):
        """
        Test NumberOfRuns splitting type with very large SliceSize
        """
        dqmHarvArgs = getRequestArgs()
        dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
        dqmHarvArgs["DQMHarvestUnit"] = 'multiRun'
        factory = DQMHarvestWorkloadFactory()
        DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
        DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
        splitArgs = DQMHarvWorkload.startPolicyParameters()
        inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()

        for task in DQMHarvWorkload.taskIterator():
            units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
            self.assertEqual(1, len(units))
            for unit in units:
                self.assertEqual(1, unit['Jobs'])
                self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
                self.assertEqual(task, unit['Task'])
                self.assertEqual(unit['Inputs'].keys(), [inputDataset])
                self.assertEqual(4855, unit['NumberOfLumis'])
                self.assertEqual(72, unit['NumberOfFiles'])
                self.assertEqual(743201, unit['NumberOfEvents'])
コード例 #19
0
ファイル: DQMHarvest_t.py プロジェクト: huohuo21/WMCore
    def testDQMHarvestFailed(self):
        """
        Build a DQMHarvest workload without a DQM config doc
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        # Read in the request
        request = json.load(open(self.jsonTemplate))
        testArguments.update(request['createRequest'])
        testArguments.update({
            "CouchURL": os.environ["COUCHURL"],
            "ConfigCacheUrl": os.environ["COUCHURL"],
            "CouchDBName": "dqmharvest_t",
            "ConfigCacheID": self.injectDQMHarvestConfig()
        })
        testArguments.pop("DQMConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        self.assertRaises(WMSpecFactoryException, factory.validateSchema, testArguments)
        return
コード例 #20
0
def getRequestArgs():
    """ Returns default values defined in the spec workload """
    return DQMHarvestWorkloadFactory.getTestArguments()
コード例 #21
0
ファイル: DQMHarvest_t.py プロジェクト: lucacopa/WMCore
    def testDQMHarvest(self):
        """
        Build a DQMHarvest workload
        """
        testArguments = DQMHarvestWorkloadFactory.getTestArguments()
        # Read in the request
        request = json.load(open(self.jsonTemplate))
        testArguments.update(request['createRequest'])
        testArguments.update({
            "CouchURL": os.environ["COUCHURL"],
            "ConfigCacheUrl": os.environ["COUCHURL"],
            "CouchDBName": "dqmharvest_t",
            "DQMConfigCacheID": self.injectDQMHarvestConfig()
        })
        testArguments.pop("ConfigCacheID", None)

        factory = DQMHarvestWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "harvesting")
        self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
        self.assertEqual(testWorkload.getAcquisitionEra(),
                         "CMSSW_7_3_1_patch1")
        self.assertEqual(testWorkload.getProcessingString(),
                         "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertFalse(testWorkload.getPrepID(), "PrepId does not match")
        self.assertEqual(testWorkload.getCMSSWVersions(),
                         ['CMSSW_7_3_1_patch1'])

        # test workload attributes
        self.assertEqual(testWorkload.processingString,
                         "GR_R_73_V0A_TEST_RelVal_jetHT2012c")
        self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_3_1_patch1")
        self.assertEqual(testWorkload.processingVersion, 1)
        self.assertEqual(sorted(testWorkload.lumiList.keys()),
                         ['139788', '139790', '144011'])
        self.assertEqual(
            sorted(testWorkload.lumiList.values()),
            [[[5, 10], [15, 20], [25, 30]], [[25, 75], [125, 175], [275, 325]],
             [[50, 100], [110, 125]]])
        self.assertEqual(testWorkload.data.policies.start.policyName,
                         "DatasetBlock")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 2)
        self.assertEqual(
            sorted(tasks),
            ['EndOfRunDQMHarvest', 'EndOfRunDQMHarvestLogCollect'])

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "EndOfRunDQMHarvest")
        self.assertEqual(task.getPathName(),
                         "/TestWorkload/EndOfRunDQMHarvest")
        self.assertEqual(task.taskType(), "Harvesting", "Wrong task type")
        self.assertEqual(task.jobSplittingAlgorithm(), "Harvest",
                         "Wrong job splitting algo")
        self.assertFalse(task.inputLocationFlag(), "Wrong input location flag")
        self.assertEqual(sorted(task.inputRunWhitelist()), [
            138923, 138924, 138934, 138937, 139788, 139789, 139790, 144011,
            144083, 144084, 144086
        ])

        self.assertEqual(sorted(task.listAllStepNames()),
                         ['cmsRun1', 'logArch1', 'upload1'])
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertEqual(task.getStep("logArch1").stepType(), "LogArchive")
        self.assertEqual(task.getStep("upload1").stepType(), "DQMUpload")

        #print "ALAN inspect %s" % inspect.getmembers(allTasks)
        #for property, value in vars(allTasks).iteritems():
        #    print property, ": ", value
        return