def testMultiTaskProcessingWorkload(self): """Multi Task Processing Workflow""" datasets = [] tasks, count = 0, 0 for task in MultiTaskProcessingWorkload.taskIterator(): tasks += 1 inputDataset = task.inputDataset() datasets.append("/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier)) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in MultiTaskProcessingWorkload.taskIterator(): units, _ = Block(**self.splitArgs)(MultiTaskProcessingWorkload, task) self.assertEqual(58, len(units)) blocks = [] # fill with blocks as we get work units for them for unit in units: self.assertTrue(1 <= unit['Jobs']) self.assertEqual(MultiTaskProcessingWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual( len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(datasets[0]))) count += 1 self.assertEqual(tasks, count)
def testMultiTaskProcessingWorkload(self): """Multi Task Processing Workflow""" datasets = [] tasks, count = 0, 0 for task in MultiTaskProcessingWorkload.taskIterator(): tasks += 1 inputDataset = task.inputDataset() datasets.append("/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier)) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} for task in MultiTaskProcessingWorkload.taskIterator(): units, _ = Block(**self.splitArgs)(MultiTaskProcessingWorkload, task) self.assertEqual(58, len(units)) blocks = [] # fill with blocks as we get work units for them for unit in units: self.assertTrue(1 <= unit['Jobs']) self.assertEqual(MultiTaskProcessingWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(datasets[0]))) count += 1 self.assertEqual(tasks, count)
def testMultiTaskProcessingWorkload(self): """Multi Task Processing Workflow""" splitArgs = dict(SliceType='NumberOfFiles', SliceSize=5) datasets = [] tasks, count = 0, 0 for task in MultiTaskProcessingWorkload.taskIterator(): tasks += 1 inputDataset = task.getInputDatasetPath() datasets.append(inputDataset) for task in MultiTaskProcessingWorkload.taskIterator(): units, _, _ = Dataset(**splitArgs)(MultiTaskProcessingWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(22, unit['Jobs']) self.assertEqual(MultiTaskProcessingWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [datasets[count]]) count += 1 self.assertEqual(tasks, count)
def testMultiTaskProcessingWorkload(self): """Multi Task Processing Workflow""" datasets = [] tasks, count = 0, 0 for task in MultiTaskProcessingWorkload.taskIterator(): tasks += 1 inputDataset = task.inputDataset() datasets.append("/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier)) for task in MultiTaskProcessingWorkload.taskIterator(): units, _ = Dataset(**self.splitArgs)(MultiTaskProcessingWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(22, unit['Jobs']) self.assertEqual(MultiTaskProcessingWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [datasets[count]]) count += 1 self.assertEqual(tasks, count)
def testPileupData(self): """ _testPileupData_ Check that every workqueue element split contains the pile up data if it is present in the workload. """ for task in MultiTaskProcessingWorkload.taskIterator(): units, _ = Block(**self.splitArgs)(MultiTaskProcessingWorkload, task) self.assertEqual(58, len(units)) for unit in units: pileupData = unit["PileupData"] self.assertEqual(len(pileupData), 1) self.assertItemsEqual(pileupData.values()[0], ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']) return
def testPileupData(self): """ _testPileupData_ Check that every workqueue element split contains the pile up data if it is present in the workload. """ for task in MultiTaskProcessingWorkload.taskIterator(): units, _ = Block(**self.splitArgs)(MultiTaskProcessingWorkload, task) self.assertEqual(Globals.GlobalParams.numOfBlocksPerDataset(), len(units)) for unit in units: pileupData = unit["PileupData"] self.assertEqual(len(pileupData), 1) self.assertEqual(pileupData.values()[0], ["T2_XX_SiteC"]) return
def testPileupData(self): """ _testPileupData_ Check that every workqueue element split contains the pile up data if it is present in the workload. """ for task in MultiTaskProcessingWorkload.taskIterator(): units, _, _ = Block(**self.splitArgs)(MultiTaskProcessingWorkload, task) self.assertEqual(58, len(units)) for unit in units: pileupData = unit["PileupData"] self.assertEqual(len(pileupData), 1) self.assertItemsEqual(pileupData.values()[0], ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']) return