def test_deprecated(caplog): with caplog.at_level(logging.WARNING): CollectorComposite() assert len(caplog.records) == 1 assert caplog.records[0].levelname == 'WARNING' assert 'alphatwirl.loop.deprecated' in caplog.records[0].name assert 'deprecated. replaced with alphatwirl.datasetloop.CollectorComposite' in caplog.records[ 0].msg
def test_deprecated_option(caplog): with caplog.at_level(logging.WARNING, logger='alphatwirl'): CollectorComposite(progressReporter=None) assert len(caplog.records) == 2 assert caplog.records[0].levelname == 'WARNING' assert 'alphatwirl.loop.deprecated' in caplog.records[0].name assert 'deprecated. replaced with alphatwirl.datasetloop.CollectorComposite' in caplog.records[ 0].msg assert caplog.records[1].levelname == 'WARNING' assert 'alphatwirl.loop.deprecated' in caplog.records[1].name assert 'deprecated. "progressReporter=None" is given.' in caplog.records[ 1].msg
def _configure(self, datasets, reader_collector_pairs): dataset_readers = DatasetReaderComposite() reader_top = ReaderComposite() collector_top = CollectorComposite() for r, c in reader_collector_pairs: reader_top.add(r) collector_top.add(c) eventLoopRunner = MPEventLoopRunner(self.parallel.communicationChannel) eventBuilderConfigMaker = EventBuilderConfigMaker( self.nevents_per_block, treename_of_files_map=self._treename_of_files(datasets), ) datasetIntoEventBuildersSplitter = DatasetIntoEventBuildersSplitter( EventBuilder=EventBuilder, eventBuilderConfigMaker=eventBuilderConfigMaker, maxEvents=self.max_blocks_per_dataset, maxEventsPerRun=self.max_blocks_per_process, maxFiles=self.max_files_per_dataset, maxFilesPerRun=self.max_files_per_process, ) eventReader = EventDatasetReader( eventLoopRunner=eventLoopRunner, reader=reader_top, collector=collector_top, split_into_build_events=datasetIntoEventBuildersSplitter) dataset_readers.add(eventReader) if self.parallel_mode not in ('multiprocessing', ): loop = ResumableDatasetLoop(datasets=datasets, reader=dataset_readers, workingarea=self.parallel.workingarea) else: loop = DatasetLoop(datasets=datasets, reader=dataset_readers) return loop
def test_collect(): """ 1:composite |- 2:composite | |- 3:leaf | |- 4:leaf | |- 5:leaf """ ## build collector collector1 = CollectorComposite() collector2 = CollectorComposite() collector3 = MockCollector('result3') collector4 = MockCollector('result4') collector5 = MockCollector('result5') collector1.add(collector2) collector2.add(collector3) collector2.add(collector4) collector1.add(collector5) ## build reader reader1 = ReaderComposite() reader2 = ReaderComposite() reader3 = MockReader() reader4 = MockReader() reader5 = MockReader() reader1.add(reader2) reader2.add(reader3) reader2.add(reader4) reader1.add(reader5) ## copy readers reader1_copy1 = copy.deepcopy(reader1) reader1_copy2 = copy.deepcopy(reader1) reader1_copy3 = copy.deepcopy(reader1) reader1_copy4 = copy.deepcopy(reader1) ## build data set dataset1 = MockDataset() dataset2 = MockDataset() dataset3 = MockDataset() assert collector3.collected is None assert collector4.collected is None assert collector5.collected is None dataset_readers_list = [ (dataset1, (reader1_copy1, reader1_copy2, reader1_copy3)), (dataset2, ()), (dataset3, (reader1_copy4, )), ] assert [['result3', 'result4'], 'result5'] == collector1.collect(dataset_readers_list) assert [ (dataset1, (reader1_copy1.readers[0].readers[0], reader1_copy2.readers[0].readers[0], reader1_copy3.readers[0].readers[0])), (dataset2, ()), (dataset3, (reader1_copy4.readers[0].readers[0], )), ] == collector3.collected assert [ (dataset1, (reader1_copy1.readers[0].readers[1], reader1_copy2.readers[0].readers[1], reader1_copy3.readers[0].readers[1])), (dataset2, ()), (dataset3, (reader1_copy4.readers[0].readers[1], )), ] == collector4.collected assert [ (dataset1, (reader1_copy1.readers[1], reader1_copy2.readers[1], reader1_copy3.readers[1])), (dataset2, ()), (dataset3, (reader1_copy4.readers[1], )), ] == collector5.collected
def test_repr(): obj = CollectorComposite() repr(obj)
def test_two(self): """ 1:composite |- 3:composite | |- 4:counter | |- 5:counter | |- 7:counter |- 8:counter """ keyComposer4 = KeyValueComposer(('var4', ), (Echo(), )) counts4 = Count() reader4 = Reader(keyComposer4, counts4) collector4 = Collector(MockResultsCombinationMethod()) keyComposer5 = KeyValueComposer(('var5', ), (Echo(), )) counts5 = Count() reader5 = Reader(keyComposer5, counts5) collector5 = Collector(MockResultsCombinationMethod()) keyComposer7 = KeyValueComposer(('var7', ), (Echo(), )) counts7 = Count() reader7 = Reader(keyComposer7, counts7) collector7 = Collector(MockResultsCombinationMethod()) keyComposer8 = KeyValueComposer(('var8', ), (Echo(), )) counts8 = Count() reader8 = Reader(keyComposer8, counts8) collector8 = Collector(MockResultsCombinationMethod()) reader3 = ReaderComposite() reader3.add(reader4) reader3.add(reader5) collector3 = CollectorComposite() collector3.add(collector4) collector3.add(collector5) reader1 = ReaderComposite() reader1.add(reader3) reader1.add(reader7) reader1.add(reader8) collector1 = CollectorComposite() collector1.add(collector3) collector1.add(collector7) collector1.add(collector8) reader1_ds1 = copy.deepcopy(reader1) reader1_ds2 = copy.deepcopy(reader1) reader3_ds1 = reader1_ds1.readers[0] reader4_ds1 = reader3_ds1.readers[0] reader5_ds1 = reader3_ds1.readers[1] reader7_ds1 = reader1_ds1.readers[1] reader8_ds1 = reader1_ds1.readers[2] self.assertIsInstance(reader1_ds1, ReaderComposite) self.assertIsInstance(reader3_ds1, ReaderComposite) self.assertIsInstance(reader4_ds1, Reader) self.assertIsInstance(reader5_ds1, Reader) self.assertIsInstance(reader7_ds1, Reader) self.assertIsInstance(reader8_ds1, Reader) self.assertIsNot(reader1, reader1_ds1) self.assertIsNot(reader3, reader3_ds1) self.assertIsNot(reader4, reader4_ds1) self.assertIsNot(reader5, reader5_ds1) self.assertIsNot(reader7, reader7_ds1) self.assertIsNot(reader8, reader8_ds1) reader3_ds2 = reader1_ds2.readers[0] reader4_ds2 = reader3_ds2.readers[0] reader5_ds2 = reader3_ds2.readers[1] reader7_ds2 = reader1_ds2.readers[1] reader8_ds2 = reader1_ds2.readers[2] self.assertIsInstance(reader1_ds2, ReaderComposite) self.assertIsInstance(reader3_ds2, ReaderComposite) self.assertIsInstance(reader4_ds2, Reader) self.assertIsInstance(reader5_ds2, Reader) self.assertIsInstance(reader7_ds2, Reader) self.assertIsInstance(reader8_ds2, Reader) self.assertIsNot(reader1, reader1_ds2) self.assertIsNot(reader3, reader3_ds2) self.assertIsNot(reader4, reader4_ds2) self.assertIsNot(reader5, reader5_ds2) self.assertIsNot(reader7, reader7_ds2) self.assertIsNot(reader8, reader8_ds2)
def test_collect(): """ 1:composite |- 2:composite | |- 3:leaf | |- 4:leaf | |- 5:leaf """ ## build collector collector1 = CollectorComposite() collector2 = CollectorComposite() collector3 = MockCollector('result3') collector4 = MockCollector('result4') collector5 = MockCollector('result5') collector1.add(collector2) collector2.add(collector3) collector2.add(collector4) collector1.add(collector5) ## build reader reader1 = ReaderComposite() reader2 = ReaderComposite() reader3 = MockReader() reader4 = MockReader() reader5 = MockReader() reader1.add(reader2) reader2.add(reader3) reader2.add(reader4) reader1.add(reader5) ## copy readers reader1_copy1 = copy.deepcopy(reader1) reader1_copy2 = copy.deepcopy(reader1) reader1_copy3 = copy.deepcopy(reader1) reader1_copy4 = copy.deepcopy(reader1) ## build data set dataset1 = MockDataset() dataset2 = MockDataset() dataset3 = MockDataset() assert collector3.collected is None assert collector4.collected is None assert collector5.collected is None dataset_readers_list = [ (dataset1, (reader1_copy1, reader1_copy2, reader1_copy3)), (dataset2, ( )), (dataset3, (reader1_copy4, )), ] assert [ ['result3', 'result4'], 'result5' ] == collector1.collect(dataset_readers_list) assert [ (dataset1, (reader1_copy1.readers[0].readers[0], reader1_copy2.readers[0].readers[0], reader1_copy3.readers[0].readers[0])), (dataset2, ( )), (dataset3, (reader1_copy4.readers[0].readers[0], )), ] == collector3.collected assert [ (dataset1, (reader1_copy1.readers[0].readers[1], reader1_copy2.readers[0].readers[1], reader1_copy3.readers[0].readers[1])), (dataset2, ( )), (dataset3, (reader1_copy4.readers[0].readers[1], )), ] == collector4.collected assert [ (dataset1, (reader1_copy1.readers[1], reader1_copy2.readers[1], reader1_copy3.readers[1])), (dataset2, ( )), (dataset3, (reader1_copy4.readers[1], )), ] == collector5.collected