def test_init_with_readers():
reader1 = mock.Mock()
reader2 = mock.Mock()
readers = [reader1, reader2]
obj = ReaderComposite(readers=readers)
assert readers is not obj.readers
assert readers == obj.readers
def _configure(self, datasets, reader_collector_pairs):
dataset_readers = DatasetReaderComposite()
reader_top = ReaderComposite()
collector_top = CollectorComposite()
for r, c in reader_collector_pairs:
reader_top.add(r)
collector_top.add(c)
eventLoopRunner = MPEventLoopRunner(self.parallel.communicationChannel)
eventBuilderConfigMaker = EventBuilderConfigMaker(
self.nevents_per_block,
treename_of_files_map=self._treename_of_files(datasets),
)
datasetIntoEventBuildersSplitter = DatasetIntoEventBuildersSplitter(
EventBuilder=EventBuilder,
eventBuilderConfigMaker=eventBuilderConfigMaker,
maxEvents=self.max_blocks_per_dataset,
maxEventsPerRun=self.max_blocks_per_process,
maxFiles=self.max_files_per_dataset,
maxFilesPerRun=self.max_files_per_process,
)
eventReader = EventDatasetReader(
eventLoopRunner=eventLoopRunner,
reader=reader_top,
collector=collector_top,
split_into_build_events=datasetIntoEventBuildersSplitter)
dataset_readers.add(eventReader)
if self.parallel_mode not in ('multiprocessing', ):
loop = ResumableDatasetLoop(datasets=datasets,
reader=dataset_readers,
workingarea=self.parallel.workingarea)
else:
loop = DatasetLoop(datasets=datasets, reader=dataset_readers)
return loop
def test_collect():
"""
1:composite
|- 2:composite
| |- 3:leaf
| |- 4:leaf
|
|- 5:leaf
"""
## build collector
collector1 = CollectorComposite()
collector2 = CollectorComposite()
collector3 = MockCollector('result3')
collector4 = MockCollector('result4')
collector5 = MockCollector('result5')
collector1.add(collector2)
collector2.add(collector3)
collector2.add(collector4)
collector1.add(collector5)
## build reader
reader1 = ReaderComposite()
reader2 = ReaderComposite()
reader3 = MockReader()
reader4 = MockReader()
reader5 = MockReader()
reader1.add(reader2)
reader2.add(reader3)
reader2.add(reader4)
reader1.add(reader5)
## copy readers
reader1_copy1 = copy.deepcopy(reader1)
reader1_copy2 = copy.deepcopy(reader1)
reader1_copy3 = copy.deepcopy(reader1)
reader1_copy4 = copy.deepcopy(reader1)
## build data set
dataset1 = MockDataset()
dataset2 = MockDataset()
dataset3 = MockDataset()
assert collector3.collected is None
assert collector4.collected is None
assert collector5.collected is None
dataset_readers_list = [
(dataset1, (reader1_copy1, reader1_copy2, reader1_copy3)),
(dataset2, ()),
(dataset3, (reader1_copy4, )),
]
assert [['result3', 'result4'],
'result5'] == collector1.collect(dataset_readers_list)
assert [
(dataset1, (reader1_copy1.readers[0].readers[0],
reader1_copy2.readers[0].readers[0],
reader1_copy3.readers[0].readers[0])),
(dataset2, ()),
(dataset3, (reader1_copy4.readers[0].readers[0], )),
] == collector3.collected
assert [
(dataset1, (reader1_copy1.readers[0].readers[1],
reader1_copy2.readers[0].readers[1],
reader1_copy3.readers[0].readers[1])),
(dataset2, ()),
(dataset3, (reader1_copy4.readers[0].readers[1], )),
] == collector4.collected
assert [
(dataset1, (reader1_copy1.readers[1], reader1_copy2.readers[1],
reader1_copy3.readers[1])),
(dataset2, ()),
(dataset3, (reader1_copy4.readers[1], )),
] == collector5.collected
def test_two(self):
"""
1:composite
|- 3:composite
| |- 4:counter
| |- 5:counter
|
|- 7:counter
|- 8:counter
"""
keyComposer4 = KeyValueComposer(('var4', ), (Echo(), ))
counts4 = Count()
reader4 = Reader(keyComposer4, counts4)
collector4 = Collector(MockResultsCombinationMethod())
keyComposer5 = KeyValueComposer(('var5', ), (Echo(), ))
counts5 = Count()
reader5 = Reader(keyComposer5, counts5)
collector5 = Collector(MockResultsCombinationMethod())
keyComposer7 = KeyValueComposer(('var7', ), (Echo(), ))
counts7 = Count()
reader7 = Reader(keyComposer7, counts7)
collector7 = Collector(MockResultsCombinationMethod())
keyComposer8 = KeyValueComposer(('var8', ), (Echo(), ))
counts8 = Count()
reader8 = Reader(keyComposer8, counts8)
collector8 = Collector(MockResultsCombinationMethod())
reader3 = ReaderComposite()
reader3.add(reader4)
reader3.add(reader5)
collector3 = CollectorComposite()
collector3.add(collector4)
collector3.add(collector5)
reader1 = ReaderComposite()
reader1.add(reader3)
reader1.add(reader7)
reader1.add(reader8)
collector1 = CollectorComposite()
collector1.add(collector3)
collector1.add(collector7)
collector1.add(collector8)
reader1_ds1 = copy.deepcopy(reader1)
reader1_ds2 = copy.deepcopy(reader1)
reader3_ds1 = reader1_ds1.readers[0]
reader4_ds1 = reader3_ds1.readers[0]
reader5_ds1 = reader3_ds1.readers[1]
reader7_ds1 = reader1_ds1.readers[1]
reader8_ds1 = reader1_ds1.readers[2]
self.assertIsInstance(reader1_ds1, ReaderComposite)
self.assertIsInstance(reader3_ds1, ReaderComposite)
self.assertIsInstance(reader4_ds1, Reader)
self.assertIsInstance(reader5_ds1, Reader)
self.assertIsInstance(reader7_ds1, Reader)
self.assertIsInstance(reader8_ds1, Reader)
self.assertIsNot(reader1, reader1_ds1)
self.assertIsNot(reader3, reader3_ds1)
self.assertIsNot(reader4, reader4_ds1)
self.assertIsNot(reader5, reader5_ds1)
self.assertIsNot(reader7, reader7_ds1)
self.assertIsNot(reader8, reader8_ds1)
reader3_ds2 = reader1_ds2.readers[0]
reader4_ds2 = reader3_ds2.readers[0]
reader5_ds2 = reader3_ds2.readers[1]
reader7_ds2 = reader1_ds2.readers[1]
reader8_ds2 = reader1_ds2.readers[2]
self.assertIsInstance(reader1_ds2, ReaderComposite)
self.assertIsInstance(reader3_ds2, ReaderComposite)
self.assertIsInstance(reader4_ds2, Reader)
self.assertIsInstance(reader5_ds2, Reader)
self.assertIsInstance(reader7_ds2, Reader)
self.assertIsInstance(reader8_ds2, Reader)
self.assertIsNot(reader1, reader1_ds2)
self.assertIsNot(reader3, reader3_ds2)
self.assertIsNot(reader4, reader4_ds2)
self.assertIsNot(reader5, reader5_ds2)
self.assertIsNot(reader7, reader7_ds2)
self.assertIsNot(reader8, reader8_ds2)
def test_two(self):
"""
1:composite
|- 3:composite
| |- 4:counter
| |- 5:counter
|
|- 7:counter
|- 8:counter
"""
keyComposer4 = KeyValueComposer(('var4', ), (Echo(), ))
counts4 = Count()
reader4 = Reader(keyComposer4, counts4)
collector4 = Collector(MockResultsCombinationMethod())
keyComposer5 = KeyValueComposer(('var5', ), (Echo(), ))
counts5 = Count()
reader5 = Reader(keyComposer5, counts5)
collector5 = Collector(MockResultsCombinationMethod())
keyComposer7 = KeyValueComposer(('var7', ), (Echo(), ))
counts7 = Count()
reader7 = Reader(keyComposer7, counts7)
collector7 = Collector(MockResultsCombinationMethod())
keyComposer8 = KeyValueComposer(('var8', ), (Echo(), ))
counts8 = Count()
reader8 = Reader(keyComposer8, counts8)
collector8 = Collector(MockResultsCombinationMethod())
reader3 = ReaderComposite()
reader3.add(reader4)
reader3.add(reader5)
collector3 = CollectorComposite()
collector3.add(collector4)
collector3.add(collector5)
reader1 = ReaderComposite()
reader1.add(reader3)
reader1.add(reader7)
reader1.add(reader8)
collector1 = CollectorComposite()
collector1.add(collector3)
collector1.add(collector7)
collector1.add(collector8)
reader1_ds1 = copy.deepcopy(reader1)
reader1_ds2 = copy.deepcopy(reader1)
reader3_ds1 = reader1_ds1.readers[0]
reader4_ds1 = reader3_ds1.readers[0]
reader5_ds1 = reader3_ds1.readers[1]
reader7_ds1 = reader1_ds1.readers[1]
reader8_ds1 = reader1_ds1.readers[2]
self.assertIsInstance(reader1_ds1, ReaderComposite)
self.assertIsInstance(reader3_ds1, ReaderComposite)
self.assertIsInstance(reader4_ds1, Reader)
self.assertIsInstance(reader5_ds1, Reader)
self.assertIsInstance(reader7_ds1, Reader)
self.assertIsInstance(reader8_ds1, Reader)
self.assertIsNot(reader1, reader1_ds1)
self.assertIsNot(reader3, reader3_ds1)
self.assertIsNot(reader4, reader4_ds1)
self.assertIsNot(reader5, reader5_ds1)
self.assertIsNot(reader7, reader7_ds1)
self.assertIsNot(reader8, reader8_ds1)
reader3_ds2 = reader1_ds2.readers[0]
reader4_ds2 = reader3_ds2.readers[0]
reader5_ds2 = reader3_ds2.readers[1]
reader7_ds2 = reader1_ds2.readers[1]
reader8_ds2 = reader1_ds2.readers[2]
self.assertIsInstance(reader1_ds2, ReaderComposite)
self.assertIsInstance(reader3_ds2, ReaderComposite)
self.assertIsInstance(reader4_ds2, Reader)
self.assertIsInstance(reader5_ds2, Reader)
self.assertIsInstance(reader7_ds2, Reader)
self.assertIsInstance(reader8_ds2, Reader)
self.assertIsNot(reader1, reader1_ds2)
self.assertIsNot(reader3, reader3_ds2)
self.assertIsNot(reader4, reader4_ds2)
self.assertIsNot(reader5, reader5_ds2)
self.assertIsNot(reader7, reader7_ds2)
self.assertIsNot(reader8, reader8_ds2)
def test_collect():
"""
1:composite
|- 2:composite
| |- 3:leaf
| |- 4:leaf
|
|- 5:leaf
"""
## build collector
collector1 = CollectorComposite()
collector2 = CollectorComposite()
collector3 = MockCollector('result3')
collector4 = MockCollector('result4')
collector5 = MockCollector('result5')
collector1.add(collector2)
collector2.add(collector3)
collector2.add(collector4)
collector1.add(collector5)
## build reader
reader1 = ReaderComposite()
reader2 = ReaderComposite()
reader3 = MockReader()
reader4 = MockReader()
reader5 = MockReader()
reader1.add(reader2)
reader2.add(reader3)
reader2.add(reader4)
reader1.add(reader5)
## copy readers
reader1_copy1 = copy.deepcopy(reader1)
reader1_copy2 = copy.deepcopy(reader1)
reader1_copy3 = copy.deepcopy(reader1)
reader1_copy4 = copy.deepcopy(reader1)
## build data set
dataset1 = MockDataset()
dataset2 = MockDataset()
dataset3 = MockDataset()
assert collector3.collected is None
assert collector4.collected is None
assert collector5.collected is None
dataset_readers_list = [
(dataset1, (reader1_copy1, reader1_copy2, reader1_copy3)),
(dataset2, ( )),
(dataset3, (reader1_copy4, )),
]
assert [
['result3', 'result4'],
'result5'
] == collector1.collect(dataset_readers_list)
assert [
(dataset1, (reader1_copy1.readers[0].readers[0], reader1_copy2.readers[0].readers[0], reader1_copy3.readers[0].readers[0])),
(dataset2, ( )),
(dataset3, (reader1_copy4.readers[0].readers[0], )),
] == collector3.collected
assert [
(dataset1, (reader1_copy1.readers[0].readers[1], reader1_copy2.readers[0].readers[1], reader1_copy3.readers[0].readers[1])),
(dataset2, ( )),
(dataset3, (reader1_copy4.readers[0].readers[1], )),
] == collector4.collected
assert [
(dataset1, (reader1_copy1.readers[1], reader1_copy2.readers[1], reader1_copy3.readers[1])),
(dataset2, ( )),
(dataset3, (reader1_copy4.readers[1], )),
] == collector5.collected
def test_event_nested_composite():
"""
composite1
|- composite2
| |- reader1
| |- reader2
|- reader3
"""
obj1 = ReaderComposite()
obj2 = ReaderComposite()
reader1 = mock.Mock()
reader2 = mock.Mock()
reader3 = mock.Mock()
obj1.add(obj2)
obj2.add(reader1)
obj2.add(reader2)
obj1.add(reader3)
events = mock.Mock()
obj1.begin(events)
assert [mock.call(events)] == reader1.begin.call_args_list
assert [mock.call(events)] == reader2.begin.call_args_list
assert [mock.call(events)] == reader3.begin.call_args_list
event1 = mock.Mock()
obj1.event(event1)
event2 = mock.Mock()
obj1.event(event2)
assert [mock.call(event1),
mock.call(event2)], reader1.events.call_args_list
assert [mock.call(event1),
mock.call(event2)], reader2.events.call_args_list
assert [mock.call(event1),
mock.call(event2)], reader3.events.call_args_list
obj1.end()
assert [mock.call()] == reader1.end.call_args_list
assert [mock.call()] == reader2.end.call_args_list
assert [mock.call()] == reader3.end.call_args_list
def obj():
return ReaderComposite()
def test_event_nested_composite(self):
"""
composite1
|- composite2
| |- reader1
| |- reader2
|- reader3
"""
composite1 = ReaderComposite()
composite2 = ReaderComposite()
reader1 = MockReader()
reader2 = MockReader()
reader3 = MockReader()
composite1.add(composite2)
composite2.add(reader1)
composite2.add(reader2)
composite1.add(reader3)
events = MockEvent()
composite1.begin(events)
self.assertIs(events, reader1._beganWith)
self.assertIs(events, reader2._beganWith)
self.assertIs(events, reader3._beganWith)
event1 = MockEvent()
composite1.event(event1)
event2 = MockEvent()
composite1.event(event2)
self.assertEqual([event1, event2], reader1._events)
self.assertEqual([event1, event2], reader2._events)
self.assertEqual([event1, event2], reader3._events)
composite1.end()
self.assertTrue(reader1._ended)
self.assertTrue(reader2._ended)
self.assertTrue(reader3._ended)
def test_event_two_readers_two_events(self):
"""
composite
|- reader1
|- reader2
"""
composite = ReaderComposite()
reader1 = MockReader()
reader2 = MockReader()
composite.add(reader1)
composite.add(reader2)
events = MockEvent()
composite.begin(events)
self.assertIs(events, reader1._beganWith)
self.assertIs(events, reader2._beganWith)
event1 = MockEvent()
composite.event(event1)
event2 = MockEvent()
composite.event(event2)
self.assertEqual([event1, event2], reader1._events)
self.assertEqual([event1, event2], reader2._events)
composite.end()
self.assertTrue(reader1._ended)
self.assertTrue(reader2._ended)
def test_no_begin_end(self):
"""
composite
|- reader1
|- reader2 (without begin end)
|- reader3
"""
composite = ReaderComposite()
reader1 = MockReader()
reader2 = MockReader_without_begin_end()
reader3 = MockReader()
composite.add(reader1)
composite.add(reader2)
composite.add(reader3)
events = MockEvent()
composite.begin(events)
self.assertIs(events, reader1._beganWith)
event1 = MockEvent()
composite.event(event1)
event2 = MockEvent()
composite.event(event2)
self.assertEqual([event1, event2], reader1._events)
self.assertEqual([event1, event2], reader2._events)
self.assertEqual([event1, event2], reader3._events)
composite.end()
self.assertTrue(reader1._ended)
self.assertTrue(reader3._ended)
def test_return_False(self):
"""
composite
|- reader1 (return None)
|- reader2 (return True)
|- reader3 (return False)
|- reader4
"""
composite = ReaderComposite()
reader1 = MockReader_with_return()
reader2 = MockReader_with_return()
reader3 = MockReader_with_return()
reader4 = MockReader_with_return()
composite.add(reader1)
composite.add(reader2)
composite.add(reader3)
composite.add(reader4)
events = MockEvent()
composite.begin(events)
reader1._ret = None
reader2._ret = True
reader3._ret = False
event1 = MockEvent()
ret = composite.event(event1)
self.assertEqual([
event1,
], reader1._events)
self.assertEqual([
event1,
], reader2._events)
self.assertEqual([
event1,
], reader3._events)
self.assertEqual([], reader4._events)
self.assertIsNone(ret)
composite.end()