def test_merge_in_order_one_data():
ndata = 1
map_ = OrderedDict([(1000, None)])
data = [MockData(name='data0')]
merge_in_order(map_, 1000, data[0])
assert [ ] == data[0].merged
def let_reader_read(file_loaders, reader, parallel):
eventLoopRunner = alphatwirl.loop.MPEventLoopRunner(
parallel.communicationChannel)
eventLoopRunner.begin()
njobs = len(file_loaders)
eventLoops = []
for i, file_loader in enumerate(file_loaders):
reader_copy = copy.deepcopy(reader)
eventLoop = alphatwirl.loop.EventLoop(file_loader, reader_copy,
'{} / {}'.format(i, njobs))
eventLoops.append(eventLoop)
runids = eventLoopRunner.run_multiple(eventLoops)
# e.g., [0, 1, 2]
runid_reader_map = collections.OrderedDict([(i, None) for i in runids])
# e.g., OrderedDict([(0, None), (1, None), (2, None)])
runids_towait = runids[:]
while runids_towait:
runid, reader_returned = eventLoopRunner.receive_one()
merge_in_order(runid_reader_map, runid, reader_returned)
runids_towait.remove(runid)
if runid_reader_map:
# assert 1 == len(runid_reader_map)
reader = list(runid_reader_map.values())[0]
return reader.collect()
def test_merge_in_order_no_merge_method(orders):
ndata = 6
map_ = OrderedDict([(i, None) for i in range(ndata)])
data = [MockDataWoMerge(name='data{}'.format(i)) for i in range(ndata)]
for i in orders:
merge_in_order(map_, i, data[i])
assert [ ] == data[0].merged
def test_merge_in_order_six_data(orders):
ndata = 6
map_ = OrderedDict([(i, None) for i in range(ndata)])
data = [MockData(name='data{}'.format(i)) for i in range(ndata)]
for i in orders:
merge_in_order(map_, i, data[i])
assert ['data1', 'data2', 'data3', 'data4', 'data5'] == data[0].merged
def _merge(self, runid, reader):
dataset = self.runid_dataset_map[runid]
runid_reader_map = self.dataset_runid_reader_map[dataset]
merge_in_order(runid_reader_map, runid, reader)