def test_dispatch_multiprocessing(backend):
""" Check that using pre_dispatch Parallel does indeed dispatch items
lazily.
"""
manager = mp.Manager()
queue = manager.list()
def producer():
for i in range(6):
queue.append('Produced %i' % i)
yield i
Parallel(n_jobs=2, batch_size=1, pre_dispatch=3,
backend=backend)(delayed(consumer)(queue, 'any')
for _ in producer())
queue_contents = list(queue)
assert queue_contents[0] == 'Produced 0'
# Only 3 tasks are pre-dispatched out of 6. The 4th task is dispatched only
# after any of the first 3 jobs have completed.
first_consumption_index = queue_contents[:4].index('Consumed any')
assert first_consumption_index > -1
produced_3_index = queue_contents.index('Produced 3') # 4th task produced
assert produced_3_index > first_consumption_index
assert len(queue) == 12
def check_dispatch_multiprocessing(backend):
""" Check that using pre_dispatch Parallel does indeed dispatch items
lazily.
"""
if mp is None:
raise SkipTest()
manager = mp.Manager()
queue = manager.list()
def producer():
for i in range(6):
queue.append('Produced %i' % i)
yield i
Parallel(n_jobs=2, batch_size=1, pre_dispatch=3,
backend=backend)(delayed(consumer)(queue, 'any')
for _ in producer())
# Only 3 tasks are dispatched out of 6. The 4th task is dispatched only
# after any of the first 3 jobs have completed.
first_four = list(queue)[:4]
# The the first consumption event can sometimes happen before the end of
# the dispatching, hence, pop it before introspecting the "Produced" events
first_four.remove('Consumed any')
assert_equal(first_four, ['Produced 0', 'Produced 1', 'Produced 2'])
assert len(queue) == 12
