def __init__(self, target, *args, parallel=5, **kwargs):
super().__init__()
self.output = FiniteQueue()
kwargs['_output'] = self.output
self.threads = [
generate_on_thread(target, *args, **kwargs)
for _ in range(parallel)
]
if kwargs.get('immediate', True):
self.start()
def test_queue_ends_with_QUEUE_END(self):
test_queue = FiniteQueue()
test_queue.put('First')
test_queue.end()
assert test_queue.get() == 'First'
assert test_queue.get() is FiniteQueue.QUEUE_END
# We should keep getting QUEUE_END from now on, too.
assert test_queue.get() is FiniteQueue.QUEUE_END
def test_queue_is_iterable(self):
test_queue = FiniteQueue()
test_queue.put('First')
test_queue.end()
result = list(test_queue)
assert result == ['First']
class generate_parallel(threading.Thread):
"""
Run a generator on a multiple separate threads, placing the combined
results in to a FiniteQueue. The queue is available via `instance.output`.
Examples
--------
Create some random numbers on a separate thread.
>>> def yield_numbers():
>>> for number in range(2)
>>> yield number
>>>
>>> for number in generate_parallel(yield_numbers, parallel=3).output:
>>> print(number)
0
0
0
1
1
1
"""
def __init__(self, target, *args, parallel=5, **kwargs):
super().__init__()
self.output = FiniteQueue()
kwargs['_output'] = self.output
self.threads = [
generate_on_thread(target, *args, **kwargs)
for _ in range(parallel)
]
if kwargs.get('immediate', True):
self.start()
def run(self):
for thread in self.threads:
thread.join()
self.output.end()
def __init__(self,
target,
*args,
immediate=True,
cancel=None,
_output=None,
**kwargs):
super().__init__()
self.should_end = _output is None
self.output = _output or FiniteQueue()
self.cancel = cancel
self.target = target
self.args = args
self.kwargs = kwargs
if immediate:
self.start()
def test_queue_can_be_safely_read_from_multiple_threads(self):
# We want to make sure that a thread can't get stuck waiting for the
# next item on a queue that is already ended.
test_queue = FiniteQueue()
results = queue.SimpleQueue()
def read_one_item():
try:
results.put(test_queue.get(timeout=1))
except queue.Empty as error:
results.put(error)
threads = [threading.Thread(target=read_one_item) for i in range(3)]
[thread.start() for thread in threads]
test_queue.put('First')
test_queue.end()
[thread.join() for thread in threads]
assert results.get() == 'First'
assert results.get() is FiniteQueue.QUEUE_END
assert results.get() is FiniteQueue.QUEUE_END