def get_results(self, timeout=None):
"""Returns results from worker pool
:param timeout: If None, will block forever, otherwise will raise TimeoutWaitingForResultError exception
if no data received within the timeout (in seconds)
:return: arguments passed to publish_func(...) by a worker. If no more results are anticipated, EmptyResultError
is raised.
"""
while True:
# If there is no more work to do, raise an EmptyResultError
if self._ventilated_items == self._ventilated_items_processed:
# We also need to check if we are using a ventilator and if it is completed
if not self._ventilator or self._ventilator.completed():
raise EmptyResultError()
socks = self._results_receiver_poller.poll(
timeout * 1e3 if timeout else None)
if not socks:
raise TimeoutWaitingForResultError()
result = self._results_receiver.recv_pyobj(0)
if isinstance(result, VentilatedItemProcessedMessage):
self._ventilated_items_processed += 1
if self._ventilator:
self._ventilator.processed_item()
continue
if isinstance(result, Exception):
self.stop()
self.join()
raise result
else:
return result
def get_results(self):
"""Returns results
The processing is done on the get_results caller thread if the results queue is empty
:return: arguments passed to publish_func(...) by a worker
"""
if self._results_queue:
# We have already calculated result. Just return it
return self._results_queue.pop(0)
else:
# If we don't have any tasks waiting for processing, then indicate empty queue
while self._ventilator_queue or (self._ventilator and
not self._ventilator.completed()):
# To prevent a race condition of the ventilator working but not yet placing an item
# on the ventilator queue. We block until something is on the ventilator queue.
while not self._ventilator_queue:
sleep(.1)
# If we do have some tasks, then process a task from the head of a queue
args, kargs = self._ventilator_queue.pop(0)
self._worker.process(*args, **kargs)
if self._ventilator:
self._ventilator.processed_item()
if self._results_queue:
return self._results_queue.pop(0)
raise EmptyResultError()
def get_results(self):
"""Returns results from worker pool or re-raise worker's exception if any happen in worker thread.
:param timeout: If None, will block forever, otherwise will raise :class:`.TimeoutWaitingForResultError`
exception if no data received within the timeout (in seconds)
:return: arguments passed to ``publish_func(...)`` by a worker. If no more results are anticipated,
:class:`.EmptyResultError`.
"""
while True:
# If there is no more work to do, raise an EmptyResultError
if self._results_queue.empty(
) and self._ventilated_items == self._ventilated_items_processed:
# We also need to check if we are using a ventilator and if it is completed
if not self._ventilator or self._ventilator.completed():
raise EmptyResultError()
try:
result = self._results_queue.get(
timeout=_VERIFY_END_OF_VENTILATION_PERIOD)
if isinstance(result, VentilatedItemProcessedMessage):
self._ventilated_items_processed += 1
if self._ventilator:
self._ventilator.processed_item()
continue
elif isinstance(result, Exception):
self.stop()
self.join()
raise result
else:
return result
except queue.Empty:
continue
def get_results(self):
"""Returns results from worker pool
:param timeout: If None, will block forever, otherwise will raise :class:`.TimeoutWaitingForResultError`
exception if no data received within the timeout (in seconds)
:return: arguments passed to ``publish_func(...)`` by a worker. If no more results are anticipated,
:class:`.EmptyResultError` is raised.
"""
while True:
# If there is no more work to do, raise an EmptyResultError
logger.debug(
'ventilated_items=%d ventilated_items_processed=%d ventilator.completed=%s',
self._ventilated_items, self._ventilated_items_processed,
str(self._ventilator.completed())
if self._ventilator else 'N/A')
if self._ventilated_items == self._ventilated_items_processed:
# We also need to check if we are using a ventilator and if it is completed
if not self._ventilator or self._ventilator.completed():
logger.debug(
'ventilator reported it has completed. Reporting end of results'
)
raise EmptyResultError()
logger.debug('get_results polling on the next result')
socks = self._results_receiver_poller.poll(
_VERIFY_END_OF_VENTILATION_PERIOD * 1e3)
if not socks:
continue
# Result message is a tuple containing data payload and possible exception (or None).
# By specifying pyarrow_serialize=True, we may choose to use pyarrow serializer which is faster, but
# does not support all data types correctly.
fast_serialized, pickle_serialized = self._results_receiver.recv_multipart(
copy=self._zmq_copy_buffers)
pickle_serialized = pickle.loads(pickle_serialized)
if pickle_serialized:
logger.debug('get_results a pickled message %s',
type(pickle_serialized))
if isinstance(pickle_serialized,
VentilatedItemProcessedMessage):
self._ventilated_items_processed += 1
if self._ventilator:
self._ventilator.processed_item()
elif isinstance(pickle_serialized, Exception):
self.stop()
self.join()
raise pickle_serialized
else:
logger.debug('get_results received new results')
if self._zmq_copy_buffers:
deserialized_result = self._serializer.deserialize(
fast_serialized)
else:
deserialized_result = self._serializer.deserialize(
fast_serialized.buffer)
return deserialized_result
def get_results(self):
"""Returns results from worker pool
:param timeout: If None, will block forever, otherwise will raise :class:`.TimeoutWaitingForResultError`
exception if no data received within the timeout (in seconds)
:return: arguments passed to ``publish_func(...)`` by a worker. If no more results are anticipated,
:class:`.EmptyResultError` is raised.
"""
while True:
# If there is no more work to do, raise an EmptyResultError
if self._ventilated_items == self._ventilated_items_processed:
# We also need to check if we are using a ventilator and if it is completed
if not self._ventilator or self._ventilator.completed():
raise EmptyResultError()
socks = self._results_receiver_poller.poll(
_VERIFY_END_OF_VENTILATION_PERIOD * 1e3)
if not socks:
continue
result = self._results_receiver.recv_pyobj(0)
if isinstance(result, VentilatedItemProcessedMessage):
self._ventilated_items_processed += 1
if self._ventilator:
self._ventilator.processed_item()
continue
if isinstance(result, Exception):
self.stop()
self.join()
raise result
else:
if self._serializer:
deserialized_result = self._serializer.deserialize(result)
else:
deserialized_result = result
return deserialized_result