def _handle_tasks_insert(self, batch_size=None):
"""Convert all Async's into tasks, then insert them into queues."""
if self._tasks_inserted:
raise errors.ContextAlreadyStartedError(
"This Context has already had its tasks inserted.")
task_map = self._get_tasks_by_queue()
# QUESTION: Should the persist happen before or after the task
# insertion? I feel like this is something that will alter the
# behavior of the tasks themselves by adding a callback (check context
# complete) to each Async's callback stack.
# If we are able to and there is a reason to persist... persist.
callbacks = self._options.get('callbacks')
if self._persistence_engine and callbacks:
self.persist()
retry_transient = self._options.get('retry_transient_errors', True)
retry_delay = self._options.get('retry_delay', RETRY_SLEEP_SECS)
for queue, tasks in task_map.iteritems():
for batch in _task_batcher(tasks, batch_size=batch_size):
inserted = self._insert_tasks(
batch,
queue=queue,
retry_transient_errors=retry_transient,
retry_delay=retry_delay)
if isinstance(inserted, (int, long)):
# Don't blow up on insert_tasks that don't return counts.
self._insert_success_count += inserted
self._insert_failed_count += len(batch) - inserted
def add(self, target, args=None, kwargs=None, **options):
"""Add an Async job to this context.
Takes an Async object or the arguments to construct an Async
object as arguments. Returns the newly added Async object.
"""
from furious. async import Async
from furious.batcher import Message
if self._tasks_inserted:
raise errors.ContextAlreadyStartedError(
"This Context has already had its tasks inserted.")
if not isinstance(target, (Async, Message)):
target = Async(target, args, kwargs, **options)
target.update_options(_context_id=self.id)
if self.persist_async_results:
target.update_options(persist_result=True)
self._tasks.append(target)
self._options['_task_ids'].append(target.id)
return target
def _handle_tasks_insert(self, batch_size=None):
"""Convert all Async's into tasks, then insert them into queues."""
if self._tasks_inserted:
raise errors.ContextAlreadyStartedError(
"This Context has already had its tasks inserted.")
task_map = self._get_tasks_by_queue()
for queue, tasks in task_map.iteritems():
for batch in _task_batcher(tasks, batch_size=batch_size):
inserted = self._insert_tasks(batch, queue=queue)
if isinstance(inserted, (int, long)):
# Don't blow up on insert_tasks that don't return counts.
self._insert_success_count += inserted
self._insert_failed_count += len(batch) - inserted