def test_chain_replaced_with_a_chain_and_an_error_callback(self, manager):
if not manager.app.conf.result_backend.startswith('redis'):
raise pytest.skip('Requires redis result backend.')
redis_connection = get_redis_connection()
redis_connection.delete('redis-echo')
link_msg = 'Internal chain errback'
c = chain(identity.s('Hello '),
replace_with_chain_which_raises.s(link_msg=link_msg),
add.s(' will never be seen :('))
res = c.delay()
with pytest.raises(ValueError):
res.get(timeout=TIMEOUT)
expected_msgs = {
link_msg,
}
while expected_msgs:
maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT)
if maybe_key_msg is None:
raise TimeoutError('redis-echo')
_, msg = maybe_key_msg
msg = msg.decode()
expected_msgs.remove(msg) # KeyError if `msg` is not in here
# There should be no more elements - block momentarily
assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None
redis_connection.delete('redis-echo')
def test_chain_with_cb_replaced_with_chain_with_cb(self, manager):
if not manager.app.conf.result_backend.startswith('redis'):
raise pytest.skip('Requires redis result backend.')
redis_connection = get_redis_connection()
redis_connection.delete('redis-echo')
link_msg = 'Internal chain callback'
c = chain(
identity.s('Hello '),
# The replacement chain will pass its args though
replace_with_chain.s(link_msg=link_msg),
add.s('world'),
)
c.link(redis_echo.s())
res = c.delay()
assert res.get(timeout=TIMEOUT) == 'Hello world'
expected_msgs = {link_msg, 'Hello world'}
while expected_msgs:
maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT)
if maybe_key_msg is None:
raise TimeoutError('redis-echo')
_, msg = maybe_key_msg
msg = msg.decode()
expected_msgs.remove(msg) # KeyError if `msg` is not in here
# There should be no more elements - block momentarily
assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None
redis_connection.delete('redis-echo')
def wait_for(self, task_id, timeout=None, propagate=True, interval=0.5):
"""Wait for task and return its result.
If the task raises an exception, this exception
will be re-raised by :func:`wait_for`.
If `timeout` is not :const:`None`, this raises the
:class:`celery.exceptions.TimeoutError` exception if the operation
takes longer than `timeout` seconds.
"""
time_elapsed = 0.0
while 1:
status = self.get_status(task_id)
if status == states.SUCCESS:
return self.get_result(task_id)
elif status in states.PROPAGATE_STATES:
result = self.get_result(task_id)
if propagate:
raise result
return result
# avoid hammering the CPU checking status.
time.sleep(interval)
time_elapsed += interval
if timeout and time_elapsed >= timeout:
raise TimeoutError('The operation timed out.')
def get_many(self, task_ids, timeout=None, interval=0.5):
ids = set(task_ids)
cached_ids = set()
for task_id in ids:
try:
cached = self._cache[task_id]
except KeyError:
pass
else:
if cached['status'] in states.READY_STATES:
yield bytes_to_str(task_id), cached
cached_ids.add(task_id)
ids ^= cached_ids
iterations = 0
while ids:
keys = list(ids)
r = self._mget_to_results(
self.mget([self.get_key_for_task(k) for k in keys]), keys)
self._cache.update(r)
ids ^= set(map(bytes_to_str, r))
for key, value in r.iteritems():
yield bytes_to_str(key), value
if timeout and iterations * interval >= timeout:
raise TimeoutError('Operation timed out (%s)' % (timeout, ))
time.sleep(interval) # don't busy loop.
iterations += 1
def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True,
READY_STATES=states.READY_STATES):
interval = 0.5 if interval is None else interval
ids = task_ids if isinstance(task_ids, set) else set(task_ids)
cached_ids = set()
cache = self._cache
for task_id in ids:
try:
cached = cache[task_id]
except KeyError:
pass
else:
if cached['status'] in READY_STATES:
yield bytes_to_str(task_id), cached
cached_ids.add(task_id)
ids.difference_update(cached_ids)
iterations = 0
while ids:
keys = list(ids)
r = self._mget_to_results(self.mget([self.get_key_for_task(k)
for k in keys]), keys)
cache.update(r)
ids.difference_update(set(bytes_to_str(v) for v in r))
for key, value in items(r):
yield bytes_to_str(key), value
if timeout and iterations * interval >= timeout:
raise TimeoutError('Operation timed out ({0})'.format(timeout))
time.sleep(interval) # don't busy loop.
iterations += 1
def test_subsequent_syncs_when_job_complete(self):
# First sync, return a timout. Ensure that the async_task_id gets set
cache_id = restore_cache_key(ASYNC_RESTORE_CACHE_KEY_PREFIX, self.user.user_id)
with mock.patch('casexml.apps.phone.restore.get_async_restore_payload') as task:
delay = mock.MagicMock()
delay.id = 'random_task_id'
delay.get = mock.MagicMock(side_effect=TimeoutError()) # task not finished
task.delay.return_value = delay
restore_config = self._restore_config(async=True)
initial_payload = restore_config.get_payload()
self.assertIsNotNone(restore_config.cache.get(cache_id))
self.assertIsInstance(initial_payload, AsyncRestoreResponse)
# new synclog should not have been created
self.assertIsNone(restore_config.restore_state.current_sync_log)
# Second sync, don't timeout (can't use AsyncResult in tests, so mock
# the return value).
file_restore_response = mock.MagicMock(return_value=FileRestoreResponse())
with mock.patch.object(AsyncResult, 'get', file_restore_response) as get_result:
with mock.patch.object(AsyncResult, 'status', ASYNC_RESTORE_SENT):
subsequent_restore = self._restore_config(async=True)
self.assertIsNotNone(restore_config.cache.get(cache_id))
subsequent_restore.get_payload()
# if the task actually ran, the cache should now not have the task id,
# however, the task is not run in this test. See `test_completed_task_deletes_cache`
# self.assertIsNone(restore_config.cache.get(cache_id))
get_result.assert_called_with(timeout=1)
def wait_for(self,
task_id,
timeout=None,
cache=True,
propagate=True,
**kwargs):
cached_meta = self._cache.get(task_id)
if cache and cached_meta and \
cached_meta["status"] in states.READY_STATES:
meta = cached_meta
else:
try:
meta = self.consume(task_id, timeout=timeout)
except socket.timeout:
raise TimeoutError("The operation timed out.")
state = meta["status"]
if state == states.SUCCESS:
return meta["result"]
elif state in states.PROPAGATE_STATES:
if propagate:
raise self.exception_to_python(meta["result"])
return meta["result"]
else:
return self.wait_for(task_id, timeout, cache)
def wait_for(self,
task_id,
timeout=None,
interval=0.5,
no_ack=True,
on_interval=None):
"""Wait for task and return its result.
If the task raises an exception, this exception
will be re-raised by :func:`wait_for`.
Raises:
celery.exceptions.TimeoutError:
If `timeout` is not :const:`None`, and the operation
takes longer than `timeout` seconds.
"""
self._ensure_not_eager()
time_elapsed = 0.0
while 1:
meta = self.get_task_meta(task_id)
if meta['status'] in states.READY_STATES:
return meta
if on_interval:
on_interval()
# avoid hammering the CPU checking status.
time.sleep(interval)
time_elapsed += interval
if timeout and time_elapsed >= timeout:
raise TimeoutError('The operation timed out.')
def test_chain_with_eb_replaced_with_chain_with_eb(self, manager):
if not manager.app.conf.result_backend.startswith('redis'):
raise pytest.skip('Requires redis result backend.')
redis_connection = get_redis_connection()
redis_connection.delete('redis-echo')
inner_link_msg = 'Internal chain errback'
outer_link_msg = 'External chain errback'
c = chain(
identity.s('Hello '),
# The replacement chain will pass its args though
replace_with_chain_which_raises.s(link_msg=inner_link_msg),
add.s('world'),
)
c.link_error(redis_echo.s(outer_link_msg))
res = c.delay()
with pytest.raises(ValueError):
res.get(timeout=TIMEOUT)
expected_msgs = {inner_link_msg, outer_link_msg}
while expected_msgs:
# Shorter timeout here because we expect failure
timeout = min(5, TIMEOUT)
maybe_key_msg = redis_connection.blpop('redis-echo', timeout)
if maybe_key_msg is None:
raise TimeoutError('redis-echo')
_, msg = maybe_key_msg
msg = msg.decode()
expected_msgs.remove(msg) # KeyError if `msg` is not in here
# There should be no more elements - block momentarily
assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None
redis_connection.delete('redis-echo')
def try_while(fun, reason='Timed out', timeout=10, interval=0.5):
time_start = time()
for iterations in count(0):
if time() - time_start >= timeout:
raise TimeoutError()
ret = fun()
if ret:
return ret
def test_async_call_raise(self):
app = MagicMock()
app.send_task.side_effect = TimeoutError("whatever")
path, name = "p_test", "n_test"
expected = "Celery error - %s" % name
with self.assertRaises(IrmaTaskError) as context:
module.async_call(app, path, name)
self.assertEqual(str(context.exception), expected)
def wait_for(self, task_id, timeout=None, cache=True):
if task_id in self._cache:
meta = self._cache[task_id]
else:
try:
meta = self.consume(task_id, timeout=timeout)
except socket.timeout:
raise TimeoutError("The operation timed out.")
if meta["status"] == states.SUCCESS:
return meta["result"]
elif meta["status"] in states.PROPAGATE_STATES:
raise self.exception_to_python(meta["result"])
def test_restore_then_sync_on_same_synclog_returns_async_restore_response(self, task):
delay = mock.MagicMock()
delay.id = 'random_task_id'
delay.get = mock.MagicMock(side_effect=TimeoutError()) # task not finished
task.delay.return_value = delay
restore_config = self._restore_config(async=True)
initial_payload = restore_config.get_payload()
self.assertIsInstance(initial_payload, AsyncRestoreResponse)
subsequent_restore = self._restore_config(async=True)
subsequent_payload = subsequent_restore.get_payload()
self.assertIsInstance(subsequent_payload, AsyncRestoreResponse)
def _wait_for_pending(self, result, timeout=None, interval=0.5,
no_ack=True, on_interval=None, callback=None,
on_message=None, propagate=True):
prev_on_m, self.on_message = self.on_message, on_message
try:
for _ in self.drain_events_until(
result.on_ready, timeout=timeout,
on_interval=on_interval):
yield
time.sleep(0)
except socket.timeout:
raise TimeoutError('The operation timed out.')
finally:
self.on_message = prev_on_m
def wait_for(self, task_id, timeout=None, cache=True,
no_ack=True, on_interval=None,
READY_STATES=states.READY_STATES,
PROPAGATE_STATES=states.PROPAGATE_STATES,
**kwargs):
cached_meta = self._cache.get(task_id)
if cache and cached_meta and \
cached_meta['status'] in READY_STATES:
return cached_meta
else:
try:
return self.consume(task_id, timeout=timeout, no_ack=no_ack,
on_interval=on_interval)
except socket.timeout:
raise TimeoutError('The operation timed out.')
def _wait_for_pending(self, result,
timeout=None, on_interval=None, on_message=None,
**kwargs):
self.on_wait_for_pending(result, timeout=timeout, **kwargs)
prev_on_m, self.on_message = self.on_message, on_message
try:
for _ in self.drain_events_until(
result.on_ready, timeout=timeout,
on_interval=on_interval):
yield
sleep(0)
except socket.timeout:
raise TimeoutError('The operation timed out.')
finally:
self.on_message = prev_on_m
def join(self, timeout=None, propagate=True, interval=0.5):
"""Gathers the results of all tasks in the taskset,
and returns a list ordered by the order of the set.
.. note::
This can be an expensive operation for result store
backends that must resort to polling (e.g. database).
You should consider using :meth:`join_native` if your backend
supports it.
.. warning::
Waiting for subtasks may lead to deadlocks.
Please see :ref:`task-synchronous-subtasks`.
:keyword timeout: The number of seconds to wait for results before
the operation times out.
:keyword propagate: If any of the subtasks raises an exception, the
exception will be re-raised.
:keyword interval: Time to wait (in seconds) before retrying to
retrieve a result from the set. Note that this
does not have any effect when using the AMQP
result store backend, as it does not use polling.
:raises celery.exceptions.TimeoutError: if `timeout` is not
:const:`None` and the operation takes longer than `timeout`
seconds.
"""
time_start = time.time()
remaining = None
results = []
for subtask in self.subtasks:
remaining = None
if timeout:
remaining = timeout - (time.time() - time_start)
if remaining <= 0.0:
raise TimeoutError("join operation timed out")
results.append(
subtask.wait(timeout=remaining,
propagate=propagate,
interval=interval))
return results
def do_wait():
try:
time_elapsed = 0.0
while True:
self.jobmgr._p_jar.sync()
status = self.get_status(task_id)
if status in states.READY_STATES:
result = self.get_result(task_id)
result_queue.put((status, result))
return
# avoid hammering the CPU checking status.
time.sleep(interval)
time_elapsed += interval
if timeout and time_elapsed >= timeout:
raise TimeoutError("The operation timed out.")
finally:
self.reset()
def get_many(
self,
task_ids,
timeout=None,
interval=0.5,
no_ack=True,
on_message=None,
on_interval=None,
max_iterations=None,
READY_STATES=states.READY_STATES,
):
interval = 0.5 if interval is None else interval
ids = task_ids if isinstance(task_ids, set) else set(task_ids)
cached_ids = set()
cache = self._cache
for task_id in ids:
try:
cached = cache[task_id]
except KeyError:
pass
else:
if cached["status"] in READY_STATES:
yield bytes_to_str(task_id), cached
cached_ids.add(task_id)
ids.difference_update(cached_ids)
iterations = 0
while ids:
keys = list(ids)
r = self._mget_to_results(
self.mget([self.get_key_for_task(k) for k in keys]), keys,
READY_STATES)
cache.update(r)
ids.difference_update({bytes_to_str(v) for v in r})
for key, value in items(r):
if on_message is not None:
on_message(value)
yield bytes_to_str(key), value
if timeout and iterations * interval >= timeout:
raise TimeoutError("Operation timed out ({0})".format(timeout))
if on_interval:
on_interval()
time.sleep(interval) # don't busy loop.
iterations += 1
if max_iterations and iterations >= max_iterations:
break
def wait_for_background_jobs_completion(as_at_date,
minutes_to_wait=5,
poll_interval=5):
tries_remaining = int((60 * minutes_to_wait) / poll_interval)
while tries_remaining:
cache_sightings = PeriodicTaskWithTTL.objects.get(
name='cache_sightings')
cache_backlinks = PeriodicTaskWithTTL.objects.get(
name='cache_backlinks')
if cache_backlinks.last_run_at > as_at_date and cache_sightings.last_run_at > as_at_date:
return
else:
sleep(poll_interval)
tries_remaining -= 1
raise TimeoutError(
'Timeout waiting for sightings and backlinks jobs to complete. Will retry in 24 hours.'
)
def wait_for(self, task_id, timeout=None, cache=True, propagate=True,
READY_STATES=states.READY_STATES,
PROPAGATE_STATES=states.PROPAGATE_STATES,
**kwargs):
cached_meta = self._cache.get(task_id)
if cache and cached_meta and \
cached_meta['status'] in READY_STATES:
meta = cached_meta
else:
try:
meta = self.consume(task_id, timeout=timeout)
except socket.timeout:
raise TimeoutError('The operation timed out.')
if meta['status'] in PROPAGATE_STATES and propagate:
raise self.exception_to_python(meta['result'])
# consume() always returns READY_STATE.
return meta['result']
def apply_async_and_get_result(self,
args=None,
kwargs=None,
timeout=None,
propagate=True,
**options):
"""
Apply task in an asynchronous way, wait defined timeout and get AsyncResult or TimeoutError
:param args: task args
:param kwargs: task kwargs
:param timeout: timout in seconds to wait for result
:param propagate: propagate or not exceptions from celery task
:param options: apply_async method options
:return: AsyncResult or TimeoutError
"""
result = self.apply_async(args=args, kwargs=kwargs, **options)
if timeout is None or timeout > 0:
return result.get(timeout=timeout, propagate=propagate)
else:
raise TimeoutError('The operation timed out.')
def iterate(self, timeout=None, propagate=True, interval=0.5):
"""Deprecated method, use :meth:`get` with a callback argument."""
elapsed = 0.0
results = OrderedDict(
(result.id, copy(result)) for result in self.results)
while results:
removed = set()
for task_id, result in items(results):
if result.ready():
yield result.get(timeout=timeout and timeout - elapsed,
propagate=propagate)
removed.add(task_id)
else:
if result.backend.subpolling_interval:
time.sleep(result.backend.subpolling_interval)
for task_id in removed:
results.pop(task_id, None)
time.sleep(interval)
elapsed += interval
if timeout and elapsed >= timeout:
raise TimeoutError('The operation timed out')
def iterate(self, timeout=None, propagate=True, interval=0.5):
"""Iterate over the return values of the tasks as they finish
one by one.
:raises: The exception if any of the tasks raised an exception.
"""
elapsed = 0.0
results = dict(
(result.task_id, copy(result)) for result in self.results)
while results:
removed = set()
for task_id, result in results.iteritems():
yield result.get(timeout=timeout and timeout - elapsed,
propagate=propagate,
interval=0.0)
removed.add(task_id)
for task_id in removed:
results.pop(task_id, None)
time.sleep(interval)
elapsed += interval
if timeout and elapsed >= timeout:
raise TimeoutError("The operation timed out")
def join(self,
timeout=None,
propagate=True,
interval=0.5,
callback=None,
no_ack=True,
on_message=None,
disable_sync_subtasks=True,
on_interval=None):
"""Gather the results of all tasks as a list in order.
Note:
This can be an expensive operation for result store
backends that must resort to polling (e.g., database).
You should consider using :meth:`join_native` if your backend
supports it.
Warning:
Waiting for tasks within a task may lead to deadlocks.
Please see :ref:`task-synchronous-subtasks`.
Arguments:
timeout (float): The number of seconds to wait for results
before the operation times out.
propagate (bool): If any of the tasks raises an exception,
the exception will be re-raised when this flag is set.
interval (float): Time to wait (in seconds) before retrying to
retrieve a result from the set. Note that this does not have
any effect when using the amqp result store backend,
as it does not use polling.
callback (Callable): Optional callback to be called for every
result received. Must have signature ``(task_id, value)``
No results will be returned by this function if a callback
is specified. The order of results is also arbitrary when a
callback is used. To get access to the result object for
a particular id you'll have to generate an index first:
``index = {r.id: r for r in gres.results.values()}``
Or you can create new result objects on the fly:
``result = app.AsyncResult(task_id)`` (both will
take advantage of the backend cache anyway).
no_ack (bool): Automatic message acknowledgment (Note that if this
is set to :const:`False` then the messages
*will not be acknowledged*).
disable_sync_subtasks (bool): Disable tasks to wait for sub tasks
this is the default configuration. CAUTION do not enable this
unless you must.
Raises:
celery.exceptions.TimeoutError: if ``timeout`` isn't
:const:`None` and the operation takes longer than ``timeout``
seconds.
"""
if disable_sync_subtasks:
assert_will_not_block()
time_start = monotonic()
remaining = None
if on_message is not None:
raise ImproperlyConfigured(
'Backend does not support on_message callback')
results = []
for result in self.results:
remaining = None
if timeout:
remaining = timeout - (monotonic() - time_start)
if remaining <= 0.0:
raise TimeoutError('join operation timed out')
value = result.get(
timeout=remaining,
propagate=propagate,
interval=interval,
no_ack=no_ack,
on_interval=on_interval,
)
if callback:
callback(result.id, value)
else:
results.append(value)
return results
def on_timeout():
raise TimeoutError("The operation timed out.")
def test_run_task(self):
try:
clean_media.delay().get(timeout=2)
except TimeoutError as e:
message = str(e) + ' Is the Celery worker running?'
raise TimeoutError(message)
