def test_chain_replaced_with_a_chain_and_an_error_callback(self, manager): if not manager.app.conf.result_backend.startswith('redis'): raise pytest.skip('Requires redis result backend.') redis_connection = get_redis_connection() redis_connection.delete('redis-echo') link_msg = 'Internal chain errback' c = chain(identity.s('Hello '), replace_with_chain_which_raises.s(link_msg=link_msg), add.s(' will never be seen :(')) res = c.delay() with pytest.raises(ValueError): res.get(timeout=TIMEOUT) expected_msgs = { link_msg, } while expected_msgs: maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) if maybe_key_msg is None: raise TimeoutError('redis-echo') _, msg = maybe_key_msg msg = msg.decode() expected_msgs.remove(msg) # KeyError if `msg` is not in here # There should be no more elements - block momentarily assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None redis_connection.delete('redis-echo')
def test_chain_with_cb_replaced_with_chain_with_cb(self, manager): if not manager.app.conf.result_backend.startswith('redis'): raise pytest.skip('Requires redis result backend.') redis_connection = get_redis_connection() redis_connection.delete('redis-echo') link_msg = 'Internal chain callback' c = chain( identity.s('Hello '), # The replacement chain will pass its args though replace_with_chain.s(link_msg=link_msg), add.s('world'), ) c.link(redis_echo.s()) res = c.delay() assert res.get(timeout=TIMEOUT) == 'Hello world' expected_msgs = {link_msg, 'Hello world'} while expected_msgs: maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) if maybe_key_msg is None: raise TimeoutError('redis-echo') _, msg = maybe_key_msg msg = msg.decode() expected_msgs.remove(msg) # KeyError if `msg` is not in here # There should be no more elements - block momentarily assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None redis_connection.delete('redis-echo')
def wait_for(self, task_id, timeout=None, propagate=True, interval=0.5): """Wait for task and return its result. If the task raises an exception, this exception will be re-raised by :func:`wait_for`. If `timeout` is not :const:`None`, this raises the :class:`celery.exceptions.TimeoutError` exception if the operation takes longer than `timeout` seconds. """ time_elapsed = 0.0 while 1: status = self.get_status(task_id) if status == states.SUCCESS: return self.get_result(task_id) elif status in states.PROPAGATE_STATES: result = self.get_result(task_id) if propagate: raise result return result # avoid hammering the CPU checking status. time.sleep(interval) time_elapsed += interval if timeout and time_elapsed >= timeout: raise TimeoutError('The operation timed out.')
def get_many(self, task_ids, timeout=None, interval=0.5): ids = set(task_ids) cached_ids = set() for task_id in ids: try: cached = self._cache[task_id] except KeyError: pass else: if cached['status'] in states.READY_STATES: yield bytes_to_str(task_id), cached cached_ids.add(task_id) ids ^= cached_ids iterations = 0 while ids: keys = list(ids) r = self._mget_to_results( self.mget([self.get_key_for_task(k) for k in keys]), keys) self._cache.update(r) ids ^= set(map(bytes_to_str, r)) for key, value in r.iteritems(): yield bytes_to_str(key), value if timeout and iterations * interval >= timeout: raise TimeoutError('Operation timed out (%s)' % (timeout, )) time.sleep(interval) # don't busy loop. iterations += 1
def get_many(self, task_ids, timeout=None, interval=0.5, no_ack=True, READY_STATES=states.READY_STATES): interval = 0.5 if interval is None else interval ids = task_ids if isinstance(task_ids, set) else set(task_ids) cached_ids = set() cache = self._cache for task_id in ids: try: cached = cache[task_id] except KeyError: pass else: if cached['status'] in READY_STATES: yield bytes_to_str(task_id), cached cached_ids.add(task_id) ids.difference_update(cached_ids) iterations = 0 while ids: keys = list(ids) r = self._mget_to_results(self.mget([self.get_key_for_task(k) for k in keys]), keys) cache.update(r) ids.difference_update(set(bytes_to_str(v) for v in r)) for key, value in items(r): yield bytes_to_str(key), value if timeout and iterations * interval >= timeout: raise TimeoutError('Operation timed out ({0})'.format(timeout)) time.sleep(interval) # don't busy loop. iterations += 1
def test_subsequent_syncs_when_job_complete(self): # First sync, return a timout. Ensure that the async_task_id gets set cache_id = restore_cache_key(ASYNC_RESTORE_CACHE_KEY_PREFIX, self.user.user_id) with mock.patch('casexml.apps.phone.restore.get_async_restore_payload') as task: delay = mock.MagicMock() delay.id = 'random_task_id' delay.get = mock.MagicMock(side_effect=TimeoutError()) # task not finished task.delay.return_value = delay restore_config = self._restore_config(async=True) initial_payload = restore_config.get_payload() self.assertIsNotNone(restore_config.cache.get(cache_id)) self.assertIsInstance(initial_payload, AsyncRestoreResponse) # new synclog should not have been created self.assertIsNone(restore_config.restore_state.current_sync_log) # Second sync, don't timeout (can't use AsyncResult in tests, so mock # the return value). file_restore_response = mock.MagicMock(return_value=FileRestoreResponse()) with mock.patch.object(AsyncResult, 'get', file_restore_response) as get_result: with mock.patch.object(AsyncResult, 'status', ASYNC_RESTORE_SENT): subsequent_restore = self._restore_config(async=True) self.assertIsNotNone(restore_config.cache.get(cache_id)) subsequent_restore.get_payload() # if the task actually ran, the cache should now not have the task id, # however, the task is not run in this test. See `test_completed_task_deletes_cache` # self.assertIsNone(restore_config.cache.get(cache_id)) get_result.assert_called_with(timeout=1)
def wait_for(self, task_id, timeout=None, cache=True, propagate=True, **kwargs): cached_meta = self._cache.get(task_id) if cache and cached_meta and \ cached_meta["status"] in states.READY_STATES: meta = cached_meta else: try: meta = self.consume(task_id, timeout=timeout) except socket.timeout: raise TimeoutError("The operation timed out.") state = meta["status"] if state == states.SUCCESS: return meta["result"] elif state in states.PROPAGATE_STATES: if propagate: raise self.exception_to_python(meta["result"]) return meta["result"] else: return self.wait_for(task_id, timeout, cache)
def wait_for(self, task_id, timeout=None, interval=0.5, no_ack=True, on_interval=None): """Wait for task and return its result. If the task raises an exception, this exception will be re-raised by :func:`wait_for`. Raises: celery.exceptions.TimeoutError: If `timeout` is not :const:`None`, and the operation takes longer than `timeout` seconds. """ self._ensure_not_eager() time_elapsed = 0.0 while 1: meta = self.get_task_meta(task_id) if meta['status'] in states.READY_STATES: return meta if on_interval: on_interval() # avoid hammering the CPU checking status. time.sleep(interval) time_elapsed += interval if timeout and time_elapsed >= timeout: raise TimeoutError('The operation timed out.')
def test_chain_with_eb_replaced_with_chain_with_eb(self, manager): if not manager.app.conf.result_backend.startswith('redis'): raise pytest.skip('Requires redis result backend.') redis_connection = get_redis_connection() redis_connection.delete('redis-echo') inner_link_msg = 'Internal chain errback' outer_link_msg = 'External chain errback' c = chain( identity.s('Hello '), # The replacement chain will pass its args though replace_with_chain_which_raises.s(link_msg=inner_link_msg), add.s('world'), ) c.link_error(redis_echo.s(outer_link_msg)) res = c.delay() with pytest.raises(ValueError): res.get(timeout=TIMEOUT) expected_msgs = {inner_link_msg, outer_link_msg} while expected_msgs: # Shorter timeout here because we expect failure timeout = min(5, TIMEOUT) maybe_key_msg = redis_connection.blpop('redis-echo', timeout) if maybe_key_msg is None: raise TimeoutError('redis-echo') _, msg = maybe_key_msg msg = msg.decode() expected_msgs.remove(msg) # KeyError if `msg` is not in here # There should be no more elements - block momentarily assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None redis_connection.delete('redis-echo')
def try_while(fun, reason='Timed out', timeout=10, interval=0.5): time_start = time() for iterations in count(0): if time() - time_start >= timeout: raise TimeoutError() ret = fun() if ret: return ret
def test_async_call_raise(self): app = MagicMock() app.send_task.side_effect = TimeoutError("whatever") path, name = "p_test", "n_test" expected = "Celery error - %s" % name with self.assertRaises(IrmaTaskError) as context: module.async_call(app, path, name) self.assertEqual(str(context.exception), expected)
def wait_for(self, task_id, timeout=None, cache=True): if task_id in self._cache: meta = self._cache[task_id] else: try: meta = self.consume(task_id, timeout=timeout) except socket.timeout: raise TimeoutError("The operation timed out.") if meta["status"] == states.SUCCESS: return meta["result"] elif meta["status"] in states.PROPAGATE_STATES: raise self.exception_to_python(meta["result"])
def test_restore_then_sync_on_same_synclog_returns_async_restore_response(self, task): delay = mock.MagicMock() delay.id = 'random_task_id' delay.get = mock.MagicMock(side_effect=TimeoutError()) # task not finished task.delay.return_value = delay restore_config = self._restore_config(async=True) initial_payload = restore_config.get_payload() self.assertIsInstance(initial_payload, AsyncRestoreResponse) subsequent_restore = self._restore_config(async=True) subsequent_payload = subsequent_restore.get_payload() self.assertIsInstance(subsequent_payload, AsyncRestoreResponse)
def _wait_for_pending(self, result, timeout=None, interval=0.5, no_ack=True, on_interval=None, callback=None, on_message=None, propagate=True): prev_on_m, self.on_message = self.on_message, on_message try: for _ in self.drain_events_until( result.on_ready, timeout=timeout, on_interval=on_interval): yield time.sleep(0) except socket.timeout: raise TimeoutError('The operation timed out.') finally: self.on_message = prev_on_m
def wait_for(self, task_id, timeout=None, cache=True, no_ack=True, on_interval=None, READY_STATES=states.READY_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES, **kwargs): cached_meta = self._cache.get(task_id) if cache and cached_meta and \ cached_meta['status'] in READY_STATES: return cached_meta else: try: return self.consume(task_id, timeout=timeout, no_ack=no_ack, on_interval=on_interval) except socket.timeout: raise TimeoutError('The operation timed out.')
def _wait_for_pending(self, result, timeout=None, on_interval=None, on_message=None, **kwargs): self.on_wait_for_pending(result, timeout=timeout, **kwargs) prev_on_m, self.on_message = self.on_message, on_message try: for _ in self.drain_events_until( result.on_ready, timeout=timeout, on_interval=on_interval): yield sleep(0) except socket.timeout: raise TimeoutError('The operation timed out.') finally: self.on_message = prev_on_m
def join(self, timeout=None, propagate=True, interval=0.5): """Gathers the results of all tasks in the taskset, and returns a list ordered by the order of the set. .. note:: This can be an expensive operation for result store backends that must resort to polling (e.g. database). You should consider using :meth:`join_native` if your backend supports it. .. warning:: Waiting for subtasks may lead to deadlocks. Please see :ref:`task-synchronous-subtasks`. :keyword timeout: The number of seconds to wait for results before the operation times out. :keyword propagate: If any of the subtasks raises an exception, the exception will be re-raised. :keyword interval: Time to wait (in seconds) before retrying to retrieve a result from the set. Note that this does not have any effect when using the AMQP result store backend, as it does not use polling. :raises celery.exceptions.TimeoutError: if `timeout` is not :const:`None` and the operation takes longer than `timeout` seconds. """ time_start = time.time() remaining = None results = [] for subtask in self.subtasks: remaining = None if timeout: remaining = timeout - (time.time() - time_start) if remaining <= 0.0: raise TimeoutError("join operation timed out") results.append( subtask.wait(timeout=remaining, propagate=propagate, interval=interval)) return results
def do_wait(): try: time_elapsed = 0.0 while True: self.jobmgr._p_jar.sync() status = self.get_status(task_id) if status in states.READY_STATES: result = self.get_result(task_id) result_queue.put((status, result)) return # avoid hammering the CPU checking status. time.sleep(interval) time_elapsed += interval if timeout and time_elapsed >= timeout: raise TimeoutError("The operation timed out.") finally: self.reset()
def get_many( self, task_ids, timeout=None, interval=0.5, no_ack=True, on_message=None, on_interval=None, max_iterations=None, READY_STATES=states.READY_STATES, ): interval = 0.5 if interval is None else interval ids = task_ids if isinstance(task_ids, set) else set(task_ids) cached_ids = set() cache = self._cache for task_id in ids: try: cached = cache[task_id] except KeyError: pass else: if cached["status"] in READY_STATES: yield bytes_to_str(task_id), cached cached_ids.add(task_id) ids.difference_update(cached_ids) iterations = 0 while ids: keys = list(ids) r = self._mget_to_results( self.mget([self.get_key_for_task(k) for k in keys]), keys, READY_STATES) cache.update(r) ids.difference_update({bytes_to_str(v) for v in r}) for key, value in items(r): if on_message is not None: on_message(value) yield bytes_to_str(key), value if timeout and iterations * interval >= timeout: raise TimeoutError("Operation timed out ({0})".format(timeout)) if on_interval: on_interval() time.sleep(interval) # don't busy loop. iterations += 1 if max_iterations and iterations >= max_iterations: break
def wait_for_background_jobs_completion(as_at_date, minutes_to_wait=5, poll_interval=5): tries_remaining = int((60 * minutes_to_wait) / poll_interval) while tries_remaining: cache_sightings = PeriodicTaskWithTTL.objects.get( name='cache_sightings') cache_backlinks = PeriodicTaskWithTTL.objects.get( name='cache_backlinks') if cache_backlinks.last_run_at > as_at_date and cache_sightings.last_run_at > as_at_date: return else: sleep(poll_interval) tries_remaining -= 1 raise TimeoutError( 'Timeout waiting for sightings and backlinks jobs to complete. Will retry in 24 hours.' )
def wait_for(self, task_id, timeout=None, cache=True, propagate=True, READY_STATES=states.READY_STATES, PROPAGATE_STATES=states.PROPAGATE_STATES, **kwargs): cached_meta = self._cache.get(task_id) if cache and cached_meta and \ cached_meta['status'] in READY_STATES: meta = cached_meta else: try: meta = self.consume(task_id, timeout=timeout) except socket.timeout: raise TimeoutError('The operation timed out.') if meta['status'] in PROPAGATE_STATES and propagate: raise self.exception_to_python(meta['result']) # consume() always returns READY_STATE. return meta['result']
def apply_async_and_get_result(self, args=None, kwargs=None, timeout=None, propagate=True, **options): """ Apply task in an asynchronous way, wait defined timeout and get AsyncResult or TimeoutError :param args: task args :param kwargs: task kwargs :param timeout: timout in seconds to wait for result :param propagate: propagate or not exceptions from celery task :param options: apply_async method options :return: AsyncResult or TimeoutError """ result = self.apply_async(args=args, kwargs=kwargs, **options) if timeout is None or timeout > 0: return result.get(timeout=timeout, propagate=propagate) else: raise TimeoutError('The operation timed out.')
def iterate(self, timeout=None, propagate=True, interval=0.5): """Deprecated method, use :meth:`get` with a callback argument.""" elapsed = 0.0 results = OrderedDict( (result.id, copy(result)) for result in self.results) while results: removed = set() for task_id, result in items(results): if result.ready(): yield result.get(timeout=timeout and timeout - elapsed, propagate=propagate) removed.add(task_id) else: if result.backend.subpolling_interval: time.sleep(result.backend.subpolling_interval) for task_id in removed: results.pop(task_id, None) time.sleep(interval) elapsed += interval if timeout and elapsed >= timeout: raise TimeoutError('The operation timed out')
def iterate(self, timeout=None, propagate=True, interval=0.5): """Iterate over the return values of the tasks as they finish one by one. :raises: The exception if any of the tasks raised an exception. """ elapsed = 0.0 results = dict( (result.task_id, copy(result)) for result in self.results) while results: removed = set() for task_id, result in results.iteritems(): yield result.get(timeout=timeout and timeout - elapsed, propagate=propagate, interval=0.0) removed.add(task_id) for task_id in removed: results.pop(task_id, None) time.sleep(interval) elapsed += interval if timeout and elapsed >= timeout: raise TimeoutError("The operation timed out")
def join(self, timeout=None, propagate=True, interval=0.5, callback=None, no_ack=True, on_message=None, disable_sync_subtasks=True, on_interval=None): """Gather the results of all tasks as a list in order. Note: This can be an expensive operation for result store backends that must resort to polling (e.g., database). You should consider using :meth:`join_native` if your backend supports it. Warning: Waiting for tasks within a task may lead to deadlocks. Please see :ref:`task-synchronous-subtasks`. Arguments: timeout (float): The number of seconds to wait for results before the operation times out. propagate (bool): If any of the tasks raises an exception, the exception will be re-raised when this flag is set. interval (float): Time to wait (in seconds) before retrying to retrieve a result from the set. Note that this does not have any effect when using the amqp result store backend, as it does not use polling. callback (Callable): Optional callback to be called for every result received. Must have signature ``(task_id, value)`` No results will be returned by this function if a callback is specified. The order of results is also arbitrary when a callback is used. To get access to the result object for a particular id you'll have to generate an index first: ``index = {r.id: r for r in gres.results.values()}`` Or you can create new result objects on the fly: ``result = app.AsyncResult(task_id)`` (both will take advantage of the backend cache anyway). no_ack (bool): Automatic message acknowledgment (Note that if this is set to :const:`False` then the messages *will not be acknowledged*). disable_sync_subtasks (bool): Disable tasks to wait for sub tasks this is the default configuration. CAUTION do not enable this unless you must. Raises: celery.exceptions.TimeoutError: if ``timeout`` isn't :const:`None` and the operation takes longer than ``timeout`` seconds. """ if disable_sync_subtasks: assert_will_not_block() time_start = monotonic() remaining = None if on_message is not None: raise ImproperlyConfigured( 'Backend does not support on_message callback') results = [] for result in self.results: remaining = None if timeout: remaining = timeout - (monotonic() - time_start) if remaining <= 0.0: raise TimeoutError('join operation timed out') value = result.get( timeout=remaining, propagate=propagate, interval=interval, no_ack=no_ack, on_interval=on_interval, ) if callback: callback(result.id, value) else: results.append(value) return results
def on_timeout(): raise TimeoutError("The operation timed out.")
def test_run_task(self): try: clean_media.delay().get(timeout=2) except TimeoutError as e: message = str(e) + ' Is the Celery worker running?' raise TimeoutError(message)