def __init__(self, results, app=None, ready_barrier=None, **kwargs):
self._app = app
self._cache = None
self.results = results
self.on_ready = promise()
self._on_full = ready_barrier
if self._on_full:
self._on_full.then(promise(self.on_ready))
def test_cancel_sv(self):
p = promise()
a = promise(Mock(name='a'))
p.then(a)
p.cancel()
self.assertTrue(p.cancelled)
self.assertTrue(a.cancelled)
p.throw(KeyError())
def test_cancel_sv(self):
p = promise()
a = promise(Mock(name='a'))
p.then(a)
p.cancel()
self.assertTrue(p.cancelled)
self.assertTrue(a.cancelled)
p.throw(KeyError())
def test_svpending_raises(self):
p = promise()
a_on_error = promise(Mock(name='a_on_error'))
a = promise(Mock(name='a'), on_error=a_on_error)
p.then(a)
exc = KeyError()
a.fun.side_effect = exc
p(42)
a_on_error.fun.assert_called_with(exc)
def test_wrap(self):
cb1 = Mock()
cb2 = Mock()
x = wrap(promise(cb1))
x(1, y=2)
cb1.assert_called_with(1, y=2)
p2 = promise(cb2)
x(p2)
p2()
cb1.assert_called_with(cb2())
def test_svpending_raises(self):
p = promise()
a_on_error = promise(Mock(name='a_on_error'))
a = promise(Mock(name='a'), on_error=a_on_error)
p.then(a)
exc = KeyError()
a.fun.side_effect = exc
p(42)
a_on_error.fun.assert_called_with(exc)
def get(self, timeout=None, propagate=True, interval=0.5,
no_ack=True, follow_parents=True, callback=None, on_interval=None,
EXCEPTION_STATES=states.EXCEPTION_STATES,
PROPAGATE_STATES=states.PROPAGATE_STATES):
"""Wait until task is ready, and return its result.
.. warning::
Waiting for tasks within a task may lead to deadlocks.
Please read :ref:`task-synchronous-subtasks`.
:keyword timeout: How long to wait, in seconds, before the
operation times out.
:keyword propagate: Re-raise exception if the task failed.
:keyword interval: Time to wait (in seconds) before retrying to
retrieve the result. Note that this does not have any effect
when using the amqp result store backend, as it does not
use polling.
:keyword no_ack: Enable amqp no ack (automatically acknowledge
message). If this is :const:`False` then the message will
**not be acked**.
:keyword follow_parents: Reraise any exception raised by parent task.
:raises celery.exceptions.TimeoutError: if `timeout` is not
:const:`None` and the result does not arrive within `timeout`
seconds.
If the remote call raised an exception then that exception will
be re-raised.
"""
assert_will_not_block()
_on_interval = promise()
if follow_parents and propagate and self.parent:
on_interval = promise(self._maybe_reraise_parent_error)
self._maybe_reraise_parent_error()
if on_interval:
_on_interval.then(on_interval)
if self._cache:
if propagate:
self.maybe_throw(callback=callback)
return self.result
self.backend.add_pending_result(self)
return self.backend.wait_for_pending(
self, timeout=timeout,
interval=interval,
on_interval=_on_interval,
no_ack=no_ack,
propagate=propagate,
callback=callback,
)
def test_shallow_filter(self):
a, b = promise(Mock(name='a')), promise(Mock(name='b'))
p = promise(a, callback=b)
self.assertIsNotNone(p._svpending)
self.assertIsNone(p._lvpending)
p(30)
self.assertIsNone(p._svpending)
a.fun.assert_called_with(30)
b.fun.assert_called_with(a.fun.return_value)
c, d = Mock(name='c'), Mock(name='d')
promise(c, callback=d)(1)
c.assert_called_with(1)
d.assert_called_with(c.return_value)
def test_shallow_filter(self):
a, b = promise(Mock(name='a')), promise(Mock(name='b'))
p = promise(a, callback=b)
self.assertIsNotNone(p._svpending)
self.assertIsNone(p._lvpending)
p(30)
self.assertIsNone(p._svpending)
a.fun.assert_called_with(30)
b.fun.assert_called_with(a.fun.return_value)
c, d = Mock(name='c'), Mock(name='d')
promise(c, callback=d)(1)
c.assert_called_with(1)
d.assert_called_with(c.return_value)
def test_chained_filter(self):
a = promise(Mock(name='a'))
b = promise(Mock(name='b'))
c = promise(Mock(name='c'))
d = promise(Mock(name='d'))
p = promise(a)
p.then(b).then(c).then(d)
p(42, kw=300)
a.fun.assert_called_with(42, kw=300)
b.fun.assert_called_with(a.fun.return_value)
c.fun.assert_called_with(b.fun.return_value)
d.fun.assert_called_with(c.fun.return_value)
def test_chained_filter(self):
a = promise(Mock(name='a'))
b = promise(Mock(name='b'))
c = promise(Mock(name='c'))
d = promise(Mock(name='d'))
p = promise(a)
p.then(b).then(c).then(d)
p(42, kw=300)
a.fun.assert_called_with(42, kw=300)
b.fun.assert_called_with(a.fun.return_value)
c.fun.assert_called_with(b.fun.return_value)
d.fun.assert_called_with(c.fun.return_value)
def test_reverse(self):
callback = Mock()
x = barrier(self.ps, callback=promise(callback))
for p in self.ps:
p()
self.assertTrue(x.ready)
callback.assert_called_with()
def __init__(self, id, ret_value, state, traceback=None):
self.id = id
self._result = ret_value
self._state = state
self._traceback = traceback
self.on_ready = promise()
self.on_ready()
def _mexe(self, request, sender=None, callback=None):
callback = callback or promise()
boto.log.debug(
'HTTP %s %s/%s headers=%s body=%s',
request.host,
request.path,
request.headers,
request.body,
)
conn = self.get_http_connection(
request.host,
request.port,
self.is_secure,
)
request.authorize(connection=self)
if callable(sender):
sender(conn, request.method, request.path, request.body,
request.headers, callback)
else:
conn.request(request.method, request.path, request.body,
request.headers)
conn.getresponse(callback=callback)
return callback
def test_evaluate(self):
x = barrier(self.ps)
x()
self.assertFalse(x.ready)
x()
self.assertFalse(x.ready)
x.add(promise())
x()
self.assertFalse(x.ready)
x()
self.assertTrue(x.ready)
x()
x()
with self.assertRaises(ValueError):
x.add(promise())
def test_cancel(self):
x = barrier(self.ps)
x.cancel()
for p in self.ps:
p()
x.add(promise())
x.throw(KeyError())
self.assertFalse(x.ready)
def test_lvpending_raises(self):
p = promise()
a_on_error = promise(Mock(name='a_on_error'))
a = promise(Mock(name='a'), on_error=a_on_error)
b_on_error = promise(Mock(name='b_on_error'))
b = promise(Mock(name='a'), on_error=b_on_error)
p.then(a)
p.then(b)
exc = KeyError()
a.fun.side_effect = exc
a.then(Mock(name='foobar'))
a.then(Mock(name='foozi'))
p(42)
a_on_error.fun.assert_called_with(exc)
b.fun.assert_called_with(42)
def test_lvpending_raises(self):
p = promise()
a_on_error = promise(Mock(name='a_on_error'))
a = promise(Mock(name='a'), on_error=a_on_error)
b_on_error = promise(Mock(name='b_on_error'))
b = promise(Mock(name='a'), on_error=b_on_error)
p.then(a)
p.then(b)
exc = KeyError()
a.fun.side_effect = exc
a.then(Mock(name='foobar'))
a.then(Mock(name='foozi'))
p(42)
a_on_error.fun.assert_called_with(exc)
b.fun.assert_called_with(42)
def _schedule_queue(self, queue):
if queue in self._active_queues:
if self.qos.can_consume():
self._get_bulk_async(
queue, callback=promise(self._loop1, (queue, )),
)
else:
self._loop1(queue)
def _schedule_queue(self, queue):
if queue in self._active_queues:
if self.qos.can_consume():
self._get_bulk_async(
queue,
callback=promise(self._loop1, (queue, )),
)
else:
self._loop1(queue)
def __init__(self, id, backend=None,
task_name=None, # deprecated
app=None, parent=None):
if id is None:
raise ValueError(
'AsyncResult requires valid id, not {0}'.format(type(id)))
self.app = app_or_default(app or self.app)
self.id = id
self.backend = backend or self.app.backend
self.parent = parent
self.on_ready = promise(self._on_fulfilled)
self._cache = None
def test_deep_filter(self):
a = promise(Mock(name='a'))
b1, b2, b3 = (
promise(Mock(name='a1')),
promise(Mock(name='a2')),
promise(Mock(name='a3')),
)
p = promise(a)
p.then(b1)
self.assertIsNone(p._lvpending)
self.assertIsNotNone(p._svpending)
p.then(b2)
self.assertIsNotNone(p._lvpending)
self.assertIsNone(p._svpending)
p.then(b3)
p(42)
a.fun.assert_called_with(42)
b1.fun.assert_called_with(a.fun.return_value)
b2.fun.assert_called_with(a.fun.return_value)
b3.fun.assert_called_with(a.fun.return_value)
def test_transform(self):
callback = Mock()
def filter_key_value(key, filter_, mapping):
return filter_(mapping[key])
x = transform(filter_key_value, promise(callback), 'Value', int)
x({'Value': 303})
callback.assert_called_with(303)
with self.assertRaises(KeyError):
x({})
def test_deep_filter(self):
a = promise(Mock(name='a'))
b1, b2, b3 = (
promise(Mock(name='a1')),
promise(Mock(name='a2')),
promise(Mock(name='a3')),
)
p = promise(a)
p.then(b1)
self.assertIsNone(p._lvpending)
self.assertIsNotNone(p._svpending)
p.then(b2)
self.assertIsNotNone(p._lvpending)
self.assertIsNone(p._svpending)
p.then(b3)
p(42)
a.fun.assert_called_with(42)
b1.fun.assert_called_with(a.fun.return_value)
b2.fun.assert_called_with(a.fun.return_value)
b3.fun.assert_called_with(a.fun.return_value)
def on_task_received(message):
# payload will only be set for v1 protocol, since v2
# will defer deserializing the message body to the pool.
payload = None
try:
type_ = message.headers['task'] # protocol v2
except TypeError:
return on_unknown_message(None, message)
except KeyError:
try:
payload = message.decode()
except Exception as exc:
return self.on_decode_error(message, exc)
try:
type_, payload = payload['task'], payload # protocol v1
except (TypeError, KeyError):
return on_unknown_message(payload, message)
try:
strategy = strategies[type_]
except KeyError as exc:
return on_unknown_task(payload, message, exc)
else:
try:
strategy(
message,
payload,
promise(call_soon, (message.ack_log_error, )),
promise(call_soon, (message.reject_log_error, )),
callbacks,
)
except InvalidTaskError as exc:
return on_invalid_task(payload, message, exc)
except MemoryError:
raise
except Exception as exc:
# XXX handle as internal error?
return on_invalid_task(payload, message, exc)
def on_task_received(message):
# payload will only be set for v1 protocol, since v2
# will defer deserializing the message body to the pool.
payload = None
try:
type_ = message.headers['task'] # protocol v2
except TypeError:
return on_unknown_message(None, message)
except KeyError:
try:
payload = message.decode()
except Exception as exc:
return self.on_decode_error(message, exc)
try:
type_, payload = payload['task'], payload # protocol v1
except (TypeError, KeyError):
return on_unknown_message(payload, message)
try:
strategy = strategies[type_]
except KeyError as exc:
return on_unknown_task(payload, message, exc)
else:
try:
strategy(
message, payload,
promise(call_soon, (message.ack_log_error,)),
promise(call_soon, (message.reject_log_error,)),
callbacks,
)
except InvalidTaskError as exc:
return on_invalid_task(payload, message, exc)
except MemoryError:
raise
except Exception as exc:
# XXX handle as internal error?
return on_invalid_task(payload, message, exc)
def __init__(self, url, method='GET', on_ready=None, on_timeout=None,
on_stream=None, on_prepare=None, on_header=None,
headers=None, **kwargs):
self.url = url
self.method = method or self.method
self.on_ready = maybe_promise(on_ready) or promise()
self.on_timeout = maybe_promise(on_timeout)
self.on_stream = maybe_promise(on_stream)
self.on_prepare = maybe_promise(on_prepare)
self.on_header = maybe_promise(on_header)
if kwargs:
for k, v in items(kwargs):
setattr(self, k, v)
if not isinstance(headers, Headers):
headers = Headers(headers or {})
self.headers = headers
def test_throw_from_cb(self):
ae = promise(Mock(name='ae'))
a = Mock(name='a')
be = promise(Mock(name='be'))
b = promise(Mock(name='b'), on_error=be)
ce = promise(Mock(name='ce'))
c = promise(Mock(name='c'), on_error=ce)
exc = a.side_effect = KeyError()
p1 = promise(a, on_error=ae)
p1.then(b)
self.assertTrue(p1._svpending)
p1(42)
p1.on_error.fun.assert_called_with(exc)
p2 = promise(a)
p2.then(b).then(c)
p2(42)
de = promise(Mock(name='de'))
d = promise(Mock(name='d'), on_error=de)
p2.then(d)
de.fun.assert_called_with(exc)
def test_throw_from_cb(self):
ae = promise(Mock(name='ae'))
a = Mock(name='a')
be = promise(Mock(name='be'))
b = promise(Mock(name='b'), on_error=be)
ce = promise(Mock(name='ce'))
c = promise(Mock(name='c'), on_error=ce)
exc = a.side_effect = KeyError()
p1 = promise(a, on_error=ae)
p1.then(b)
self.assertTrue(p1._svpending)
p1(42)
p1.on_error.fun.assert_called_with(exc)
p2 = promise(a)
p2.then(b).then(c)
p2(42)
de = promise(Mock(name='de'))
d = promise(Mock(name='d'), on_error=de)
p2.then(d)
de.fun.assert_called_with(exc)
def _mexe(self, request, sender=None, callback=None):
callback = callback or promise()
boto.log.debug(
'HTTP %s %s/%s headers=%s body=%s',
request.host, request.path,
request.headers, request.body,
)
conn = self.get_http_connection(
request.host, request.port, self.is_secure,
)
request.authorize(connection=self)
if callable(sender):
sender(conn, request.method, request.path, request.body,
request.headers, callback)
else:
conn.request(request.method, request.path,
request.body, request.headers)
conn.getresponse(callback=callback)
return callback
def test_chained(self):
def add(x, y):
return x + y
def pow2(x):
return x ** 2
adder = Mock(name='adder')
adder.side_effect = add
power = Mock(name='multiplier')
power.side_effect = pow2
final = Mock(name='final')
p = promise()
p.then(adder).then(power).then(final)
p(42, 42)
self.assertEqual(p.value, ((42, 42), {}))
adder.assert_called_with(42, 42)
power.assert_called_with(84)
final.assert_called_with(7056)
def test_chained(self):
def add(x, y):
return x + y
def pow2(x):
return x ** 2
adder = Mock(name='adder')
adder.side_effect = add
power = Mock(name='multiplier')
power.side_effect = pow2
final = Mock(name='final')
p = promise()
p.then(adder).then(power).then(final)
p(42, 42)
self.assertEqual(p.value, ((42, 42), {}))
adder.assert_called_with(42, 42)
power.assert_called_with(84)
final.assert_called_with(7056)
def test_cancel(self):
on_error = promise(Mock(name='on_error'))
p = promise(on_error=on_error)
a, b, c = (
promise(Mock(name='a')),
promise(Mock(name='b')),
promise(Mock(name='c')),
)
a2 = promise(Mock(name='a1'))
p.then(a).then(b).then(c)
p.then(a2)
p.cancel()
p(42)
self.assertTrue(p.cancelled)
self.assertTrue(a.cancelled)
self.assertTrue(a2.cancelled)
self.assertTrue(b.cancelled)
self.assertTrue(c.cancelled)
self.assertTrue(on_error.cancelled)
d = promise(Mock(name='d'))
p.then(d)
self.assertTrue(d.cancelled)
def test_cancel(self):
on_error = promise(Mock(name='on_error'))
p = promise(on_error=on_error)
a, b, c = (
promise(Mock(name='a')),
promise(Mock(name='b')),
promise(Mock(name='c')),
)
a2 = promise(Mock(name='a1'))
p.then(a).then(b).then(c)
p.then(a2)
p.cancel()
p(42)
self.assertTrue(p.cancelled)
self.assertTrue(a.cancelled)
self.assertTrue(a2.cancelled)
self.assertTrue(b.cancelled)
self.assertTrue(c.cancelled)
self.assertTrue(on_error.cancelled)
d = promise(Mock(name='d'))
p.then(d)
self.assertTrue(d.cancelled)
def test_signal(self):
callback = Mock(name='callback')
a = promise()
a.then(callback)
a(42)
callback.assert_called_with(42)
def read(self, size, callback=None):
callback = callback or promise()
_pending.append((size, callback))
return callback
def test_promise(self):
self.assertIsInstance(promise(lambda x: x), Thenable)
def test_empty_promise(self):
p = promise()
p(42)
x = Mock(name='x')
p.then(x)
x.assert_called_with(42)
def test_signal(self):
callback = Mock(name='callback')
a = promise()
a.then(callback)
a(42)
callback.assert_called_with(42)
def test_repr(self):
self.assertTrue(repr(promise()))
def test_with_partial_args_and_args(self):
m = Mock(name='m')
p = promise(m, (1, 2, 3), {'foobar': 2})
p(4, 5, bazbar=3)
m.assert_called_with(1, 2, 3, 4, 5, foobar=2, bazbar=3)
def test_with_partial_args(self):
m = Mock(name='m')
p = promise(m, (1, 2, 3), {'foobar': 2})
p()
m.assert_called_with(1, 2, 3, foobar=2)
def test_empty_promise(self):
p = promise()
p(42)
x = Mock(name='x')
p.then(x)
x.assert_called_with(42)
def test_promise(self):
self.assertIsInstance(promise(lambda x: x), Thenable)
def test_repr(self):
self.assertTrue(repr(promise()))
def read(self, size, callback=None):
callback = callback or promise()
_pending.append((size, callback))
return callback
def call_soon(self, callback, *args):
if not isinstance(callback, Thenable):
callback = promise(callback, args)
self._ready.add(callback)
return callback
def test_with_partial_args_and_args(self):
m = Mock(name='m')
p = promise(m, (1, 2, 3), {'foobar': 2})
p(4, 5, bazbar=3)
m.assert_called_with(1, 2, 3, 4, 5, foobar=2, bazbar=3)
def call_soon(self, callback, *args):
if not isinstance(callback, Thenable):
callback = promise(callback, args)
self._ready.add(callback)
return callback
def test_with_partial_args(self):
m = Mock(name='m')
p = promise(m, (1, 2, 3), {'foobar': 2})
p()
m.assert_called_with(1, 2, 3, foobar=2)
