def testNestedhandle(self):
handle = lambda value : reduce(lambda x,y: x+y, value)\
if isinstance(value, list) else value
d = MultiDeferred(handle_value=handle)
d.append((a for a in range(1,11)))
r = maybe_async(d.lock())
self.assertFalse(is_async(r))
self.assertEqual(r, [55])
handle = lambda value: 'c'*value
d = MultiDeferred(handle_value=handle)
d.append((a for a in range(1,11)))
r = maybe_async(d.lock())
self.assertFalse(is_async(r))
self.assertTrue(is_failure(r[0]))
def do_stream(self, request):
# stream the body
body = self.body.render(request)
# the body has asynchronous components
# delay the header untl later
if is_async(body):
yield self._html(request, body)
head = self.head.render(request)
#
# header not ready (this should never occur really)
if is_async(head):
yield self._html(request, body, head)
else:
yield self._template % (self.flatatt(), head, body)
def do_stream(self, request):
# stream the body
body = self.body.render(request)
# the body has asynchronous components
# delay the header untl later
if is_async(body):
yield self._html(request, body)
head = self.head.render(request)
#
# header not ready (this should never occur really)
if is_async(head):
yield self._html(request, body, head)
else:
yield self._template % (self.flatatt(), head, body)
def __iter__(self):
request = self.request
rpc = request['rpc']
status_code = 200
try:
result = rpc.process(request)
except Exception as e:
result = as_failure(e)
handler = rpc.handler
result = maybe_async(result)
while is_async(result):
yield b''
result = maybe_async(result)
try:
if is_failure(result):
e = result.trace[1]
status_code = getattr(e, 'status', 400)
log_failure(result)
result = handler.dumps(rpc.id, rpc.version, error=e)
else:
result = handler.dumps(rpc.id, rpc.version, result=result)
except Exception as e:
LOGGER.error('Could not serialize', exc_info=True)
status_code = 500
result = handler.dumps(rpc.id, rpc.version, error=e)
content = to_bytes(result)
response = WsgiResponse(status_code, content,
content_type=handler.content_type)
for c in self.start(response):
yield c
def main(loop, syspath, params, stask):
logger = LOGGER
try:
from pq import TaskApp
sys.path[:] = json.loads(syspath)
params = json.loads(params)
params.update({'python_path': False,
'parse_console': False})
producer = TaskApp(**params).backend
logger = producer.logger
yield from producer.ready()
task = producer._pubsub.load(stask)
JobClass = producer.registry.get(task.name)
if not JobClass:
raise RuntimeError('%s not in registry' % task.name)
job = JobClass(producer, task)
result = yield from job.green_pool.submit(job, **task.kwargs)
if is_async(result):
result = yield from result
sys.stdout.write({'cpubound_result': result})
except Exception:
exc_info = sys.exc_info()
error = str(exc_info[1])
stacktrace = traceback.format_tb(exc_info[2])
sys.stderr.write({'cpubound_failure': (error, stacktrace)})
msg = '%s\n%s' % (error, ''.join(stacktrace))
logger.error(msg)
def testNested(self):
d = MultiDeferred()
# add a generator
d.append((a for a in range(1,11)))
r = maybe_async(d.lock())
self.assertTrue(d.locked)
self.assertFalse(is_async(r))
self.assertEqual(r, [[1,2,3,4,5,6,7,8,9,10]])
def __call__(self):
try:
r = self._call()
except Exception as e:
r = e
if pulsar.is_async(r):
return r.add_callback(self)
else:
return self.callback(r)
def stream_mapping(value, request):
result = {}
async = False
for key, value in value.items():
if isinstance(value, String):
value = value.render(request)
if is_async(value):
async = True
result[key] = value
return multi_async(result) if async else result
def __call__(self, *args, **kwargs):
data, raw = self._get_data(*args, **kwargs)
body = self._json.dumps(data).encode('latin-1')
# Always make sure the content-type is application/json
self.http.headers['content-type'] = 'application/json'
resp = self.http.post(self.__url, data=body)
if is_async(resp):
return resp.add_callback(lambda r: self._end_call(r, raw))
else:
return self._end_call(resp, raw)
def stream_mapping(value, request):
result = {}
async = False
for key, value in value.items():
if isinstance(value, String):
value = value.render(request)
if is_async(value):
async = True
result[key] = value
return multi_async(result) if async else result
def parse(self, mem_limit=None, **kw):
mem_limit = mem_limit or DEFAULT_MAXSIZE
if self.content_length > mem_limit:
raise HttpException("Request to big. Increase MAXMEM.", status=LARGE_BODY_CODE)
inp = self.environ.get("wsgi.input") or BytesIO()
data = inp.read()
if is_async(data):
return async(self._async(data))
else:
return self._ready(data)
def _html(self, request, body, head=None):
'''Asynchronous rendering
'''
if head is None:
body = yield from body
head = self.head.render(request)
if is_async(head):
head = yield from head
return self._template % (self.flatatt(), head, body)
def rpc_server_info(self, request):
'''Return a dictionary of information regarding the server and workers.
It invokes the :meth:`extra_server_info` for adding custom
information.
'''
info = yield from pulsar.send('arbiter', 'info')
info = self.extra_server_info(request, info)
if is_async(info):
info = yield from info
return info
def testPingArbiter(self):
worker = pulsar.get_actor()
yield self.async.assertEqual(send('arbiter', 'ping'), 'pong')
yield self.async.assertEqual(send('arbiter', 'ping'), 'pong')
result = worker.send('arbiter', 'notify', worker.info())
if is_async(result):
yield outcome
result = outcome.result
self.assertTrue(result>0)
yield self.async.assertEqual(send('arbiter', 'ping'), 'pong')
yield self.async.assertEqual(send('arbiter', 'echo', 'ciao'), 'ciao')
def rpc_server_info(self, request):
'''Return a dictionary of information regarding the server and workers.
It invokes the :meth:`extra_server_info` for adding custom
information.
'''
info = yield from pulsar.send('arbiter', 'info')
info = self.extra_server_info(request, info)
if is_async(info):
info = yield from info
return info
def _html(self, request, body, head=None):
'''Asynchronous rendering
'''
if head is None:
body = yield from body
head = self.head.render(request)
if is_async(head):
head = yield from head
return self._template % (self.flatatt(), head, body)
def __call__(self, client, keys=None, args=None):
'''Execute the script, passing any required ``args``
'''
if self.sha not in client.store.loaded_scripts:
sha = yield from client.immediate_execute('SCRIPT', 'LOAD',
self.script)
self.sha = sha.decode('utf-8')
client.store.loaded_scripts.add(self.sha)
result = client.evalsha(self.sha, keys, args)
if is_async(result):
result = yield from result
return result
def __call__(self, client, keys=None, args=None):
'''Execute the script, passing any required ``args``
'''
if self.sha not in client.store.loaded_scripts:
sha = yield from client.immediate_execute('SCRIPT', 'LOAD',
self.script)
self.sha = sha.decode('utf-8')
client.store.loaded_scripts.add(self.sha)
result = client.evalsha(self.sha, keys, args)
if is_async(result):
result = yield from result
return result
def _(*args, **kwargs):
green = GreenletWorker(callable)
# switch to the new greenlet
result = green.switch(*args, **kwargs)
# back to the parent
while is_async(result):
# keep on switching back to the greenlet if we get a Future
try:
result = green.switch((yield from result))
except Exception as exc:
result = green.throw(exc)
return green.switch(result)
def _(*args, **kwargs):
green = GreenletWorker(callable)
# switch to the new greenlet
result = green.switch(*args, **kwargs)
# back to the parent
while is_async(result):
# keep on switching back to the greenlet if we get a Future
try:
result = green.switch((yield from result))
except Exception as exc:
result = green.throw(exc)
return green.switch(result)
def _green_task(self, greenlet, task):
# Run in the main greenlet of the event-loop thread
while task is not _DONE:
# switch to the greenlet to start the task
task = greenlet.switch(task)
# if an asynchronous result is returned, yield from
while is_async(task):
try:
task = yield from task
except Exception as exc:
# This call can return an asynchronous component
task = greenlet.throw(exc)
def as_frame(self, connection, body):
'''Build a websocket server frame from body.'''
body = maybe_async(body)
if is_async(body):
return body.addBoth(lambda b: self.as_frame(connection, b))
if is_failure(body):
# We have a failure. shut down connection
body.log()
body = Frame.close('Server error')
elif not isinstance(body, Frame):
# If the body is not a frame, build a final frame from it.
body = Frame(body or '', version=connection.protocol.version,
final=True)
return body.msg
def _green_task(self, greenlet, task):
# Run in the main greenlet of the event-loop thread
while task is not _DONE:
# switch to the greenlet to start the task
task = greenlet.switch(task)
# if an asynchronous result is returned, yield from
while is_async(task):
try:
task = yield from task
except Exception as exc:
# This call can return an asynchronous component
task = greenlet.throw(exc)
def wait_for_body_middleware(environ, start_response=None):
'''Use this middleware to wait for the full body.
This middleware wait for the full body to be received before letting
other middleware to be processed.
Useful when using synchronous web-frameworks such as :django:`django <>`.
'''
if environ['wsgi.input']:
stream = environ['wsgi.input']
chunk = stream.read()
if is_async(chunk):
chunk = yield from chunk
environ['wsgi.input'] = BytesIO(chunk)
def testDeferredCallback(self):
d = Deferred()
d.add_callback(lambda r : Cbk(r))
self.assertFalse(d.called)
result = d.callback('ciao')
self.assertTrue(d.called)
self.assertEqual(d.paused,1)
self.assertTrue(is_async(result))
self.assertEqual(len(result._callbacks),1)
self.assertFalse(result.called)
result.set_result('luca')
self.assertTrue(result.called)
self.assertEqual(result.result,('ciao','luca'))
self.assertEqual(d.paused,0)
def wait_for_body_middleware(environ, start_response=None):
'''Use this middleware to wait for the full body.
This middleware wait for the full body to be received before letting
other middleware to be processed.
Useful when using synchronous web-frameworks such as :django:`django <>`.
'''
if environ['wsgi.input']:
stream = environ['wsgi.input']
chunk = stream.read()
if is_async(chunk):
chunk = yield from chunk
environ['wsgi.input'] = BytesIO(chunk)
def _green_task(self, green, task):
# Coroutine executing the in main greenlet
# This coroutine is executed for every task put into the queue
while task is not _DONE:
# switch to the greenlet to start the task
task = green.switch(task)
# if an asynchronous result is returned, yield from
while is_async(task):
try:
task = yield from task
except Exception as exc:
# This call can return an asynchronous component
task = green.throw(exc)
def _green_task(self, green, task):
# Coroutine executing the in main greenlet
# This coroutine is executed for every task put into the queue
while task is not _DONE:
# switch to the greenlet to start the task
task = green.switch(task)
# if an asynchronous result is returned, yield from
while is_async(task):
try:
task = yield from task
except Exception as exc:
# This call can return an asynchronous component
task = green.throw(exc)
def response_generator(self, response, wsgi_response):
response = maybe_async(response)
while is_async(response):
yield b''
response = maybe_async(response)
stream_content = None
if is_failure(response):
wsgi_response.status_code = 500
else:
wsgi_response.status_code = response.status_code
wsgi_response.headers.update(response.headers)
stream_content = response.stream()
wsgi_response.start()
if stream_content:
for content in stream_content:
yield content
def _consume(self, job, kwargs):
concurrency = job.get_concurrency()
if concurrency == models.ASYNC_IO:
result = job(**kwargs)
assert is_async(result), "ASYNC_IO tasks not asynchronous"
return result
elif concurrency == models.GREEN_IO:
return self.green_pool.submit(job, **kwargs)
elif concurrency == models.THREAD_IO:
return job._loop.run_in_executor(None, lambda: job(**kwargs))
elif concurrency == models.CPUBOUND:
return self._consume_in_subprocess(job, kwargs)
else:
raise ImproperlyConfigured('invalid concurrency "%d"' %
concurrency)
def render(self, request=None, callback=None):
'''Render this string.
This method returns a string or a :class:`~asyncio.Future` which
results in a string. On the other hand, the callable method of
a :class:`.String` **always** returns a :class:`~asyncio.Future`.
'''
stream = []
async = False
for data in self.stream(request):
if is_async(data):
async = True
stream.append(data)
if not callback:
callback = self.to_string
if async:
return chain_future(multi_async(stream), callback=callback)
else:
return callback(stream)
def wsgi_iterator(gen, encoding=None):
encoding = encoding or "utf-8"
for data in gen:
data = maybe_async(data)
while is_async(data):
yield b""
data = maybe_async(data)
if data is NOT_DONE:
yield b""
elif data is None:
continue
elif is_failure(data):
log_failure(data)
else:
if isinstance(data, bytes):
yield data
elif isinstance(data, string_type):
yield data.encode(encoding)
else:
for b in wsgi_iterator(data, encoding):
yield b
def render(self, request=None, callback=None):
'''Render this string.
This method returns a string or a :class:`~asyncio.Future` which
results in a string. On the other hand, the callable method of
a :class:`.String` **always** returns a :class:`~asyncio.Future`.
'''
stream = []
async = False
for data in self.stream(request):
if is_async(data):
async = True
stream.append(data)
if not callback:
callback = self.to_string
if async:
return chain_future(multi_async(stream), callback=callback)
else:
return callback(stream)
def start(self, worker):
'''Called by the :class:`pulsar.Worker` *worker* when the task
start its execution. If no timeout has occured the task will switch to
a ``STARTED`` :attr:`Task.status` and invoke the :meth:`on_start`
callback.'''
job = registry[self.name]
result = None
try:
if self.maybe_revoked():
yield self.on_timeout(worker)
else:
self.status = STARTED
self.time_start = datetime.now()
yield self.on_start(worker)
consumer = TaskConsumer(self, worker, job)
result = maybe_async(job(consumer, *self.args, **self.kwargs))
if is_async(result):
yield result
result = maybe_async(result)
except Exception as e:
result = as_failure(e)
finally:
yield self.finish(worker, result)
def ascoro(result=None):
if is_async(result):
result = yield from result
return result
def _write_streamed_data(self, transport):
for data in self.data:
if is_async(data):
data = yield from data
self._write_body_data(transport, data)
self._write_body_data(transport, b'', True)
def _(*args, **kwargs):
args = yield from multi_async(args)
result = getattr(self.test, name)(*args, **kwargs)
if is_async(result):
result = yield from result
return result
def _(*args, **kwargs):
args = yield from multi_async(args)
result = getattr(self.test, name)(*args, **kwargs)
if is_async(result):
result = yield from result
return result
