def do_stream(self, request):
# stream the body
body = self.body.render(request)
# the body has asynchronous components
# delay the header until later
if isawaitable(body):
yield self._html(request, body)
head = self.head.render(request)
#
# header not ready (this should never occur really)
if isawaitable(head):
yield self._html(request, body, head)
else:
yield self._template % (self.flatatt(), head, body)
async def _run_safe(self, test, method_name, test_timeout, error=None):
exc = None
self._check_abort()
try:
method = getattr(test, method_name)
coro = method()
# a coroutine
if isawaitable(coro):
test_timeout = get_test_timeout(method, test_timeout)
exc = await asyncio.wait_for(
store_trace(coro),
test_timeout,
loop=self._loop
)
elif isgenerator(coro):
raise InvalidTestFunction('test function returns a generator')
except SkipTest as x:
self.runner.addSkip(test, str(x))
exc = None
except Exception as x:
exc = TestFailure(x)
if exc and not error:
error = exc
self.add_failure(test, error, expecting_failure(method))
return error
def emit(self, record):
"""Emit record to slack channel using pycurl to avoid recurrence
event logging (log logged record)
"""
if record.message.startswith(MESSAGE): # avoid cyrcular emit
return
cfg = self.app.config
managers = cfg['SLACK_LINK_NAMES']
text = ''
data = {}
if managers:
text = ' '.join(('@%s' % m for m in managers))
text = '%s\n\n' % text
data['link_names'] = 1
context_factory = cfg['LOG_CONTEXT_FACTORY']
data['text'] = text
if context_factory:
ctx = context_factory(self)
data['text'] += "\n" + context_text_formatter(ctx)
data['text'] += "```\n%s\n```" % self.format(record)
sessions = self.app.http()
response = sessions.post(self.webhook_url,
data=json.dumps(data))
if isawaitable(response):
ensure_future(self._emit(response), loop=sessions._loop)
else:
sessions._loop.call_soon(self._raise_error, response)
async def _run(self, method, test_timeout):
self._check_abort()
coro = method()
# a coroutine
if isawaitable(coro):
test_timeout = get_test_timeout(method, test_timeout)
await asyncio.wait_for(coro, test_timeout, loop=self._loop)
elif isgenerator(coro):
raise InvalidTestFunction('test function returns a generator')
async def _run(self, method, test_timeout):
self._check_abort()
coro = method()
# a coroutine
if isawaitable(coro):
test_timeout = get_test_timeout(method, test_timeout)
await asyncio.wait_for(coro, test_timeout, loop=self._loop)
elif isgenerator(coro):
raise InvalidTestFunction('test function returns a generator')
def parse(self, mem_limit=None, **kw):
if self.content_length > self.limit:
raise_large_body_error(self.limit)
inp = self.environ.get('wsgi.input') or BytesIO()
data = inp.read()
if isawaitable(data):
return ensure_future(self._async(data))
else:
return self._ready(data)
async def read(self, n=None):
'''Read all content'''
if self._streamed:
return b''
buffer = []
for body in self:
if isawaitable(body):
body = await body
buffer.append(body)
return b''.join(buffer)
def stream_mapping(value, request):
result = {}
async = False
for key, value in value.items():
if isinstance(value, String):
value = value.render(request)
if isawaitable(value):
async = True
result[key] = value
return multi_async(result) if async else result
def parse(self, mem_limit=None, **kw):
if self.content_length > self.limit:
raise_large_body_error(self.limit)
inp = self.environ.get("wsgi.input") or BytesIO()
data = inp.read()
if isawaitable(data):
return ensure_future(self._async(data))
else:
return self._ready(data)
async def _html(self, request, body, head=None):
'''Asynchronous rendering
'''
if head is None:
body = await body
head = self.head.render(request)
if isawaitable(head):
head = await head
return self._template % (self.flatatt(), head, body)
async def rpc_server_info(self, request):
'''Return a dictionary of information regarding the server and workers.
It invokes the :meth:`extra_server_info` for adding custom
information.
'''
info = await pulsar.send('arbiter', 'info')
info = self.extra_server_info(request, info)
if isawaitable(info):
info = await info
return info
def __call__(self, argv, **params):
if not self.app.callable.command:
self.app.callable.command = self.name
app = self.pulsar_app(argv)
app.cfg.daemon = False
app()
# Make sure the wsgi handler is created
assert self.app.wsgi_handler()
result = maybe_green(self.app, self.run, app.cfg, **params)
if isawaitable(result) and not self.app._loop.is_running():
result = self.app._loop.run_until_complete(result)
return result
def __call__(self, argv, **params):
if not self.app.callable.command:
self.app.callable.command = self.name
app = self.pulsar_app(argv)
app.cfg.daemon = False
app()
# Make sure the wsgi handler is created
assert self.app.wsgi_handler()
result = maybe_green(self.app, self.run, app.cfg, **params)
if isawaitable(result) and not self.app._loop.is_running():
result = self.app._loop.run_until_complete(result)
return result
def _data_and_files(self, data=True, files=True, stream=None, future=None):
if future is None:
data_files = parse_form_data(self.environ, stream=stream)
if isawaitable(data_files):
return chain_future(
data_files,
partial(self._data_and_files, data, files, stream))
else:
data_files = future
self.cache.data_and_files = data_files
return self.data_and_files(data, files, stream)
def _data_and_files(self, data=True, files=True, stream=None, future=None):
if future is None:
data_files = parse_form_data(self.environ, stream=stream)
if isawaitable(data_files):
return chain_future(
data_files,
partial(self._data_and_files, data, files, stream))
else:
data_files = future
self.cache.data_and_files = data_files
return self.data_and_files(data, files, stream)
def parse(self, mem_limit=None, **kw):
mem_limit = mem_limit or DEFAULT_MAXSIZE
if self.content_length > mem_limit:
raise HttpException("Request to big. Increase MAXMEM.",
status=LARGE_BODY_CODE)
inp = self.environ.get('wsgi.input') or BytesIO()
data = inp.read()
if isawaitable(data):
return ensure_future(self._async(data))
else:
return self._ready(data)
async def __call__(self, client, keys=None, args=None):
'''Execute the script, passing any required ``args``
'''
if self.sha not in client.store.loaded_scripts:
sha = await client.immediate_execute('SCRIPT', 'LOAD', self.script)
self.sha = sha.decode('utf-8')
client.store.loaded_scripts.add(self.sha)
result = client.evalsha(self.sha, keys, args)
if isawaitable(result):
result = await result
return result
async def __call__(self, client, keys=None, args=None):
'''Execute the script, passing any required ``args``
'''
if self.sha not in client.store.loaded_scripts:
sha = await client.immediate_execute('SCRIPT', 'LOAD', self.script)
self.sha = sha.decode('utf-8')
client.store.loaded_scripts.add(self.sha)
result = client.evalsha(self.sha, keys, args)
if isawaitable(result):
result = await result
return result
def parse(self, mem_limit=None, **kw):
mem_limit = mem_limit or DEFAULT_MAXSIZE
if self.content_length > mem_limit:
raise HttpException("Request to big. Increase MAXMEM.",
status=LARGE_BODY_CODE)
inp = self.environ.get('wsgi.input') or BytesIO()
data = inp.read()
if isawaitable(data):
return ensure_future(self._async(data))
else:
return self._ready(data)
async def _async_call(self, environ, start_response):
response = None
try:
for middleware in self.middleware:
response = middleware(environ, start_response)
if isawaitable(response):
response = await response
if response is not None:
break
if response is None:
raise Http404
except Exception as exc:
response = handle_wsgi_error(environ, exc)
if isinstance(response, WsgiResponse) and not response.started:
for middleware in self.response_middleware:
response = middleware(environ, response) or response
if isawaitable(response):
response = await response
response.start(start_response)
return response
async def _async_call(self, environ, start_response):
response = None
try:
for middleware in self.middleware:
response = middleware(environ, start_response)
if isawaitable(response):
response = await response
if response is not None:
break
if response is None:
raise Http404
except Exception as exc:
response = handle_wsgi_error(environ, exc)
if isinstance(response, WsgiResponse) and not response.started:
for middleware in self.response_middleware:
response = middleware(environ, response) or response
if isawaitable(response):
response = await response
response.start(start_response)
return response
async def _(*args, **kwargs):
green = greenlet(callable)
# switch to the new greenlet
result = green.switch(*args, **kwargs)
# back to the parent
while isawaitable(result):
# keep on switching back to the greenlet if we get a Future
try:
result = green.switch((await result))
except Exception as exc:
result = green.throw(exc)
return green.switch(result)
async def wait_for_body_middleware(environ, start_response=None):
'''Use this middleware to wait for the full body.
This middleware wait for the full body to be received before letting
other middleware to be processed.
Useful when using synchronous web-frameworks such as :django:`django <>`.
'''
if environ['wsgi.input']:
stream = environ['wsgi.input']
chunk = stream.read()
if isawaitable(chunk):
chunk = await chunk
environ['wsgi.input'] = BytesIO(chunk)
async def _green_task(self, green, task):
# Coroutine executing the in main greenlet
# This coroutine is executed for every task put into the queue
while task is not _DONE:
# switch to the greenlet to start the task
task = green.switch(task)
# if an asynchronous result is returned, await
while isawaitable(task):
try:
task = await task
except Exception as exc:
# This call can return an asynchronous component
task = green.throw(exc)
async def _run_safe(self, test, method_name, test_timeout, error=None):
self._check_abort()
try:
method = getattr(test, method_name)
coro = method()
# a coroutine
if isawaitable(coro):
test_timeout = get_test_timeout(method, test_timeout)
await asyncio.wait_for(coro, test_timeout, loop=self._loop)
elif isgenerator(coro):
raise InvalidTestFunction('test function returns a generator')
except SkipTest as exc:
self.runner.addSkip(test, str(exc))
except Exception as exc:
if not error:
error = TestFailure(exc)
self.add_failure(test, error, expecting_failure(method))
return error
def _run_safe(self, test, method_name, test_timeout, error=None):
self._check_abort()
try:
method = getattr(test, method_name)
coro = method()
# a coroutine
if isawaitable(coro):
test_timeout = get_test_timeout(method, test_timeout)
yield from asyncio.wait_for(coro, test_timeout,
loop=self._loop)
elif isgenerator(coro):
raise InvalidTestFunction('test function returns a generator')
except SkipTest as exc:
self.runner.addSkip(test, str(exc))
except Exception as exc:
if not error:
error = TestFailure(exc)
self.add_failure(test, error, expecting_failure(method))
return error
def render(self, request=None, callback=None):
'''Render this string.
This method returns a string or a :class:`~asyncio.Future` which
results in a string. On the other hand, the callable method of
a :class:`.String` **always** returns a :class:`~asyncio.Future`.
'''
stream = []
async = False
for data in self.stream(request):
if isawaitable(data):
async = True
stream.append(data)
if not callback:
callback = self.to_string
if async:
return chain_future(multi_async(stream), callback=callback)
else:
return callback(stream)
def start_request(request, conn):
response = conn.current_consumer()
# bind request-specific events
response.bind_events(**request.inp_params)
if request.auth:
response.bind_event('pre_request', request.auth)
if request.stream:
response.bind_event('data_processed', response.raw)
response.start(request)
yield from response.events['on_headers']
else:
response.bind_event('data_processed', response_content)
response.start(request)
yield from response.on_finished
if hasattr(response.request_again, '__call__'):
response = response.request_again(response)
if isawaitable(response):
response = yield from response
return response
async def start_request(request, conn):
response = conn.current_consumer()
# bind request-specific events
response.bind_events(**request.inp_params)
if request.auth:
response.bind_event('pre_request', request.auth)
if request.stream:
response.bind_event('data_processed', response.raw)
response.start(request)
await response.events['on_headers']
else:
response.bind_event('data_processed', response_content)
response.start(request)
await response.on_finished
if hasattr(response.request_again, '__call__'):
response = response.request_again(response)
if isawaitable(response):
response = await response
return response
async def _response(self, environ):
exc_info = None
response = None
done = False
alive = self.cfg.keep_alive or 15
while not done:
done = True
try:
if exc_info is None:
if (not environ.get('HTTP_HOST')
and environ['SERVER_PROTOCOL'] != 'HTTP/1.0'):
raise BadRequest
response = self.wsgi_callable(environ, self.start_response)
if isawaitable(response):
response = await wait_for(response, alive)
else:
response = handle_wsgi_error(environ, exc_info)
if isawaitable(response):
response = await wait_for(response, alive)
#
if exc_info:
self.start_response(response.status,
response.get_headers(), exc_info)
#
# Do the actual writing
loop = self._loop
start = loop.time()
for chunk in response:
if isawaitable(chunk):
chunk = await wait_for(chunk, alive)
start = loop.time()
result = self.write(chunk)
if isawaitable(result):
await wait_for(result, alive)
start = loop.time()
else:
time_in_loop = loop.time() - start
if time_in_loop > MAX_TIME_IN_LOOP:
self.logger.debug(
'Released the event loop after %.3f seconds',
time_in_loop)
await sleep(0.1, loop=self._loop)
start = loop.time()
#
# make sure we write headers and last chunk if needed
self.write(b'', True)
# client disconnected, end this connection
except (IOError, AbortWsgi):
self.finished()
except Exception:
if wsgi_request(environ).cache.handle_wsgi_error:
self.keep_alive = False
self._write_headers()
self.connection.close()
self.finished()
else:
done = False
exc_info = sys.exc_info()
else:
log_wsgi_info(self.logger.info, environ, self.status)
self.finished()
if not self.keep_alive:
self.logger.debug('No keep alive, closing connection %s',
self.connection)
self.connection.close()
finally:
close_object(response)
def _(*args, **kwargs):
args = yield from as_gather(*args)
result = getattr(self.test, name)(*args, **kwargs)
if isawaitable(result):
result = yield from result
return result
async def _write_streamed_data(self, transport):
for data in self.body:
if isawaitable(data):
data = await data
self._write_body_data(transport, data)
self._write_body_data(transport, b'', True)
async def _response(self, environ):
exc_info = None
response = None
done = False
alive = self.cfg.keep_alive or 15
while not done:
done = True
try:
if exc_info is None:
if (not environ.get('HTTP_HOST') and
environ['SERVER_PROTOCOL'] != 'HTTP/1.0'):
raise BadRequest
response = self.wsgi_callable(environ, self.start_response)
if isawaitable(response):
response = await wait_for(response, alive)
else:
response = handle_wsgi_error(environ, exc_info)
if isawaitable(response):
response = await wait_for(response, alive)
#
if exc_info:
self.start_response(response.status,
response.get_headers(), exc_info)
#
# Do the actual writing
loop = self._loop
start = loop.time()
for chunk in response:
if isawaitable(chunk):
chunk = await wait_for(chunk, alive)
start = loop.time()
result = self.write(chunk)
if isawaitable(result):
await wait_for(result, alive)
start = loop.time()
else:
time_in_loop = loop.time() - start
if time_in_loop > MAX_TIME_IN_LOOP:
self.logger.debug(
'Released the event loop after %.3f seconds',
time_in_loop)
await sleep(0.1, loop=self._loop)
start = loop.time()
#
# make sure we write headers and last chunk if needed
self.write(b'', True)
# client disconnected, end this connection
except (IOError, AbortWsgi):
self.finished()
except Exception:
if wsgi_request(environ).cache.handle_wsgi_error:
self.keep_alive = False
self._write_headers()
self.connection.close()
self.finished()
else:
done = False
exc_info = sys.exc_info()
else:
log_wsgi_info(self.logger.info, environ, self.status)
self.finished()
if not self.keep_alive:
self.logger.debug('No keep alive, closing connection %s',
self.connection)
self.connection.close()
finally:
close_object(response)
async def _(*args, **kwargs):
args = await as_gather(*args)
result = getattr(self.test, name)(*args, **kwargs)
if isawaitable(result):
result = await result
return result
async def _write_streamed_data(self, transport):
for data in self.body:
if isawaitable(data):
data = await data
self._write_body_data(transport, data)
self._write_body_data(transport, b'', True)
def _write_streamed_data(self, transport):
for data in self.data:
if isawaitable(data):
data = yield from data
self._write_body_data(transport, data)
self._write_body_data(transport, b'', True)