def _try_special_request(self, environ, request):
static_mount = '/__piecrust_static/'
if request.path.startswith(static_mount):
rel_req_path = request.path[len(static_mount):]
mount = os.path.join(os.path.dirname(__file__), 'resources',
'server')
full_path = os.path.join(mount, rel_req_path)
try:
response = self._make_wrapped_file_response(
environ, request, full_path)
return response
except OSError:
pass
debug_mount = '/__piecrust_debug/'
if request.path.startswith(debug_mount):
rel_req_path = request.path[len(debug_mount):]
if rel_req_path == 'pipeline_status':
from piecrust.server.procloop import (
PipelineStatusServerSideEventProducer)
provider = PipelineStatusServerSideEventProducer(
self._proc_loop.status_queue)
it = ClosingIterator(provider.run(), [provider.close])
response = Response(it)
response.headers['Cache-Control'] = 'no-cache'
if 'text/event-stream' in request.accept_mimetypes:
response.mimetype = 'text/event-stream'
response.direct_passthrough = True
response.implicit_sequence_conversion = False
return response
return None
def __call__(self, environ, start_response):
cookie = parse_cookie(environ.get('HTTP_COOKIE', ''))
sid = cookie.get(self.cookie_name, None)
if sid is None:
session = self.store.new()
else:
session = self.store.get(sid)
environ[self.environ_key] = session
def injecting_start_response(status, headers, exc_info=None):
if session.should_save:
self.store.save(session)
headers.append(('Set-Cookie',
dump_cookie(self.cookie_name,
session.sid,
self.cookie_age,
self.cookie_expires,
self.cookie_path,
self.cookie_domain,
self.cookie_secure,
self.cookie_httponly,
samesite=self.cookie_samesite)))
return start_response(status, headers, exc_info)
return ClosingIterator(self.app(environ, injecting_start_response),
lambda: self.store.save_if_modified(session))
def __call__(self, environ, start_response):
body = list(self.app(environ, start_response))
def _log():
self.log_callback(Request(environ), body)
return ClosingIterator(body, _log)
def __call__(self, environ, after_response):
iterator = self.application(environ, after_response)
try:
return ClosingIterator(iterator, [self.after_response_ext.flush])
except Exception:
traceback.print_exc()
return iterator
def inner(environ, after_response):
iterator = wsgi_app(environ, after_response)
try:
return ClosingIterator(iterator, [callback])
except:
traceback.print_exc()
return iterator
def __call__(self, environ, start_response):
"""
We must case-mangle the Set-Cookie header name or AWS will use only a
single one of these headers.
"""
def encode_response(status, headers, exc_info=None):
"""
This makes the 'set-cookie' headers name lowercase,
all the non-cookie headers should be sent unharmed.
Related: https://github.com/Miserlou/Zappa/issues/1965
"""
new_headers = [header for header in headers
if ((type(header[0]) != str) or (header[0].lower() != 'set-cookie'))]
cookie_headers = [(header[0].lower(), header[1]) for header in headers
if ((type(header[0]) == str) and (header[0].lower() == "set-cookie"))]
new_headers = new_headers + cookie_headers
return start_response(status, new_headers, exc_info)
# Call the application with our modifier
response = self.application(environ, encode_response)
# Return the response as a WSGI-safe iterator
return ClosingIterator(response)
class Plnt(object):
def __init__(self, database_uri):
self.database_engine = create_engine(database_uri)
self._dispatch = local_manager.middleware(self.dispatch_request)
self._dispatch = SharedDataMiddleware(self._dispatch,
{'/shared': SHARED_DATA})
def init_database(self):
metadata.create_all(self.database_engine)
def bind_to_context(self):
local.application = self
def dispatch_request(self, environ, start_response):
self.bind_to_context()
local.request = request = Request(environ, start_response)
local.url_adapter = adapter = url_map.bind_to_environ(environ)
try:
endpoint, values = adapter.match(request.path)
response = endpoints[endpoint](request, **values)
except HTTPException, e:
response = e
return ClosingIterator(response(environ, start_response),
session.remove)
def _make_response(sijax_response):
"""Takes a Sijax response object and returns a
valid Flask response object."""
from types import GeneratorType
if isinstance(sijax_response, GeneratorType):
# Streaming response using a generator (non-JSON response).
# Upon returning a response, Flask would automatically destroy
# the request data and uploaded files - done by `flask.ctx.RequestContext.auto_pop()`
# We can't allow that, since the user-provided callback we're executing
# from within the generator may want to access request data/files.
# That's why we'll tell Flask to preserve the context and we'll clean up ourselves.
request.environ['flask._preserve_context'] = True
# Clean-up code taken from `flask.testing.TestingClient`
def clean_up_context():
top = _request_ctx_stack.top
if top is not None and top.preserved:
top.pop()
# As per the WSGI specification, `close()` would be called on iterator responses.
# Let's wrap the iterator in another one, which will forward that `close()` call to our clean-up callback.
response = Response(ClosingIterator(sijax_response, clean_up_context),
direct_passthrough=True)
else:
# Non-streaming response - a single JSON string
response = Response(sijax_response)
return response
def run_wsgi_app(app, environ, buffered=False):
environ = _get_environ(environ)
response = []
buffer = []
def start_response(status, headers, exc_info=None):
if exc_info is not None:
raise exc_info[0], exc_info[1], exc_info[2]
response[:] = [status, headers]
return buffer.append
app_iter = app(environ, start_response)
if buffered:
close_func = getattr(app_iter, 'close', None)
try:
app_iter = list(app_iter)
finally:
if close_func is not None:
close_func()
else:
while not response:
buffer.append(app_iter.next())
if buffer:
close_func = getattr(app_iter, 'close', None)
app_iter = chain(buffer, app_iter)
if close_func is not None:
app_iter = ClosingIterator(app_iter, close_func)
return (app_iter, response[0], response[1])
def __call__(self, environ, after_response):
iterator = self.application(environ, after_response)
# noinspection PyBroadException
try:
return ClosingIterator(iterator, [self.after_response_ext.flush])
except Exception:
print_exc()
return iterator
def inner(environ, start_response):
iterator = wsgi_app(environ, start_response)
apps_stats = environ.get('appstats.apps_stats')
tasks_stats = environ.get('appstats.tasks_stats')
if not (apps_stats or tasks_stats):
return iterator
return ClosingIterator(
iterator, lambda: add_stats(apps_stats, tasks_stats, apps_counters,
tasks_counters))
def run_wsgi_app(app, environ, buffered=False):
"""Return a tuple in the form (app_iter, status, headers) of the
application output. This works best if you pass it an application that
returns an iterator all the time.
Sometimes applications may use the `write()` callable returned
by the `start_response` function. This tries to resolve such edge
cases automatically. But if you don't get the expected output you
should set `buffered` to `True` which enforces buffering.
If passed an invalid WSGI application the behavior of this function is
undefined. Never pass non-conforming WSGI applications to this function.
:param app: the application to execute.
:param buffered: set to `True` to enforce buffering.
:return: tuple in the form ``(app_iter, status, headers)``
"""
environ = _get_environ(environ)
response = []
buffer = []
def start_response(status, headers, exc_info=None):
if exc_info is not None:
reraise(*exc_info)
response[:] = [status, headers]
return buffer.append
app_rv = app(environ, start_response)
close_func = getattr(app_rv, 'close', None)
app_iter = iter(app_rv)
# when buffering we emit the close call early and convert the
# application iterator into a regular list
if buffered:
try:
app_iter = list(app_iter)
finally:
if close_func is not None:
close_func()
# otherwise we iterate the application iter until we have a response, chain
# the already received data with the already collected data and wrap it in
# a new `ClosingIterator` if we need to restore a `close` callable from the
# original return value.
else:
for item in app_iter:
buffer.append(item)
if response:
break
if buffer:
app_iter = chain(buffer, app_iter)
if close_func is not None and app_iter is not app_rv:
app_iter = ClosingIterator(app_iter, close_func)
return app_iter, response[0], Headers(response[1])
def dispatch_request(self, environ, start_response):
self.bind_to_context()
local.request = request = Request(environ, start_response)
local.url_adapter = adapter = url_map.bind_to_environ(environ)
try:
endpoint, values = adapter.match(request.path)
response = endpoints[endpoint](request, **values)
except HTTPException as e:
response = e
return ClosingIterator(response(environ, start_response),
session.remove)
def serve_full_file_request(request, headers, file, callbacks=[]):
headers.update({
'Content-Length': file.length,
'Accept-Ranges': 'bytes',
})
file_iterator = wrap_file(request.environ, file)
response = Response(ClosingIterator(file_iterator, callbacks=callbacks),
direct_passthrough=True,
mimetype=file.content_type,
headers=headers)
response.last_modified = file.uploadDate
response.set_etag(file.md5)
return response
def _startSSEProvider(self, request, start_response):
from piecrust.serving.procloop import (
PipelineStatusServerSentEventProducer)
provider = PipelineStatusServerSentEventProducer(
self._proc_loop)
it = provider.run()
response = Response(it, mimetype='text/event-stream')
response.headers['Cache-Control'] = 'no-cache'
response.headers['Last-Event-ID'] = \
self._proc_loop.last_status_id
return ClosingIterator(
response(request.environ, start_response),
[provider.close])
def run_wsgi_app(app, environ, buffered=False):
"""返回一个应用输出的元组形式 (app_iter, status, headers)。如果你通过应用
返回一个迭代器他将会工作的更好。
有时应用可以使用 `start_ewsponse` 返回的 `write()` 回调函数。这将会自动解
决边界情况。如果没有得到预期输出，你应该将 `buffered` 设为 `True` 执行
buffering
如果传入一个错误的应用，这个函数将会是未定义的。不要给这个函数传入一个不标准
的 WSGI 应用。
:param app: 要执行的应用。
:param buffered: 设为 `True` 来执行 buffering.
:return: 元组形式 ``(app_iter, status, headers)``
"""
environ = _get_environ(environ)
response = []
buffer = []
def start_response(status, headers, exc_info=None):
if exc_info is not None:
reraise(*exc_info)
response[:] = [status, headers]
return buffer.append
app_iter = app(environ, start_response)
# when buffering we emit the close call early and convert the
# application iterator into a regular list
if buffered:
close_func = getattr(app_iter, 'close', None)
try:
app_iter = list(app_iter)
finally:
if close_func is not None:
close_func()
# otherwise we iterate the application iter until we have
# a response, chain the already received data with the already
# collected data and wrap it in a new `ClosingIterator` if
# we have a close callable.
else:
while not response:
buffer.append(next(app_iter))
if buffer:
close_func = getattr(app_iter, 'close', None)
app_iter = chain(buffer, app_iter)
if close_func is not None:
app_iter = ClosingIterator(app_iter, close_func)
return app_iter, response[0], Headers(response[1])
def __call__(self, environ, start_response):
"""
A note about the zappa cookie: Only 1 cookie can be passed through API
Gateway. Hence all cookies are packed into a special cookie, the
zappa cookie. There are a number of problems with this:
* updates of single cookies, when there are multiple present results
in deletion of the ones that are not being updated.
* expiration of cookies. The client no longer knows when cookies
expires.
The first is solved by unpacking the zappa cookie on each request and
saving all incoming cookies. The response Set-Cookies are then used
to update the saved cookies, which are packed and set as the zappa
cookie.
The second is solved by filtering cookies on their expiration time,
only passing cookies that are still valid to the WSGI app.
"""
self.start_response = start_response
# Parse cookies from the WSGI environment
parsed = parse_cookie(environ)
# Decode the special zappa cookie if present in the request
if 'zappa' in parsed:
# Save the parsed cookies. We need to send them back on every update.
self.decode_zappa_cookie(parsed['zappa'])
# Since the client doesn't know it has old cookies,
# manual expire them.
self.filter_expired_cookies()
# Set the WSGI environment cookie to be the decoded value.
environ[u'HTTP_COOKIE'] = self.cookie_environ_string()
else:
# No cookies were previously set
self.request_cookies = dict()
# Call the application with our modifier
response = self.application(environ, self.encode_response)
# If we have a redirect, smash in our response content.
if self.redirect_content:
response = [self.redirect_content for item in response]
self.redirect_content = None # Make sure that nothing is cached from a previous request
# Return the response as a WSGI-safe iterator
return ClosingIterator(response)
def dispatch(self, environ, start_response):
local.application = self
request = Request(environ)
local.url_adapter = adapter = url_map.bind_to_environ(environ)
try:
endpoint, values = adapter.match()
handler = getattr(views, endpoint)
response = handler(request, **values)
except NotFound:
response = views.not_found(request)
response.status_code = 404
except HTTPException as e:
response = e
return ClosingIterator(response(environ, start_response),
[local_manager.cleanup])
def run_wsgi_app(app, environ, buffered=False):
"""Return a tuple in the form (app_iter, status, headers) of the
application output. This works best if you pass it an application that
returns an iterator all the time.
Sometimes applications may use the `write()` callable returned
by the `start_response` function. This tries to resolve such edge
cases automatically. But if you don't get the expected output you
should set `buffered` to `True` which enforces buffering.
If passed an invalid WSGI application the behavior of this function is
undefined. Never pass non-conforming WSGI applications to this function.
:param app: the application to execute.
:param buffered: set to `True` to enforce buffering.
:return: tuple in the form ``(app_iter, status, headers)``
"""
environ = _get_environ(environ)
response = []
buffer = []
def start_response(status, headers, exc_info=None):
if exc_info is not None:
raise exc_info[0], exc_info[1], exc_info[2]
response[:] = [status, headers]
return buffer.append
app_iter = app(environ, start_response)
if buffered:
close_func = getattr(app_iter, 'close', None)
try:
app_iter = list(app_iter)
finally:
if close_func is not None:
close_func()
else:
while not response:
buffer.append(app_iter.next())
if buffer:
close_func = getattr(app_iter, 'close', None)
app_iter = chain(buffer, app_iter)
if close_func is not None:
app_iter = ClosingIterator(app_iter, close_func)
return (app_iter, response[0], response[1])
def __call__(self, environ, start_response):
"""
We must case-mangle the Set-Cookie header name or AWS will use only a
single one of these headers.
"""
def encode_response(status, headers, exc_info=None):
"""
Create an APIGW-acceptable version of our cookies.
We have to use a bizarre hack that turns multiple Set-Cookie headers into
their case-permutated format, ex:
Set-cookie:
sEt-cookie:
seT-cookie:
To get around an API Gateway limitation.
This is weird, but better than our previous hack of creating a Base58-encoded
supercookie.
"""
# All the non-cookie headers should be sent unharmed.
# The main app can send 'set-cookie' headers in any casing
# Related: https://github.com/Miserlou/Zappa/issues/990
new_headers = [
header for header in headers if ((type(header[0]) != str) or (
header[0].lower() != 'set-cookie'))
]
cookie_headers = [
header for header in headers if ((type(header[0]) == str) and (
header[0].lower() == "set-cookie"))
]
for header, new_name in zip(cookie_headers,
all_casings("Set-Cookie")):
new_headers.append((new_name, header[1]))
return start_response(status, new_headers, exc_info)
# Call the application with our modifier
response = self.application(environ, encode_response)
# Return the response as a WSGI-safe iterator
return ClosingIterator(response)
def serve_partial_file_request(request,
headers,
file,
start,
end,
callbacks=[]):
# Note: byte positions are inclusive!
headers.update({
'Content-Length':
end - start,
'Content-Range':
'bytes %i-%i/%i' % (start, end - 1, file.length),
})
file_iterator = LimitedFileWrapper(file, start, end)
return Response(ClosingIterator(file_iterator, callbacks=callbacks),
direct_passthrough=True,
mimetype=file.content_type,
headers=headers,
status=206)
def dispatch(self, application, environ, start_response):
session = None
if self.enabled:
session = self._get_session(environ)
environ['request.session'] = session
if session is None:
return application(environ, start_response)
def injecting_start_response(status, headers, exc_info=None):
if session.expired:
headers.append(
('Set-Cookie', self._construct_cookie(session, True)))
self.store.delete(session)
elif session.should_save:
self.store.save(session)
headers.append(('Set-Cookie', self._construct_cookie(session)))
return start_response(status, headers, exc_info)
return ClosingIterator(application(environ, injecting_start_response),
lambda: self.store.save_if_modified(session))
def __call__(self, environ, start_response):
sid = self._get_session_id(environ)
if sid is None:
session = self.store.new()
else:
session = self.store.get(sid)
environ[self.environ_key] = session
def injecting_start_response(status, headers, exc_info=None):
if session.should_save:
self.store.save(session)
headers.append(("Set-Cookie", self._dump_cookie(session)))
return start_response(status, headers, exc_info)
return ClosingIterator(
self.app(environ, injecting_start_response),
lambda: self.store.save_if_modified(session),
)
def __call__(self, environ, start_response) -> ClosingIterator:
body__ = self.request_body(environ)
def _start_response(status, response_headers, *args):
self.start_response(status, response_headers)
return start_response(status, response_headers, *args)
response_chunks = self.finish_response(
self.app(environ, _start_response))
request = Request(environ)
parased_raw_params: Dict[str, List[str]] = parse.parse_qs(
parse.urlparse(request.url).query)
params: Dict[str, str] = {}
# Type correction
for k, v in parased_raw_params.items():
params[k] = v[0]
HttpMessage.send(
self.logger,
request=HttpRequestImpl(
method=request.method,
url=str(request.url),
headers=dict(request.headers),
params=params,
body=body__,
),
response=HttpResponseImpl(
status=self.status,
body=str(self.response[0].decode()) if self.response else None,
headers=dict(self.response_headers),
),
interval=str(self.interval),
)
return ClosingIterator(response_chunks)
def dispatch_request(self, environ, start_response):
"""Dispatch an incoming request."""
# set up all the stuff we want to have for this request. That is
# creating a request object, propagating the application to the
# current context and instanciating the database session.
self.bind_to_context()
request = Request(environ)
request.bind_to_context()
# get the current action from the url and normalize the page name
# which is just the request path
action_name = request.args.get('action') or 'show'
page_name = u'_'.join(
[x for x in request.path.strip('/').split() if x])
# redirect to the Main_Page if the user requested the index
if not page_name:
response = redirect(href('Main_Page'))
# check special pages
elif page_name.startswith('Special:'):
if page_name[8:] not in pages:
response = page_not_found(request, page_name)
else:
response = pages[page_name[8:]](request)
# get the callback function for the requested action from the
# action module. It's "on_" + the action name. If it doesn't
# exists call the missing_action method from the same module.
else:
action = getattr(actions, 'on_' + action_name, None)
if action is None:
response = actions.missing_action(request, action_name)
else:
response = action(request, page_name)
# make sure the session is removed properly
return ClosingIterator(response(environ, start_response),
session.remove)
def __call__(self, environ, start_response):
"""
We must case-mangle the Set-Cookie header name or AWS will use only a
single one of these headers.
"""
def encode_response(status, headers, exc_info=None):
"""
Create an APIGW-acceptable version of our cookies.
We have to use a bizarre hack that turns multiple Set-Cookie headers into
their case-permutated format, ex:
Set-cookie:
sEt-cookie:
seT-cookie:
To get around an API Gateway limitation.
This is weird, but better than our previous hack of creating a Base58-encoded
supercookie.
"""
# All the non-cookie headers should be sent unharmed.
new_headers = [(header[0], header[1]) for header in headers if
header[0] != 'Set-Cookie']
cookie_headers = [x for x in headers if x[0] == "Set-Cookie"]
for header, new_name in zip(cookie_headers,
all_casings("Set-Cookie")):
new_headers.append((new_name, header[1]))
return start_response(status, new_headers, exc_info)
# Call the application with our modifier
response = self.application(environ, encode_response)
# Return the response as a WSGI-safe iterator
return ClosingIterator(response)
def __call__(self, environ, start_response):
"""
A note about the zappa cookie: Only 1 cookie can be passed through API
Gateway. Hence all cookies are packed into a special cookie, the
zappa cookie. There are a number of problems with this:
* updates of single cookies, when there are multiple present results
in deletion of the ones that are not being updated.
* expiration of cookies. The client no longer knows when cookies
expires.
The first is solved by unpacking the zappa cookie on each request and
saving all incoming cookies. The response Set-Cookies are then used
to update the saved cookies, which are packed and set as the zappa
cookie.
The second is solved by filtering cookies on their expiration time,
only passing cookies that are still valid to the WSGI app.
"""
self.start_response = start_response
# Parse cookies from the WSGI environment
parsed = parse_cookie(environ)
parsed.pop('zappa', None)
self.request_cookies = parsed
environ[u'HTTP_COOKIE'] = parsed
# Call the application with our modifier
response = self.application(environ, self.encode_response)
# If we have a redirect, smash in our response content.
if self.redirect_content:
response = [self.redirect_content for item in response]
self.redirect_content = None # Make sure that nothing is cached from a previous request
# Return the response as a WSGI-safe iterator
return ClosingIterator(response)
def application(environ, start_response):
return ClosingIterator(app(environ, start_response), self.cleanup)
class Slicer(object):
def __init__(self, config=None):
"""Create a WSGI server for providing OLAP web service. You might provide ``config``
as ``ConfigParser`` object.
"""
self.config = config
self.initialize_logger()
self.context = create_slicer_context(config)
self.model = self.context["model"]
self.locales = self.context["locales"]
self.backend = self.context["backend"]
self.model_localizations = {}
if self.locales is None:
if self.model.locale:
self.locales = [self.model.locale]
else:
self.locales = []
## Create workspace
self.logger.info("using backend '%s'" % self.context["backend_name"])
self.workspace = self.backend.create_workspace(
self.model, **self.context["workspace_options"])
def initialize_logger(self):
# Configure logger
self.logger = cubes.common.get_logger()
if self.config.has_option("server", "log"):
formatter = logging.Formatter(
fmt='%(asctime)s %(levelname)s %(message)s')
handler = logging.FileHandler(self.config.get("server", "log"))
handler.setFormatter(formatter)
self.logger.addHandler(handler)
if self.config.has_option("server", "log_level"):
level_str = self.config.get("server", "log_level").lower()
levels = {
"info": logging.INFO,
"debug": logging.DEBUG,
"warn": logging.WARN,
"error": logging.ERROR
}
if level_str not in levels:
self.logger.warn(
"Unknown logging level '%s', keeping default" % level_str)
else:
self.logger.setLevel(levels[level_str])
self.logger.debug("loading model")
def wsgi_app(self, environ, start_response):
request = Request(environ)
urls = rules.bind_to_environ(environ)
try:
endpoint, params = urls.match()
(ctrl_class, action) = endpoint
response = self.dispatch(ctrl_class, action, request, params)
except HTTPException, e:
response = e
return ClosingIterator(response(environ, start_response),
[local_manager.cleanup])
class Shorty(object):
def __init__(self, config):
local.application = self
self.database_engine = create_engine(config['db_uri'],
convert_unicode=True)
self.dispatch = SharedDataMiddleware(self.dispatch,
{'/static': STATIC_PATH})
def init_database(self):
metadata.create_all(self.database_engine)
def dispatch(self, environ, start_response):
local.application = self
request = Request(environ)
local.url_adapter = adapter = url_map.bind_to_environ(environ)
try:
endpoint, values = adapter.match()
handler = getattr(views, endpoint)
response = handler(request, **values)
except NotFound, e:
response = views.not_found(request)
response.status_code = 404
except HTTPException, e:
response = e
return ClosingIterator(response(environ, start_response),
[session.remove, local_manager.cleanup])
def __call__(self, environ, start_response):
return self.dispatch(environ, start_response)
