def __call__(self, environ: Any, start_response: Callable[...,
Any]) -> Any:
if self.app.debug:
headers = list(EnvironHeaders(environ).items()) # type: ignore
for k, v in headers:
print(f'{k}: {v}')
return self.wsgi_app(environ, start_response)
def test_multiple_values(self, request_factory, request):
request = request_factory('GET',
'/',
subdomain='www',
query_string='a=b&a=c')
openapi_request = FlaskOpenAPIRequest(request)
path = {}
query = ImmutableMultiDict([
('a', 'b'),
('a', 'c'),
])
headers = EnvironHeaders(request.environ)
cookies = {}
assert openapi_request.parameters == RequestParameters(
path=path,
query=query,
header=headers,
cookie=cookies,
)
assert openapi_request.method == request.method.lower()
assert openapi_request.full_url_pattern == \
urljoin(request.host_url, request.path)
assert openapi_request.body == request.data
assert openapi_request.mimetype == request.mimetype
def from_environ(cls, environ, **kwargs):
"""Turn an environ dict back into a builder. Any extra kwargs
override the args extracted from the environ.
.. versionadded:: 0.15
"""
headers = Headers(EnvironHeaders(environ))
out = {
"path": environ["PATH_INFO"],
"base_url": cls._make_base_url(
environ["wsgi.url_scheme"],
headers.pop("Host"),
environ["SCRIPT_NAME"],
),
"query_string": environ["QUERY_STRING"],
"method": environ["REQUEST_METHOD"],
"input_stream": environ["wsgi.input"],
"content_type": headers.pop("Content-Type", None),
"content_length": headers.pop("Content-Length", None),
"errors_stream": environ["wsgi.errors"],
"multithread": environ["wsgi.multithread"],
"multiprocess": environ["wsgi.multiprocess"],
"run_once": environ["wsgi.run_once"],
"headers": headers,
}
out.update(kwargs)
return cls(**out)
def test_multiple_values(self, request_factory, request):
request = request_factory('GET',
'/',
subdomain='www',
query_string='a=b&a=c')
openapi_request = FlaskOpenAPIRequest(request)
path = {}
query = ImmutableMultiDict([
('a', 'b'),
('a', 'c'),
])
headers = EnvironHeaders(request.environ)
cookies = {}
assert openapi_request.parameters == {
'path': path,
'query': query,
'header': headers,
'cookie': cookies,
}
assert openapi_request.host_url == request.host_url
assert openapi_request.path == request.path
assert openapi_request.method == request.method.lower()
assert openapi_request.path_pattern == request.path
assert openapi_request.body == request.data
assert openapi_request.mimetype == request.mimetype
def _base_response_with_common_cors_headers(incoming_headers: EnvironHeaders):
response = Response()
response.headers.pop("content-type")
response.headers.add("Access-Control-Allow-Origin",
incoming_headers.get("origin", default="*"))
response.headers.add(
"Access-Control-Allow-Headers",
incoming_headers.get("access-control-request-headers", default="*"))
response.headers.add("Access-Control-Allow-Credentials", "true")
response.headers.add(
"Access-Control-Expose-Headers",
incoming_headers.get("access-control-expose-headers", default="*"))
return response
def get_headers(self, environ=None):
headers = []
if environ:
headers = list(EnvironHeaders(environ))
headers.append(('Location', self.location))
return headers
def test_environ_headers_counts():
"""Ensure that the EnvironHeaders count correctly."""
# this happens in multiple WSGI servers because they
# use a vary naive way to convert the headers;
broken_env = {
'HTTP_CONTENT_TYPE': 'text/html',
'CONTENT_TYPE': 'text/html',
'HTTP_CONTENT_LENGTH': '0',
'CONTENT_LENGTH': '0',
'HTTP_ACCEPT': '*',
'wsgi.version': (1, 0)
}
headers = EnvironHeaders(broken_env)
assert headers
assert len(headers) == 3
assert sorted(headers) == [('Accept', '*'), ('Content-Length', '0'),
('Content-Type', 'text/html')]
assert not EnvironHeaders({'wsgi.version': (1, 0)})
assert len(EnvironHeaders({'wsgi.version': (1, 0)})) == 0
def __getitem__(self, key, _get_mode=False):
"""
Sanitized __getitem__
:param key: the key for the value
:type key: str
:param _get_mode: is a no-op for this class as there is no index but
used because get() calls it.
:type _get_mode: bool
"""
val = EnvironHeaders.__getitem__(self, key, _get_mode=_get_mode)
return self.sanitize_input(val)
def __getitem__(self, key, _get_mode=False):
"""
Sanitized __getitem__
:param key: the key for the value
:type key: str
:param _get_mode: is a no-op for this class as there is no index but
used because get() calls it.
:type _get_mode: bool
"""
val = EnvironHeaders.__getitem__(self, key, _get_mode=_get_mode)
return self.sanitize_input(val)
def __call__(self, environ, start_response):
if self.enabled:
self.headers = EnvironHeaders(environ)
should_log = True
if self.pi_filter is not None and self.pi_filter not in environ[
'PATH_INFO']:
should_log = False
if self.rm_filter is not None and environ['REQUEST_METHOD'].lower(
) not in [x.lower() for x in tolist(self.rm_filter)]:
should_log = False
if should_log:
wsgi_input = self.replace_wsgi_input(environ)
fname = '%s_%s' % (time.time(), randchars())
fh = open(path.join(self.log_dir, fname), 'wb+')
try:
fh.write(pformat(environ))
fh.write('\n')
fh.write(wsgi_input.read())
wsgi_input.seek(0)
finally:
fh.close()
return self.application(environ, start_response)
def test_immutable_structures():
"""Test immutable structures"""
l = ImmutableList([1, 2, 3])
assert_raises(TypeError, l.__delitem__, 0)
assert_raises(TypeError, l.__delslice__, 0, 1)
assert_raises(TypeError, l.__iadd__, [1, 2])
assert_raises(TypeError, l.__setitem__, 0, 1)
assert_raises(TypeError, l.__setslice__, 0, 1, [2, 3])
assert_raises(TypeError, l.append, 42)
assert_raises(TypeError, l.insert, 0, 32)
assert_raises(TypeError, l.pop)
assert_raises(TypeError, l.extend, [2, 3])
assert_raises(TypeError, l.reverse)
assert_raises(TypeError, l.sort)
assert l == [1, 2, 3]
d = ImmutableDict(foo=23, bar=42)
assert_raises(TypeError, d.setdefault, 'baz')
assert_raises(TypeError, d.update, {2: 3})
assert_raises(TypeError, d.popitem)
assert_raises(TypeError, d.__delitem__, 'foo')
assert_raises(TypeError, d.clear)
assert d == dict(foo=23, bar=42)
d = ImmutableDict.fromkeys([1, 2])
assert d[1] == d[2] == None
d = ImmutableMultiDict(d)
assert_raises(TypeError, d.add, 'fuss', 44)
assert_raises(TypeError, d.popitemlist)
assert_raises(TypeError, d.poplist, 'foo')
assert_raises(TypeError, d.setlist, 'tadaa', [1, 2])
assert_raises(TypeError, d.setlistdefault, 'tadaa')
d = ImmutableMultiDict.fromkeys([1, 2])
assert d[1] == d[2] == None
d = EnvironHeaders({'HTTP_X_FOO': 'test'})
assert_raises(TypeError, d.__delitem__, 0)
assert_raises(TypeError, d.add, 42)
assert_raises(TypeError, d.pop, 'x-foo')
assert_raises(TypeError, d.popitem)
assert_raises(TypeError, d.setdefault, 'foo', 42)
assert dict(d.items()) == {'X-Foo': 'test'}
assert_raises(TypeError, d.copy)
def authorize(cls, headers: EnvironHeaders) -> bool:
key = headers.get("Authorization")
# If incoming request did not provide a key
if not key:
# Case where no key provided when one was required
if cls._CONF:
LoggingUtils.info(
"Request denied - a key was required but no key was provided",
color=LoggingUtils.YELLOW)
return False
# Case where no key was provided and none were required
elif not cls._CONF:
LoggingUtils.info(
"Request authorized - no key was required, and no key was provided",
color=LoggingUtils.GREEN)
return True
else:
# Case where a key was provided and one was required
if cls._CONF:
for user, password in cls._CONF.items():
if key == password:
# Found a matching key, so return True
LoggingUtils.info(
"Request authorized - a key was required, and a matching key was provided",
color=LoggingUtils.GREEN)
LoggingUtils.info(
"Matching key was sent from {}".format(user))
return True
# If no matching key was found, return false
LoggingUtils.info(
"Request denied - a key was required, but a nonmatching key was provided",
color=LoggingUtils.YELLOW)
# Case where a key was provided but one was not required
elif not cls._CONF:
LoggingUtils.info(
"Request denied - a key was not required, but one was provided",
color=LoggingUtils.YELLOW)
return False
def test_url_rule(self, request_factory, request):
request = request_factory('GET', '/browse/12/', subdomain='kb')
openapi_request = FlaskOpenAPIRequest(request)
path = {'id': 12}
query = ImmutableMultiDict([])
headers = EnvironHeaders(request.environ)
cookies = {}
assert openapi_request.parameters == RequestParameters(
path=path,
query=query,
header=headers,
cookie=cookies,
)
assert openapi_request.method == request.method.lower()
assert openapi_request.full_url_pattern == \
urljoin(request.host_url, '/browse/{id}/')
assert openapi_request.body == request.data
assert openapi_request.mimetype == request.mimetype
def test_url_rule(self, request_factory, request):
request = request_factory('GET', '/browse/12/', subdomain='kb')
openapi_request = FlaskOpenAPIRequest(request)
path = {'id': 12}
query = ImmutableMultiDict([])
headers = EnvironHeaders(request.environ)
cookies = {}
assert openapi_request.parameters == {
'path': path,
'query': query,
'header': headers,
'cookie': cookies,
}
assert openapi_request.host_url == request.host_url
assert openapi_request.path == request.path
assert openapi_request.method == request.method.lower()
assert openapi_request.path_pattern == request.url_rule.rule
assert openapi_request.body == request.data
assert openapi_request.mimetype == request.mimetype
def create_filter_chain(headers: EnvironHeaders):
expansions = ExpansionsFilter(headers.get(Filter._include_expansions_header_name))
players = PlayersFilter(headers.get(Filter._players_count_header_name),
headers.get(Filter._use_recommended_players_count_header_name))
duration = DurationFilter(headers.get(Filter._min_duration_header_name),
headers.get(Filter._max_duration_header_name))
complexity = ComplexityFilter(headers.get(
Filter._max_complexity_duration_header_name))
mechanics = MechanicFilter(headers.get(
Filter._mechanics_header_name))
rating = RatingFilter(headers.get(Filter._min_rating_header_name))
expansions.set_successor(players) \
.set_successor(duration) \
.set_successor(complexity) \
.set_successor(mechanics) \
.set_successor(rating)
return expansions
def test_simple(self, request_factory, request):
request = request_factory('GET', '/', subdomain='www')
openapi_request = FlaskOpenAPIRequest(request)
path = {}
query = ImmutableMultiDict([])
headers = EnvironHeaders(request.environ)
cookies = {}
assert openapi_request.parameters == RequestParameters(
path=path,
query=query,
header=headers,
cookie=cookies,
)
assert openapi_request.host_url == request.host_url
assert openapi_request.path == request.path
assert openapi_request.method == request.method.lower()
assert openapi_request.path_pattern == request.path
assert openapi_request.body == request.data
assert openapi_request.mimetype == request.mimetype
def in_headers(self):
return EnvironHeaders(self.environ)
def application(environ, start_response):
headers = list(EnvironHeaders(environ).items())
headers[:] = [(k, v) for k, v in headers
if not is_hop_by_hop_header(k) and k.lower() not in (
'content-length', 'host')]
headers.append(('Connection', 'close'))
if opts['host'] == '<auto>':
headers.append(('Host', target.ascii_host))
elif opts['host'] is None:
headers.append(('Host', environ['HTTP_HOST']))
else:
headers.append(('Host', opts['host']))
headers.extend(opts['headers'].items())
remote_path = path
if opts['remove_prefix']:
remote_path = '%s/%s' % (target.path.rstrip('/'),
remote_path[len(prefix):].lstrip('/'))
content_length = environ.get('CONTENT_LENGTH')
chunked = False
if content_length not in ('', None):
headers.append(('Content-Length', content_length))
elif content_length is not None:
headers.append(('Transfer-Encoding', 'chunked'))
chunked = True
try:
if target.scheme == 'http':
con = httplib.HTTPConnection(target.ascii_host,
target.port or 80,
timeout=self.timeout)
elif target.scheme == 'https':
con = httplib.HTTPSConnection(target.ascii_host,
target.port or 443,
timeout=self.timeout,
context=opts['ssl_context'])
con.connect()
con.putrequest(environ['REQUEST_METHOD'],
url_quote(remote_path),
skip_host=True)
for k, v in headers:
if k.lower() == 'connection':
v = 'close'
con.putheader(k, v)
con.endheaders()
stream = get_input_stream(environ)
while 1:
data = stream.read(self.chunk_size)
if not data:
break
if chunked:
con.send(b'%x\r\n%s\r\n' % (len(data), data))
else:
con.send(data)
resp = con.getresponse()
except socket.error:
from werkzeug.exceptions import BadGateway
return BadGateway()(environ, start_response)
start_response(
'%d %s' % (resp.status, resp.reason),
[(k.title(), v)
for k, v in resp.getheaders() if not is_hop_by_hop_header(k)])
def read():
while 1:
try:
data = resp.read(self.chunk_size)
except socket.error:
break
if not data:
break
yield data
return read()
class HttpRequestLogger(object):
"""
Logs the full HTTP request to text files for debugging purposes
Note: should only be used low-request applications and/or with filters.
Example (<project>/applications.py):
def make_wsgi(profile='Default'):
config.appinit(settingsmod, profile)
app = WSGIApp()
<...snip...>
app = HttpRequestLogger(app, enabled=True, path_info_filter='files/add',
request_method_filter='post')
return app
"""
def __init__(self,
application,
enabled=False,
path_info_filter=None,
request_method_filter=None):
self.log_dir = path.join(settings.dirs.logs, 'http_requests')
mkdirs(self.log_dir)
self.application = application
self.enabled = enabled
self.pi_filter = path_info_filter
self.rm_filter = request_method_filter
def __call__(self, environ, start_response):
if self.enabled:
self.headers = EnvironHeaders(environ)
should_log = True
if self.pi_filter is not None and self.pi_filter not in environ[
'PATH_INFO']:
should_log = False
if self.rm_filter is not None and environ['REQUEST_METHOD'].lower(
) not in [x.lower() for x in tolist(self.rm_filter)]:
should_log = False
if should_log:
wsgi_input = self.replace_wsgi_input(environ)
fname = '%s_%s' % (time.time(), randchars())
fh = open(path.join(self.log_dir, fname), 'wb+')
try:
fh.write(pformat(environ))
fh.write('\n')
fh.write(wsgi_input.read())
wsgi_input.seek(0)
finally:
fh.close()
return self.application(environ, start_response)
def replace_wsgi_input(self, environ):
content_length = self.headers.get('content-length', type=int)
limited_stream = LimitedStream(environ['wsgi.input'], content_length)
if content_length is not None and content_length > 1024 * 500:
wsgi_input = TemporaryFile('wb+')
else:
wsgi_input = StringIO()
wsgi_input.write(limited_stream.read())
wsgi_input.seek(0)
environ['wsgi.input'] = wsgi_input
return environ['wsgi.input']
def __call__(self, environ, start_response):
# this middleware only used for (/call/<dbot_address>/<path:uri>
url_map = Map(
[Rule('/call/<dbot_address>/<path:uri>', endpoint='proxy')])
urls = url_map.bind_to_environ(environ)
try:
endpoint, kwargs = urls.match()
except (NotFound, MethodNotAllowed) as e:
return self.app(environ, start_response)
try:
method = environ['REQUEST_METHOD']
if method == 'OPTIONS':
# if CORS first request, return response
return self.app(environ, start_response)
dbot_address = kwargs['dbot_address']
dbot_address = to_checksum_address(dbot_address)
uri = kwargs['uri']
dbot_service = dbot.get_service(dbot_address)
if not dbot_service:
raise InvalidUsage('dbot address not found', status_code=404)
# Ensure the URI is an approved API endpoint, else abort
price, proxy_uri = _get_endpoint(dbot_service, uri, method)
if price < 0:
logger.error(
"({}: {}) is not an approved API endpoint.".format(
method, uri))
raise InvalidUsage(
"({}: {}) is not an approved API endpoint.".format(
method, uri),
status_code=404)
environ['PATH_INFO'] = '/call/{}/{}'.format(
dbot_address, remove_slash_prefix(proxy_uri))
if price > 0:
# Balance Proof check
ret, code, headers = dbot_service.paywall.access(
price, EnvironHeaders(environ))
if code != 200:
return _make_response(environ, start_response, ret, code,
headers)
logger.info("Balance Proof check is ok.")
middleware_cls = dbot_service.middleware
if middleware_cls:
# load dbot's middleware in runtime
logger.info('init middleware({}) for dbot({})'.format(
middleware_cls, dbot_address))
middleware = middleware_cls(self.app)
return middleware(environ, start_response)
else:
return self.app(environ, start_response)
except InvalidUsage as e:
logger.error('{}'.format(e.message))
return _make_response(environ, start_response,
'{}'.format(e.to_dict()), e.status_code)
except Exception as e:
logger.error('{}'.format(e))
return _make_response(environ, start_response, '{}'.format(e), 500)
def __iter__(self):
"""
Sanitized __iter__
"""
for val in EnvironHeaders.__iter__(self):
yield self.sanitize_input(val)
def __iter__(self):
"""
Sanitized __iter__
"""
for val in EnvironHeaders.__iter__(self):
yield self.sanitize_input(val)
def application(environ, start_response):
headers = list(EnvironHeaders(environ).items())
headers[:] = [(k, v) for k, v in headers
if not is_hop_by_hop_header(k) and k.lower() not in (
"content-length", "host")]
headers.append(("Connection", "close"))
if opts["host"] == "<auto>":
headers.append(("Host", target.ascii_host))
elif opts["host"] is None:
headers.append(("Host", environ["HTTP_HOST"]))
else:
headers.append(("Host", opts["host"]))
headers.extend(opts["headers"].items())
remote_path = path
if opts["remove_prefix"]:
remote_path = "%s/%s" % (
target.path.rstrip("/"),
remote_path[len(prefix):].lstrip("/"),
)
content_length = environ.get("CONTENT_LENGTH")
chunked = False
if content_length not in ("", None):
headers.append(("Content-Length", content_length))
elif content_length is not None:
headers.append(("Transfer-Encoding", "chunked"))
chunked = True
try:
if target.scheme == "http":
con = httplib.HTTPConnection(target.ascii_host,
target.port or 80,
timeout=self.timeout)
elif target.scheme == "https":
con = httplib.HTTPSConnection(
target.ascii_host,
target.port or 443,
timeout=self.timeout,
context=opts["ssl_context"],
)
con.connect()
con.putrequest(environ["REQUEST_METHOD"],
url_quote(remote_path),
skip_host=True)
for k, v in headers:
if k.lower() == "connection":
v = "close"
con.putheader(k, v)
con.endheaders()
stream = get_input_stream(environ)
while 1:
data = stream.read(self.chunk_size)
if not data:
break
if chunked:
con.send(b"%x\r\n%s\r\n" % (len(data), data))
else:
con.send(data)
resp = con.getresponse()
except socket.error:
from werkzeug.exceptions import BadGateway
return BadGateway()(environ, start_response)
start_response(
"%d %s" % (resp.status, resp.reason),
[(k.title(), v)
for k, v in resp.getheaders() if not is_hop_by_hop_header(k)],
)
def read():
while 1:
try:
data = resp.read(self.chunk_size)
except socket.error:
break
if not data:
break
yield data
return read()
def __call__(self, environ, start_response):
if self.app.debug:
headers = list(EnvironHeaders(environ).items())
for k, v in headers:
print(f'{k}: {v}')
return self.wsgi_app(environ, start_response)
def build(cls, environ: WSGIEnviron):
return cls(EnvironHeaders(environ))
def is_authorized(self, headers: EnvironHeaders) -> bool:
return self.authorization == headers.get('Authorization')
def create_field_reduction(headers: EnvironHeaders) -> 'FieldReduction':
fields = headers.get(FieldReduction._BggFieldReductionHeader)
return FieldReduction(fields.split(",") if fields else None)
