def TestOneInput(data): fdp = atheris.FuzzedDataProvider(data) whttp.parse_content_range_header(fdp.ConsumeUnicode(100)) whttp.parse_range_header(fdp.ConsumeUnicode(100)) whttp.parse_set_header(fdp.ConsumeUnicode(100)) whttp.parse_etags(fdp.ConsumeUnicode(100)) whttp.parse_if_range_header(fdp.ConsumeUnicode(100)) whttp.parse_dict_header(fdp.ConsumeUnicode(100))
def index():
cache_connection = cache.Cache()
current_theme = session.get("theme") or app.config.get(
"DEFAULT_THEME") or "stock"
response_cache_key = "firehose-%d-%s-render" % (get_slip_bitmask(),
current_theme)
cached_response_body = cache_connection.get(response_cache_key)
etag_value = "%s-%f" % (response_cache_key, time.time())
etag_cache_key = "%s-etag" % response_cache_key
if cached_response_body:
etag_header = request.headers.get("If-None-Match")
current_etag = cache_connection.get(etag_cache_key)
if etag_header:
parsed_etag = parse_etags(etag_header)
if parsed_etag.contains_weak(current_etag):
return make_response("", 304)
cached_response = make_response(cached_response_body)
cached_response.set_etag(current_etag, weak=True)
cached_response.headers["Cache-Control"] = "public,must-revalidate"
return cached_response
greeting = open("deploy-configs/index-greeting.html").read()
threads = Firehose().get_impl()
tag_styles = get_tags(threads)
template = render_template("index.html",
greeting=greeting,
threads=threads,
tag_styles=tag_styles)
uncached_response = make_response(template)
uncached_response.set_etag(etag_value, weak=True)
uncached_response.headers["Cache-Control"] = "public,must-revalidate"
cache_connection.set(response_cache_key, template)
cache_connection.set(etag_cache_key, etag_value)
return uncached_response
def serve_path(file):
site = current_site
while site:
try:
sf = StaticFile.q.get_by(site=site, path=file)
except NoData:
site = site.parent
if not site:
raise
else:
break
if parse_etags(request.environ.get('HTTP_IF_NONE_MATCH')).contains(
sf.hash):
r = Response("", mimetype=str(sf.mimetype))
r.status_code = 304
remove_entity_headers(r.headers)
else:
r = Response(sf.content, mimetype=str(sf.mimetype))
r.set_etag(sf.hash)
r.headers[b'Cache-Control'] = 'public'
r.headers[b'Expiry'] = http_date(
datetime.utcnow() + timedelta(0, current_app.config.STATIC_EXPIRE))
r.headers[b'Last-Modified'] = http_date(sf.modified)
return r
def response(self, request, data, etag=None, cache_policy=None):
"""Renders `data` to a JSON response.
An ETag may be specified. When it is not specified one will be generated
based on the data.
The caching policy can be optionally configured. By default it takes the
policy from the controller object: `cache_policy`.
"""
if etag is None and data is not None:
etag = self.etag(data)
# FIXME: Check content-type headers
if data is None:
if etag is None:
raise TypeError('response requires an etag when '
'the response body is None')
resp = Response(status=304, content_type='application/json')
else:
# Avoid sending the resource when an ETag matches
request_etags = parse_etags(
request.environ.get('HTTP_IF_NONE_MATCH'))
if request_etags.contains(etag):
resp = Response(status=304, content_type='application/json')
# Render the given data to a response object
else:
resp = Response(self.data_encoder.encode(data), content_type='application/json')
resp.headers['ETag'] = quote_etag(etag)
if cache_policy is None:
cache_policy = self.cache_policy
return cache_policy(resp)
def test_etags(self):
assert http.quote_etag("foo") == '"foo"'
assert http.quote_etag("foo", True) == 'w/"foo"'
assert http.unquote_etag('"foo"') == ("foo", False)
assert http.unquote_etag('w/"foo"') == ("foo", True)
es = http.parse_etags('"foo", "bar", w/"baz", blar')
assert sorted(es) == ["bar", "blar", "foo"]
assert "foo" in es
assert "baz" not in es
assert es.contains_weak("baz")
assert "blar" in es
assert es.contains_raw('w/"baz"')
assert es.contains_raw('"foo"')
assert sorted(es.to_header().split(", ")) == ['"bar"', '"blar"', '"foo"', 'w/"baz"']
def test_etags(self):
assert http.quote_etag('foo') == '"foo"'
assert http.quote_etag('foo', True) == 'W/"foo"'
assert http.unquote_etag('"foo"') == ('foo', False)
assert http.unquote_etag('W/"foo"') == ('foo', True)
es = http.parse_etags('"foo", "bar", W/"baz", blar')
assert sorted(es) == ['bar', 'blar', 'foo']
assert 'foo' in es
assert 'baz' not in es
assert es.contains_weak('baz')
assert 'blar' in es
assert es.contains_raw('W/"baz"')
assert es.contains_raw('"foo"')
assert sorted(es.to_header().split(', ')) == ['"bar"', '"blar"', '"foo"', 'W/"baz"']
def test_etags(self):
assert http.quote_etag('foo') == '"foo"'
assert http.quote_etag('foo', True) == 'w/"foo"'
assert http.unquote_etag('"foo"') == ('foo', False)
assert http.unquote_etag('w/"foo"') == ('foo', True)
es = http.parse_etags('"foo", "bar", w/"baz", blar')
assert sorted(es) == ['bar', 'blar', 'foo']
assert 'foo' in es
assert 'baz' not in es
assert es.contains_weak('baz')
assert 'blar' in es
assert es.contains_raw('w/"baz"')
assert es.contains_raw('"foo"')
assert sorted(es.to_header().split(', ')) == ['"bar"', '"blar"', '"foo"', 'w/"baz"']
def view(thread_id):
thread = db.session.query(Thread).filter(Thread.id == thread_id).one()
thread.views += 1
db.session.add(thread)
db.session.commit()
current_theme = session.get("theme") or app.config.get("DEFAULT_THEME") or "stock"
response_cache_key = "thread-%d-%d-%s-render" % (thread_id, get_slip_bitmask(), current_theme)
cache_connection = cache.Cache()
view_key = "thread-%d-views" % thread_id
cached_views = cache_connection.get(view_key)
fetch_from_cache = True
if cached_views is None:
fetch_from_cache = False
else:
cached_views = int(cached_views)
if fetch_from_cache and (thread.views / cached_views) >= app.config.get("CACHE_VIEW_RATIO", 0):
fetch_from_cache = False
etag_value = "%s-%f" % (response_cache_key, time.time())
etag_cache_key = "%s-etag" % response_cache_key
if fetch_from_cache:
etag_header = request.headers.get("If-None-Match")
current_etag = cache_connection.get(etag_cache_key)
if etag_header:
parsed_etag = parse_etags(etag_header)
if parsed_etag.contains_weak(current_etag):
return make_response("", 304)
cache_response_body = cache_connection.get(response_cache_key)
if cache_response_body is not None:
cached_response = make_response(cache_response_body)
cached_response.set_etag(current_etag, weak=True)
cached_response.headers["Cache-Control"] = "public,must-revalidate"
return cached_response
posts = ThreadPosts().retrieve(thread_id)
render_for_threads(posts)
board = db.session.query(Board).get(thread.board)
num_posters = db.session.query(Poster).filter(Poster.thread == thread_id).count()
num_media = thread.num_media()
reply_urls = _get_reply_urls(posts)
template = render_template("thread.html", thread_id=thread_id, board=board, posts=posts, num_views=thread.views,
num_media=num_media, num_posters=num_posters, reply_urls=reply_urls)
uncached_response = make_response(template)
uncached_response.set_etag(etag_value, weak=True)
uncached_response.headers["Cache-Control"] = "public,must-revalidate"
cache_connection.set(view_key, str(thread.views))
cache_connection.set(response_cache_key, template)
cache_connection.set(etag_cache_key, etag_value)
return uncached_response
def test_etags(self):
assert http.quote_etag("foo") == '"foo"'
assert http.quote_etag("foo", True) == 'W/"foo"'
assert http.unquote_etag('"foo"') == ("foo", False)
assert http.unquote_etag('W/"foo"') == ("foo", True)
es = http.parse_etags('"foo", "bar", W/"baz", blar')
assert sorted(es) == ["bar", "blar", "foo"]
assert "foo" in es
assert "baz" not in es
assert es.contains_weak("baz")
assert "blar" in es
assert es.contains_raw('W/"baz"')
assert es.contains_raw('"foo"')
assert sorted(es.to_header().split(", ")) == [
'"bar"',
'"blar"',
'"foo"',
'W/"baz"',
]
def catalog(board_id):
current_theme = session.get("theme") or app.config.get(
"DEFAULT_THEME") or "stock"
response_cache_key = "board-%d-%d-%s-render" % (
board_id, get_slip_bitmask(), current_theme)
cache_connection = cache.Cache()
cached_response_body = cache_connection.get(response_cache_key)
etag_value = "%s-%f" % (response_cache_key, time.time())
etag_cache_key = "%s-etag" % response_cache_key
if cached_response_body:
etag_header = request.headers.get("If-None-Match")
current_etag = cache_connection.get(etag_cache_key)
if etag_header:
parsed_etag = parse_etags(etag_header)
if parsed_etag.contains_weak(current_etag):
return make_response("", 304)
cached_response = make_response(cached_response_body)
cached_response.set_etag(current_etag, weak=True)
cached_response.headers["Cache-Control"] = "public,must-revalidate"
return cached_response
threads = BoardCatalog().retrieve(board_id)
board = db.session.query(Board).get(board_id)
board_name = board.name
render_for_catalog(threads)
tag_styles = get_tags(threads)
catalog_data = {}
catalog_data["tag_styles"] = tag_styles
for thread in threads:
del thread["last_updated"]
catalog_data["threads"] = threads
extra_data = {}
if app.config.get("CAPTCHA_METHOD") == "CAPTCHOULI":
extra_data = renderer.captchouli_to_json(captchouli.request_captcha())
template = renderer.render_catalog(catalog_data, board_name, board_id,
extra_data)
uncached_response = make_response(template)
uncached_response.set_etag(etag_value, weak=True)
uncached_response.headers["Cache-Control"] = "public,must-revalidate"
cache_connection.set(response_cache_key, template)
cache_connection.set(etag_cache_key, etag_value)
return uncached_response
def download(request,oid,name=None):
obj = Object.by_oid(oid)
r = Response(obj.content, mimetype=obj.mimetype)
if name:
n = obj.name
if obj.mime.ext:
n += "."+obj.mime.ext
assert n == name
if parse_etags(request.environ.get('HTTP_IF_NONE_MATCH')).contains(obj.hash):
r = Response("", mimetype=obj.mimetype)
r.status_code = 304
remove_entity_headers(r.headers)
else:
r = Response(obj.content, mimetype=obj.mimetype)
r.set_etag(obj.hash)
r.headers['Cache-Control']='public'
r.headers['Expiry']=http_date(datetime.utcnow()+timedelta(999))
r.headers['Last-Modified']=http_date(obj.timestamp)
return r
def download(request, oid, name=None):
obj = Object.by_oid(oid)
r = Response(obj.content, mimetype=obj.mimetype)
if name:
n = obj.name
if obj.mime.ext:
n += "." + obj.mime.ext
assert n == name
if parse_etags(request.environ.get('HTTP_IF_NONE_MATCH')).contains(
obj.hash):
r = Response("", mimetype=obj.mimetype)
r.status_code = 304
remove_entity_headers(r.headers)
else:
r = Response(obj.content, mimetype=obj.mimetype)
r.set_etag(obj.hash)
r.headers['Cache-Control'] = 'public'
r.headers['Expiry'] = http_date(datetime.utcnow() + timedelta(999))
r.headers['Last-Modified'] = http_date(obj.timestamp)
return r
def serve_path(file):
site = current_site
while site:
try:
sf = StaticFile.q.get_by(site=site, path=file)
except NoData:
site = site.parent
if not site:
raise
else:
break
if parse_etags(request.environ.get('HTTP_IF_NONE_MATCH')).contains(sf.hash):
r = Response("", mimetype=str(sf.mimetype))
r.status_code = 304
remove_entity_headers(r.headers)
else:
r = Response(sf.content, mimetype=str(sf.mimetype))
r.set_etag(sf.hash)
r.headers[b'Cache-Control']='public'
r.headers[b'Expiry']=http_date(datetime.utcnow()+timedelta(0,current_app.config.STATIC_EXPIRE))
r.headers[b'Last-Modified']=http_date(sf.modified)
return r
def catalog(board_id):
current_theme = session.get("theme") or app.config.get(
"DEFAULT_THEME") or "stock"
response_cache_key = "board-%d-%d-%s-render" % (
board_id, get_slip_bitmask(), current_theme)
cache_connection = cache.Cache()
cached_response_body = cache_connection.get(response_cache_key)
etag_value = "%s-%f" % (response_cache_key, time.time())
etag_cache_key = "%s-etag" % response_cache_key
if cached_response_body:
etag_header = request.headers.get("If-None-Match")
current_etag = cache_connection.get(etag_cache_key)
if etag_header:
parsed_etag = parse_etags(etag_header)
if parsed_etag.contains_weak(current_etag):
return make_response("", 304)
cached_response = make_response(cached_response_body)
cached_response.set_etag(current_etag, weak=True)
cached_response.headers["Cache-Control"] = "public,must-revalidate"
return cached_response
threads = BoardCatalog().retrieve(board_id)
board = db.session.query(Board).get(board_id)
board_name = board.name
render_for_catalog(threads)
tag_styles = get_tags(threads)
template = render_template("catalog.html",
threads=threads,
board=board,
board_name=board_name,
tag_styles=tag_styles)
uncached_response = make_response(template)
uncached_response.set_etag(etag_value, weak=True)
uncached_response.headers["Cache-Control"] = "public,must-revalidate"
cache_connection.set(response_cache_key, template)
cache_connection.set(etag_cache_key, etag_value)
return uncached_response
def test_etags_nonzero(self):
etags = http.parse_etags('w/"foo"')
self.assert_(bool(etags))
self.assert_(etags.contains_raw('w/"foo"'))
def test_etags_nonzero(self):
etags = http.parse_etags('W/"foo"')
assert bool(etags)
assert etags.contains_raw('W/"foo"')
def make_conditional(
self,
request_or_environ: "WSGIEnvironment",
accept_ranges: t.Union[bool, str] = False,
complete_length: t.Optional[int] = None,
) -> "Response":
"""Make the response conditional to the request. This method works
best if an etag was defined for the response already. The `add_etag`
method can be used to do that. If called without etag just the date
header is set.
This does nothing if the request method in the request or environ is
anything but GET or HEAD.
For optimal performance when handling range requests, it's recommended
that your response data object implements `seekable`, `seek` and `tell`
methods as described by :py:class:`io.IOBase`. Objects returned by
:meth:`~werkzeug.wsgi.wrap_file` automatically implement those methods.
It does not remove the body of the response because that's something
the :meth:`__call__` function does for us automatically.
Returns self so that you can do ``return resp.make_conditional(req)``
but modifies the object in-place.
:param request_or_environ: a request object or WSGI environment to be
used to make the response conditional
against.
:param accept_ranges: This parameter dictates the value of
`Accept-Ranges` header. If ``False`` (default),
the header is not set. If ``True``, it will be set
to ``"bytes"``. If ``None``, it will be set to
``"none"``. If it's a string, it will use this
value.
:param complete_length: Will be used only in valid Range Requests.
It will set `Content-Range` complete length
value and compute `Content-Length` real value.
This parameter is mandatory for successful
Range Requests completion.
:raises: :class:`~werkzeug.exceptions.RequestedRangeNotSatisfiable`
if `Range` header could not be parsed or satisfied.
.. versionchanged:: 2.0
Range processing is skipped if length is 0 instead of
raising a 416 Range Not Satisfiable error.
"""
environ = _get_environ(request_or_environ)
if environ["REQUEST_METHOD"] in ("GET", "HEAD"):
# if the date is not in the headers, add it now. We however
# will not override an already existing header. Unfortunately
# this header will be overriden by many WSGI servers including
# wsgiref.
if "date" not in self.headers:
self.headers["Date"] = http_date()
accept_ranges = _clean_accept_ranges(accept_ranges)
is206 = self._process_range_request(environ, complete_length,
accept_ranges)
if not is206 and not is_resource_modified(
environ,
self.headers.get("etag"),
None,
self.headers.get("last-modified"),
):
if parse_etags(environ.get("HTTP_IF_MATCH")):
self.status_code = 412
else:
self.status_code = 304
if (self.automatically_set_content_length
and "content-length" not in self.headers):
length = self.calculate_content_length()
if length is not None:
self.headers["Content-Length"] = length
return self
def if_none_match(self):
"""An object containing all the etags in the `If-None-Match` header."""
return parse_etags(self.environ.get('HTTP_IF_NONE_MATCH'))
def if_match(self):
return parse_etags(self.environ.get("HTTP_IF_MATCH"))
def if_none_match(self):
return parse_etags(self.environ.get('HTTP_IF_NONE_MATCH'))
def view(thread_id):
thread = db.session.query(Thread).filter(Thread.id == thread_id).one()
thread.views += 1
db.session.add(thread)
db.session.commit()
current_theme = session.get("theme") or app.config.get(
"DEFAULT_THEME") or "stock"
response_cache_key = "thread-%d-%d-%s-render" % (
thread_id, get_slip_bitmask(), current_theme)
cache_connection = cache.Cache()
view_key = "thread-%d-views" % thread_id
cached_views = cache_connection.get(view_key)
fetch_from_cache = True
if cached_views is None:
fetch_from_cache = False
else:
cached_views = int(cached_views)
if fetch_from_cache and (thread.views / cached_views) >= app.config.get(
"CACHE_VIEW_RATIO", 0):
fetch_from_cache = False
etag_value = "%s-%f" % (response_cache_key, time.time())
etag_cache_key = "%s-etag" % response_cache_key
if fetch_from_cache:
etag_header = request.headers.get("If-None-Match")
current_etag = cache_connection.get(etag_cache_key)
if etag_header:
parsed_etag = parse_etags(etag_header)
if parsed_etag.contains_weak(current_etag):
return make_response("", 304)
cache_response_body = cache_connection.get(response_cache_key)
if cache_response_body is not None:
cached_response = make_response(cache_response_body)
cached_response.set_etag(current_etag, weak=True)
cached_response.headers["Cache-Control"] = "public,must-revalidate"
return cached_response
posts = ThreadPosts().retrieve(thread_id)
render_for_threads(posts)
board = db.session.query(Board).get(thread.board)
num_posters = db.session.query(Poster).filter(
Poster.thread == thread_id).count()
num_media = thread.num_media()
reply_urls = _get_reply_urls(posts)
thread_data = {}
for post in posts:
post["datetime"] = post["datetime"].strftime(
"%a, %d %b %Y %H:%M:%S UTC")
if post["media"]:
post["media_url"] = storage.get_media_url(post["media"],
post["media_ext"])
post["thumb_url"] = storage.get_thumb_url(post["media"])
thread_data["posts"] = posts
extra_data = {}
if app.config.get("CAPTCHA_METHOD") == "CAPTCHOULI":
extra_data = renderer.captchouli_to_json(captchouli.request_captcha())
template = renderer.render_thread(thread_data, thread_id, extra_data)
uncached_response = make_response(template)
uncached_response.set_etag(etag_value, weak=True)
uncached_response.headers["Cache-Control"] = "public,must-revalidate"
cache_connection.set(view_key, str(thread.views))
cache_connection.set(response_cache_key, template)
cache_connection.set(etag_cache_key, etag_value)
return uncached_response
def if_none_match(self) -> ETags:
return parse_etags(self.headers.get("If-None-Match"))
def if_none_match(self):
return parse_etags(self.environ.get('HTTP_IF_NONE_MATCH'))
def test_etags_nonzero(self):
etags = http.parse_etags('w/"foo"')
self.assert_true(bool(etags))
self.assert_true(etags.contains_raw('w/"foo"'))
def test_etags_nonzero(self):
etags = http.parse_etags('W/"foo"')
assert bool(etags)
assert etags.contains_raw('W/"foo"')
