def test_no_compress_compressed_response(self):
"""
Compression isn't performed on responses that are already compressed.
"""
self.resp['Content-Encoding'] = 'deflate'
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(r.content, self.compressible_string)
self.assertEqual(r.get('Content-Encoding'), 'deflate')
def test_compress_streaming_response(self):
"""
Compression is performed on responses with streaming content.
"""
r = GZipMiddleware().process_response(self.req, self.stream_resp)
self.assertEqual(self.decompress(b''.join(r)), b''.join(self.sequence))
self.assertEqual(r.get('Content-Encoding'), 'gzip')
self.assertFalse(r.has_header('Content-Length'))
def test_compress_streaming_response(self):
"""
Tests that compression is performed on responses with streaming content.
"""
r = GZipMiddleware().process_response(self.req, self.stream_resp)
self.assertEqual(self.decompress(b"".join(r)), b"".join(self.sequence))
self.assertEqual(r.get("Content-Encoding"), "gzip")
self.assertFalse(r.has_header("Content-Length"))
def test_no_compress_incompressible_response(self):
"""
Compression isn't performed on responses with incompressible content.
"""
self.resp.content = self.incompressible_string
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(r.content, self.incompressible_string)
self.assertIsNone(r.get('Content-Encoding'))
def test_compress_response(self):
"""
Tests that compression is performed on responses with compressible content.
"""
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(self.decompress(r.content), self.compressible_string)
self.assertEqual(r.get("Content-Encoding"), "gzip")
self.assertEqual(r.get("Content-Length"), str(len(r.content)))
def test_no_compress_compressed_response(self):
"""
Tests that compression isn't performed on responses that are already compressed.
"""
self.resp["Content-Encoding"] = "deflate"
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(r.content, self.compressible_string)
self.assertEqual(r.get("Content-Encoding"), "deflate")
def test_compress_streaming_response_unicode(self):
"""
Tests that compression is performed on responses with streaming Unicode content.
"""
r = GZipMiddleware().process_response(self.req, self.stream_resp_unicode)
self.assertEqual(self.decompress(b''.join(r)), b''.join(x.encode('utf-8') for x in self.sequence_unicode))
self.assertEqual(r.get('Content-Encoding'), 'gzip')
self.assertFalse(r.has_header('Content-Length'))
def test_no_compress_uncompressible_response(self):
"""
Tests that compression isn't performed on responses with uncompressible content.
"""
self.resp.content = self.uncompressible_string
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(r.content, self.uncompressible_string)
self.assertEqual(r.get("Content-Encoding"), None)
def test_no_compress_ie_js_requests(self):
"""
Tests that compression isn't performed on JavaScript requests from Internet Explorer.
"""
self.req.META['HTTP_USER_AGENT'] = 'Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)'
self.resp['Content-Type'] = 'application/javascript; charset=UTF-8'
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(r.content, self.compressible_string)
self.assertEqual(r.get('Content-Encoding'), None)
def test_compress_non_200_response(self):
"""
Tests that compression is performed on responses with a status other than 200.
See #10762.
"""
self.resp.status_code = 404
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(self.decompress(r.content), self.compressible_string)
self.assertEqual(r.get("Content-Encoding"), "gzip")
def records(request):
query = request.GET.copy()
records = None
record_filter = query.pop("filter", ["none"])[0].lower()
record_format = query.pop("format", ["json"])[0].lower()
record_sort = query.pop("sort", ["none"])[0].lower()
cached_records = query.pop("cached", ["true"])[0].lower() in ("yes", "true")
compress_records = query.pop("compress", ["true"])[0].lower() in ("yes", "true")
pretty_records = query.pop("pretty", ["false"])[0].lower() in ("yes", "true")
cache_key = "UI_REQUEST(%s)" % hash_to_query(query)
geocode_records = query.pop("geocode", ["false"])[0].lower() in ("yes", "true")
remap_records = query.pop("remap", ["false"])[0].lower() in ("yes", "true")
if config.UI_CACHE_REQUESTS and cached_records:
records = models.cache_get_records(cache_key)
if records is None:
logger.info("Not using UI cache for %s" % cache_key)
records = models.query_ls(query=query, cached_records=cached_records)
if geocode_records:
records = models.geocode_records(records)
if remap_records:
records = models.remap_records(records)
if config.UI_CACHE_REQUESTS:
models.cache_set_records(cache_key, records, config.UI_CACHE_TIMEOUT)
else:
logger.info("Using UI cache for %s" % cache_key)
if record_filter in ("default",):
records = models.filter_default(records)
if record_sort not in ("none",):
records = sorted(records, key=lambda v: v.get(record_sort, ""))
if record_format in ("html",):
context = {}
if pretty_records:
context = {"records": records, "pretty": True}
else:
context = {"records": records, "pretty": False}
response = render(request, "servicesDirectory/records.html", context)
else:
content = ""
if pretty_records or record_format in ("pretty",):
content = json.dumps(records, sort_keys=True, indent=4)
else:
content = json.dumps(records)
response = HttpResponse(content, content_type="application/json")
if compress_records:
gzip = GZipMiddleware()
return gzip.process_response(request, response)
else:
return response
def test_streaming_compression(self):
"""
Tests that iterators as response content return a compressed stream without consuming
the whole response.content while doing so.
See #24158.
"""
self.resp.content = self.iterator_as_content
r = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(self.decompress(''.join(r.content)), self.compressible_string)
self.assertEqual(r.get('Content-Encoding'), 'gzip')
self.assertEqual(r.get('Content-Length'), None)
def test_compress_file_response(self):
"""
Compression is performed on FileResponse.
"""
with open(__file__, "rb") as file1:
file_resp = FileResponse(file1)
file_resp["Content-Type"] = "text/html; charset=UTF-8"
r = GZipMiddleware().process_response(self.req, file_resp)
with open(__file__, "rb") as file2:
self.assertEqual(self.decompress(b"".join(r)), file2.read())
self.assertEqual(r.get("Content-Encoding"), "gzip")
self.assertIsNot(r.file_to_stream, file1)
def test_compress_file_response(self):
"""
Compression is performed on FileResponse.
"""
with open(__file__, 'rb') as file1:
file_resp = FileResponse(file1)
file_resp['Content-Type'] = 'text/html; charset=UTF-8'
r = GZipMiddleware().process_response(self.req, file_resp)
with open(__file__, 'rb') as file2:
self.assertEqual(self.decompress(b''.join(r)), file2.read())
self.assertEqual(r.get('Content-Encoding'), 'gzip')
self.assertIsNot(r.file_to_stream, file1)
def test_compress_streaming_response(self):
"""
Compression is performed on responses with streaming content.
"""
def get_stream_response(request):
resp = StreamingHttpResponse(self.sequence)
resp["Content-Type"] = "text/html; charset=UTF-8"
return resp
r = GZipMiddleware(get_stream_response)(self.req)
self.assertEqual(self.decompress(b"".join(r)), b"".join(self.sequence))
self.assertEqual(r.get("Content-Encoding"), "gzip")
self.assertFalse(r.has_header("Content-Length"))
def test_compress_file_response(self):
"""
Compression is performed on FileResponse.
"""
open_file = lambda: open(__file__, 'rb')
with open_file() as file1:
file_resp = FileResponse(file1)
file_resp['Content-Type'] = 'text/html; charset=UTF-8'
r = GZipMiddleware().process_response(self.req, file_resp)
with open_file() as file2:
self.assertEqual(self.decompress(b''.join(r)), file2.read())
self.assertEqual(r.get('Content-Encoding'), 'gzip')
self.assertIsNot(r.file_to_stream, file1)
def test_compress_deterministic(self):
"""
Compression results are the same for the same content and don't
include a modification time (since that would make the results
of compression non-deterministic and prevent
ConditionalGetMiddleware from recognizing conditional matches
on gzipped content).
"""
r1 = GZipMiddleware().process_response(self.req, self.resp)
r2 = GZipMiddleware().process_response(self.req, self.resp)
self.assertEqual(r1.content, r2.content)
self.assertEqual(self.get_mtime(r1.content), 0)
self.assertEqual(self.get_mtime(r2.content), 0)
def test_compress_streaming_response(self):
"""
Compression is performed on responses with streaming content.
"""
def get_stream_response(request):
resp = StreamingHttpResponse(self.sequence)
resp['Content-Type'] = 'text/html; charset=UTF-8'
return resp
r = GZipMiddleware(get_stream_response)(self.req)
self.assertEqual(self.decompress(b''.join(r)), b''.join(self.sequence))
self.assertEqual(r.get('Content-Encoding'), 'gzip')
self.assertFalse(r.has_header('Content-Length'))
def test_compress_response(self):
"""
ETag is changed after gzip compression is performed.
"""
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
response = GZipMiddleware().process_response(request,
CommonMiddleware().process_response(request, HttpResponse(self.compressible_string)))
gzip_etag = response.get('ETag')
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='')
response = GZipMiddleware().process_response(request,
CommonMiddleware().process_response(request, HttpResponse(self.compressible_string)))
nogzip_etag = response.get('ETag')
self.assertNotEqual(gzip_etag, nogzip_etag)
def test_middleware_wont_compress_if_response_is_already_compressed(self):
fake_request = FakeRequestAcceptsBrotli()
response_content = UTF8_LOREM_IPSUM_IN_CZECH
fake_response = FakeResponse(content=response_content)
brotli_middleware = BrotliMiddleware()
django_gzip_middleware = GZipMiddleware()
gzip_response = django_gzip_middleware.process_response(
fake_request, fake_response)
brotli_response = brotli_middleware.process_response(
fake_request, gzip_response)
self.assertEqual(
response_content,
gzip.decompress(brotli_response.content).decode(encoding='utf-8'))
def test_compress_file_response(self):
"""
Compression is performed on FileResponse.
"""
with open(__file__, "rb") as file1:
def get_response(req):
file_resp = FileResponse(file1)
file_resp["Content-Type"] = "text/html; charset=UTF-8"
return file_resp
r = GZipMiddleware(get_response)(self.req)
with open(__file__, "rb") as file2:
self.assertEqual(self.decompress(b"".join(r)), file2.read())
self.assertEqual(r.get("Content-Encoding"), "gzip")
self.assertIsNot(r.file_to_stream, file1)
class GzipHttpOnlyMiddleware(object):
'''
Wraps django's default GZIP middleware to only GZIP HTTP Requests, not HTTPS
thus sidestepping the BREACH vulnerability on HTTPS pages and
getting gzip performance improvements on HTTP pages
Overview of BREACH and django:
http://stacks.11craft.com/what-is-breach-how-can-we-protect-django-projects-against-it.html
Alternate BREACH mitigation:
https://github.com/lpomfrey/django-debreach
https://github.com/jsocol/django-ratelimit
'''
def __init__(self, *args, **kwargs):
self.gzip_middleware = GZipMiddleware(*args, **kwargs)
def process_response(self, request, response):
if CONVOY_CONSERVATIVE_MSIE_GZIP:
if "msie" in request.META.get('HTTP_USER_AGENT', '').lower():
return response
if hasattr(request, 'scheme') and request.scheme is 'https':
# scheme property wasn't added until late 2013
# https://code.djangoproject.com/ticket/7603
return response
if request.is_secure():
return response
if request.META.get("HTTP_X_FORWARDED_PROTO", "") == 'https':
return response
return self.gzip_middleware.process_response(request, response)
def test_middleware_wont_compress_if_client_not_accept(self):
fake_request = FakeLegacyRequest()
response_content = UTF8_LOREM_IPSUM_IN_CZECH
fake_response = FakeResponse(content=response_content)
compression_middleware = CompressionMiddleware()
response = compression_middleware.process_response(
fake_request, fake_response)
django_gzip_middleware = GZipMiddleware()
gzip_response = django_gzip_middleware.process_response(
fake_request, fake_response)
self.assertEqual(response_content,
response.content.decode(encoding='utf-8'))
self.assertEqual(response.get('Vary'), 'Accept-Encoding')
def test_weak_etag_not_modified(self):
"""
GZipMiddleware doesn't modify a weak ETag.
"""
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
response = HttpResponse(self.compressible_string)
response['ETag'] = 'W/"eggs"'
gzip_response = GZipMiddleware().process_response(request, response)
self.assertEqual(gzip_response['ETag'], 'W/"eggs"')
def test_strong_etag_modified(self):
"""
GZipMiddleware makes a strong ETag weak.
"""
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
response = HttpResponse(self.compressible_string)
response['ETag'] = '"eggs"'
gzip_response = GZipMiddleware().process_response(request, response)
self.assertEqual(gzip_response['ETag'], 'W/"eggs"')
def gzip_action(func):
gz = GZipMiddleware()
@wraps(func)
def wrapper(self, request, *args, **kwargs):
resp = func(self, request, *args, **kwargs)
return gz.process_response(request, resp)
if getattr(settings, 'GZIP_MODELS'):
return wrapper
return func
def test_strong_etag_modified(self):
"""
GZipMiddleware makes a strong ETag weak.
"""
def get_response(req):
response = HttpResponse(self.compressible_string)
response.headers["ETag"] = '"eggs"'
return response
request = self.rf.get("/", HTTP_ACCEPT_ENCODING="gzip, deflate")
gzip_response = GZipMiddleware(get_response)(request)
self.assertEqual(gzip_response.headers["ETag"], 'W/"eggs"')
def test_weak_etag_not_modified(self):
"""
GZipMiddleware doesn't modify a weak ETag.
"""
def get_response(req):
response = HttpResponse(self.compressible_string)
response.headers['ETag'] = 'W/"eggs"'
return response
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
gzip_response = GZipMiddleware(get_response)(request)
self.assertEqual(gzip_response.headers['ETag'], 'W/"eggs"')
def wrapped_function(request, layer_name, z, x, y):
x, y, z = int(x), int(y), int(z)
tile_cache_path = os.path.realpath(settings.STATIC_ROOT + '/tiles')
layer = Layer.objects.get(name=layer_name)
name = layer.query_hash()
tile_path = tile_cache_path + '/{}/{}/{}/{}.pbf'.format(name, z, x, y)
if not os.path.isfile(tile_path):
response = func(request, name, z, x, y)
else:
print "Sending cached tile..."
response = HttpResponse(content_type='application/x-protobuf')
with open(tile_path, 'rb') as f:
response.write(f.read())
gzip_middleware = GZipMiddleware()
response = gzip_middleware.process_response(request, response)
return response
def test_etag_match(self):
"""
GZipMiddleware allows 304 Not Modified responses.
"""
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
response = GZipMiddleware().process_response(
request,
ConditionalGetMiddleware().process_response(request, HttpResponse(self.compressible_string))
)
gzip_etag = response['ETag']
next_request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate', HTTP_IF_NONE_MATCH=gzip_etag)
next_response = ConditionalGetMiddleware().process_response(
next_request,
HttpResponse(self.compressible_string)
)
self.assertEqual(next_response.status_code, 304)
def test_etag_match(self):
"""
GZipMiddleware allows 304 Not Modified responses.
"""
def get_response(req):
response = HttpResponse(self.compressible_string)
return response
def get_cond_response(req):
return ConditionalGetMiddleware(get_response)(req)
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
response = GZipMiddleware(get_cond_response)(request)
gzip_etag = response.headers['ETag']
next_request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate', HTTP_IF_NONE_MATCH=gzip_etag)
next_response = ConditionalGetMiddleware(get_response)(next_request)
self.assertEqual(next_response.status_code, 304)
def test_compress_response(self):
"""
Tests that ETag is changed after gzip compression is performed.
"""
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
response = GZipMiddleware().process_response(request,
CommonMiddleware().process_response(request,
HttpResponse(self.compressible_string)))
gzip_etag = response.get('ETag')
request = self.rf.get('/', HTTP_ACCEPT_ENCODING='')
response = GZipMiddleware().process_response(request,
CommonMiddleware().process_response(request,
HttpResponse(self.compressible_string)))
nogzip_etag = response.get('ETag')
self.assertNotEqual(gzip_etag, nogzip_etag)
def gzip_response_content(request, response):
GZipMiddleware().process_response(request, response)
def get(self, request, *args, **kwargs):
meetings = self.get_meetings()
meetings_dict = []
# Eager load all regions to reference below.
regions = Region.objects.all().prefetch_related("children")
for meeting in meetings:
meeting_types = list(
meeting.types.values_list("spec_code", flat=True))
group_info = ""
if len(meeting.district):
location = f"{meeting.meeting_location.title} (D{meeting.district})"
group_info = f"D{meeting.district}"
else:
location = meeting.meeting_location.title
gso_number = getattr(meeting.group, "gso_number", None)
if gso_number and len(gso_number):
group_info += f" / GSO #{gso_number}"
region_ancestors = list(
regions.get(
id=meeting.meeting_location.region.id).get_ancestors(
include_self=True).values_list("name", flat=True))
notes = meeting.details
meetings_dict.append({
"name":
meeting.title,
"slug":
meeting.slug,
"notes":
notes,
"updated":
f"{meeting.last_published_at if meeting.last_published_at else datetime.datetime.now():%Y-%m-%d %H:%M:%S}",
"url":
f"{settings.BASE_URL}{meeting.url_path}",
"day":
meeting.day_of_week,
"time":
f"{meeting.start_time:%H:%M}",
"end_time":
f"{meeting.end_time:%H:%M}",
"conference_url":
meeting.conference_url,
"conference_phone":
meeting.conference_phone,
"types":
meeting_types,
"location":
location,
"formatted_address":
meeting.meeting_location.formatted_address,
"latitude":
meeting.meeting_location.lat,
"longitude":
meeting.meeting_location.lng,
"regions":
region_ancestors,
"group":
group_info,
"paypal":
f"https://paypal.me/{meeting.paypal}"
if len(meeting.paypal) else "",
"venmo":
meeting.venmo,
})
response = GZipMiddleware().process_response(
request,
HttpResponse(json.dumps(meetings_dict),
content_type="application/json"))
return response
def patch_gzip_response(response, request):
if not response.status_code == 200:
return # nocv
GZipMiddleware.process_response(None, request, response)
def full_process_response(request, response, gzip=False):
if gzip:
response = GZipMiddleware().process_response(request, response)
response = EsiHeaderMiddleware().process_response(request, response)
response = IncludeEsiMiddleware().process_response(request, response)
return response
import gzip, json, base64, jwt
from datetime import datetime, timedelta, date
from django.shortcuts import get_object_or_404
from django.forms.models import model_to_dict
from django.utils import timezone
from rest_framework import viewsets, generics
from rest_framework.response import Response
from rest_framework.decorators import action
from .models import ItemDb
from .serializers import ItemDbSerializer
from django.middleware.gzip import GZipMiddleware
from django.http import FileResponse, HttpResponse
gzip_middleware = GZipMiddleware()
def data_to_gzip(data):
file_name = 'gz-warehouse/items.gz'
with gzip.GzipFile(file_name, 'w') as fout:
fout.write(json.dumps(data).encode('utf-8'))
return file_name
class ItemsList(generics.ListAPIView):
# queryset = ItemDb.objects.all()
def get(self, request):
items = ItemDb.objects.all()
serializer = ItemDbSerializer(items, many=True)
res_data = list(serializer.data)
gzip_file_name = data_to_gzip(res_data)
def __init__(self, *args, **kwargs):
self.gzip_middleware = GZipMiddleware(*args, **kwargs)