def process_request(self, request):
request._cache_key_prefix = key_prefix = self.get_key_prefix(
request,
*request.resolver_match.args,
**request.resolver_match.kwargs
)
with patch(self, 'key_prefix', key_prefix):
response = super(CacheMiddleware, self).process_request(request)
# check if we should return "304 Not Modified"
response = response and get_conditional_response(request, response)
# setting cache age
if response and 'Expires' in response:
max_age = get_cache_max_age(response.get('Cache-Control'))
if max_age:
expires = http.parse_http_date(response['Expires'])
timeout = expires - int(time.time())
response['Age'] = age = max_age - timeout
# check cache age limit provided by client
age_limit = get_cache_max_age(request.META.get('HTTP_CACHE_CONTROL'))
if age_limit is None and request.META.get('HTTP_PRAGMA') == 'no-cache':
age_limit = 0
if age_limit is not None:
min_age = self.cache_min_age
if min_age is None:
min_age = getattr(settings, 'DJANGOCACHE_MIN_AGE', 0)
age_limit = max(min_age, age_limit)
if age >= age_limit:
request._cache_update_cache = True
return None
return response
def dynamic_css(request, css):
if not _dynamic_cssmap.has_key(css):
raise Http404("CSS not found")
files = _dynamic_cssmap[css]
resp = HttpResponse(content_type="text/css")
# We honor if-modified-since headers by looking at the most recently
# touched CSS file.
latestmod = 0
for fn in files:
try:
stime = os.stat(fn).st_mtime
if latestmod < stime:
latestmod = stime
except OSError:
# If we somehow referred to a file that didn't exist, or
# one that we couldn't access.
raise Http404("CSS (sub) not found")
if request.META.has_key("HTTP_IF_MODIFIED_SINCE"):
# This code is mostly stolen from django :)
matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", request.META.get("HTTP_IF_MODIFIED_SINCE"), re.IGNORECASE)
header_mtime = parse_http_date(matches.group(1))
# We don't do length checking, just the date
if int(latestmod) <= header_mtime:
return HttpResponseNotModified(mimetype="text/css")
resp["Last-Modified"] = http_date(latestmod)
for fn in files:
with open(fn) as f:
resp.write("/* %s */\n" % fn)
resp.write(f.read())
resp.write("\n")
return resp
def sendfile(request, filename, **kwargs):
# Respect the If-Modified-Since header.
parseresult = urlparse(filename)
if parseresult.scheme:
with contextlib.closing(urllib2.urlopen(filename)) as result:
headers = result.headers.dict
data = result.read()
lastmodified = parse_http_date(headers['last-modified'])
contentlength = int(headers['content-length'])
response = HttpResponse(data)
else:
statobj = os.stat(filename)
lastmodified = statobj[stat.ST_MTIME]
contentlength = statobj[stat.ST_SIZE]
response = HttpResponse(File(open(filename, 'rb')).chunks())
if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
lastmodified, contentlength):
return HttpResponseNotModified()
response["Last-Modified"] = http_date(lastmodified)
return response
def should_regenerate(self, response):
""" Check if this page was originally generated less than LOCAL_POSTCHECK seconds ago """
if response.has_header('Last-Modified'):
last_modified = parse_http_date(response['Last-Modified'])
next_regen = last_modified + settings.BETTERCACHE_LOCAL_POSTCHECK
return time.time() > next_regen
def is_expired(self, url):
"""
If the cached response for the given URL is expired based on
Cache-Control or Expires headers, returns True.
"""
if url not in self:
return True
headers = self.get_headers(url)
# First check if the Cache-Control header has set a max-age
if 'cache-control' in headers:
cache_control = parse_cache_control_header(headers['cache-control'])
if 'max-age' in cache_control:
max_age = int(cache_control['max-age'])
response_age = int(
os.stat(self._header_cache_file_path(url)).st_mtime)
current_timestamp = int(time.time())
return current_timestamp - response_age >= max_age
# Alternatively, try the Expires header
if 'expires' in headers:
expires_date = timezone.datetime.utcfromtimestamp(
parse_http_date(headers['expires']))
expires_date = timezone.make_aware(expires_date, timezone.utc)
current_date = timezone.now()
return current_date > expires_date
# If there is no cache freshness date consider the item expired
return True
def was_modified_since(header=None, mtime=0, size=0):
"""
Was something modified since the user last downloaded it?
header
This is the value of the If-Modified-Since header. If this is None,
I'll just return True.
mtime
This is the modification time of the item we're talking about.
size
This is the size of the item we're talking about.
"""
try:
if header is None:
raise ValueError
matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header,
re.IGNORECASE)
header_mtime = parse_http_date(matches.group(1))
header_len = matches.group(3)
if header_len and int(header_len) != size:
raise ValueError
if int(mtime) > header_mtime:
raise ValueError
except (AttributeError, ValueError, OverflowError):
return True
return False
def _serve_file(request, stat, rel_path, full_path):
since = request.META.get('HTTP_IF_MODIFIED_SINCE')
mimetype = _guess_mimetype(rel_path)
if since and parse_http_date(since) > stat.st_mtime:
return HttpResponseNotModified(mimetype=mimetype)
response = HttpResponse(open(full_path, 'rb').read(), mimetype=mimetype)
response["Last-Modified"] = http_date(stat.st_mtime)
return response
def test_cache_headers(self):
"""
Should have appropriate Cache-Control and Expires headers.
"""
with self.activate('en-US'):
resp = self.client.get(reverse('tabzilla'))
self.assertEqual(resp['cache-control'], 'max-age=43200') # 12h
now_date = floor(time.time())
exp_date = parse_http_date(resp['expires'])
self.assertAlmostEqual(now_date + 43200, exp_date, delta=2)
def _test_cache_headers(self, view, hours):
"""
Should have appropriate Cache-Control and Expires headers.
"""
test_request = self.rf.get('/hi-there-dude/')
resp = view(test_request)
num_seconds = hours * 60 * 60
self.assertEqual(resp['cache-control'], 'max-age=%d' % num_seconds)
now_date = floor(time.time())
exp_date = parse_http_date(resp['expires'])
self.assertAlmostEqual(now_date + num_seconds, exp_date, delta=2)
def get_media(request, path):
pathname = os.path.join(settings.MEDIA_ROOT, path)
# Get the Media and check the permissions
media = get_object_or_404(Media, path=pathname)
if not media.is_visible_to(request.user):
# If the user is not logged-in, redirect to the login page
if not request.user.is_authenticated():
return redirect_to_login(request.get_full_path(),
settings.LOGIN_URL,
REDIRECT_FIELD_NAME)
else:
return HttpResponseForbidden()
# Get a thumbnails if requested
size_str = request.GET.get('size', None)
if size_str == 'small' or size_str == 'medium':
# Set the size
if size_str == 'small':
size = (280, 210)
else:
size = (800, 600)
smallpath = os.path.join(settings.CACHE_ROOT, size_str, path)
# TODO: check that the thumbnails is youger than the original image
if not os.path.isfile(smallpath):
# Create the destination directory and thumbnail
mkdir(os.path.dirname(smallpath))
if not create_thumbnail(media, smallpath, size):
raise Http404
pathname = smallpath
# Stat the file to grab metadata
stats = os.stat(pathname)
# Check if the client has the media in his cache
if_modified_since = request.META.get("HTTP_IF_MODIFIED_SINCE")
if if_modified_since:
if_modified_since = parse_http_date(if_modified_since)
if if_modified_since >= int(stats.st_mtime):
return HttpResponseNotModified()
# Stream the file
# FIXME: this will be wrong for video thumbnails
mime = mimetypes.guess_type(pathname)
response = FileResponse(open(pathname, 'rb'),
content_type=mime[0] if mime[0] else 'text/plain')
# Set the headers
response['Content-Length'] = stats.st_size
response['Last-Modified'] = http_date(stats.st_mtime)
return response
def process_response(self, request, response):
if 'Expires' in response:
# Replace 'max-age' value of 'Cache-Control' header by one
# calculated from the 'Expires' header's date.
# This is necessary because of Django's `FetchFromCacheMiddleware`
# gets 'Cache-Control' header from the cache
# where 'max-age' corresponds to the moment original response
# was generated and thus may be already stale for the current time
expires = parse_http_date(response['Expires'])
timeout = expires - int(now().timestamp())
patch_cache_control(response, max_age=timeout)
key_prefix = getattr(request, '_cache_key_prefix', self.key_prefix)
with patch(self, 'key_prefix', key_prefix):
return super().process_response(request, response)
def has_uncacheable_headers(self, response):
""" Should this response be cached based on it's headers
broken out from should_cache for flexibility
"""
cc_dict = get_header_dict(response, 'Cache-Control')
if cc_dict:
if cc_dict.has_key('max-age') and cc_dict['max-age'] == '0':
return True
if cc_dict.has_key('no-cache'):
return True
if cc_dict.has_key('private'):
return True
if response.has_header('Expires'):
if parse_http_date(response['Expires']) < time.time():
return True
return False
def has_uncacheable_headers(self, response):
""" Should this response be cached based on it's headers
broken out from should_cache for flexibility
"""
cc_dict = get_header_dict(response, "Cache-Control")
if cc_dict:
if "max-age" in cc_dict and cc_dict["max-age"] == "0":
return True
if "no-cache" in cc_dict:
return True
if "private" in cc_dict:
return True
if response.has_header("Expires"):
if parse_http_date(response["Expires"]) < time.time():
return True
return False
def test_read_model_mixin_must_return_last_modified_header_as_now(self):
dummier = DummierModel.objects.create(name='my dummier')
class DummierResource(ModelResource):
model = DummierModel
request = self.req.get('/dummiers')
mixin = ReadModelMixin()
mixin.resource = DummierResource
response = mixin.get(request, dummier.id)
self.assertEquals(dummier.name, response.cleaned_content.name)
self.assertTrue(response.headers.has_key('Last-Modified'))
last_modified = response.headers['Last-Modified']
# no error should occur, meaning we have the right format
last_modified_datetime = parse_http_date(last_modified)
self.assertEqual(last_modified_datetime, time.mktime(datetime(2012, 12, 12, 6, 6, 6).timetuple()))
def parse_datetime(datestr):
'''Turns a ISO8601 or RFC1123 date string representation to a Python
datetime instance.'''
try:
datestr = float(datestr)
return datetime.fromtimestamp(datestr)
except ValueError:
pass
try:
datestr = str(datestr)
if 'GMT' in datestr:
return datetime.fromtimestamp(parse_http_date(datestr))
return datetime.strptime(datestr, ISO8601_DATEFORMAT)
except Exception, e:
raise ValueError('Unable to parse date \'%s\' (reason: %s)' %
(datestr, repr(e)))
def is_precondition_failed(self, request, response, *args, **kwargs):
# ETags are enabled. Check for conditional request headers. The current
# ETag value is used for the conditional requests. After the request
# method handler has been processed, the new ETag will be calculated.
if self.use_etags and 'HTTP_IF_MATCH' in request.META:
request_etag = parse_etags(request.META['HTTP_IF_MATCH'])[0]
etag = self.get_etag(request, request_etag)
if request_etag != etag:
return True
# Last-Modified date enabled. check for conditional request headers. The
# current modification datetime value is used for the conditional
# requests. After the request method handler has been processed, the new
# Last-Modified datetime will be returned.
if self.use_last_modified and 'HTTP_IF_UNMODIFIED_SINCE' in request.META:
last_modified = self.get_last_modified(request, *args, **kwargs)
known_last_modified = EPOCH_DATE + timedelta(seconds=parse_http_date(request.META['HTTP_IF_UNMODIFIED_SINCE']))
if known_last_modified != last_modified:
return True
return False
def test_parsing_year_less_than_70(self):
parsed = parse_http_date('Sun Nov 6 08:49:37 0050')
self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(2050, 11, 6, 8, 49, 37))
def test_parsing_asctime(self):
parsed = parse_http_date("Sun Nov 6 08:49:37 1994")
self.assertEqual(
datetime.fromtimestamp(parsed, timezone.utc),
datetime(1994, 11, 6, 8, 49, 37, tzinfo=timezone.utc),
)
def test_parsing_rfc1123(self):
parsed = http.parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT')
self.assertEqual(datetime.utcfromtimestamp(parsed),
datetime(1994, 11, 6, 8, 49, 37))
def testParsingAsctime(self):
parsed = parse_http_date("Sun Nov 6 08:49:37 1994")
self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 06, 8, 49, 37))
def testParsingRfc1123(self):
parsed = http.parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT')
self.assertEqual(datetime.utcfromtimestamp(parsed),
datetime(1994, 11, 6, 8, 49, 37))
def testParsingRfc850(self):
parsed = http.parse_http_date('Sunday, 06-Nov-94 08:49:37 GMT')
self.assertEqual(datetime.utcfromtimestamp(parsed),
datetime(1994, 11, 6, 8, 49, 37))
def test_parsing_asctime_nonascii_digits(self):
"""Non-ASCII unicode decimals raise an error."""
with self.assertRaises(ValueError):
parse_http_date("Sun Nov 6 08:49:37 １９９４")
with self.assertRaises(ValueError):
parse_http_date("Sun Nov １２ 08:49:37 1994")
def test_parsing_year_less_than_70(self):
parsed = parse_http_date("Sun Nov 6 08:49:37 0037")
self.assertEqual(
datetime.fromtimestamp(parsed, timezone.utc),
datetime(2037, 11, 6, 8, 49, 37, tzinfo=timezone.utc),
)
def test_parsing_asctime(self):
parsed = http.parse_http_date('Sun Nov 6 08:49:37 1994')
self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def testParsingRfc850(self):
parsed = parse_http_date("Sunday, 06-Nov-94 08:49:37 GMT")
self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 06, 8, 49, 37))
def process_request(self, request, *args, **kwargs):
# Initilize a new response for this request. Passing the response along
# the request cycle allows for gradual modification of the headers.
response = HttpResponse()
# TODO keep track of a list of request headers used to
# determine the resource representation for the 'Vary'
# header.
# ### 503 Service Unavailable
# The server does not need to be unavailable for a resource to be
# unavailable...
if self.is_service_unavailable(request, response, *args, **kwargs):
response.status_code = codes.service_unavailable
return response
# ### 414 Request URI Too Long _(not implemented)_
# This should be be handled upstream by the Web server
# ### 400 Bad Request _(not implemented)_
# Note that many services respond with this code when entities are
# unprocessable. This should really be a 422 Unprocessable Entity
# Most actualy bad requests are handled upstream by the Web server
# when parsing the HTTP message
# ### 401 Unauthorized
# Check if the request is authorized to access this resource.
if self.is_unauthorized(request, response, *args, **kwargs):
response.status_code = codes.unauthorized
return response
# ### 403 Forbidden
# Check if this resource is forbidden for the request.
if self.is_forbidden(request, response, *args, **kwargs):
response.status_code = codes.forbidden
return response
# ### 501 Not Implemented _(not implemented)_
# This technically refers to a service-wide response for an
# unimplemented request method, again this is upstream.
# ### 429 Too Many Requests
# Both `rate_limit_count` and `rate_limit_seconds` must be none
# falsy values to be checked.
if self.rate_limit_count and self.rate_limit_seconds:
if self.is_too_many_requests(request, response, *args, **kwargs):
response.status_code = codes.too_many_requests
return response
# ### 405 Method Not Allowed
if self.is_method_not_allowed(request, response, *args, **kwargs):
response.status_code = codes.method_not_allowed
return response
# ### 406 Not Acceptable
# Checks Accept and Accept-* headers
if self.is_not_acceptable(request, response, *args, **kwargs):
response.status_code = codes.not_acceptable
return response
# ### Process an _OPTIONS_ request
# Enough processing has been performed to allow an OPTIONS request.
if request.method == methods.OPTIONS and 'OPTIONS' in self.allowed_methods:
return self.options(request, response)
# ## Request Entity Checks
# Only perform these checks if the request has supplied a body.
if 'CONTENT_LENGTH' in request.META and request.META['CONTENT_LENGTH']:
# ### 415 Unsupported Media Type
# Check if the entity `Content-Type` supported for decoding.
if self.is_unsupported_media_type(request, response, *args, **kwargs):
response.status_code = codes.unsupported_media_type
return response
# ### 413 Request Entity Too Large
# Check if the entity is too large for processing
if self.is_request_entity_too_large(request, response, *args, **kwargs):
response.status_code = codes.request_entity_too_large
return response
# ### 404 Not Found
# Check if this resource exists. Note, if this requires a database
# lookup or some other expensive lookup, the relevant object may
# be _attached_ to the request or response object to be used
# dowstream in the handler. This prevents multiple database
# hits or filesystem lookups.
if self.is_not_found(request, response, *args, **kwargs):
response.status_code = codes.not_found
return response
# ### 410 Gone
# Check if this resource used to exist, but does not anymore. A common
# strategy for this when dealing with this in a database context is to
# have an `archived` or `deleted` flag that can be used associated with
# the given lookup key while the rest of the content in the row may be
# deleted.
if self.is_gone(request, response, *args, **kwargs):
response.status_code = codes.gone
return response
# ### 428 Precondition Required
# Prevents the "lost udpate" problem and requires client to confirm
# the state of the resource has not changed since the last `GET`
# request. This applies to `PUT` and `PATCH` requests.
if request.method == methods.PUT or request.method == methods.PATCH:
if self.is_precondition_required(request, response, *args, **kwargs):
return UncacheableResponse(status=codes.precondition_required)
# ### 412 Precondition Failed
# Conditional requests applies to GET, HEAD, PUT, and PATCH.
# For GET and HEAD, the request checks the either the entity changed
# since the last time it requested it, `If-Modified-Since`, or if the
# entity tag (ETag) has changed, `If-None-Match`.
if request.method == methods.PUT or request.method == methods.PATCH:
if self.is_precondition_failed(request, response, *args, **kwargs):
return UncacheableResponse(status=codes.precondition_failed)
# Check for conditional GET or HEAD request
if request.method == methods.GET or request.method == methods.HEAD:
# Check Etags before Last-Modified...
if self.use_etags and 'HTTP_IF_NONE_MATCH' in request.META:
# Parse request Etags (only one is currently supported)
request_etag = parse_etags(request.META['HTTP_IF_NONE_MATCH'])[0]
# Check if the request Etag is valid. The current Etag is
# supplied to enable strategies where the etag does not need
# to be used to regenerate the Etag. This may include
# generating an MD5 of the resource and storing it as a key
# in memcache.
etag = self.get_etag(request, response, request_etag, *args, **kwargs)
# Nothing has changed, simply return
if request_etag == etag:
response.status_code = codes.not_modified
return response
if self.use_last_modified and 'HTTP_IF_MODIFIED_SINCE' in request.META:
# Get the last known modified date from the client, compare it
# to the last modified date of the resource
last_modified = self.get_last_modified(request, *args, **kwargs)
known_last_modified = EPOCH_DATE + timedelta(seconds=parse_http_date(request.META['HTTP_IF_MODIFIED_SINCE']))
if known_last_modified >= last_modified:
response.status_code = codes.not_modified
return response
# Check if there is a request body
if 'CONTENT_LENGTH' in request.META and request.META['CONTENT_LENGTH']:
content_type = request._content_type
if content_type in serializers:
request.data = serializers.decode(content_type, request.body)
"""
def test_parsing_asctime(self):
parsed = parse_http_date('Sun Nov 6 08:49:37 1994')
self.assertEqual(datetime.utcfromtimestamp(parsed),
datetime(1994, 11, 6, 8, 49, 37))
def test_parsing_rfc850(self):
parsed = http.parse_http_date('Sunday, 06-Nov-94 08:49:37 GMT')
self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def test_parsing_year_less_than_70(self):
parsed = parse_http_date('Sun Nov 6 08:49:37 0037')
self.assertEqual(datetime.utcfromtimestamp(parsed),
datetime(2037, 11, 6, 8, 49, 37))
def test_parsing_rfc1123(self):
parsed = parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT')
self.assertEqual(
datetime.fromtimestamp(parsed, timezone.utc),
datetime(1994, 11, 6, 8, 49, 37, tzinfo=timezone.utc),
)
import re
import os
from pathlib import Path
from django.utils.http import parse_http_date
parser = argparse.ArgumentParser(description="tpts.py")
parser.add_argument("path")
args = parser.parse_args()
p = Path(args.path)
m = re.match(r'(^[^\.]*)\.(.*$)', str(p.name))
filename = m.group(1)
ext = re.sub("[\-\.].*", "", m.group(2))
ext = ext.replace('_large', '')
url = 'https://pbs.twimg.com/media/' + filename + "." + ext + ":orig"
print(url)
r = requests.head(url)
if "Last-Modified" in r.headers:
lm = parse_http_date(r.headers["Last-Modified"])
cl = int(r.headers["Content-Length"])
if p.stat().st_size < cl:
r = requests.get(url)
with p.open("wb") as f:
f.write(r.content)
nf = p.parent / Path(filename + "." + ext + "-orig." + ext)
p.rename(nf)
os.utime(str(nf), (lm, lm))
