def process_request(self, request): request._cache_key_prefix = key_prefix = self.get_key_prefix( request, *request.resolver_match.args, **request.resolver_match.kwargs ) with patch(self, 'key_prefix', key_prefix): response = super(CacheMiddleware, self).process_request(request) # check if we should return "304 Not Modified" response = response and get_conditional_response(request, response) # setting cache age if response and 'Expires' in response: max_age = get_cache_max_age(response.get('Cache-Control')) if max_age: expires = http.parse_http_date(response['Expires']) timeout = expires - int(time.time()) response['Age'] = age = max_age - timeout # check cache age limit provided by client age_limit = get_cache_max_age(request.META.get('HTTP_CACHE_CONTROL')) if age_limit is None and request.META.get('HTTP_PRAGMA') == 'no-cache': age_limit = 0 if age_limit is not None: min_age = self.cache_min_age if min_age is None: min_age = getattr(settings, 'DJANGOCACHE_MIN_AGE', 0) age_limit = max(min_age, age_limit) if age >= age_limit: request._cache_update_cache = True return None return response
def dynamic_css(request, css): if not _dynamic_cssmap.has_key(css): raise Http404("CSS not found") files = _dynamic_cssmap[css] resp = HttpResponse(content_type="text/css") # We honor if-modified-since headers by looking at the most recently # touched CSS file. latestmod = 0 for fn in files: try: stime = os.stat(fn).st_mtime if latestmod < stime: latestmod = stime except OSError: # If we somehow referred to a file that didn't exist, or # one that we couldn't access. raise Http404("CSS (sub) not found") if request.META.has_key("HTTP_IF_MODIFIED_SINCE"): # This code is mostly stolen from django :) matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", request.META.get("HTTP_IF_MODIFIED_SINCE"), re.IGNORECASE) header_mtime = parse_http_date(matches.group(1)) # We don't do length checking, just the date if int(latestmod) <= header_mtime: return HttpResponseNotModified(mimetype="text/css") resp["Last-Modified"] = http_date(latestmod) for fn in files: with open(fn) as f: resp.write("/* %s */\n" % fn) resp.write(f.read()) resp.write("\n") return resp
def sendfile(request, filename, **kwargs): # Respect the If-Modified-Since header. parseresult = urlparse(filename) if parseresult.scheme: with contextlib.closing(urllib2.urlopen(filename)) as result: headers = result.headers.dict data = result.read() lastmodified = parse_http_date(headers['last-modified']) contentlength = int(headers['content-length']) response = HttpResponse(data) else: statobj = os.stat(filename) lastmodified = statobj[stat.ST_MTIME] contentlength = statobj[stat.ST_SIZE] response = HttpResponse(File(open(filename, 'rb')).chunks()) if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'), lastmodified, contentlength): return HttpResponseNotModified() response["Last-Modified"] = http_date(lastmodified) return response
def should_regenerate(self, response): """ Check if this page was originally generated less than LOCAL_POSTCHECK seconds ago """ if response.has_header('Last-Modified'): last_modified = parse_http_date(response['Last-Modified']) next_regen = last_modified + settings.BETTERCACHE_LOCAL_POSTCHECK return time.time() > next_regen
def is_expired(self, url): """ If the cached response for the given URL is expired based on Cache-Control or Expires headers, returns True. """ if url not in self: return True headers = self.get_headers(url) # First check if the Cache-Control header has set a max-age if 'cache-control' in headers: cache_control = parse_cache_control_header(headers['cache-control']) if 'max-age' in cache_control: max_age = int(cache_control['max-age']) response_age = int( os.stat(self._header_cache_file_path(url)).st_mtime) current_timestamp = int(time.time()) return current_timestamp - response_age >= max_age # Alternatively, try the Expires header if 'expires' in headers: expires_date = timezone.datetime.utcfromtimestamp( parse_http_date(headers['expires'])) expires_date = timezone.make_aware(expires_date, timezone.utc) current_date = timezone.now() return current_date > expires_date # If there is no cache freshness date consider the item expired return True
def was_modified_since(header=None, mtime=0, size=0): """ Was something modified since the user last downloaded it? header This is the value of the If-Modified-Since header. If this is None, I'll just return True. mtime This is the modification time of the item we're talking about. size This is the size of the item we're talking about. """ try: if header is None: raise ValueError matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, re.IGNORECASE) header_mtime = parse_http_date(matches.group(1)) header_len = matches.group(3) if header_len and int(header_len) != size: raise ValueError if int(mtime) > header_mtime: raise ValueError except (AttributeError, ValueError, OverflowError): return True return False
def _serve_file(request, stat, rel_path, full_path): since = request.META.get('HTTP_IF_MODIFIED_SINCE') mimetype = _guess_mimetype(rel_path) if since and parse_http_date(since) > stat.st_mtime: return HttpResponseNotModified(mimetype=mimetype) response = HttpResponse(open(full_path, 'rb').read(), mimetype=mimetype) response["Last-Modified"] = http_date(stat.st_mtime) return response
def test_cache_headers(self): """ Should have appropriate Cache-Control and Expires headers. """ with self.activate('en-US'): resp = self.client.get(reverse('tabzilla')) self.assertEqual(resp['cache-control'], 'max-age=43200') # 12h now_date = floor(time.time()) exp_date = parse_http_date(resp['expires']) self.assertAlmostEqual(now_date + 43200, exp_date, delta=2)
def _test_cache_headers(self, view, hours): """ Should have appropriate Cache-Control and Expires headers. """ test_request = self.rf.get('/hi-there-dude/') resp = view(test_request) num_seconds = hours * 60 * 60 self.assertEqual(resp['cache-control'], 'max-age=%d' % num_seconds) now_date = floor(time.time()) exp_date = parse_http_date(resp['expires']) self.assertAlmostEqual(now_date + num_seconds, exp_date, delta=2)
def get_media(request, path): pathname = os.path.join(settings.MEDIA_ROOT, path) # Get the Media and check the permissions media = get_object_or_404(Media, path=pathname) if not media.is_visible_to(request.user): # If the user is not logged-in, redirect to the login page if not request.user.is_authenticated(): return redirect_to_login(request.get_full_path(), settings.LOGIN_URL, REDIRECT_FIELD_NAME) else: return HttpResponseForbidden() # Get a thumbnails if requested size_str = request.GET.get('size', None) if size_str == 'small' or size_str == 'medium': # Set the size if size_str == 'small': size = (280, 210) else: size = (800, 600) smallpath = os.path.join(settings.CACHE_ROOT, size_str, path) # TODO: check that the thumbnails is youger than the original image if not os.path.isfile(smallpath): # Create the destination directory and thumbnail mkdir(os.path.dirname(smallpath)) if not create_thumbnail(media, smallpath, size): raise Http404 pathname = smallpath # Stat the file to grab metadata stats = os.stat(pathname) # Check if the client has the media in his cache if_modified_since = request.META.get("HTTP_IF_MODIFIED_SINCE") if if_modified_since: if_modified_since = parse_http_date(if_modified_since) if if_modified_since >= int(stats.st_mtime): return HttpResponseNotModified() # Stream the file # FIXME: this will be wrong for video thumbnails mime = mimetypes.guess_type(pathname) response = FileResponse(open(pathname, 'rb'), content_type=mime[0] if mime[0] else 'text/plain') # Set the headers response['Content-Length'] = stats.st_size response['Last-Modified'] = http_date(stats.st_mtime) return response
def process_response(self, request, response): if 'Expires' in response: # Replace 'max-age' value of 'Cache-Control' header by one # calculated from the 'Expires' header's date. # This is necessary because of Django's `FetchFromCacheMiddleware` # gets 'Cache-Control' header from the cache # where 'max-age' corresponds to the moment original response # was generated and thus may be already stale for the current time expires = parse_http_date(response['Expires']) timeout = expires - int(now().timestamp()) patch_cache_control(response, max_age=timeout) key_prefix = getattr(request, '_cache_key_prefix', self.key_prefix) with patch(self, 'key_prefix', key_prefix): return super().process_response(request, response)
def has_uncacheable_headers(self, response): """ Should this response be cached based on it's headers broken out from should_cache for flexibility """ cc_dict = get_header_dict(response, 'Cache-Control') if cc_dict: if cc_dict.has_key('max-age') and cc_dict['max-age'] == '0': return True if cc_dict.has_key('no-cache'): return True if cc_dict.has_key('private'): return True if response.has_header('Expires'): if parse_http_date(response['Expires']) < time.time(): return True return False
def has_uncacheable_headers(self, response): """ Should this response be cached based on it's headers broken out from should_cache for flexibility """ cc_dict = get_header_dict(response, "Cache-Control") if cc_dict: if "max-age" in cc_dict and cc_dict["max-age"] == "0": return True if "no-cache" in cc_dict: return True if "private" in cc_dict: return True if response.has_header("Expires"): if parse_http_date(response["Expires"]) < time.time(): return True return False
def test_read_model_mixin_must_return_last_modified_header_as_now(self): dummier = DummierModel.objects.create(name='my dummier') class DummierResource(ModelResource): model = DummierModel request = self.req.get('/dummiers') mixin = ReadModelMixin() mixin.resource = DummierResource response = mixin.get(request, dummier.id) self.assertEquals(dummier.name, response.cleaned_content.name) self.assertTrue(response.headers.has_key('Last-Modified')) last_modified = response.headers['Last-Modified'] # no error should occur, meaning we have the right format last_modified_datetime = parse_http_date(last_modified) self.assertEqual(last_modified_datetime, time.mktime(datetime(2012, 12, 12, 6, 6, 6).timetuple()))
def parse_datetime(datestr): '''Turns a ISO8601 or RFC1123 date string representation to a Python datetime instance.''' try: datestr = float(datestr) return datetime.fromtimestamp(datestr) except ValueError: pass try: datestr = str(datestr) if 'GMT' in datestr: return datetime.fromtimestamp(parse_http_date(datestr)) return datetime.strptime(datestr, ISO8601_DATEFORMAT) except Exception, e: raise ValueError('Unable to parse date \'%s\' (reason: %s)' % (datestr, repr(e)))
def is_precondition_failed(self, request, response, *args, **kwargs): # ETags are enabled. Check for conditional request headers. The current # ETag value is used for the conditional requests. After the request # method handler has been processed, the new ETag will be calculated. if self.use_etags and 'HTTP_IF_MATCH' in request.META: request_etag = parse_etags(request.META['HTTP_IF_MATCH'])[0] etag = self.get_etag(request, request_etag) if request_etag != etag: return True # Last-Modified date enabled. check for conditional request headers. The # current modification datetime value is used for the conditional # requests. After the request method handler has been processed, the new # Last-Modified datetime will be returned. if self.use_last_modified and 'HTTP_IF_UNMODIFIED_SINCE' in request.META: last_modified = self.get_last_modified(request, *args, **kwargs) known_last_modified = EPOCH_DATE + timedelta(seconds=parse_http_date(request.META['HTTP_IF_UNMODIFIED_SINCE'])) if known_last_modified != last_modified: return True return False
def test_parsing_year_less_than_70(self): parsed = parse_http_date('Sun Nov 6 08:49:37 0050') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(2050, 11, 6, 8, 49, 37))
def test_parsing_asctime(self): parsed = parse_http_date("Sun Nov 6 08:49:37 1994") self.assertEqual( datetime.fromtimestamp(parsed, timezone.utc), datetime(1994, 11, 6, 8, 49, 37, tzinfo=timezone.utc), )
def test_parsing_rfc1123(self): parsed = http.parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def testParsingAsctime(self): parsed = parse_http_date("Sun Nov 6 08:49:37 1994") self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 06, 8, 49, 37))
def testParsingRfc1123(self): parsed = http.parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def testParsingRfc850(self): parsed = http.parse_http_date('Sunday, 06-Nov-94 08:49:37 GMT') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def test_parsing_asctime_nonascii_digits(self): """Non-ASCII unicode decimals raise an error.""" with self.assertRaises(ValueError): parse_http_date("Sun Nov 6 08:49:37 1994") with self.assertRaises(ValueError): parse_http_date("Sun Nov 12 08:49:37 1994")
def test_parsing_year_less_than_70(self): parsed = parse_http_date("Sun Nov 6 08:49:37 0037") self.assertEqual( datetime.fromtimestamp(parsed, timezone.utc), datetime(2037, 11, 6, 8, 49, 37, tzinfo=timezone.utc), )
def test_parsing_asctime(self): parsed = http.parse_http_date('Sun Nov 6 08:49:37 1994') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def testParsingRfc850(self): parsed = parse_http_date("Sunday, 06-Nov-94 08:49:37 GMT") self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 06, 8, 49, 37))
def process_request(self, request, *args, **kwargs): # Initilize a new response for this request. Passing the response along # the request cycle allows for gradual modification of the headers. response = HttpResponse() # TODO keep track of a list of request headers used to # determine the resource representation for the 'Vary' # header. # ### 503 Service Unavailable # The server does not need to be unavailable for a resource to be # unavailable... if self.is_service_unavailable(request, response, *args, **kwargs): response.status_code = codes.service_unavailable return response # ### 414 Request URI Too Long _(not implemented)_ # This should be be handled upstream by the Web server # ### 400 Bad Request _(not implemented)_ # Note that many services respond with this code when entities are # unprocessable. This should really be a 422 Unprocessable Entity # Most actualy bad requests are handled upstream by the Web server # when parsing the HTTP message # ### 401 Unauthorized # Check if the request is authorized to access this resource. if self.is_unauthorized(request, response, *args, **kwargs): response.status_code = codes.unauthorized return response # ### 403 Forbidden # Check if this resource is forbidden for the request. if self.is_forbidden(request, response, *args, **kwargs): response.status_code = codes.forbidden return response # ### 501 Not Implemented _(not implemented)_ # This technically refers to a service-wide response for an # unimplemented request method, again this is upstream. # ### 429 Too Many Requests # Both `rate_limit_count` and `rate_limit_seconds` must be none # falsy values to be checked. if self.rate_limit_count and self.rate_limit_seconds: if self.is_too_many_requests(request, response, *args, **kwargs): response.status_code = codes.too_many_requests return response # ### 405 Method Not Allowed if self.is_method_not_allowed(request, response, *args, **kwargs): response.status_code = codes.method_not_allowed return response # ### 406 Not Acceptable # Checks Accept and Accept-* headers if self.is_not_acceptable(request, response, *args, **kwargs): response.status_code = codes.not_acceptable return response # ### Process an _OPTIONS_ request # Enough processing has been performed to allow an OPTIONS request. if request.method == methods.OPTIONS and 'OPTIONS' in self.allowed_methods: return self.options(request, response) # ## Request Entity Checks # Only perform these checks if the request has supplied a body. if 'CONTENT_LENGTH' in request.META and request.META['CONTENT_LENGTH']: # ### 415 Unsupported Media Type # Check if the entity `Content-Type` supported for decoding. if self.is_unsupported_media_type(request, response, *args, **kwargs): response.status_code = codes.unsupported_media_type return response # ### 413 Request Entity Too Large # Check if the entity is too large for processing if self.is_request_entity_too_large(request, response, *args, **kwargs): response.status_code = codes.request_entity_too_large return response # ### 404 Not Found # Check if this resource exists. Note, if this requires a database # lookup or some other expensive lookup, the relevant object may # be _attached_ to the request or response object to be used # dowstream in the handler. This prevents multiple database # hits or filesystem lookups. if self.is_not_found(request, response, *args, **kwargs): response.status_code = codes.not_found return response # ### 410 Gone # Check if this resource used to exist, but does not anymore. A common # strategy for this when dealing with this in a database context is to # have an `archived` or `deleted` flag that can be used associated with # the given lookup key while the rest of the content in the row may be # deleted. if self.is_gone(request, response, *args, **kwargs): response.status_code = codes.gone return response # ### 428 Precondition Required # Prevents the "lost udpate" problem and requires client to confirm # the state of the resource has not changed since the last `GET` # request. This applies to `PUT` and `PATCH` requests. if request.method == methods.PUT or request.method == methods.PATCH: if self.is_precondition_required(request, response, *args, **kwargs): return UncacheableResponse(status=codes.precondition_required) # ### 412 Precondition Failed # Conditional requests applies to GET, HEAD, PUT, and PATCH. # For GET and HEAD, the request checks the either the entity changed # since the last time it requested it, `If-Modified-Since`, or if the # entity tag (ETag) has changed, `If-None-Match`. if request.method == methods.PUT or request.method == methods.PATCH: if self.is_precondition_failed(request, response, *args, **kwargs): return UncacheableResponse(status=codes.precondition_failed) # Check for conditional GET or HEAD request if request.method == methods.GET or request.method == methods.HEAD: # Check Etags before Last-Modified... if self.use_etags and 'HTTP_IF_NONE_MATCH' in request.META: # Parse request Etags (only one is currently supported) request_etag = parse_etags(request.META['HTTP_IF_NONE_MATCH'])[0] # Check if the request Etag is valid. The current Etag is # supplied to enable strategies where the etag does not need # to be used to regenerate the Etag. This may include # generating an MD5 of the resource and storing it as a key # in memcache. etag = self.get_etag(request, response, request_etag, *args, **kwargs) # Nothing has changed, simply return if request_etag == etag: response.status_code = codes.not_modified return response if self.use_last_modified and 'HTTP_IF_MODIFIED_SINCE' in request.META: # Get the last known modified date from the client, compare it # to the last modified date of the resource last_modified = self.get_last_modified(request, *args, **kwargs) known_last_modified = EPOCH_DATE + timedelta(seconds=parse_http_date(request.META['HTTP_IF_MODIFIED_SINCE'])) if known_last_modified >= last_modified: response.status_code = codes.not_modified return response # Check if there is a request body if 'CONTENT_LENGTH' in request.META and request.META['CONTENT_LENGTH']: content_type = request._content_type if content_type in serializers: request.data = serializers.decode(content_type, request.body)
"""
def test_parsing_asctime(self): parsed = parse_http_date('Sun Nov 6 08:49:37 1994') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def test_parsing_rfc850(self): parsed = http.parse_http_date('Sunday, 06-Nov-94 08:49:37 GMT') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))
def test_parsing_year_less_than_70(self): parsed = parse_http_date('Sun Nov 6 08:49:37 0037') self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(2037, 11, 6, 8, 49, 37))
def test_parsing_rfc1123(self): parsed = parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT') self.assertEqual( datetime.fromtimestamp(parsed, timezone.utc), datetime(1994, 11, 6, 8, 49, 37, tzinfo=timezone.utc), )
import re import os from pathlib import Path from django.utils.http import parse_http_date parser = argparse.ArgumentParser(description="tpts.py") parser.add_argument("path") args = parser.parse_args() p = Path(args.path) m = re.match(r'(^[^\.]*)\.(.*$)', str(p.name)) filename = m.group(1) ext = re.sub("[\-\.].*", "", m.group(2)) ext = ext.replace('_large', '') url = 'https://pbs.twimg.com/media/' + filename + "." + ext + ":orig" print(url) r = requests.head(url) if "Last-Modified" in r.headers: lm = parse_http_date(r.headers["Last-Modified"]) cl = int(r.headers["Content-Length"]) if p.stat().st_size < cl: r = requests.get(url) with p.open("wb") as f: f.write(r.content) nf = p.parent / Path(filename + "." + ext + "-orig." + ext) p.rename(nf) os.utime(str(nf), (lm, lm))