def learn_cache_key(request, response, tags=(), cache_timeout=None, key_prefix=None, cache=None): # patched """ Learns what headers to take into account for some request path from the response object. It stores those headers in a global path registry so that later access to that path will know what headers to take into account without building the response object itself. The headers are named in the Vary header of the response, but we want to prevent response generation. The list of headers to use for cache key generation is stored in the same cache as the pages themselves. If the cache ages some data out of the cache, this just means that we have to build the response once to get at the Vary header and so at the list of headers to use for the cache key. """ if key_prefix is None: key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX if cache_timeout is None: cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS cache_key = _generate_cache_header_key(key_prefix, request) if cache is None: cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS) if response.has_header('Vary'): headerlist = ['HTTP_'+header.upper().replace('-', '_') for header in cc_delim_re.split(response['Vary'])] cache.set(cache_key, headerlist, tags, cache_timeout) # patched return _generate_cache_key(request, request.method, headerlist, key_prefix) else: # if there is no Vary header, we still need a cache key # for the request.get_full_path() cache.set(cache_key, [], tags, cache_timeout) # patched return _generate_cache_key(request, request.method, [], key_prefix)
def warmup_cache(self): cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS) cache_header_key = _generate_cache_header_key(settings.CACHE_MIDDLEWARE_KEY_PREFIX, self.request) cache.set(cache_header_key, 'header dummy cache', 30) cache_key = get_cache_key(self.request, settings.CACHE_MIDDLEWARE_KEY_PREFIX, 'GET', cache=cache) cache.set(cache_key, self.RESPONSE_CACHED_VALUE, 30)
def set_cached_data(request, key_prefix, data): """Function to save data to the cache based on the request.""" cache_header_key = _generate_cache_header_key(key_prefix, request) if cache.get(cache_header_key) is None: cache.set(cache_header_key, [], None) cache_key = get_cache_key(request, key_prefix) cache.set(cache_key, data)
def invalidate_cached_entry(self): url = urlparse(self.get_absolute_url()) rf = RequestFactory(SERVER_NAME=url.netloc, HTTP_X_FORWARDED_PROTOCOL=url.scheme) is_secure = url.scheme == "https" request = rf.get(url.path, secure=is_secure) request.LANGUAGE_CODE = "en" cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] cache_key = _generate_cache_header_key(settings.CACHE_MIDDLEWARE_KEY_PREFIX, request) cache.delete(cache_key)
def _custom_vary_on(report): """ signature is intentionally restricted to a single argument to prevent @request_cache() from decorating a method that has non-self args """ return [ _generate_cache_header_key(CACHE_PREFIX, report.request), report.request.domain, report.request.couch_user._id, ]
def get_orderless_cache_key(request, key_prefix=None, method='GET', cache=None): if key_prefix is None: key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX cache_key = _generate_cache_header_key(key_prefix, request) if cache is None: cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] headerlist = cache.get(cache_key, None) if headerlist is not None: return _generate_orderless_cache_key(request, method, headerlist, key_prefix) else: return None
def get_cache_key_parameters(request): result = [] key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX result.append("key_prefix: %s" % key_prefix) cache_key = _generate_cache_header_key(key_prefix, request) headerlist = cache.get(cache_key, None) if headerlist: for header in headerlist: value = request.META.get(header, None) if value is not None: result.append("%s = %s" % (header, value)) return result
def invalidate_cached_entry(self): url = urlparse(self.get_absolute_url()) rf = RequestFactory( SERVER_NAME=url.netloc, HTTP_X_FORWARDED_PROTOCOL=url.scheme, ) is_secure = url.scheme == 'https' request = rf.get(url.path, secure=is_secure) request.LANGUAGE_CODE = 'en' cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] cache_key = _generate_cache_header_key(settings.CACHE_MIDDLEWARE_KEY_PREFIX, request) cache.delete(cache_key)
def test_update_cache(self): cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS) cache.clear() request = RequestFactory().get('/') cache_header_key = _generate_cache_header_key(settings.CACHE_MIDDLEWARE_KEY_PREFIX, request) cache.set(cache_header_key, 'header dummy cache', 30) cache_key = get_cache_key(request, settings.CACHE_MIDDLEWARE_KEY_PREFIX, 'GET', cache=cache) self.assertEqual(cache.get(cache_key), None, 'Cache is empty') update_cache('/') self.assertNotEqual(cache.get(cache_key), None, 'Cache is not empty')
def get_cached_data(request, key_prefix, data, serializer): """Function to get serialized data from the cache based on the request.""" cache_header_key = _generate_cache_header_key(key_prefix, request) if cache.get(cache_header_key) is None: cache.set(cache_header_key, [], None) cache_key = get_cache_key(request, key_prefix) cached_data = cache.get(cache_key, None) cache_used = True if not cached_data and data is not None: cache_used = False cached_data = serializer(data, many=True).data cache.set(cache_key, cached_data) return cached_data, cache_used
def handle(self, *args, **options): # Deletes hard-wired 'main_results_table' fragment from main results args = hashlib.md5(u':'.join([urlquote(u'')])) cache_key = 'template.cache.%s.%s' % ('main_results_table', args.hexdigest()) self.stdout.write(u'Was there a page fragment cache? {0}\n'.format(cache.has_key(cache_key))) cache.delete(cache_key) self.stdout.write(u'Cleared the main_results_table fragment!\n') # Hard-wire delete of page URL /ballot/results/full/ request = HttpRequest() key_prefix = '' request.path = '/ballot/results/full/' key = _generate_cache_header_key(key_prefix, request) self.stdout.write(u'Was there a URL cache? {0}\n'.format(cache.has_key(key))) cache.set(key, None, 0) self.stdout.write(u'Cleared the /ballot/results/full/ URL!\n')
def learn_cache_key(request, response, tags=(), cache_timeout=None, key_prefix=None, cache=None): # patched """ Learns what headers to take into account for some request URL from the response object. It stores those headers in a global URL registry so that later access to that URL will know what headers to take into account without building the response object itself. The headers are named in the Vary header of the response, but we want to prevent response generation. The list of headers to use for cache key generation is stored in the same cache as the pages themselves. If the cache ages some data out of the cache, this just means that we have to build the response once to get at the Vary header and so at the list of headers to use for the cache key. """ if key_prefix is None: key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX if cache_timeout is None: cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS cache_key = _generate_cache_header_key(key_prefix, request) if cache is None: cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] if response.has_header('Vary'): is_accept_language_redundant = settings.USE_I18N or settings.USE_L10N # If i18n or l10n are used, the generated cache key will be suffixed # with the current locale. Adding the raw value of Accept-Language is # redundant in that case and would result in storing the same content # under multiple keys in the cache. See #18191 for details. headerlist = [] for header in cc_delim_re.split(response['Vary']): header = header.upper().replace('-', '_') if header == 'ACCEPT_LANGUAGE' and is_accept_language_redundant: continue headerlist.append('HTTP_' + header) headerlist.sort() cache.set(cache_key, headerlist, tags, cache_timeout) # patched return _generate_cache_key(request, request.method, headerlist, key_prefix) else: # if there is no Vary header, we still need a cache key # for the request.build_absolute_uri() cache.set(cache_key, [], tags, cache_timeout) # patched return _generate_cache_key(request, request.method, [], key_prefix)
def add_page_cache(request, group='default', key_prefix=None): """增加视图缓存key,清除缓存时使用,不支持reponse has_header 'Vary'""" group = group or 'page' page_cache = _page_cache_dict() if group not in page_cache: page_cache[group] = {} if request.get_full_path() in page_cache[group]: return if key_prefix is None: key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX headerkey = _generate_cache_header_key(key_prefix, request) cachekey = _generate_cache_key(request, request.method, [], key_prefix) page_cache[group][request.get_full_path()] = { 'header': headerkey, 'cache': cachekey, } _save_page_cache(page_cache)
def learn_orderless_cache_key(request, response, cache_timeout=None, key_prefix=None, cache=None): """ Learns what headers to take into account for some request URL from the response object. It stores those headers in a global URL registry so that later access to that URL will know what headers to take into account without building the response object itself. The headers are named in the Vary header of the response, but we want to prevent response generation. The list of headers to use for cache key generation is stored in the same cache as the pages themselves. If the cache ages some data out of the cache, this just means that we have to build the response once to get at the Vary header and so at the list of headers to use for the cache key. """ if key_prefix is None: key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX if cache_timeout is None: cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS cache_key = _generate_cache_header_key(key_prefix, request) if cache is None: cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] if response.has_header('Vary'): is_accept_language_redundant = settings.USE_I18N or settings.USE_L10N # If i18n or l10n are used, the generated cache key will be suffixed # with the current locale. Adding the raw value of Accept-Language is # redundant in that case and would result in storing the same content # under multiple keys in the cache. See #18191 for details. headerlist = [] for header in cc_delim_re.split(response['Vary']): header = header.upper().replace('-', '_') if header == 'ACCEPT_LANGUAGE' and is_accept_language_redundant: continue headerlist.append('HTTP_' + header) headerlist.sort() cache.set(cache_key, headerlist, cache_timeout) return _generate_orderless_cache_key(request, request.method, headerlist, key_prefix) else: # if there is no Vary header, we still need a cache key # for the request.build_absolute_uri() cache.set(cache_key, [], cache_timeout) return _generate_orderless_cache_key(request, request.method, [], key_prefix)
def learn_cache_key(request, response, tags=(), cache_timeout=None, key_prefix=None, cache=None): # patched """ Learns what headers to take into account for some request path from the response object. It stores those headers in a global path registry so that later access to that path will know what headers to take into account without building the response object itself. The headers are named in the Vary header of the response, but we want to prevent response generation. The list of headers to use for cache key generation is stored in the same cache as the pages themselves. If the cache ages some data out of the cache, this just means that we have to build the response once to get at the Vary header and so at the list of headers to use for the cache key. """ if key_prefix is None: key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX if cache_timeout is None: cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS cache_key = _generate_cache_header_key(key_prefix, request) if cache is None: cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS) if response.has_header('Vary'): headerlist = [ 'HTTP_' + header.upper().replace('-', '_') for header in cc_delim_re.split(response['Vary']) ] cache.set(cache_key, headerlist, tags, cache_timeout) # patched return _generate_cache_key(request, request.method, headerlist, key_prefix) else: # if there is no Vary header, we still need a cache key # for the request.get_full_path() cache.set(cache_key, [], tags, cache_timeout) # patched return _generate_cache_key(request, request.method, [], key_prefix)
def invalidate_cache_for_path(request_path): """ invalidates cache based on request.path """ # use a dummy request object so we can call django's _generate_cache_header_key class Request(object): def __init__(self, request_path): self.request_path = request_path def get_full_path(self): return self.request_path request = Request(request_path) cache_header_key = _generate_cache_header_key(settings.CACHE_MIDDLEWARE_KEY_PREFIX, request) # cache must be invalidated for all available languages if settings.USE_I18N: base_key = cache_header_key[:cache_header_key.rfind('.')] for code, name in settings.LANGUAGES: key = "%s.%s" % (base_key, code) cache.delete(key) else: cache.delete(cache_header_key)
def save(self, *args, **kwargs): # Code Smell; hard-wiring these URLs for clearing view caches on save() # Better way ('cept for maybe the signal part ... ): # http://stackoverflow.com/questions/2268417/expire-a-view-cache-in-django request = HttpRequest() key_prefix = '' json_views = ( '/ballot/json/', '/ballot/json/eugspr/', '/ballot/json/laneco/', '/ballot/json/region/', '/ballot/json/laneme/', '/ballot/json/stater/', '/ballot/json/statem/', '/ballot/json/topset/', ) for json_view in json_views: request.path = json_view key = _generate_cache_header_key(key_prefix, request) if cache.has_key(key): cache.set(key, None, 0) super(CCOnSaveBase, self).save(*args, **kwargs)
def invalidate_cache_for_path(request_path): """ invalidates cache based on request.path """ # use a dummy request object so we can call django's _generate_cache_header_key class Request(object): def __init__(self, request_path): self.request_path = request_path def get_full_path(self): return self.request_path request = Request(request_path) cache_header_key = _generate_cache_header_key( settings.CACHE_MIDDLEWARE_KEY_PREFIX, request) # cache must be invalidated for all available languages if settings.USE_I18N: base_key = cache_header_key[:cache_header_key.rfind('.')] for code, name in settings.LANGUAGES: key = "%s.%s" % (base_key, code) cache.delete(key) else: cache.delete(cache_header_key)
def process_request(self, request): """ Checks whether the page is already cached and returns the cached version if available. """ if self.cache_anonymous_only: assert hasattr( request, "user" ), "The Django cache middleware with CACHE_MIDDLEWARE_ANONYMOUS_ONLY=True requires authentication middleware to be installed. Edit your MIDDLEWARE_CLASSES setting to insert 'django.contrib.auth.middleware.AuthenticationMiddleware' before the CacheMiddleware." # we define these attributes for a possible later use in views request.path_cache_key = _generate_cache_header_key(self.key_prefix, request) request.cache_query_string = self.cache_query_string if not request.method in ("GET", "HEAD") or (request.GET and not self.cache_query_string): request._cache_update_cache = False return None # Don't bother checking the cache. if self.cache_anonymous_only and request.user.is_authenticated(): request._cache_update_cache = False return None # Don't cache requests from authenticated users. cache_key = get_request_cache_key(request, self.key_prefix) if cache_key is None: request._cache_update_cache = True return None # No cache information available, need to rebuild. cache_key = add_suffix_to_cache_key(cache_key, request, cache_query_string=self.cache_query_string) response = cache.get(cache_key, None) if response is None: request._cache_update_cache = True return None # No cache information available, need to rebuild. request._cache_update_cache = False return response
def expire_cache(path, args=[], GET={}, HOSTNAME="127.0.0.1:8000", cache_name=None, isview=True, lang_code=None, method='GET'): if cache_name is None: cache_name = DEFAULT_CACHE_ALIAS cache = get_cache(cache_name) key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX print(key_prefix) request = HttpRequest() fake_meta = {'HTTP_HOST':HOSTNAME,} print(fake_meta['HTTP_HOST']) request.META = fake_meta request.GET = GET if isview: request.path = reverse(path, args=args) else: request.path = path language_code = lang_code or getattr(settings, 'LANGUAGE_CODE') if language_code: request.LANGUAGE_CODE = language_code header_key = _generate_cache_header_key(key_prefix, request) if not header_key: return False print(header_key) headerlist = cache.get(header_key, None) print(headerlist) if headerlist is not None: cache.set(header_key, None, 0) page_key = _generate_cache_key(request, method, headerlist, key_prefix) if not page_key: return False print(cache.get(page_key)) cache.set(page_key, None, 0) return True
def get_request_cache_key(request, key_prefix=None): """ Based on django.utils.cache.get_cache_key function Returns a cache key based on the request path. It can be used in the request phase because it pulls the list of headers to take into account from the global path registry and uses those to build a cache key to check against. If there is no headerlist stored, the page needs to be rebuilt, so this function returns None. This function has been refactored to be able to call get_view_cache_key in a separately way. This is needed to define a request.path_cache_key attribute (see cmsutils.middleware.I18NFetchFromCacheMiddleware). """ if key_prefix is None: key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX cache_key = _generate_cache_header_key(key_prefix, request) headerlist = cache.get(cache_key, None) if headerlist is not None: return _generate_cache_key(request, headerlist, key_prefix) else: return None
def cache_key(self): # went source diving for this in django, it does exactly # what we want here, though is marked 'private'. seemed # better to import than to copy it and all dependencies # elsewhere return _generate_cache_header_key(self.CACHE_PREFIX, self.request)
def debug(request): ''' created to debug code Args: request: ''' UKCL = request.GET["UKCL"] return HttpResponse("OK: " + UKCL) try: from django.core import management # management.call_command('migrate', "--database=materialized", interactive=False) # management.call_command('migrate', 'ap', interactive=False) # management.call_command('migrate', interactive=False) management.call_command('migrate', 'ap', interactive=False) # funziona? management.call_command('migrate', "knowledge_server 0003_initial_data --database=materialized", interactive=False) return HttpResponse("OK") import scrapy class DmozItem(scrapy.Item): title = scrapy.Field() link = scrapy.Field() desc = scrapy.Field() class DmozSpider(scrapy.Spider): name = "dmoz" allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/", ] def parse(self, response): for href in response.css( "ul.directory.dir-col > li > a::attr('href')"): url = response.urljoin(href.extract()) yield scrapy.Request(url, callback=self.parse_dir_contents) def parse_dir_contents(self, response): for sel in response.xpath('//ul/li'): item = DmozItem() item['title'] = sel.xpath('a/text()').extract() item['link'] = sel.xpath('a/@href').extract() item['desc'] = sel.xpath('text()').extract() yield item return HttpResponse('OK') ar = ApiResponse() ar.content = {"DataSet": "Versions"} ar.status = ApiResponse.success return HttpResponse(ar.json(), content_type="application/json") # TODO: AGGIORNARE SU STACKOVERFLOW: http://stackoverflow.com/questions/8784400/clearing-specific-cache-in-django from licenses.models import License db_alias = 'default' ccbysa40 = License.objects.using(db_alias).get( short_name="CC-BY-SA-4.0") dssModelMetadataFields = DataSetStructure.get_from_name( DataSetStructure.model_metadata_DSN, db_alias) dssDataSetStructureStructureNode = DataSetStructure.get_from_name( DataSetStructure.dataset_structure_DSN, db_alias) dssOrganizationKS = DataSetStructure.get_from_name( DataSetStructure.organization_DSN, db_alias) for ds in DataSet.objects.using(db_alias).filter( dataset_structure=dssModelMetadataFields): ds.licenses.add(ccbysa40) ds.save() for ds in DataSet.objects.using(db_alias).filter( dataset_structure=dssDataSetStructureStructureNode): ds.licenses.add(ccbysa40) ds.save() for ds in DataSet.objects.using(db_alias).filter( dataset_structure=dssOrganizationKS): ds.licenses.add(ccbysa40) ds.save() db_alias = 'materialized' ccbysa40 = License.objects.using(db_alias).get( short_name="CC-BY-SA-4.0") dssModelMetadataFields = DataSetStructure.get_from_name( DataSetStructure.model_metadata_DSN, db_alias) dssDataSetStructureStructureNode = DataSetStructure.get_from_name( DataSetStructure.dataset_structure_DSN, db_alias) dssOrganizationKS = DataSetStructure.get_from_name( DataSetStructure.organization_DSN, db_alias) for ds in DataSet.objects.using(db_alias).filter( dataset_structure=dssModelMetadataFields): ds.licenses.add(ccbysa40) ds.save() for ds in DataSet.objects.using(db_alias).filter( dataset_structure=dssDataSetStructureStructureNode): ds.licenses.add(ccbysa40) ds.save() for ds in DataSet.objects.using(db_alias).filter( dataset_structure=dssOrganizationKS): ds.licenses.add(ccbysa40) ds.save() return HttpResponse("OK ") from django.core.cache import cache from django.utils.cache import get_cache_key, _generate_cache_header_key from django.utils.encoding import escape_uri_path from django.http import HttpRequest new_request = HttpRequest() new_request.path = 'root.beta.thekoa.org/oks/api/ks_info/JSON/' ##this path works new_request.META['SERVER_PORT'] = request.META['SERVER_PORT'] new_request.META['SERVER_NAME'] = request.META['SERVER_NAME'] key = _generate_cache_header_key("", new_request) if cache.has_key(key): cache.delete(key) full_path = 'http://root.beta.thekoa.org/oks/api/datasets/http%253A%252F%252Froot.beta.thekoa.org%252Fknowledge_server%252FDataSetStructure%252F4/JSON/' import hashlib from django.utils.encoding import force_bytes, iri_to_uri from django.utils.cache import _i18n_cache_key_suffix # code from _generate_cache_header_key url = hashlib.md5(force_bytes(iri_to_uri(full_path))) cache_key = 'views.decorators.cache.cache_header.%s.%s' % ( "", url.hexdigest()) key = _i18n_cache_key_suffix(request, cache_key) if cache.has_key(key): cache.delete(key) return HttpResponse("OK ") # d = DataSet.objects.get(pk=1) # s = d.shallow_structure() # rct = d.root_content_type # # # for structure_child_node in s.root_node.child_nodes.all(): # mm = structure_child_node.sn_model_metadata(d) # print(mm.name) dssContinentState = DataSetStructure() dssContinentState.name = "Test Continent-SubContinent-State" dssContinentState.SetNotNullFields() dssContinentState.save() return HttpResponse("OK ") except Exception as ex: logger.error("views.debug: " + str(ex)) return HttpResponse(str(ex))