def get_cached_urls(self):
"""
Create the cached urls set.
Merges the assets, the urls, removes the network urls and the external urls
See :meth:`BaseAppCache.get_urls`, :meth:`get_network_urls`
"""
if not self._cached:
self._cached = self.get_urls()
for appcache in self.registry:
self._cached.update(appcache.get_assets(self.request))
self._cached.update(self._external_appcaches['cached'])
self._cached.difference_update(self.get_network_urls())
if get_setting('DISCARD_EXTERNAL'):
self._cached = filter(
lambda url: not is_external_url(url, self.request),
self._cached)
if get_setting('EXCLUDE_URL'):
new = set()
for url in self._cached:
if not any(
url.startswith(excluded)
for excluded in get_setting('EXCLUDE_URL')):
new.add(url)
self._cached = new
return self._cached
def _fetch_url(self, client, url):
"""
Scrape a single URL and fetches assets
"""
if not is_external_url(url):
response = client.get(url, data={"appcache_analyze":1}, LANGUAGE_CODE=self.language)
if response.status_code == 200:
self.add_appcache(response.appcache)
elif response.status_code == 302:
self._fetch_url(client, response['Location'])
else:
print("Unrecognized code %s for %s\n" % (
response.status_code, url))
def _fetch_url(self, client, url):
"""
Scrape a single URL and fetches assets
"""
if not is_external_url(url):
response = client.get(url,
data={"appcache_analyze": 1},
LANGUAGE_CODE=self.language)
if response.status_code == 200:
self.add_appcache(response.appcache)
elif response.status_code == 302:
self._fetch_url(client, response['Location'])
else:
print("Unrecognized code %s for %s\n" %
(response.status_code, url))
def _add_language(self, request, urls):
""" For django CMS 2.3 we need to manually add language code to the
urls returned by the appcache classes
:return: list of urls
"""
if DJANGOCMS_2_3:
new_urls = []
for url in urls:
if not is_external_url(url, request):
new_urls.append("/%s%s" % (request.LANGUAGE_CODE, url))
else:
new_urls.append(url)
return new_urls
else:
return urls
def _add_language(self, request, urls):
""" For django CMS 2.3 we need to manually add language code to the
urls returned by the appcache classes
:return: list of urls
"""
if DJANGOCMS_2_3:
new_urls = []
for url in urls:
if not is_external_url(url, request):
new_urls.append("/%s%s" % (request.LANGUAGE_CODE, url))
else:
new_urls.append(url)
return new_urls
else:
return urls
def get_cached_urls(self):
"""
Create the cached urls set.
Merges the assets, the urls, removes the network urls and the external urls
See :meth:`BaseAppCache.get_urls`, :meth:`get_network_urls`
"""
if not self._cached:
self._cached = self.get_urls()
for appcache in self.registry:
self._cached.update(appcache.get_assets(self.request))
self._cached.update(self._external_appcaches['cached'])
self._cached.difference_update(self.get_network_urls())
if get_setting('DISCARD_EXTERNAL'):
self._cached = filter(lambda url: not is_external_url(
url, self.request), self._cached)
if get_setting('EXCLUDE_URL'):
new = set()
for url in self._cached:
if not any(url.startswith(excluded) for excluded in get_setting('EXCLUDE_URL')):
new.add(url)
self._cached = new
return self._cached
