def submission_listing_cache_key(*args, **kw):
ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
if ns_key is None:
ns_key = random.randint(1, 10000)
memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
full_path = args[0].get_full_path()
username = args[0].user.username
return "demos_%s:%s" % (ns_key, hashlib.md5(full_path + username).hexdigest())
def submission_listing_cache_key(*args, **kw):
ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
if ns_key is None:
ns_key = random.randint(1, 10000)
memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
full_path = args[0].get_full_path()
username = args[0].user.username
return 'demos_%s:%s' % (ns_key,
hashlib.md5(full_path + username).hexdigest())
def _invalidate_submission_listing_helper_cache():
"""Invalidate the cache for submission_listing helper used in templates"""
# TODO: Does this belong in helpers.py? Better done with a model save event
# subscription?
ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
if ns_key is None:
ns_key = random.randint(1, 10000)
memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
else:
memcache.incr(DEMOS_CACHE_NS_KEY)
def _invalidate_submission_listing_helper_cache():
"""Invalidate the cache for submission_listing helper used in templates"""
# TODO: Does this belong in helpers.py? Better done with a model save event
# subscription?
ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
if ns_key is None:
ns_key = random.randint(1, 10000)
memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
else:
memcache.incr(DEMOS_CACHE_NS_KEY)
def test_home_community_stats(client, db, cleared_memcache):
stats = {
'contributors': 'so many, like more than 10,000',
'locales': 'lots, maybe fifty'
}
memcache.set('community_stats', stats)
response = client.get(reverse('home'), follow=True)
assert response.status_code == 200
assert_shared_cache_header(response)
assert stats['contributors'] in response.content
assert stats['locales'] in response.content
def wrapper(*args, **kw):
if type(key_fn) is str:
cache_key = key_fn
else:
cache_key = key_fn(*args, **kw)
out = memcache.get(cache_key)
if out is None:
context = f(*args, **kw)
t = get_template(template).render(context)
out = jinja2.Markup(t)
memcache.set(cache_key, out, expires)
return out
def bitly_shorten(url):
"""Attempt to shorten a given URL through bit.ly / mzl.la"""
cache_key = "bitly:%s" % hashlib.md5(smart_str(url)).hexdigest()
short_url = memcache.get(cache_key)
if short_url is None:
try:
short_url = bitly.shorten(url)["url"]
memcache.set(cache_key, short_url, 60 * 60 * 24 * 30 * 12)
except (bitly_api.BitlyError, KeyError):
# Just in case the bit.ly service fails or the API key isn't
# configured, fall back to using the original URL.
return url
return short_url
def wrapper(*args, **kw):
if type(key_fn) is str:
cache_key = key_fn
else:
cache_key = key_fn(*args, **kw)
out = memcache.get(cache_key)
if out is None:
context = f(*args, **kw)
t = jingo.env.get_template(template).render(context)
out = jinja2.Markup(t)
memcache.set(cache_key, out, expires)
return out
def bitly_shorten(url):
"""Attempt to shorten a given URL through bit.ly / mzl.la"""
cache_key = 'bitly:%s' % hashlib.md5(smart_str(url)).hexdigest()
short_url = memcache.get(cache_key)
if short_url is None:
try:
short_url = bitly.shorten(url)['url']
memcache.set(cache_key, short_url, 60 * 60 * 24 * 30 * 12)
except (bitly_api.BitlyError, KeyError):
# Just in case the bit.ly service fails or the API key isn't
# configured, fall back to using the original URL.
return url
return short_url
def get_url_remaps(self, locale):
cache_key = URL_REMAPS_CACHE_KEY_TMPL % locale
remaps = memcache.get(cache_key)
if not remaps:
qs = (self.filter(document__locale=locale,
url_root__isnull=False).exclude(url_root=''))
remaps = [{
'original_path': '/docs/%s' % zone.document.slug,
'new_path': '/%s' % zone.url_root
} for zone in qs]
memcache.set(cache_key, remaps)
return remaps
def update_community_stats():
cursor = connection.cursor()
try:
cursor.execute(
"""
SELECT count(creator_id)
FROM
(SELECT DISTINCT creator_id
FROM wiki_revision
WHERE created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS contributors
"""
)
contributors = cursor.fetchone()
cursor.execute(
"""
SELECT count(locale)
FROM
(SELECT DISTINCT wd.locale
FROM wiki_document wd,
wiki_revision wr
WHERE wd.id = wr.document_id
AND wr.created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS locales
"""
)
locales = cursor.fetchone()
finally:
cursor.close()
community_stats = {}
try:
community_stats["contributors"] = contributors[0]
community_stats["locales"] = locales[0]
except IndexError:
community_stats = None
# storing a None value in cache allows a better check for
# emptiness in the view
if 0 in community_stats.values():
community_stats = None
memcache.set("community_stats", community_stats, 86400)
def update_community_stats():
cursor = connection.cursor()
try:
cursor.execute("""
SELECT count(creator_id)
FROM
(SELECT DISTINCT creator_id
FROM wiki_revision
WHERE created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS contributors
""")
contributors = cursor.fetchone()
cursor.execute("""
SELECT count(locale)
FROM
(SELECT DISTINCT wd.locale
FROM wiki_document wd,
wiki_revision wr
WHERE wd.id = wr.document_id
AND wr.created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS locales
""")
locales = cursor.fetchone()
finally:
cursor.close()
community_stats = {}
try:
community_stats['contributors'] = contributors[0]
community_stats['locales'] = locales[0]
except IndexError:
community_stats = None
# storing a None value in cache allows a better check for
# emptiness in the view
if 0 in community_stats.values():
community_stats = None
memcache.set('community_stats', community_stats, 86400)
def get(document, cache_control, base_url, timeout=None):
"""Perform a kumascript GET request for a document locale and slug."""
if not cache_control:
# Default to the configured max-age for cache control.
max_age = constance.config.KUMASCRIPT_MAX_AGE
cache_control = 'max-age=%s' % max_age
if not base_url:
site = Site.objects.get_current()
base_url = 'http://%s' % site.domain
if not timeout:
timeout = constance.config.KUMASCRIPT_TIMEOUT
document_locale = document.locale
document_slug = document.slug
max_age = constance.config.KUMASCRIPT_MAX_AGE
# 1063580 - Kumascript converts template name calls to lower case and bases
# caching keys off of that.
document_slug_for_kumascript = document_slug
if document.is_template:
document_slug_for_kumascript = _format_slug_for_request(document_slug)
body, errors = None, None
try:
url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
url = unicode(url_tmpl).format(path=u'%s/%s' %
(document_locale,
document_slug_for_kumascript))
cache_keys = build_cache_keys(document_slug, document_locale)
etag_key, modified_key, body_key, errors_key = cache_keys
headers = {
'X-FireLogger': '1.2',
'Cache-Control': cache_control,
}
# Create the file interface
files = []
for attachment in document.attachments.all():
files.append(_get_attachment_metadata_dict(attachment))
# Assemble some KumaScript env vars
# TODO: See dekiscript vars for future inspiration
# http://developer.mindtouch.com/en/docs/DekiScript/Reference/
# Wiki_Functions_and_Variables
path = document.get_absolute_url()
# TODO: Someday merge with _get_document_for_json in views.py
# where most of this is duplicated code.
env_vars = dict(
path=path,
url=urljoin(base_url, path),
id=document.pk,
revision_id=document.current_revision.pk,
locale=document.locale,
title=document.title,
files=files,
attachments=files, # Just for sake of verbiage?
slug=document.slug,
tags=list(document.tags.values_list('name', flat=True)),
review_tags=list(document.current_revision
.review_tags
.values_list('name', flat=True)),
modified=time.mktime(document.modified.timetuple()),
cache_control=cache_control,
)
add_env_headers(headers, env_vars)
# Set up for conditional GET, if we have the details cached.
cached_meta = memcache.get_many([etag_key, modified_key])
if etag_key in cached_meta:
headers['If-None-Match'] = cached_meta[etag_key]
if modified_key in cached_meta:
headers['If-Modified-Since'] = cached_meta[modified_key]
# Finally, fire off the request.
response = requests.get(url, headers=headers, timeout=timeout)
if response.status_code == 304:
# Conditional GET was a pass, so use the cached content.
result = memcache.get_many([body_key, errors_key])
body = result.get(body_key, '').decode('utf-8')
errors = result.get(errors_key, None)
elif response.status_code == 200:
body = process_body(response)
errors = process_errors(response)
# Cache the request for conditional GET, but use the max_age for
# the cache timeout here too.
headers = response.headers
memcache.set(etag_key, headers.get('etag'), timeout=max_age)
memcache.set(modified_key, headers.get('last-modified'), timeout=max_age)
memcache.set(body_key, body.encode('utf-8'), timeout=max_age)
if errors:
memcache.set(errors_key, errors, timeout=max_age)
elif response.status_code is None:
errors = KUMASCRIPT_TIMEOUT_ERROR
else:
errors = [
{
"level": "error",
"message": "Unexpected response from Kumascript service: %s" %
response.status_code,
"args": ["UnknownError"],
},
]
except Exception, exc:
# Last resort: Something went really haywire. Kumascript server died
# mid-request, or something. Try to report at least some hint.
errors = [
{
"level": "error",
"message": "Kumascript service failed unexpectedly: %s" % exc,
"args": ["UnknownError"],
},
]
def get(document, cache_control, base_url, timeout=None):
"""Perform a kumascript GET request for a document locale and slug."""
if not cache_control:
# Default to the configured max-age for cache control.
max_age = config.KUMASCRIPT_MAX_AGE
cache_control = 'max-age=%s' % max_age
if not base_url:
site = Site.objects.get_current()
base_url = 'http://%s' % site.domain
if not timeout:
timeout = config.KUMASCRIPT_TIMEOUT
document_locale = document.locale
document_slug = document.slug
max_age = config.KUMASCRIPT_MAX_AGE
# 1063580 - Kumascript converts template name calls to lower case and bases
# caching keys off of that.
document_slug_for_kumascript = document_slug
body, errors = None, None
try:
url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
url = unicode(url_tmpl).format(path=u'%s/%s' %
(document_locale,
document_slug_for_kumascript))
cache_keys = build_cache_keys(document_slug, document_locale)
etag_key, modified_key, body_key, errors_key = cache_keys
headers = {
'X-FireLogger': '1.2',
'Cache-Control': cache_control,
}
# Create the file interface
files = []
for attachment in document.files.select_related('current_revision'):
files.append(_get_attachment_metadata_dict(attachment))
# Assemble some KumaScript env vars
# TODO: See dekiscript vars for future inspiration
# http://developer.mindtouch.com/en/docs/DekiScript/Reference/
# Wiki_Functions_and_Variables
path = document.get_absolute_url()
# TODO: Someday merge with _get_document_for_json in views.py
# where most of this is duplicated code.
env_vars = dict(
path=path,
url=urljoin(base_url, path),
id=document.pk,
revision_id=document.current_revision.pk,
locale=document.locale,
title=document.title,
files=files,
attachments=files, # Just for sake of verbiage?
slug=document.slug,
tags=list(document.tags.names()),
review_tags=list(document.current_revision.review_tags.names()),
modified=time.mktime(document.modified.timetuple()),
cache_control=cache_control,
)
add_env_headers(headers, env_vars)
# Set up for conditional GET, if we have the details cached.
cached_meta = memcache.get_many([etag_key, modified_key])
if etag_key in cached_meta:
headers['If-None-Match'] = cached_meta[etag_key]
if modified_key in cached_meta:
headers['If-Modified-Since'] = cached_meta[modified_key]
# Finally, fire off the request.
response = requests.get(url, headers=headers, timeout=timeout)
if response.status_code == 304:
# Conditional GET was a pass, so use the cached content.
result = memcache.get_many([body_key, errors_key])
body = result.get(body_key, '').decode('utf-8')
errors = result.get(errors_key, None)
elif response.status_code == 200:
body = process_body(response)
errors = process_errors(response)
# Cache the request for conditional GET, but use the max_age for
# the cache timeout here too.
headers = response.headers
memcache.set(etag_key, headers.get('etag'), timeout=max_age)
memcache.set(modified_key, headers.get('last-modified'), timeout=max_age)
memcache.set(body_key, body.encode('utf-8'), timeout=max_age)
if errors:
memcache.set(errors_key, errors, timeout=max_age)
elif response.status_code is None:
errors = KUMASCRIPT_TIMEOUT_ERROR
else:
errors = [
{
"level": "error",
"message": "Unexpected response from Kumascript service: %s" %
response.status_code,
"args": ["UnknownError"],
},
]
except Exception as exc:
# Last resort: Something went really haywire. Kumascript server died
# mid-request, or something. Try to report at least some hint.
errors = [
{
"level": "error",
"message": "Kumascript service failed unexpectedly: %s" % exc,
"args": ["UnknownError"],
},
]
return (body, errors)
def reset_url_remaps(self, locale, cache_key=None):
if cache_key is None:
cache_key = self.model.cache_key(locale)
remaps = self.build_url_remaps(locale)
memcache.set(cache_key, remaps, timeout=60 * 60 * 24)
return remaps
