def test_cache_get_or_set():
# Compatibility test, since cache_get_or_set is a 1:1 backport from
# Django 1.11, their unittests apply.
def some_function():
some_function.call_count += 1
return 'something' # Needed for cache_get_or_set() to work.
some_function.call_count = 0
cache_get_or_set('my-key', some_function)
cache_get_or_set('my-key', some_function)
assert some_function.call_count == 1
def test_cache_get_or_set():
# Compatibility test, since cache_get_or_set is a 1:1 backport from
# Django 1.11, their unittests apply.
def some_function():
some_function.call_count += 1
return 'something' # Needed for cache_get_or_set() to work.
some_function.call_count = 0
cache_get_or_set('my-key', some_function)
cache_get_or_set('my-key', some_function)
assert some_function.call_count == 1
def _category_personas(qs, limit):
def fetch_personas():
return randslice(qs, limit=limit)
# TODO: .query_key comes from cache-machine, find replacement
key = 'cat-personas:' + qs.query_key()
return cache_get_or_set(key, fetch_personas)
def addon_bayesian_rating(*addons, **kw):
def addon_aggregates():
return Addon.objects.valid().aggregate(rating=Avg('average_rating'),
reviews=Avg('total_ratings'))
log.info('[%s@%s] Updating bayesian ratings.' %
(len(addons), addon_bayesian_rating.rate_limit))
avg = cache_get_or_set('task.bayes.avg', addon_aggregates, 60 * 60 * 60)
# Rating can be NULL in the DB, so don't update it if it's not there.
if avg['rating'] is None:
return
mc = avg['reviews'] * avg['rating']
for addon in Addon.objects.filter(id__in=addons):
if addon.average_rating is None:
# Ignoring addons with no average rating.
continue
# Update the addon bayesian_rating atomically using F objects (unless
# it has no reviews, in which case directly set it to 0).
qs = Addon.objects.filter(id=addon.id)
if addon.total_ratings:
num = mc + F('total_ratings') * F('average_rating')
denom = avg['reviews'] + F('total_ratings')
qs.update(bayesian_rating=num / denom)
else:
qs.update(bayesian_rating=0)
def addon_bayesian_rating(*addons, **kw):
def addon_aggregates():
return Addon.objects.valid().aggregate(rating=Avg('average_rating'),
reviews=Avg('total_ratings'))
log.info('[%s@%s] Updating bayesian ratings.' %
(len(addons), addon_bayesian_rating.rate_limit))
avg = cache_get_or_set(make_key('task.bayes.avg'), addon_aggregates,
60 * 60 * 60)
# Rating can be NULL in the DB, so don't update it if it's not there.
if avg['rating'] is None:
return
mc = avg['reviews'] * avg['rating']
for addon in Addon.objects.no_cache().filter(id__in=addons):
if addon.average_rating is None:
# Ignoring addons with no average rating.
continue
# Update the addon bayesian_rating atomically using F objects (unless
# it has no reviews, in which case directly set it to 0).
qs = Addon.objects.filter(id=addon.id)
if addon.total_ratings:
num = mc + F('total_ratings') * F('average_rating')
denom = avg['reviews'] + F('total_ratings')
qs.update(bayesian_rating=num / denom)
else:
qs.update(bayesian_rating=0)
def side_nav(context, addon_type, category=None):
app = context['request'].APP.id
cat = str(category.id) if category else 'all'
cache_key = make_key(
'side-nav-%s-%s-%s' % (app, addon_type, cat),
# We have potentially very long names in the cache-key,
# normalize to not hit any memcached key-limits
normalize=True)
return cache_get_or_set(cache_key,
lambda: _side_nav(context, addon_type, category))
def side_nav(context, addon_type, category=None):
app = context['request'].APP.id
cat = str(category.id) if category else 'all'
cache_key = make_key(
'side-nav-%s-%s-%s' % (app, addon_type, cat),
# We have potentially very long names in the cache-key,
# normalize to not hit any memcached key-limits
normalize=True)
return cache_get_or_set(
cache_key, lambda: _side_nav(context, addon_type, category))
def get_versions(order=('application', 'version_int')):
def fetch_versions():
apps = amo.APP_USAGE
versions = {app.id: [] for app in apps}
qs = list(AppVersion.objects.order_by(*order)
.filter(application__in=versions)
.values_list('application', 'version'))
for app, version in qs:
versions[app].append(version)
return apps, versions
return cache_get_or_set('getv' + ':'.join(order), fetch_versions)
def get_entries(self, obj):
commit = self._get_commit(obj)
result = OrderedDict()
def _fetch_entries():
for entry_wrapper in self.repo.iter_tree(commit.tree):
entry = entry_wrapper.tree_entry
path = force_text(entry_wrapper.path)
blob = entry_wrapper.blob
is_directory = entry.type == 'tree'
mime, encoding = mimetypes.guess_type(entry.name)
is_binary = (
self.is_binary(path, mime, blob)
if not is_directory else False)
sha_hash = (
get_sha256(io.BytesIO(memoryview(blob)))
if not is_directory else '')
commit_tzinfo = FixedOffset(commit.commit_time_offset)
commit_time = datetime.fromtimestamp(
float(commit.commit_time),
commit_tzinfo)
result[path] = {
'binary': is_binary,
'depth': path.count(os.sep),
'directory': is_directory,
'filename': force_text(entry.name),
'sha256': sha_hash,
'mimetype': mime or 'application/octet-stream',
'path': path,
'size': blob.size if blob is not None else None,
'modified': commit_time,
}
return result
# Given that this is a very expensive operation we have a two-fold
# cache, one that is stored on this instance for very-fast retrieval
# to support other method calls on this serializer
# and another that uses memcached for regular caching
if hasattr(self, '_entries'):
return self._entries
self._entries = cache_get_or_set(
'reviewers:fileentriesserializer:entries:{}'.format(commit.hex),
_fetch_entries,
# Store information about this commit for 24h which should be
# enough to cover regular review-times but not overflow our
# cache
60 * 60 * 24)
return self._entries
def get_versions(order=('application', 'version_int')):
def fetch_versions():
apps = amo.APP_USAGE
versions = {app.id: [] for app in apps}
qs = list(
AppVersion.objects.order_by(*order).filter(
application__in=versions).values_list('application',
'version'))
for app, version in qs:
versions[app].append(version)
return apps, versions
return cache_get_or_set('getv' + ':'.join(order), fetch_versions)
def get_entries(self, obj):
# Given that this is a very expensive operation we have a two-fold
# cache, one that is stored on this instance for very-fast retrieval
# to support other method calls on this serializer
# and another that uses memcached for regular caching
if hasattr(self, '_entries'):
return self._entries
commit = self._get_commit(obj)
result = OrderedDict()
def _fetch_entries():
tree = self.repo.get_root_tree(commit)
for entry_wrapper in self.repo.iter_tree(tree):
entry = entry_wrapper.tree_entry
path = force_text(entry_wrapper.path)
blob = entry_wrapper.blob
sha_hash = (
get_sha256(io.BytesIO(memoryview(blob)))
if not entry.type == 'tree' else '')
commit_tzinfo = FixedOffset(commit.commit_time_offset)
commit_time = datetime.fromtimestamp(
float(commit.commit_time),
commit_tzinfo)
mimetype, entry_mime_category = self.get_entry_mime_type(
entry, blob)
result[path] = {
'depth': path.count(os.sep),
'filename': force_text(entry.name),
'sha256': sha_hash,
'mime_category': entry_mime_category,
'mimetype': mimetype,
'path': path,
'size': blob.size if blob is not None else None,
'modified': commit_time,
}
return result
self._entries = cache_get_or_set(
'reviewers:fileentriesserializer:entries:{}'.format(commit.hex),
_fetch_entries,
# Store information about this commit for 24h which should be
# enough to cover regular review-times but not overflow our
# cache
60 * 60 * 24)
return self._entries
def get_versions(order=('application', 'version_int')):
def fetch_versions():
if waffle.switch_is_active('disallow-thunderbird-and-seamonkey'):
apps = amo.APP_USAGE_FIREFOXES_ONLY
else:
apps = amo.APP_USAGE
versions = {app.id: [] for app in apps}
qs = list(AppVersion.objects.order_by(*order)
.filter(application__in=versions)
.values_list('application', 'version'))
for app, version in qs:
versions[app].append(version)
return apps, versions
return cache_get_or_set('getv' + ':'.join(order), fetch_versions)
def blocked(cls, name):
"""
Check to see if a given name is in the (cached) deny list.
Return True if the name contains one of the denied terms.
"""
name = name.lower()
qs = cls.objects.all()
def fetch_names():
return [n.lower() for n in qs.values_list('name', flat=True)]
blocked_list = cache_get_or_set('denied-name:blocked', fetch_names)
return any(n in name for n in blocked_list)
def get_files(self):
"""
Returns an OrderedDict, ordered by the filename of all the files in the
addon-file. Full of all the useful information you'll need to serve
this file, build templates etc.
"""
if self._files:
return self._files
if not self.is_extracted():
extract_file(self)
self._files = cache_get_or_set(self._cache_key(), self._get_files)
return self._files
def blocked(cls, name):
"""
Check to see if a given name is in the (cached) deny list.
Return True if the name contains one of the denied terms.
"""
name = name.lower()
qs = cls.objects.all()
def fetch_names():
return [n.lower() for n in qs.values_list('name', flat=True)]
blocked_list = cache_get_or_set('denied-name:blocked', fetch_names)
return any(n in name for n in blocked_list)
def get_files(self):
"""
Returns an OrderedDict, ordered by the filename of all the files in the
addon-file. Full of all the useful information you'll need to serve
this file, build templates etc.
"""
if self._files:
return self._files
if not self.is_extracted():
extract_file(self)
self._files = cache_get_or_set(self._cache_key(), self._get_files)
return self._files
def get_versions(order=('application', 'version_int')):
def fetch_versions():
if waffle.switch_is_active('disallow-thunderbird-and-seamonkey'):
apps = amo.APP_USAGE_FIREFOXES_ONLY
else:
apps = amo.APP_USAGE
versions = {app.id: [] for app in apps}
qs = list(
AppVersion.objects.order_by(*order).filter(
application__in=versions).values_list('application',
'version'))
for app, version in qs:
versions[app].append(version)
return apps, versions
return cache_get_or_set('getv' + ':'.join(order), fetch_versions)
def version_detail(request, addon, version_num):
# TODO: Does setting this in memcachd even make sense?
# This is specific to an add-ons version so the chance of this hitting
# the cache and not missing seems quite bad to me (cgrebs)
def _fetch():
qs = _version_list_qs(addon)
return list(qs.values_list('version', flat=True))
ids = cache_get_or_set(
u'version-detail:{}:{}'.format(addon.id, version_num),
_fetch)
url = reverse('addons.versions', args=[addon.slug])
if version_num in ids:
page = 1 + ids.index(version_num) / PER_PAGE
to = urlparams(url, 'version-%s' % version_num, page=page)
return http.HttpResponseRedirect(to)
else:
raise http.Http404()
def get_addons(self):
addons = self.collection.addons.public()
kw = {
'addon_type': 'ALL',
'limit': self.limit,
'app': self.request.APP,
'platform': self.platform,
'version': self.version,
'compat_mode': self.compat_mode
}
def fetch_and_filter_addons():
return addon_filter(addons, **kw)
# The cache-key can be very long, let's normalize it to make sure
# we never hit the 250-char limit of memcached.
cache_key = make_key(
'collections-promo-get-addons:{}'.format(repr(kw)),
normalize=True)
return cache_get_or_set(cache_key, fetch_and_filter_addons)
def get_addons(self):
addons = self.collection.addons.public()
kw = {
'addon_type': 'ALL',
'limit': self.limit,
'app': self.request.APP,
'platform': self.platform,
'version': self.version,
'compat_mode': self.compat_mode
}
def fetch_and_filter_addons():
return addon_filter(addons, **kw)
# The cache-key can be very long, let's normalize it to make sure
# we never hit the 250-char limit of memcached.
cache_key = make_key('collections-promo-get-addons:{}'.format(
repr(kw)),
normalize=True)
return cache_get_or_set(cache_key, fetch_and_filter_addons)
def version_detail(request, addon, version_num):
# TODO: Does setting this in memcachd even make sense?
# This is specific to an add-ons version so the chance of this hitting
# the cache and not missing seems quite bad to me (cgrebs)
def _fetch():
qs = _version_list_qs(addon)
return list(qs.values_list('version', flat=True))
cache_key = make_key(
u'version-detail:{}:{}'.format(addon.id, version_num),
normalize=True)
ids = cache_get_or_set(cache_key, _fetch)
url = reverse('addons.versions', args=[addon.slug])
if version_num in ids:
page = 1 + ids.index(version_num) / PER_PAGE
to = urlparams(url, 'version-%s' % version_num, page=page)
return http.HttpResponseRedirect(to)
else:
raise http.Http404()
def _category_personas(qs, limit):
def fetch_personas():
return randslice(qs, limit=limit)
key = make_key('cat-personas:' + str(qs.query), normalize=True)
return cache_get_or_set(key, fetch_personas)
def site_nav(context):
app = context['request'].APP.id
cache_key = make_key('site-nav-%s' % app, normalize=True)
return cache_get_or_set(cache_key, lambda: _site_nav(context))
def site_nav(context):
app = context['request'].APP.id
cache_key = make_key('site-nav-%s' % app, normalize=True)
return cache_get_or_set(cache_key, lambda: _site_nav(context))
def _category_personas(qs, limit):
def fetch_personas():
return randslice(qs, limit=limit)
key = make_key('cat-personas:' + str(qs.query), normalize=True)
return cache_get_or_set(key, fetch_personas)
