def get_items(apiver, app, appver=None):
# Collapse multiple blocklist items (different version ranges) into one
# item and collapse each item's apps.
addons = (BlocklistItem.objects.no_cache()
.select_related('details')
.prefetch_related('prefs')
.filter(Q(app__guid__isnull=True) | Q(app__guid=app))
.order_by('-modified')
.extra(select={'app_guid': 'blapps.guid',
'app_min': 'blapps.min',
'app_max': 'blapps.max'}))
items, details = {}, {}
for guid, rows in sorted_groupby(addons, 'guid'):
rows = list(rows)
rr = []
prefs = []
for id, rs in sorted_groupby(rows, 'id'):
rs = list(rs)
rr.append(rs[0])
prefs.extend(p.pref for p in rs[0].prefs.all())
rs[0].apps = [App(r.app_guid, r.app_min, r.app_max)
for r in rs if r.app_guid]
os = [r.os for r in rr if r.os]
items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
rows[0].block_id, prefs)
details[guid] = sorted(rows, key=attrgetter('id'))[0]
return items, details
def get_items(apiver, app, appver=None):
# Collapse multiple blocklist items (different version ranges) into one
# item and collapse each item's apps.
addons = (
BlocklistItem.objects.no_cache()
.select_related("details")
.prefetch_related("prefs")
.filter(Q(app__guid__isnull=True) | Q(app__guid=app))
.order_by("-modified")
.extra(select={"app_guid": "blapps.guid", "app_min": "blapps.min", "app_max": "blapps.max"})
)
items, details = {}, {}
for guid, rows in sorted_groupby(addons, "guid"):
rows = list(rows)
rr = []
prefs = []
for id, rs in sorted_groupby(rows, "id"):
rs = list(rs)
rr.append(rs[0])
prefs.extend(p.pref for p in rs[0].prefs.all())
rs[0].apps = [App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid]
os = [r.os for r in rr if r.os]
items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified, rows[0].block_id, prefs)
details[guid] = sorted(rows, key=attrgetter("id"))[0]
return items, details
def collections(self):
features = self.features()
lang = translation.to_language(translation.get_language())
locale = Q(locale='') | Q(locale=lang)
promos = (CollectionPromo.objects.filter(locale).filter(
collection_feature__in=features).transform(
CollectionPromo.transformer))
groups = sorted_groupby(promos, 'collection_feature_id')
# We key by feature_id and locale, so we can favor locale specific
# promos.
promo_dict = {}
for feature_id, v in groups:
promo = v.next()
key = (feature_id, translation.to_language(promo.locale))
promo_dict[key] = promo
rv = {}
# If we can, we favor locale specific collections.
for feature in features:
key = (feature.id, lang)
if key not in promo_dict:
key = (feature.id, '')
if key not in promo_dict:
continue
# We only want to see public add-ons on the front page.
c = promo_dict[key].collection
c.public_addons = c.addons.all() & Addon.objects.public()
rv[feature] = c
return rv
def transformer_promoted(cls, versions):
"""Attach the promoted approvals to the versions."""
if not versions:
return
PromotedApproval = versions[0].promoted_approvals.model
ids = {v.id for v in versions}
approvals = list(
PromotedApproval.objects.filter(version_id__in=ids).values_list(
'version_id', 'group_id', 'application_id', named=True))
approval_dict = {
version_id: list(groups)
for version_id, groups in sorted_groupby(approvals, 'version_id')
}
for version in versions:
v_id = version.id
groups = [(
PROMOTED_GROUPS_BY_ID.get(approval.group_id),
APP_IDS.get(approval.application_id),
) for approval in approval_dict.get(v_id, [])
if approval.group_id in PROMOTED_GROUPS_BY_ID]
version.approved_for_groups = groups
def __init__(self, *args, **kw):
self.addon = kw.pop('addon')
self.request = kw.pop('request', None)
super(BaseCategoryFormSet, self).__init__(*args, **kw)
self.initial = []
apps = sorted(self.addon.compatible_apps.keys(), key=lambda x: x.id)
# Drop any apps that don't have appropriate categories.
qs = Category.objects.filter(type=self.addon.type)
app_cats = {k: list(v) for k, v in sorted_groupby(qs, 'application')}
for app in list(apps):
if app and not app_cats.get(app.id):
apps.remove(app)
if not app_cats:
apps = []
for app in apps:
cats = self.addon.app_categories.get(app, [])
self.initial.append({'categories': [c.id for c in cats]})
for app, form in zip(apps, self.forms):
key = app.id if app else None
form.request = self.request
form.initial['application'] = key
form.app = app
cats = sorted(app_cats[key], key=lambda x: x.name)
form.fields['categories'].choices = [(c.id, c.name) for c in cats]
# If this add-on is featured for this application, category
# changes are forbidden.
if not acl.action_allowed(self.request,
amo.permissions.ADDONS_EDIT):
form.disabled = (app and self.addon.is_featured(app))
def collections(self):
features = self.features()
lang = translation.to_language(translation.get_language())
locale = Q(locale='') | Q(locale=lang)
promos = (CollectionPromo.objects.filter(locale)
.filter(collection_feature__in=features)
.transform(CollectionPromo.transformer))
groups = sorted_groupby(promos, 'collection_feature_id')
# We key by feature_id and locale, so we can favor locale specific
# promos.
promo_dict = {}
for feature_id, v in groups:
promo = v.next()
key = (feature_id, translation.to_language(promo.locale))
promo_dict[key] = promo
rv = {}
# If we can, we favor locale specific collections.
for feature in features:
key = (feature.id, lang)
if key not in promo_dict:
key = (feature.id, '')
if key not in promo_dict:
continue
# We only want to see public add-ons on the front page.
c = promo_dict[key].collection
c.public_addons = c.addons.all() & Addon.objects.public()
rv[feature] = c
return rv
def to_representation(self, value):
grouped = sorted_groupby(
sorted(value),
key=lambda x: getattr(amo.APP_IDS.get(x.application), 'short', ''),
)
return {
app_name: [cat.slug for cat in categories]
for app_name, categories in grouped
}
def transformer(promos):
if not promos:
return
promo_dict = dict((p.id, p) for p in promos)
q = (Collection.objects.no_cache().filter(
collectionpromo__in=promos).extra(
select={'promo_id': 'collection_promos.id'}))
for promo_id, collection in (sorted_groupby(q, 'promo_id')):
promo_dict[promo_id].collection = collection.next()
def transformer(promos):
if not promos:
return
promo_dict = dict((p.id, p) for p in promos)
q = (Collection.objects.no_cache()
.filter(collectionpromo__in=promos)
.extra(select={'promo_id': 'collection_promos.id'}))
for promo_id, collection in (sorted_groupby(q, 'promo_id')):
promo_dict[promo_id].collection = collection.next()
def get_items(apiver=None, app=None, appver=None, groupby='guid'):
# Collapse multiple blocklist items (different version ranges) into one
# item and collapse each item's apps.
if app:
app_query = Q(app__guid__isnull=True) | Q(app__guid=app)
else:
# This is useful to make the LEFT OUTER JOIN with blapps then
# used in the extra clause.
app_query = Q(app__isnull=True) | Q(app__isnull=False)
addons = (BlocklistItem.objects.no_cache().select_related(
'details').prefetch_related('prefs').filter(app_query).order_by(
'-modified').extra(
select={
'app_guid': 'blapps.guid',
'app_min': 'blapps.min',
'app_max': 'blapps.max'
}))
items, details = {}, {}
for guid, rows in sorted_groupby(addons, groupby):
rows = list(rows)
rr = []
prefs = []
for id, rs in sorted_groupby(rows, 'id'):
rs = list(rs)
rr.append(rs[0])
prefs.extend(p.pref for p in rs[0].prefs.all())
rs[0].apps = [
App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid
]
os = [r.os for r in rr if r.os]
block_id = max([r.block_id for r in rows])
items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
block_id, prefs)
details[guid] = sorted(rows, key=attrgetter('id'))[0]
return items, details
def get_items(apiver=None, app=None, appver=None, groupby='guid'):
# Collapse multiple blocklist items (different version ranges) into one
# item and collapse each item's apps.
if app:
app_query = Q(app__guid__isnull=True) | Q(app__guid=app)
else:
# This is useful to make the LEFT OUTER JOIN with blapps then
# used in the extra clause.
app_query = Q(app__isnull=True) | Q(app__isnull=False)
addons = (BlocklistItem.objects.no_cache()
.select_related('details')
.prefetch_related('prefs')
.filter(app_query)
.order_by('-modified')
.extra(select={'app_guid': 'blapps.guid',
'app_min': 'blapps.min',
'app_max': 'blapps.max'}))
items, details = {}, {}
for guid, rows in sorted_groupby(addons, groupby):
rows = list(rows)
rr = []
prefs = []
for id, rs in sorted_groupby(rows, 'id'):
rs = list(rs)
rr.append(rs[0])
prefs.extend(p.pref for p in rs[0].prefs.all())
rs[0].apps = [App(r.app_guid, r.app_min, r.app_max)
for r in rs if r.app_guid]
os = [r.os for r in rr if r.os]
block_id = max([r.block_id for r in rows])
items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
block_id, prefs)
details[guid] = sorted(rows, key=attrgetter('id'))[0]
return items, details
def category_sidebar(request, form_data, aggregations):
APP = request.APP
qatype, qcat = form_data.get('atype'), form_data.get('cat')
cats = [f['key'] for f in aggregations['categories']]
categories = Category.objects.filter(id__in=cats)
if qatype in amo.ADDON_TYPES:
categories = categories.filter(type=qatype)
# Search categories don't have an application.
categories = categories.filter(Q(application=APP.id) |
Q(type=amo.ADDON_SEARCH))
# If category is listed as a facet but type is not, then show All.
if qcat in cats and not qatype:
qatype = True
# If category is not listed as a facet NOR available for this application,
# then show All.
if qcat not in categories.values_list('id', flat=True):
qatype = qcat = None
categories = [(_atype, sorted(_cats, key=lambda x: x.name))
for _atype, _cats in sorted_groupby(categories, 'type')]
rv = []
cat_params = {'cat': None}
all_label = ugettext(u'All Add-ons')
rv = [FacetLink(all_label, {'atype': None, 'cat': None}, not qatype)]
for addon_type, cats in categories:
selected = addon_type == qatype and not qcat
# Build the linkparams.
cat_params = cat_params.copy()
cat_params.update(atype=addon_type)
link = FacetLink(amo.ADDON_TYPES[addon_type],
cat_params, selected)
link.children = [
FacetLink(c.name, dict(cat_params, cat=c.id), c.id == qcat)
for c in cats]
rv.append(link)
return rv
def category_sidebar(request, form_data, aggregations):
APP = request.APP
qatype, qcat = form_data.get('atype'), form_data.get('cat')
cats = [f['key'] for f in aggregations['categories']]
categories = Category.objects.filter(id__in=cats)
if qatype in amo.ADDON_TYPES:
categories = categories.filter(type=qatype)
# Search categories don't have an application.
categories = categories.filter(
Q(application=APP.id) | Q(type=amo.ADDON_SEARCH))
# If category is listed as a facet but type is not, then show All.
if qcat in cats and not qatype:
qatype = True
# If category is not listed as a facet NOR available for this application,
# then show All.
if qcat not in categories.values_list('id', flat=True):
qatype = qcat = None
categories = [(_atype, sorted(_cats, key=lambda x: x.name))
for _atype, _cats in sorted_groupby(categories, 'type')]
rv = []
cat_params = dict(cat=None)
all_label = _(u'All Add-ons')
rv = [FacetLink(all_label, dict(atype=None, cat=None), not qatype)]
for addon_type, cats in categories:
selected = addon_type == qatype and not qcat
# Build the linkparams.
cat_params = cat_params.copy()
cat_params.update(atype=addon_type)
link = FacetLink(amo.ADDON_TYPES[addon_type], cat_params, selected)
link.children = [
FacetLink(c.name, dict(cat_params, **dict(cat=c.id)), c.id == qcat)
for c in cats
]
rv.append(link)
return rv
def notify_compatibility(job, params):
dry_run = params['preview_only']
log.info('[@None] Starting validation email/update process for job %d.'
' dry_run=%s.' % (job.pk, dry_run))
log.info('[@None] Starting validation version bumps for job %d.' % job.pk)
version_list = updated_versions(job)
total = version_list.count()
for chunk in chunked(version_list, 100):
log.info('[%d@%d] Updating versions for job %d.' %
(len(chunk), total, job.pk))
update_maxversions.delay(chunk, job.pk, params)
log.info('[@None] Starting validation email run for job %d.' % job.pk)
updated_authors = completed_version_authors(job)
total = updated_authors.count()
for chunk in chunked(updated_authors, 100):
log.info('[%d@%d] Notifying authors for validation job %d' %
(len(chunk), total, job.pk))
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
users_addons = list(
UserProfile.objects.filter(pk__in=chunk).filter(
**_completed_versions(job, 'addons__versions')).values_list(
'pk', 'addons__pk').distinct())
users = list(
UserProfile.objects.filter(pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache().filter(
**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
}).annotate(
errors=Sum('versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict(
(u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons, lambda k: k[0]))
log.info('[%d@%d] Notifying %d authors about %d addons for '
'validation job %d' %
(len(chunk), total, len(users), len(addons.keys()), job.pk))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
notify_compatibility_chunk.delay(users, job, params)
log.info('[@None] Completed validation email/update process '
'for job %d. dry_run=%s.' % (job.pk, dry_run))
def rollup(xs):
groups = sorted_groupby(xs, 'version_id')
return dict((k, list(vs)) for k, vs in groups)
def rollup(xs):
groups = sorted_groupby(xs, 'version_id')
return dict((k, list(vs)) for k, vs in groups)
def rollup(xs):
groups = sorted_groupby(xs, 'version_id')
return {k: list(vs) for k, vs in groups}
def notify_compatibility(job, params):
dry_run = params['preview_only']
log.info('[@None] Starting validation email/update process for job %d.'
' dry_run=%s.' % (job.pk, dry_run))
log.info('[@None] Starting validation version bumps for job %d.' % job.pk)
version_list = updated_versions(job)
total = version_list.count()
for chunk in chunked(version_list, 100):
log.info('[%d@%d] Updating versions for job %d.' % (
len(chunk), total, job.pk))
update_maxversions.delay(chunk, job.pk, params)
log.info('[@None] Starting validation email run for job %d.' % job.pk)
updated_authors = completed_version_authors(job)
total = updated_authors.count()
for chunk in chunked(updated_authors, 100):
log.info('[%d@%d] Notifying authors for validation job %d'
% (len(chunk), total, job.pk))
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
users_addons = list(
UserProfile.objects.filter(pk__in=chunk)
.filter(**_completed_versions(job,
'addons__versions'))
.values_list('pk', 'addons__pk').distinct())
users = list(UserProfile.objects.filter(
pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache()
.filter(**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
})
.annotate(errors=Sum(
'versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict((u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons,
lambda k: k[0]))
log.info('[%d@%d] Notifying %d authors about %d addons for '
'validation job %d'
% (len(chunk), total, len(users), len(addons.keys()), job.pk))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
notify_compatibility_chunk.delay(users, job, params)
log.info('[@None] Completed validation email/update process '
'for job %d. dry_run=%s.' % (job.pk, dry_run))
def rollup(xs):
groups = sorted_groupby(xs, 'version_id')
return {k: list(vs) for k, vs in groups}