def get_items(apiver, app, appver=None): # Collapse multiple blocklist items (different version ranges) into one # item and collapse each item's apps. addons = (BlocklistItem.objects.no_cache() .select_related('details') .prefetch_related('prefs') .filter(Q(app__guid__isnull=True) | Q(app__guid=app)) .order_by('-modified') .extra(select={'app_guid': 'blapps.guid', 'app_min': 'blapps.min', 'app_max': 'blapps.max'})) items, details = {}, {} for guid, rows in sorted_groupby(addons, 'guid'): rows = list(rows) rr = [] prefs = [] for id, rs in sorted_groupby(rows, 'id'): rs = list(rs) rr.append(rs[0]) prefs.extend(p.pref for p in rs[0].prefs.all()) rs[0].apps = [App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid] os = [r.os for r in rr if r.os] items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified, rows[0].block_id, prefs) details[guid] = sorted(rows, key=attrgetter('id'))[0] return items, details
def get_items(apiver, app, appver=None): # Collapse multiple blocklist items (different version ranges) into one # item and collapse each item's apps. addons = ( BlocklistItem.objects.no_cache() .select_related("details") .prefetch_related("prefs") .filter(Q(app__guid__isnull=True) | Q(app__guid=app)) .order_by("-modified") .extra(select={"app_guid": "blapps.guid", "app_min": "blapps.min", "app_max": "blapps.max"}) ) items, details = {}, {} for guid, rows in sorted_groupby(addons, "guid"): rows = list(rows) rr = [] prefs = [] for id, rs in sorted_groupby(rows, "id"): rs = list(rs) rr.append(rs[0]) prefs.extend(p.pref for p in rs[0].prefs.all()) rs[0].apps = [App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid] os = [r.os for r in rr if r.os] items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified, rows[0].block_id, prefs) details[guid] = sorted(rows, key=attrgetter("id"))[0] return items, details
def collections(self): features = self.features() lang = translation.to_language(translation.get_language()) locale = Q(locale='') | Q(locale=lang) promos = (CollectionPromo.objects.filter(locale).filter( collection_feature__in=features).transform( CollectionPromo.transformer)) groups = sorted_groupby(promos, 'collection_feature_id') # We key by feature_id and locale, so we can favor locale specific # promos. promo_dict = {} for feature_id, v in groups: promo = v.next() key = (feature_id, translation.to_language(promo.locale)) promo_dict[key] = promo rv = {} # If we can, we favor locale specific collections. for feature in features: key = (feature.id, lang) if key not in promo_dict: key = (feature.id, '') if key not in promo_dict: continue # We only want to see public add-ons on the front page. c = promo_dict[key].collection c.public_addons = c.addons.all() & Addon.objects.public() rv[feature] = c return rv
def transformer_promoted(cls, versions): """Attach the promoted approvals to the versions.""" if not versions: return PromotedApproval = versions[0].promoted_approvals.model ids = {v.id for v in versions} approvals = list( PromotedApproval.objects.filter(version_id__in=ids).values_list( 'version_id', 'group_id', 'application_id', named=True)) approval_dict = { version_id: list(groups) for version_id, groups in sorted_groupby(approvals, 'version_id') } for version in versions: v_id = version.id groups = [( PROMOTED_GROUPS_BY_ID.get(approval.group_id), APP_IDS.get(approval.application_id), ) for approval in approval_dict.get(v_id, []) if approval.group_id in PROMOTED_GROUPS_BY_ID] version.approved_for_groups = groups
def __init__(self, *args, **kw): self.addon = kw.pop('addon') self.request = kw.pop('request', None) super(BaseCategoryFormSet, self).__init__(*args, **kw) self.initial = [] apps = sorted(self.addon.compatible_apps.keys(), key=lambda x: x.id) # Drop any apps that don't have appropriate categories. qs = Category.objects.filter(type=self.addon.type) app_cats = {k: list(v) for k, v in sorted_groupby(qs, 'application')} for app in list(apps): if app and not app_cats.get(app.id): apps.remove(app) if not app_cats: apps = [] for app in apps: cats = self.addon.app_categories.get(app, []) self.initial.append({'categories': [c.id for c in cats]}) for app, form in zip(apps, self.forms): key = app.id if app else None form.request = self.request form.initial['application'] = key form.app = app cats = sorted(app_cats[key], key=lambda x: x.name) form.fields['categories'].choices = [(c.id, c.name) for c in cats] # If this add-on is featured for this application, category # changes are forbidden. if not acl.action_allowed(self.request, amo.permissions.ADDONS_EDIT): form.disabled = (app and self.addon.is_featured(app))
def collections(self): features = self.features() lang = translation.to_language(translation.get_language()) locale = Q(locale='') | Q(locale=lang) promos = (CollectionPromo.objects.filter(locale) .filter(collection_feature__in=features) .transform(CollectionPromo.transformer)) groups = sorted_groupby(promos, 'collection_feature_id') # We key by feature_id and locale, so we can favor locale specific # promos. promo_dict = {} for feature_id, v in groups: promo = v.next() key = (feature_id, translation.to_language(promo.locale)) promo_dict[key] = promo rv = {} # If we can, we favor locale specific collections. for feature in features: key = (feature.id, lang) if key not in promo_dict: key = (feature.id, '') if key not in promo_dict: continue # We only want to see public add-ons on the front page. c = promo_dict[key].collection c.public_addons = c.addons.all() & Addon.objects.public() rv[feature] = c return rv
def to_representation(self, value): grouped = sorted_groupby( sorted(value), key=lambda x: getattr(amo.APP_IDS.get(x.application), 'short', ''), ) return { app_name: [cat.slug for cat in categories] for app_name, categories in grouped }
def transformer(promos): if not promos: return promo_dict = dict((p.id, p) for p in promos) q = (Collection.objects.no_cache().filter( collectionpromo__in=promos).extra( select={'promo_id': 'collection_promos.id'})) for promo_id, collection in (sorted_groupby(q, 'promo_id')): promo_dict[promo_id].collection = collection.next()
def transformer(promos): if not promos: return promo_dict = dict((p.id, p) for p in promos) q = (Collection.objects.no_cache() .filter(collectionpromo__in=promos) .extra(select={'promo_id': 'collection_promos.id'})) for promo_id, collection in (sorted_groupby(q, 'promo_id')): promo_dict[promo_id].collection = collection.next()
def get_items(apiver=None, app=None, appver=None, groupby='guid'): # Collapse multiple blocklist items (different version ranges) into one # item and collapse each item's apps. if app: app_query = Q(app__guid__isnull=True) | Q(app__guid=app) else: # This is useful to make the LEFT OUTER JOIN with blapps then # used in the extra clause. app_query = Q(app__isnull=True) | Q(app__isnull=False) addons = (BlocklistItem.objects.no_cache().select_related( 'details').prefetch_related('prefs').filter(app_query).order_by( '-modified').extra( select={ 'app_guid': 'blapps.guid', 'app_min': 'blapps.min', 'app_max': 'blapps.max' })) items, details = {}, {} for guid, rows in sorted_groupby(addons, groupby): rows = list(rows) rr = [] prefs = [] for id, rs in sorted_groupby(rows, 'id'): rs = list(rs) rr.append(rs[0]) prefs.extend(p.pref for p in rs[0].prefs.all()) rs[0].apps = [ App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid ] os = [r.os for r in rr if r.os] block_id = max([r.block_id for r in rows]) items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified, block_id, prefs) details[guid] = sorted(rows, key=attrgetter('id'))[0] return items, details
def get_items(apiver=None, app=None, appver=None, groupby='guid'): # Collapse multiple blocklist items (different version ranges) into one # item and collapse each item's apps. if app: app_query = Q(app__guid__isnull=True) | Q(app__guid=app) else: # This is useful to make the LEFT OUTER JOIN with blapps then # used in the extra clause. app_query = Q(app__isnull=True) | Q(app__isnull=False) addons = (BlocklistItem.objects.no_cache() .select_related('details') .prefetch_related('prefs') .filter(app_query) .order_by('-modified') .extra(select={'app_guid': 'blapps.guid', 'app_min': 'blapps.min', 'app_max': 'blapps.max'})) items, details = {}, {} for guid, rows in sorted_groupby(addons, groupby): rows = list(rows) rr = [] prefs = [] for id, rs in sorted_groupby(rows, 'id'): rs = list(rs) rr.append(rs[0]) prefs.extend(p.pref for p in rs[0].prefs.all()) rs[0].apps = [App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid] os = [r.os for r in rr if r.os] block_id = max([r.block_id for r in rows]) items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified, block_id, prefs) details[guid] = sorted(rows, key=attrgetter('id'))[0] return items, details
def category_sidebar(request, form_data, aggregations): APP = request.APP qatype, qcat = form_data.get('atype'), form_data.get('cat') cats = [f['key'] for f in aggregations['categories']] categories = Category.objects.filter(id__in=cats) if qatype in amo.ADDON_TYPES: categories = categories.filter(type=qatype) # Search categories don't have an application. categories = categories.filter(Q(application=APP.id) | Q(type=amo.ADDON_SEARCH)) # If category is listed as a facet but type is not, then show All. if qcat in cats and not qatype: qatype = True # If category is not listed as a facet NOR available for this application, # then show All. if qcat not in categories.values_list('id', flat=True): qatype = qcat = None categories = [(_atype, sorted(_cats, key=lambda x: x.name)) for _atype, _cats in sorted_groupby(categories, 'type')] rv = [] cat_params = {'cat': None} all_label = ugettext(u'All Add-ons') rv = [FacetLink(all_label, {'atype': None, 'cat': None}, not qatype)] for addon_type, cats in categories: selected = addon_type == qatype and not qcat # Build the linkparams. cat_params = cat_params.copy() cat_params.update(atype=addon_type) link = FacetLink(amo.ADDON_TYPES[addon_type], cat_params, selected) link.children = [ FacetLink(c.name, dict(cat_params, cat=c.id), c.id == qcat) for c in cats] rv.append(link) return rv
def category_sidebar(request, form_data, aggregations): APP = request.APP qatype, qcat = form_data.get('atype'), form_data.get('cat') cats = [f['key'] for f in aggregations['categories']] categories = Category.objects.filter(id__in=cats) if qatype in amo.ADDON_TYPES: categories = categories.filter(type=qatype) # Search categories don't have an application. categories = categories.filter( Q(application=APP.id) | Q(type=amo.ADDON_SEARCH)) # If category is listed as a facet but type is not, then show All. if qcat in cats and not qatype: qatype = True # If category is not listed as a facet NOR available for this application, # then show All. if qcat not in categories.values_list('id', flat=True): qatype = qcat = None categories = [(_atype, sorted(_cats, key=lambda x: x.name)) for _atype, _cats in sorted_groupby(categories, 'type')] rv = [] cat_params = dict(cat=None) all_label = _(u'All Add-ons') rv = [FacetLink(all_label, dict(atype=None, cat=None), not qatype)] for addon_type, cats in categories: selected = addon_type == qatype and not qcat # Build the linkparams. cat_params = cat_params.copy() cat_params.update(atype=addon_type) link = FacetLink(amo.ADDON_TYPES[addon_type], cat_params, selected) link.children = [ FacetLink(c.name, dict(cat_params, **dict(cat=c.id)), c.id == qcat) for c in cats ] rv.append(link) return rv
def notify_compatibility(job, params): dry_run = params['preview_only'] log.info('[@None] Starting validation email/update process for job %d.' ' dry_run=%s.' % (job.pk, dry_run)) log.info('[@None] Starting validation version bumps for job %d.' % job.pk) version_list = updated_versions(job) total = version_list.count() for chunk in chunked(version_list, 100): log.info('[%d@%d] Updating versions for job %d.' % (len(chunk), total, job.pk)) update_maxversions.delay(chunk, job.pk, params) log.info('[@None] Starting validation email run for job %d.' % job.pk) updated_authors = completed_version_authors(job) total = updated_authors.count() for chunk in chunked(updated_authors, 100): log.info('[%d@%d] Notifying authors for validation job %d' % (len(chunk), total, job.pk)) # There are times when you want to punch django's ORM in # the face. This may be one of those times. users_addons = list( UserProfile.objects.filter(pk__in=chunk).filter( **_completed_versions(job, 'addons__versions')).values_list( 'pk', 'addons__pk').distinct()) users = list( UserProfile.objects.filter(pk__in=set(u for u, a in users_addons))) # Annotate fails in tests when using cached results addons = (Addon.objects.no_cache().filter( **{ 'pk__in': set(a for u, a in users_addons), 'versions__files__' 'validation_results__validation_job': job }).annotate( errors=Sum('versions__files__validation_results__errors'))) addons = dict((a.id, a) for a in addons) users_addons = dict( (u, [addons[a] for u, a in row]) for u, row in sorted_groupby(users_addons, lambda k: k[0])) log.info('[%d@%d] Notifying %d authors about %d addons for ' 'validation job %d' % (len(chunk), total, len(users), len(addons.keys()), job.pk)) for u in users: addons = users_addons[u.pk] u.passing_addons = [a for a in addons if a.errors == 0] u.failing_addons = [a for a in addons if a.errors > 0] notify_compatibility_chunk.delay(users, job, params) log.info('[@None] Completed validation email/update process ' 'for job %d. dry_run=%s.' % (job.pk, dry_run))
def rollup(xs): groups = sorted_groupby(xs, 'version_id') return dict((k, list(vs)) for k, vs in groups)
def rollup(xs): groups = sorted_groupby(xs, 'version_id') return {k: list(vs) for k, vs in groups}
def notify_compatibility(job, params): dry_run = params['preview_only'] log.info('[@None] Starting validation email/update process for job %d.' ' dry_run=%s.' % (job.pk, dry_run)) log.info('[@None] Starting validation version bumps for job %d.' % job.pk) version_list = updated_versions(job) total = version_list.count() for chunk in chunked(version_list, 100): log.info('[%d@%d] Updating versions for job %d.' % ( len(chunk), total, job.pk)) update_maxversions.delay(chunk, job.pk, params) log.info('[@None] Starting validation email run for job %d.' % job.pk) updated_authors = completed_version_authors(job) total = updated_authors.count() for chunk in chunked(updated_authors, 100): log.info('[%d@%d] Notifying authors for validation job %d' % (len(chunk), total, job.pk)) # There are times when you want to punch django's ORM in # the face. This may be one of those times. users_addons = list( UserProfile.objects.filter(pk__in=chunk) .filter(**_completed_versions(job, 'addons__versions')) .values_list('pk', 'addons__pk').distinct()) users = list(UserProfile.objects.filter( pk__in=set(u for u, a in users_addons))) # Annotate fails in tests when using cached results addons = (Addon.objects.no_cache() .filter(**{ 'pk__in': set(a for u, a in users_addons), 'versions__files__' 'validation_results__validation_job': job }) .annotate(errors=Sum( 'versions__files__validation_results__errors'))) addons = dict((a.id, a) for a in addons) users_addons = dict((u, [addons[a] for u, a in row]) for u, row in sorted_groupby(users_addons, lambda k: k[0])) log.info('[%d@%d] Notifying %d authors about %d addons for ' 'validation job %d' % (len(chunk), total, len(users), len(addons.keys()), job.pk)) for u in users: addons = users_addons[u.pk] u.passing_addons = [a for a in addons if a.errors == 0] u.failing_addons = [a for a in addons if a.errors > 0] notify_compatibility_chunk.delay(users, job, params) log.info('[@None] Completed validation email/update process ' 'for job %d. dry_run=%s.' % (job.pk, dry_run))