Esempio n. 1
0
def get_items(apiver, app, appver=None):
    # Collapse multiple blocklist items (different version ranges) into one
    # item and collapse each item's apps.
    addons = (BlocklistItem.objects.no_cache()
              .select_related('details')
              .prefetch_related('prefs')
              .filter(Q(app__guid__isnull=True) | Q(app__guid=app))
              .order_by('-modified')
              .extra(select={'app_guid': 'blapps.guid',
                             'app_min': 'blapps.min',
                             'app_max': 'blapps.max'}))
    items, details = {}, {}
    for guid, rows in sorted_groupby(addons, 'guid'):
        rows = list(rows)
        rr = []
        prefs = []
        for id, rs in sorted_groupby(rows, 'id'):
            rs = list(rs)
            rr.append(rs[0])
            prefs.extend(p.pref for p in rs[0].prefs.all())
            rs[0].apps = [App(r.app_guid, r.app_min, r.app_max)
                          for r in rs if r.app_guid]
        os = [r.os for r in rr if r.os]
        items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
                             rows[0].block_id, prefs)
        details[guid] = sorted(rows, key=attrgetter('id'))[0]
    return items, details
Esempio n. 2
0
def get_items(apiver, app, appver=None):
    # Collapse multiple blocklist items (different version ranges) into one
    # item and collapse each item's apps.
    addons = (
        BlocklistItem.objects.no_cache()
        .select_related("details")
        .prefetch_related("prefs")
        .filter(Q(app__guid__isnull=True) | Q(app__guid=app))
        .order_by("-modified")
        .extra(select={"app_guid": "blapps.guid", "app_min": "blapps.min", "app_max": "blapps.max"})
    )
    items, details = {}, {}
    for guid, rows in sorted_groupby(addons, "guid"):
        rows = list(rows)
        rr = []
        prefs = []
        for id, rs in sorted_groupby(rows, "id"):
            rs = list(rs)
            rr.append(rs[0])
            prefs.extend(p.pref for p in rs[0].prefs.all())
            rs[0].apps = [App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid]
        os = [r.os for r in rr if r.os]
        items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified, rows[0].block_id, prefs)
        details[guid] = sorted(rows, key=attrgetter("id"))[0]
    return items, details
Esempio n. 3
0
    def collections(self):
        features = self.features()
        lang = translation.to_language(translation.get_language())
        locale = Q(locale='') | Q(locale=lang)
        promos = (CollectionPromo.objects.filter(locale).filter(
            collection_feature__in=features).transform(
                CollectionPromo.transformer))
        groups = sorted_groupby(promos, 'collection_feature_id')

        # We key by feature_id and locale, so we can favor locale specific
        # promos.
        promo_dict = {}
        for feature_id, v in groups:
            promo = v.next()
            key = (feature_id, translation.to_language(promo.locale))
            promo_dict[key] = promo

        rv = {}
        # If we can, we favor locale specific collections.
        for feature in features:
            key = (feature.id, lang)
            if key not in promo_dict:
                key = (feature.id, '')
                if key not in promo_dict:
                    continue

            # We only want to see public add-ons on the front page.
            c = promo_dict[key].collection
            c.public_addons = c.addons.all() & Addon.objects.public()
            rv[feature] = c

        return rv
Esempio n. 4
0
    def transformer_promoted(cls, versions):
        """Attach the promoted approvals to the versions."""
        if not versions:
            return

        PromotedApproval = versions[0].promoted_approvals.model

        ids = {v.id for v in versions}

        approvals = list(
            PromotedApproval.objects.filter(version_id__in=ids).values_list(
                'version_id', 'group_id', 'application_id', named=True))

        approval_dict = {
            version_id: list(groups)
            for version_id, groups in sorted_groupby(approvals, 'version_id')
        }
        for version in versions:
            v_id = version.id
            groups = [(
                PROMOTED_GROUPS_BY_ID.get(approval.group_id),
                APP_IDS.get(approval.application_id),
            ) for approval in approval_dict.get(v_id, [])
                      if approval.group_id in PROMOTED_GROUPS_BY_ID]
            version.approved_for_groups = groups
Esempio n. 5
0
    def __init__(self, *args, **kw):
        self.addon = kw.pop('addon')
        self.request = kw.pop('request', None)
        super(BaseCategoryFormSet, self).__init__(*args, **kw)
        self.initial = []
        apps = sorted(self.addon.compatible_apps.keys(), key=lambda x: x.id)

        # Drop any apps that don't have appropriate categories.
        qs = Category.objects.filter(type=self.addon.type)
        app_cats = {k: list(v) for k, v in sorted_groupby(qs, 'application')}
        for app in list(apps):
            if app and not app_cats.get(app.id):
                apps.remove(app)
        if not app_cats:
            apps = []

        for app in apps:
            cats = self.addon.app_categories.get(app, [])
            self.initial.append({'categories': [c.id for c in cats]})

        for app, form in zip(apps, self.forms):
            key = app.id if app else None
            form.request = self.request
            form.initial['application'] = key
            form.app = app
            cats = sorted(app_cats[key], key=lambda x: x.name)
            form.fields['categories'].choices = [(c.id, c.name) for c in cats]

            # If this add-on is featured for this application, category
            # changes are forbidden.
            if not acl.action_allowed(self.request,
                                      amo.permissions.ADDONS_EDIT):
                form.disabled = (app and self.addon.is_featured(app))
Esempio n. 6
0
    def collections(self):
        features = self.features()
        lang = translation.to_language(translation.get_language())
        locale = Q(locale='') | Q(locale=lang)
        promos = (CollectionPromo.objects.filter(locale)
                  .filter(collection_feature__in=features)
                  .transform(CollectionPromo.transformer))
        groups = sorted_groupby(promos, 'collection_feature_id')

        # We key by feature_id and locale, so we can favor locale specific
        # promos.
        promo_dict = {}
        for feature_id, v in groups:
            promo = v.next()
            key = (feature_id, translation.to_language(promo.locale))
            promo_dict[key] = promo

        rv = {}
        # If we can, we favor locale specific collections.
        for feature in features:
            key = (feature.id, lang)
            if key not in promo_dict:
                key = (feature.id, '')
                if key not in promo_dict:
                    continue

            # We only want to see public add-ons on the front page.
            c = promo_dict[key].collection
            c.public_addons = c.addons.all() & Addon.objects.public()
            rv[feature] = c

        return rv
Esempio n. 7
0
 def to_representation(self, value):
     grouped = sorted_groupby(
         sorted(value),
         key=lambda x: getattr(amo.APP_IDS.get(x.application), 'short', ''),
     )
     return {
         app_name: [cat.slug for cat in categories]
         for app_name, categories in grouped
     }
Esempio n. 8
0
    def transformer(promos):
        if not promos:
            return

        promo_dict = dict((p.id, p) for p in promos)
        q = (Collection.objects.no_cache().filter(
            collectionpromo__in=promos).extra(
                select={'promo_id': 'collection_promos.id'}))

        for promo_id, collection in (sorted_groupby(q, 'promo_id')):
            promo_dict[promo_id].collection = collection.next()
Esempio n. 9
0
    def transformer(promos):
        if not promos:
            return

        promo_dict = dict((p.id, p) for p in promos)
        q = (Collection.objects.no_cache()
             .filter(collectionpromo__in=promos)
             .extra(select={'promo_id': 'collection_promos.id'}))

        for promo_id, collection in (sorted_groupby(q, 'promo_id')):
            promo_dict[promo_id].collection = collection.next()
Esempio n. 10
0
def get_items(apiver=None, app=None, appver=None, groupby='guid'):
    # Collapse multiple blocklist items (different version ranges) into one
    # item and collapse each item's apps.

    if app:
        app_query = Q(app__guid__isnull=True) | Q(app__guid=app)
    else:
        # This is useful to make the LEFT OUTER JOIN with blapps then
        # used in the extra clause.
        app_query = Q(app__isnull=True) | Q(app__isnull=False)

    addons = (BlocklistItem.objects.no_cache().select_related(
        'details').prefetch_related('prefs').filter(app_query).order_by(
            '-modified').extra(
                select={
                    'app_guid': 'blapps.guid',
                    'app_min': 'blapps.min',
                    'app_max': 'blapps.max'
                }))

    items, details = {}, {}
    for guid, rows in sorted_groupby(addons, groupby):
        rows = list(rows)
        rr = []
        prefs = []
        for id, rs in sorted_groupby(rows, 'id'):
            rs = list(rs)
            rr.append(rs[0])
            prefs.extend(p.pref for p in rs[0].prefs.all())
            rs[0].apps = [
                App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid
            ]
        os = [r.os for r in rr if r.os]
        block_id = max([r.block_id for r in rows])
        items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
                             block_id, prefs)
        details[guid] = sorted(rows, key=attrgetter('id'))[0]
    return items, details
Esempio n. 11
0
def get_items(apiver=None, app=None, appver=None, groupby='guid'):
    # Collapse multiple blocklist items (different version ranges) into one
    # item and collapse each item's apps.

    if app:
        app_query = Q(app__guid__isnull=True) | Q(app__guid=app)
    else:
        # This is useful to make the LEFT OUTER JOIN with blapps then
        # used in the extra clause.
        app_query = Q(app__isnull=True) | Q(app__isnull=False)

    addons = (BlocklistItem.objects.no_cache()
              .select_related('details')
              .prefetch_related('prefs')
              .filter(app_query)
              .order_by('-modified')
              .extra(select={'app_guid': 'blapps.guid',
                             'app_min': 'blapps.min',
                             'app_max': 'blapps.max'}))

    items, details = {}, {}
    for guid, rows in sorted_groupby(addons, groupby):
        rows = list(rows)
        rr = []
        prefs = []
        for id, rs in sorted_groupby(rows, 'id'):
            rs = list(rs)
            rr.append(rs[0])
            prefs.extend(p.pref for p in rs[0].prefs.all())
            rs[0].apps = [App(r.app_guid, r.app_min, r.app_max)
                          for r in rs if r.app_guid]
        os = [r.os for r in rr if r.os]
        block_id = max([r.block_id for r in rows])
        items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
                             block_id, prefs)
        details[guid] = sorted(rows, key=attrgetter('id'))[0]
    return items, details
Esempio n. 12
0
def category_sidebar(request, form_data, aggregations):
    APP = request.APP
    qatype, qcat = form_data.get('atype'), form_data.get('cat')
    cats = [f['key'] for f in aggregations['categories']]
    categories = Category.objects.filter(id__in=cats)
    if qatype in amo.ADDON_TYPES:
        categories = categories.filter(type=qatype)
    # Search categories don't have an application.
    categories = categories.filter(Q(application=APP.id) |
                                   Q(type=amo.ADDON_SEARCH))

    # If category is listed as a facet but type is not, then show All.
    if qcat in cats and not qatype:
        qatype = True

    # If category is not listed as a facet NOR available for this application,
    # then show All.
    if qcat not in categories.values_list('id', flat=True):
        qatype = qcat = None

    categories = [(_atype, sorted(_cats, key=lambda x: x.name))
                  for _atype, _cats in sorted_groupby(categories, 'type')]

    rv = []
    cat_params = {'cat': None}
    all_label = ugettext(u'All Add-ons')

    rv = [FacetLink(all_label, {'atype': None, 'cat': None}, not qatype)]

    for addon_type, cats in categories:
        selected = addon_type == qatype and not qcat

        # Build the linkparams.
        cat_params = cat_params.copy()
        cat_params.update(atype=addon_type)

        link = FacetLink(amo.ADDON_TYPES[addon_type],
                         cat_params, selected)
        link.children = [
            FacetLink(c.name, dict(cat_params, cat=c.id), c.id == qcat)
            for c in cats]
        rv.append(link)
    return rv
Esempio n. 13
0
def category_sidebar(request, form_data, aggregations):
    APP = request.APP
    qatype, qcat = form_data.get('atype'), form_data.get('cat')
    cats = [f['key'] for f in aggregations['categories']]
    categories = Category.objects.filter(id__in=cats)
    if qatype in amo.ADDON_TYPES:
        categories = categories.filter(type=qatype)
    # Search categories don't have an application.
    categories = categories.filter(
        Q(application=APP.id) | Q(type=amo.ADDON_SEARCH))

    # If category is listed as a facet but type is not, then show All.
    if qcat in cats and not qatype:
        qatype = True

    # If category is not listed as a facet NOR available for this application,
    # then show All.
    if qcat not in categories.values_list('id', flat=True):
        qatype = qcat = None

    categories = [(_atype, sorted(_cats, key=lambda x: x.name))
                  for _atype, _cats in sorted_groupby(categories, 'type')]

    rv = []
    cat_params = dict(cat=None)
    all_label = _(u'All Add-ons')

    rv = [FacetLink(all_label, dict(atype=None, cat=None), not qatype)]

    for addon_type, cats in categories:
        selected = addon_type == qatype and not qcat

        # Build the linkparams.
        cat_params = cat_params.copy()
        cat_params.update(atype=addon_type)

        link = FacetLink(amo.ADDON_TYPES[addon_type], cat_params, selected)
        link.children = [
            FacetLink(c.name, dict(cat_params, **dict(cat=c.id)), c.id == qcat)
            for c in cats
        ]
        rv.append(link)
    return rv
Esempio n. 14
0
def notify_compatibility(job, params):
    dry_run = params['preview_only']

    log.info('[@None] Starting validation email/update process for job %d.'
             ' dry_run=%s.' % (job.pk, dry_run))

    log.info('[@None] Starting validation version bumps for job %d.' % job.pk)

    version_list = updated_versions(job)
    total = version_list.count()

    for chunk in chunked(version_list, 100):
        log.info('[%d@%d] Updating versions for job %d.' %
                 (len(chunk), total, job.pk))
        update_maxversions.delay(chunk, job.pk, params)

    log.info('[@None] Starting validation email run for job %d.' % job.pk)

    updated_authors = completed_version_authors(job)
    total = updated_authors.count()
    for chunk in chunked(updated_authors, 100):
        log.info('[%d@%d] Notifying authors for validation job %d' %
                 (len(chunk), total, job.pk))

        # There are times when you want to punch django's ORM in
        # the face. This may be one of those times.
        users_addons = list(
            UserProfile.objects.filter(pk__in=chunk).filter(
                **_completed_versions(job, 'addons__versions')).values_list(
                    'pk', 'addons__pk').distinct())

        users = list(
            UserProfile.objects.filter(pk__in=set(u for u, a in users_addons)))

        # Annotate fails in tests when using cached results
        addons = (Addon.objects.no_cache().filter(
            **{
                'pk__in': set(a for u, a in users_addons),
                'versions__files__'
                'validation_results__validation_job': job
            }).annotate(
                errors=Sum('versions__files__validation_results__errors')))
        addons = dict((a.id, a) for a in addons)

        users_addons = dict(
            (u, [addons[a] for u, a in row])
            for u, row in sorted_groupby(users_addons, lambda k: k[0]))

        log.info('[%d@%d] Notifying %d authors about %d addons for '
                 'validation job %d' %
                 (len(chunk), total, len(users), len(addons.keys()), job.pk))

        for u in users:
            addons = users_addons[u.pk]

            u.passing_addons = [a for a in addons if a.errors == 0]
            u.failing_addons = [a for a in addons if a.errors > 0]

        notify_compatibility_chunk.delay(users, job, params)

    log.info('[@None] Completed validation email/update process '
             'for job %d. dry_run=%s.' % (job.pk, dry_run))
Esempio n. 15
0
 def rollup(xs):
     groups = sorted_groupby(xs, 'version_id')
     return dict((k, list(vs)) for k, vs in groups)
Esempio n. 16
0
 def rollup(xs):
     groups = sorted_groupby(xs, 'version_id')
     return dict((k, list(vs)) for k, vs in groups)
Esempio n. 17
0
 def rollup(xs):
     groups = sorted_groupby(xs, 'version_id')
     return {k: list(vs) for k, vs in groups}
Esempio n. 18
0
def notify_compatibility(job, params):
    dry_run = params['preview_only']

    log.info('[@None] Starting validation email/update process for job %d.'
             ' dry_run=%s.' % (job.pk, dry_run))

    log.info('[@None] Starting validation version bumps for job %d.' % job.pk)

    version_list = updated_versions(job)
    total = version_list.count()

    for chunk in chunked(version_list, 100):
        log.info('[%d@%d] Updating versions for job %d.' % (
            len(chunk), total, job.pk))
        update_maxversions.delay(chunk, job.pk, params)

    log.info('[@None] Starting validation email run for job %d.' % job.pk)

    updated_authors = completed_version_authors(job)
    total = updated_authors.count()
    for chunk in chunked(updated_authors, 100):
        log.info('[%d@%d] Notifying authors for validation job %d'
                 % (len(chunk), total, job.pk))

        # There are times when you want to punch django's ORM in
        # the face. This may be one of those times.
        users_addons = list(
            UserProfile.objects.filter(pk__in=chunk)
                       .filter(**_completed_versions(job,
                                                     'addons__versions'))
                       .values_list('pk', 'addons__pk').distinct())

        users = list(UserProfile.objects.filter(
                     pk__in=set(u for u, a in users_addons)))

        # Annotate fails in tests when using cached results
        addons = (Addon.objects.no_cache()
                  .filter(**{
                      'pk__in': set(a for u, a in users_addons),
                      'versions__files__'
                      'validation_results__validation_job': job
                  })
                  .annotate(errors=Sum(
                      'versions__files__validation_results__errors')))
        addons = dict((a.id, a) for a in addons)

        users_addons = dict((u, [addons[a] for u, a in row])
                            for u, row in sorted_groupby(users_addons,
                                                         lambda k: k[0]))

        log.info('[%d@%d] Notifying %d authors about %d addons for '
                 'validation job %d'
                 % (len(chunk), total, len(users), len(addons.keys()), job.pk))

        for u in users:
            addons = users_addons[u.pk]

            u.passing_addons = [a for a in addons if a.errors == 0]
            u.failing_addons = [a for a in addons if a.errors > 0]

        notify_compatibility_chunk.delay(users, job, params)

    log.info('[@None] Completed validation email/update process '
             'for job %d. dry_run=%s.' % (job.pk, dry_run))
Esempio n. 19
0
 def rollup(xs):
     groups = sorted_groupby(xs, 'version_id')
     return {k: list(vs) for k, vs in groups}