def get_items(apiver, app, appver=None):
# Collapse multiple blocklist items (different version ranges) into one
# item and collapse each item's apps.
addons = (BlocklistItem.objects.no_cache()
.select_related('details')
.prefetch_related('prefs')
.filter(Q(app__guid__isnull=True) | Q(app__guid=app))
.order_by('-modified')
.extra(select={'app_guid': 'blapps.guid',
'app_min': 'blapps.min',
'app_max': 'blapps.max'}))
items, details = {}, {}
for guid, rows in sorted_groupby(addons, 'guid'):
rows = list(rows)
rr = []
prefs = []
for id, rs in sorted_groupby(rows, 'id'):
rs = list(rs)
rr.append(rs[0])
prefs.extend(p.pref for p in rs[0].prefs.all())
rs[0].apps = [App(r.app_guid, r.app_min, r.app_max)
for r in rs if r.app_guid]
os = [r.os for r in rr if r.os]
items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
rows[0].block_id, prefs)
details[guid] = sorted(rows, key=attrgetter('id'))[0]
return items, details
def build(cls):
qs = list(cls.get_objects())
# Normalize empty values.
for row in qs:
if not row['locale']:
row['locale'] = None
by_type = sorted_groupby(qs, itemgetter('type'))
by_locale = sorted_groupby(qs, itemgetter('locale'))
by_app = sorted_groupby(qs, itemgetter('application'))
pipe = cls.redis().pipeline(transaction=False)
pipe.delete(cls.by_id)
for row in qs:
pipe.sadd(cls.by_id, row['addon'])
groups = zip((cls.by_type, cls.by_locale, cls.by_app),
(by_type, by_locale, by_app))
for prefixer, group in groups:
for key, rows in group:
name = prefixer(key)
pipe.delete(name)
for row in rows:
if row['addon']:
pipe.sadd(name, row['addon'])
pipe.execute()
def build(cls):
qs = list(cls.get_objects())
# Expand any comma-separated lists of locales.
for row in list(qs):
# Normalize empty strings to None.
if row['locales'] == '':
row['locales'] = None
if row['locales']:
qs.remove(row)
for locale in row['locales'].split(','):
d = dict(row)
d['locales'] = locale.strip()
qs.append(d)
pipe = cls.redis().pipeline(transaction=False)
catapp = itemgetter('category', 'app')
for (category, app), rows in sorted_groupby(qs, catapp):
locale_getter = itemgetter('locales')
for locale, rs in sorted_groupby(rows, locale_getter):
if locale:
name = cls.by_locale(category, app, locale)
else:
name = cls.by_cat(category, app)
pipe.delete(name)
for row in rs:
if row['addon']:
pipe.sadd(name, row['addon'])
pipe.execute()
def jetpack(request):
upgrader = files.utils.JetpackUpgrader()
minver, maxver = upgrader.jetpack_versions()
form = JetpackUpgradeForm(request.POST)
if request.method == "POST":
if form.is_valid():
if "minver" in request.POST:
data = form.cleaned_data
upgrader.jetpack_versions(data["minver"], data["maxver"])
elif "upgrade" in request.POST:
if upgrader.version(maxver):
start_upgrade(minver, maxver)
elif "cancel" in request.POST:
upgrader.cancel()
return redirect("zadmin.jetpack")
else:
messages.error(request, form.errors.as_text())
jetpacks = files.utils.find_jetpacks(minver, maxver, from_builder_only=True)
upgrading = upgrader.version() # Current Jetpack version upgrading to.
repack_status = upgrader.files() # The files being repacked.
show = request.GET.get("show", upgrading or minver)
subset = filter(lambda f: not f.needs_upgrade and f.jetpack_version == show, jetpacks)
need_upgrade = filter(lambda f: f.needs_upgrade, jetpacks)
repacked = []
if upgrading:
# Group the repacked files by status for this Jetpack upgrade.
grouped_files = sorted_groupby(repack_status.values(), key=lambda f: f["status"])
for group, rows in grouped_files:
rows = sorted(list(rows), key=lambda f: f["file"])
for idx, row in enumerate(rows):
rows[idx]["file"] = File.objects.get(id=row["file"])
repacked.append((group, rows))
groups = sorted_groupby(jetpacks, "jetpack_version")
by_version = dict((version, len(list(files))) for version, files in groups)
return jingo.render(
request,
"zadmin/jetpack.html",
dict(
form=form,
upgrader=upgrader,
by_version=by_version,
upgrading=upgrading,
need_upgrade=need_upgrade,
subset=subset,
show=show,
repacked=repacked,
repack_status=repack_status,
),
)
def transformer(addons):
qs = (Category.uncached.filter(addons__in=addons)
.extra(select={'addon_id': 'addons_categories.addon_id'}))
cats = dict((addon_id, list(cs))
for addon_id, cs in sorted_groupby(qs, 'addon_id'))
for addon in addons:
addon.all_categories = cats.get(addon.id, [])
def __init__(self, *args, **kw):
self.addon = kw.pop('addon')
self.request = kw.pop('request', None)
super(BaseCategoryFormSet, self).__init__(*args, **kw)
self.initial = []
apps = sorted(self.addon.compatible_apps.keys(),
key=lambda x: x.id)
# Drop any apps that don't have appropriate categories.
qs = Category.objects.filter(type=self.addon.type)
app_cats = dict((k, list(v)) for k, v in
sorted_groupby(qs, 'application'))
for app in list(apps):
if app and not app_cats.get(app.id):
apps.remove(app)
if not app_cats:
apps = []
for app in apps:
cats = dict(self.addon.app_categories).get(app, [])
self.initial.append({'categories': [c.id for c in cats]})
for app, form in zip(apps, self.forms):
key = app.id if app else None
form.request = self.request
form.initial['application'] = key
form.app = app
cats = sorted(app_cats[key], key=lambda x: x.name)
form.fields['categories'].choices = [(c.id, c.name) for c in cats]
# If this add-on is featured for this application, category
# changes are forbidden.
if not acl.action_allowed(self.request, 'Addons', 'Edit'):
form.disabled = (app and self.addon.is_featured(app))
def __init__(self, *args, **kw):
self.addon = kw.pop('addon')
super(BaseCategoryFormSet, self).__init__(*args, **kw)
self.initial = []
apps = sorted(self.addon.compatible_apps.keys(), key=lambda x: x.id)
# Drop any apps that don't have appropriate categories.
qs = Category.objects.filter(type=self.addon.type,
application__in=[a.id for a in apps])
app_cats = dict((k, list(v))
for k, v in sorted_groupby(qs, 'application_id'))
for app in list(apps):
if not app_cats.get(app.id):
apps.remove(app)
for app in apps:
cats = dict(self.addon.app_categories).get(app, [])
self.initial.append({'categories': [c.id for c in cats]})
# Reconstruct the forms according to the initial data.
self._construct_forms()
for app, form in zip(apps, self.forms):
form.initial['application'] = app.id
form.app = app
cats = sorted(app_cats[app.id], key=lambda x: x.name)
form.fields['categories'].choices = [(c.id, c.name) for c in cats]
# If this add-on is featured for this application, category
# categories are forbidden.
form.disabled = (settings.NEW_FEATURES and
self.addon.is_featured(app))
def collections(self):
features = self.features()
lang = translation.to_language(translation.get_language())
locale = Q(locale='') | Q(locale=lang)
promos = (CollectionPromo.objects.filter(locale)
.filter(collection_feature__in=features)
.transform(CollectionPromo.transformer))
groups = sorted_groupby(promos, 'collection_feature_id')
# We key by feature_id and locale, so we can favor locale specific
# promos.
promo_dict = {}
for feature_id, v in groups:
promo = v.next()
key = (feature_id, translation.to_language(promo.locale))
promo_dict[key] = promo
rv = {}
# If we can, we favor locale specific collections.
for feature in features:
key = (feature.id, lang)
if key not in promo_dict:
key = (feature.id, '')
if key not in promo_dict:
continue
# We only want to see public add-ons on the front page.
c = promo_dict[key].collection
c.public_addons = c.addons.all() & Addon.objects.public()
rv[feature] = c
return rv
def attach_categories(addons):
"""Put all of the add-on's categories into a category_ids list."""
addon_dict = dict((a.id, a) for a in addons)
categories = (Category.objects.filter(addoncategory__addon__in=addon_dict)
.values_list('addoncategory__addon', 'id'))
for addon, cats in sorted_groupby(categories, lambda x: x[0]):
addon_dict[addon].category_ids = [c[1] for c in cats]
def scores(cls, addon_ids):
"""Get a mapping of {addon: {other_addon: score}} for each add-on."""
d = {}
q = (AddonRecommendation.objects.filter(addon__in=addon_ids)
.values('addon', 'other_addon', 'score'))
for addon, rows in sorted_groupby(q, key=lambda x: x['addon']):
d[addon] = dict((r['other_addon'], r['score']) for r in rows)
return d
def app_categories(self):
categories = sorted_groupby(order_by_translation(self.categories.all(),
'name'),
key=lambda x: x.application_id)
app_cats = []
for app, cats in categories:
app_cats.append((amo.APP_IDS[app], list(cats)))
return app_cats
def jetpack(request):
upgrader = files.utils.JetpackUpgrader()
minver, maxver = upgrader.jetpack_versions()
form = JetpackUpgradeForm(request.POST)
if request.method == 'POST':
if form.is_valid():
if 'minver' in request.POST:
data = form.cleaned_data
upgrader.jetpack_versions(data['minver'], data['maxver'])
elif 'upgrade' in request.POST:
if upgrader.version(maxver):
start_upgrade(minver, maxver)
elif 'cancel' in request.POST:
upgrader.cancel()
return redirect('zadmin.jetpack')
else:
messages.error(request, form.errors.as_text())
jetpacks = files.utils.find_jetpacks(minver, maxver)
need_upgrade = filter(lambda f: f.needs_upgrade, jetpacks)
archived = filter(lambda f: not f.needs_upgrade, jetpacks)
repacked = []
upgrading = upgrader.version() # Current Jetpack version upgrading to.
repack_status = upgrader.files() # The files being repacked.
if upgrading:
# Group the repacked files by status for this Jetpack upgrade.
grouped_files = sorted_groupby(repack_status.values(),
key=lambda f: f['status'])
for group, rows in grouped_files:
rows = sorted(list(rows), key=lambda f: f['file'])
for idx, row in enumerate(rows):
rows[idx]['file'] = File.objects.get(id=row['file'])
repacked.append((group, rows))
groups = sorted_groupby(jetpacks, 'jetpack_version')
by_version = dict((version, len(list(files))) for version, files in groups)
return jingo.render(request, 'zadmin/jetpack.html',
dict(form=form, upgrader=upgrader,
by_version=by_version, upgrading=upgrading,
need_upgrade=need_upgrade, archived=archived,
repacked=repacked, repack_status=repack_status))
def transformer(promos):
if not promos:
return
promo_dict = dict((p.id, p) for p in promos)
q = (Collection.objects.no_cache()
.filter(collectionpromo__in=promos)
.extra(select={'promo_id': 'collection_promos.id'}))
for promo_id, collection in (sorted_groupby(q, 'promo_id')):
promo_dict[promo_id].collection = collection.next()
def get_items(apiver, app, appver=None):
# Collapse multiple blocklist items (different version ranges) into one
# item and collapse each item's apps.
addons = (
BlocklistItem.uncached.select_related("details")
.filter(Q(app__guid__isnull=True) | Q(app__guid=app))
.order_by("-modified")
.extra(select={"app_guid": "blapps.guid", "app_min": "blapps.min", "app_max": "blapps.max"})
)
items, details = {}, {}
for guid, rows in sorted_groupby(addons, "guid"):
rows = list(rows)
rr = []
for id, rs in sorted_groupby(rows, "id"):
rs = list(rs)
rr.append(rs[0])
rs[0].apps = [App(r.app_guid, r.app_min, r.app_max) for r in rs if r.app_guid]
os = [r.os for r in rr if r.os]
items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified, rows[0].block_id)
details[guid] = sorted(rows, key=attrgetter("id"))[0]
return items, details
def get_items(apiver, app, appver):
# Collapse multiple blocklist items (different version ranges) into one
# item and collapse each item's apps.
addons = (BlocklistItem.uncached
.filter(Q(app__guid__isnull=True) | Q(app__guid=app))
.order_by('-modified')
.extra(select={'app_guid': 'blapps.guid',
'app_min': 'blapps.min',
'app_max': 'blapps.max'}))
items = {}
for guid, rows in sorted_groupby(addons, 'guid'):
rows = list(rows)
rr = []
for id, rs in sorted_groupby(rows, 'id'):
rs = list(rs)
rr.append(rs[0])
rs[0].apps = [App(r.app_guid, r.app_min, r.app_max)
for r in rs if r.app_guid]
os = [r.os for r in rr if r.os]
items[guid] = BlItem(rr, os[0] if os else None, rows[0].modified,
rows[0].block_id)
return items
def _site_nav(context):
# Prevent helpers from generating circular imports.
from addons.models import Category
request = context["request"]
types = amo.ADDON_EXTENSION, amo.ADDON_PERSONA
qs = Category.objects.filter(application=request.APP.id, weight__gte=0, type__in=types)
groups = utils.sorted_groupby(qs, key=attrgetter("type"))
cats = dict((key, sorted(cs, key=attrgetter("weight", "name"))) for key, cs in groups)
ctx = dict(
request=request, amo=amo, extensions=cats.get(amo.ADDON_EXTENSION, []), personas=cats.get(amo.ADDON_PERSONA, [])
)
return jinja2.Markup(env.get_template("amo/site_nav.html").render(ctx))
def _site_nav(context):
request = context['request']
types = amo.ADDON_EXTENSION, amo.ADDON_PERSONA, amo.ADDON_THEME
qs = Category.objects.filter(application=request.APP.id, weight__gte=0,
type__in=types)
groups = utils.sorted_groupby(qs, key=attrgetter('type'))
cats = dict((key, sorted(cs, key=attrgetter('weight', 'name')))
for key, cs in groups)
ctx = dict(request=request,
extensions=cats.get(amo.ADDON_EXTENSION, []),
personas=cats.get(amo.ADDON_PERSONA, []),
themes=cats.get(amo.ADDON_THEME, []))
return jinja2.Markup(env.get_template('amo/site_nav.html').render(ctx))
def attach_translations(addons):
"""Put all translations into a translations dict."""
fields = Addon._meta.translated_fields
ids = {}
for addon in addons:
addon.translations = collections.defaultdict(list)
ids.update((getattr(addon, field.attname, None), addon) for field in fields)
ids.pop(None, None)
qs = Translation.objects.filter(id__in=ids, localized_string__isnull=False).values_list(
"id", "locale", "localized_string"
)
for id, translations in sorted_groupby(qs, lambda x: x[0]):
ids[id].translations[id] = [(locale, string) for id, locale, string in translations]
def notify(request, job):
job = get_object_or_404(ValidationJob, pk=job)
notify_form = NotifyForm(request.POST, text='validation')
if not notify_form.is_valid():
messages.error(request, notify_form)
else:
for chunk in chunked(updated_versions(job), 100):
tasks.update_maxversions.delay(chunk, job.pk,
notify_form.cleaned_data)
updated_authors = completed_version_authors(job)
for chunk in chunked(updated_authors, 100):
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
# TODO: Offload most of this work to the task?
users_addons = list(
UserProfile.objects.filter(pk__in=chunk)
.filter(**_completed_versions(job,
'addons__versions'))
.values_list('pk', 'addons__pk').distinct())
users = list(UserProfile.objects.filter(
pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache()
.filter(**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
})
.annotate(errors=Sum(
'versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict((u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons,
lambda k: k[0]))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
tasks.notify_compatibility.delay(users, job,
notify_form.cleaned_data)
return redirect(reverse('zadmin.validation'))
def category_sidebar(request, query, facets):
APP = request.APP
qatype, qcat = query.get('atype'), query.get('cat')
webapp = qatype == amo.ADDON_WEBAPP
cats = [f['term'] for f in facets['categories']]
categories = Category.objects.filter(id__in=cats)
if qatype in amo.ADDON_TYPES:
categories = categories.filter(type=qatype)
if not webapp:
# Search categories don't have an application.
categories = categories.filter(Q(application=APP.id) |
Q(type=amo.ADDON_SEARCH))
# If category is listed as a facet but type is not, then show All.
if qcat in cats and not qatype:
qatype = True
# If category is not listed as a facet NOR available for this application,
# then show All.
if qcat not in categories.values_list('id', flat=True):
qatype = qcat = None
categories = [(atype, sorted(cats, key=lambda x: x.name))
for atype, cats in sorted_groupby(categories, 'type')]
rv = []
cat_params = dict(cat=None)
all_label = _(u'All Apps') if webapp else _(u'All Add-ons')
if not webapp or (webapp and not categories):
rv = [FacetLink(all_label, dict(atype=None, cat=None), not qatype)]
for addon_type, cats in categories:
selected = (webapp and not qatype) or addon_type == qatype and not qcat
# Build the linkparams.
cat_params = cat_params.copy()
if not webapp:
cat_params.update(atype=addon_type)
link = FacetLink(all_label if webapp else amo.ADDON_TYPES[addon_type],
cat_params, selected)
link.children = [FacetLink(c.name, dict(cat_params, **dict(cat=c.id)),
c.id == qcat) for c in cats]
rv.append(link)
return rv
def __init__(self, *args, **kw):
self.addon = kw.pop('addon')
self.request = kw.pop('request', None)
super(BaseCategoryFormSet, self).__init__(*args, **kw)
self.initial = []
if self.addon.type == amo.ADDON_WEBAPP:
apps = [None]
else:
apps = sorted(self.addon.compatible_apps.keys(),
key=lambda x: x.id)
# Drop any apps that don't have appropriate categories.
qs = Category.objects.filter(type=self.addon.type)
if self.addon.type != amo.ADDON_WEBAPP:
qs = qs.filter(application__in=[a.id for a in apps])
app_cats = dict((k, list(v)) for k, v in
sorted_groupby(qs, 'application_id'))
for app in list(apps):
if app and not app_cats.get(app.id):
apps.remove(app)
if not app_cats:
apps = []
for app in apps:
cats = dict(self.addon.app_categories).get(app, [])
self.initial.append({'categories': [c.id for c in cats]})
# Reconstruct the forms according to the initial data.
self._construct_forms()
for app, form in zip(apps, self.forms):
key = app.id if app else None
form.request = self.request
form.initial['application'] = key
form.app = app
cats = sorted(app_cats[key], key=lambda x: x.name)
form.fields['categories'].choices = [(c.id, c.name) for c in cats]
# If this add-on is featured for this application, category
# changes are forbidden.
if not acl.action_allowed(self.request, 'Admin', 'EditAnyAddon'):
form.disabled = (settings.NEW_FEATURES and app and
self.addon.is_featured(app))
def jetpack(request):
upgrader = files.utils.JetpackUpgrader()
minver, maxver = upgrader.jetpack_versions()
if request.method == "POST":
if "minver" in request.POST:
upgrader.jetpack_versions(request.POST["minver"], request.POST["maxver"])
elif "upgrade" in request.POST:
if upgrader.version(maxver):
start_upgrade(minver, maxver)
elif "cancel" in request.POST:
upgrader.cancel()
return redirect("zadmin.jetpack")
jetpacks = files.utils.find_jetpacks(minver, maxver)
groups = sorted_groupby(jetpacks, "jetpack_version")
by_version = dict((version, len(list(files))) for version, files in groups)
return jingo.render(
request, "zadmin/jetpack.html", dict(jetpacks=jetpacks, upgrader=upgrader, by_version=by_version)
)
def category_sidebar(request, form_data, facets):
APP = request.APP
qatype, qcat = form_data.get('atype'), form_data.get('cat')
cats = [f['term'] for f in facets['categories']]
categories = Category.objects.filter(id__in=cats)
if qatype in amo.ADDON_TYPES:
categories = categories.filter(type=qatype)
# Search categories don't have an application.
categories = categories.filter(
Q(application=APP.id) | Q(type=amo.ADDON_SEARCH))
# If category is listed as a facet but type is not, then show All.
if qcat in cats and not qatype:
qatype = True
# If category is not listed as a facet NOR available for this application,
# then show All.
if qcat not in categories.values_list('id', flat=True):
qatype = qcat = None
categories = [(_atype, sorted(_cats, key=lambda x: x.name))
for _atype, _cats in sorted_groupby(categories, 'type')]
rv = []
cat_params = dict(cat=None)
all_label = _(u'All Add-ons')
rv = [FacetLink(all_label, dict(atype=None, cat=None), not qatype)]
for addon_type, cats in categories:
selected = addon_type == qatype and not qcat
# Build the linkparams.
cat_params = cat_params.copy()
cat_params.update(atype=addon_type)
link = FacetLink(amo.ADDON_TYPES[addon_type], cat_params, selected)
link.children = [
FacetLink(c.name, dict(cat_params, **dict(cat=c.id)), c.id == qcat)
for c in cats
]
rv.append(link)
return rv
def jetpack(request):
upgrader = files.utils.JetpackUpgrader()
minver, maxver = upgrader.jetpack_versions()
if request.method == 'POST':
if 'minver' in request.POST:
upgrader.jetpack_versions(request.POST['minver'],
request.POST['maxver'])
elif 'upgrade' in request.POST:
if upgrader.version(maxver):
start_upgrade(minver, maxver)
elif 'cancel' in request.POST:
upgrader.cancel()
return redirect('zadmin.jetpack')
jetpacks = files.utils.find_jetpacks(minver, maxver)
groups = sorted_groupby(jetpacks, 'jetpack_version')
by_version = dict((version, len(list(files))) for version, files in groups)
return jingo.render(
request, 'zadmin/jetpack.html',
dict(jetpacks=jetpacks, upgrader=upgrader, by_version=by_version))
def jetpack(request):
cfg = Config.objects.get(key="jetpack_version")
upgrader = files.utils.JetpackUpgrader()
if request.method == "POST":
if request.POST.get("jetpack_version"):
cfg.value = request.POST["jetpack_version"]
cfg.save()
elif "upgrade" in request.POST:
if upgrader.version(cfg.value):
start_upgrade(cfg.value)
elif "cancel" in request.POST:
upgrader.cancel()
return redirect("zadmin.jetpack")
jetpacks = files.utils.find_jetpacks(cfg.value)
groups = sorted_groupby(jetpacks, "jetpack_version")
by_version = dict((version, len(list(files))) for version, files in groups)
return jingo.render(
request, "zadmin/jetpack.html", dict(cfg=cfg, jetpacks=jetpacks, upgrader=upgrader, by_version=by_version)
)
def category_sidebar(request, query, facets):
APP = request.APP
qatype, qcat = query.get("atype"), query.get("cat")
cats = [f["term"] for f in facets["categories"]]
categories = (
Category.objects.filter(id__in=cats)
# Search categories don't have an application.
.filter(Q(application=APP.id) | Q(type=amo.ADDON_SEARCH))
)
if qatype in amo.ADDON_TYPES:
categories = categories.filter(type=qatype)
categories = [(atype, sorted(cats, key=lambda x: x.name)) for atype, cats in sorted_groupby(categories, "type")]
rv = [FacetLink(_(u"All Add-ons"), dict(atype=None, cat=None), not qatype)]
for addon_type, cats in categories:
link = FacetLink(
amo.ADDON_TYPES[addon_type], dict(atype=addon_type, cat=None), addon_type == qatype and not qcat
)
link.children = [FacetLink(c.name, dict(atype=addon_type, cat=c.id), c.id == qcat) for c in cats]
rv.append(link)
return rv
def jetpack(request):
upgrader = files.utils.JetpackUpgrader()
minver, maxver = upgrader.jetpack_versions()
if request.method == 'POST':
if 'minver' in request.POST:
upgrader.jetpack_versions(request.POST['minver'],
request.POST['maxver'])
elif 'upgrade' in request.POST:
if upgrader.version(maxver):
start_upgrade(minver, maxver)
elif 'cancel' in request.POST:
upgrader.cancel()
return redirect('zadmin.jetpack')
jetpacks = files.utils.find_jetpacks(minver, maxver)
groups = sorted_groupby(jetpacks, 'jetpack_version')
by_version = dict((version, len(list(files))) for version, files in groups)
return jingo.render(request, 'zadmin/jetpack.html',
dict(jetpacks=jetpacks, upgrader=upgrader,
by_version=by_version))
def category_sidebar(request, query, facets):
APP = request.APP
qatype, qcat = query.get('atype'), query.get('cat')
cats = [f['term'] for f in facets['categories']]
categories = (
Category.objects.filter(id__in=cats)
# Search categories don't have an application.
.filter(Q(application=APP.id) | Q(type=amo.ADDON_SEARCH)))
if qatype in amo.ADDON_TYPES:
categories = categories.filter(type=qatype)
categories = [(atype, sorted(cats, key=lambda x: x.name))
for atype, cats in sorted_groupby(categories, 'type')]
rv = [FacetLink(_(u'All Add-ons'), dict(atype=None, cat=None), not qatype)]
for addon_type, cats in categories:
link = FacetLink(amo.ADDON_TYPES[addon_type],
dict(atype=addon_type, cat=None), addon_type == qatype
and not qcat)
link.children = [
FacetLink(c.name, dict(atype=addon_type, cat=c.id), c.id == qcat)
for c in cats
]
rv.append(link)
return rv
def notify(request, job):
job = get_object_or_404(ValidationJob, pk=job)
notify_form = NotifyForm(request.POST, text='validation')
if not notify_form.is_valid():
messages.error(request, notify_form)
else:
dry_run = notify_form.cleaned_data['preview_only']
log.info('[@None] Starting validation email/update process for job %d.'
' dry_run=%s.' % (job.pk, dry_run))
log.info('[@None] Starting validation version bumps for job %d.'
% (job.pk,))
version_list = updated_versions(job)
total = version_list.count()
for chunk in chunked(version_list, 100):
log.info('[%d@%d] Updating versions for job %d.' % (
len(chunk), total, job.pk))
tasks.update_maxversions.delay(chunk, job.pk,
notify_form.cleaned_data)
log.info('[@None] Starting validation email run for job %d.'
% (job.pk,))
updated_authors = completed_version_authors(job)
total = updated_authors.count()
for chunk in chunked(updated_authors, 100):
log.info('[%d@%d] Notifying authors for validation job %d'
% (len(chunk), total, job.pk))
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
# TODO: Offload most of this work to the task?
users_addons = list(
UserProfile.objects.filter(pk__in=chunk)
.filter(**_completed_versions(job,
'addons__versions'))
.values_list('pk', 'addons__pk').distinct())
users = list(UserProfile.objects.filter(
pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache()
.filter(**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
})
.annotate(errors=Sum(
'versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict((u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons,
lambda k: k[0]))
log.info('[%d@%d] Notifying %d authors about %d addons for '
'validation job %d'
% (len(chunk), total, len(users), len(addons.keys()),
job.pk))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
tasks.notify_compatibility.delay(users, job,
notify_form.cleaned_data)
log.info('[@None] Completed validation email/update process '
'for job %d. dry_run=%s.' % (job.pk, dry_run))
return redirect(reverse('zadmin.validation'))
def notify(request, job):
job = get_object_or_404(ValidationJob, pk=job)
notify_form = NotifyForm(request.POST, text='validation')
if not notify_form.is_valid():
messages.error(request, notify_form)
else:
dry_run = notify_form.cleaned_data['preview_only']
log.info('[@None] Starting validation email/update process for job %d.'
' dry_run=%s.' % (job.pk, dry_run))
log.info('[@None] Starting validation version bumps for job %d.'
% (job.pk,))
version_list = updated_versions(job)
total = version_list.count()
for chunk in chunked(version_list, 100):
log.info('[%d@%d] Updating versions for job %d.' % (
len(chunk), total, job.pk))
tasks.update_maxversions.delay(chunk, job.pk,
notify_form.cleaned_data)
log.info('[@None] Starting validation email run for job %d.'
% (job.pk,))
updated_authors = completed_version_authors(job)
total = updated_authors.count()
for chunk in chunked(updated_authors, 100):
log.info('[%d@%d] Notifying authors for validation job %d'
% (len(chunk), total, job.pk))
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
# TODO: Offload most of this work to the task?
users_addons = list(
UserProfile.objects.filter(pk__in=chunk)
.filter(**_completed_versions(job,
'addons__versions'))
.values_list('pk', 'addons__pk').distinct())
users = list(UserProfile.objects.filter(
pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache()
.filter(**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
})
.annotate(errors=Sum(
'versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict((u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons,
lambda k: k[0]))
log.info('[%d@%d] Notifying %d authors about %d addons for '
'validation job %d'
% (len(chunk), total, len(users), len(addons.keys()),
job.pk))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
tasks.notify_compatibility.delay(users, job,
notify_form.cleaned_data)
log.info('[@None] Completed validation email/update process '
'for job %d. dry_run=%s.' % (job.pk, dry_run))
return redirect(reverse('zadmin.validation'))
def attach_tags(addons):
addon_dict = dict((a.id, a) for a in addons)
qs = (Tag.objects.not_blacklisted().filter(addons__in=addon_dict)
.values_list('addons__id', 'tag_text'))
for addon, tags in sorted_groupby(qs, lambda x: x[0]):
addon_dict[addon].tag_list = [t[1] for t in tags]
def attach_devices(addons):
addon_dict = dict((a.id, a) for a in addons if a.type == amo.ADDON_WEBAPP)
devices = (AddonDeviceType.objects.filter(
addon__in=addon_dict).values_list('addon', 'device_type'))
for addon, device_types in sorted_groupby(devices, lambda x: x[0]):
addon_dict[addon].device_ids = [d[1] for d in device_types]
def attach_tags(addons):
addon_dict = dict((a.id, a) for a in addons)
qs = (Tag.objects.not_blacklisted().filter(
addons__in=addon_dict).values_list('addons__id', 'tag_text'))
for addon, tags in sorted_groupby(qs, lambda x: x[0]):
addon_dict[addon].tag_list = [t[1] for t in tags]
def notify_compatibility(job, params):
dry_run = params['preview_only']
log.info('[@None] Starting validation email/update process for job %d.'
' dry_run=%s.' % (job.pk, dry_run))
log.info('[@None] Starting validation version bumps for job %d.' % job.pk)
version_list = updated_versions(job)
total = version_list.count()
for chunk in chunked(version_list, 100):
log.info('[%d@%d] Updating versions for job %d.' %
(len(chunk), total, job.pk))
update_maxversions.delay(chunk, job.pk, params)
log.info('[@None] Starting validation email run for job %d.' % job.pk)
updated_authors = completed_version_authors(job)
total = updated_authors.count()
for chunk in chunked(updated_authors, 100):
log.info('[%d@%d] Notifying authors for validation job %d' %
(len(chunk), total, job.pk))
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
users_addons = list(
UserProfile.objects.filter(pk__in=chunk).filter(
**_completed_versions(job, 'addons__versions')).values_list(
'pk', 'addons__pk').distinct())
users = list(
UserProfile.objects.filter(pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache().filter(
**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
}).annotate(
errors=Sum('versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict(
(u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons, lambda k: k[0]))
log.info('[%d@%d] Notifying %d authors about %d addons for '
'validation job %d' %
(len(chunk), total, len(users), len(addons.keys()), job.pk))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
notify_compatibility_chunk.delay(users, job, params)
log.info('[@None] Completed validation email/update process '
'for job %d. dry_run=%s.' % (job.pk, dry_run))
def notify_compatibility(job, params):
dry_run = params['preview_only']
log.info('[@None] Starting validation email/update process for job %d.'
' dry_run=%s.' % (job.pk, dry_run))
log.info('[@None] Starting validation version bumps for job %d.' % job.pk)
version_list = updated_versions(job)
total = version_list.count()
for chunk in chunked(version_list, 100):
log.info('[%d@%d] Updating versions for job %d.' % (
len(chunk), total, job.pk))
update_maxversions.delay(chunk, job.pk, params)
log.info('[@None] Starting validation email run for job %d.' % job.pk)
updated_authors = completed_version_authors(job)
total = updated_authors.count()
for chunk in chunked(updated_authors, 100):
log.info('[%d@%d] Notifying authors for validation job %d'
% (len(chunk), total, job.pk))
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
users_addons = list(
UserProfile.objects.filter(pk__in=chunk)
.filter(**_completed_versions(job,
'addons__versions'))
.values_list('pk', 'addons__pk').distinct())
users = list(UserProfile.objects.filter(
pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache()
.filter(**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
})
.annotate(errors=Sum(
'versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict((u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons,
lambda k: k[0]))
log.info('[%d@%d] Notifying %d authors about %d addons for '
'validation job %d'
% (len(chunk), total, len(users), len(addons.keys()), job.pk))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
notify_compatibility_chunk.delay(users, job, params)
log.info('[@None] Completed validation email/update process '
'for job %d. dry_run=%s.' % (job.pk, dry_run))
def attach_devices(addons):
addon_dict = dict((a.id, a) for a in addons if a.type == amo.ADDON_WEBAPP)
devices = (AddonDeviceType.objects.filter(addon__in=addon_dict)
.values_list('addon', 'device_type'))
for addon, device_types in sorted_groupby(devices, lambda x: x[0]):
addon_dict[addon].device_ids = [d[1] for d in device_types]
