def find_jetpacks(minver, maxver): """ Find all jetpack files that aren't disabled. Files that should be upgraded will have needs_upgrade=True. """ from .models import File statuses = amo.VALID_STATUSES files = (File.objects.filter(jetpack_version__isnull=False, version__addon__auto_repackage=True, version__addon__status__in=statuses, version__addon__disabled_by_user=False) .exclude(status=amo.STATUS_DISABLED).no_cache() .select_related('version')) files = sorted(files, key=lambda f: (f.version.addon_id, f.version.id)) # Figure out which files need to be upgraded. for file_ in files: file_.needs_upgrade = False # If any files for this add-on are reviewed, take the last reviewed file # plus all newer files. Otherwise, only upgrade the latest file. for _group, fs in groupby(files, key=lambda f: f.version.addon_id): fs = list(fs) if any(f.status in amo.REVIEWED_STATUSES for f in fs): for file_ in reversed(fs): file_.needs_upgrade = True if file_.status in amo.REVIEWED_STATUSES: break else: fs[-1].needs_upgrade = True # Make sure only old files are marked. for file_ in [f for f in files if f.needs_upgrade]: if not (vint(minver) <= vint(file_.jetpack_version) < vint(maxver)): file_.needs_upgrade = False return files
def apps(self): """Get `AppVersion`s for the application.""" apps = ( (amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION), (amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID) ) doesnt_support_no_id = ( self.strict_min_version and (vint(self.strict_min_version) < vint(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID)) ) if self.guid is None and doesnt_support_no_id: raise forms.ValidationError( _('GUID is required for Firefox 47 and below.') ) for app, default_min_version in apps: if self.guid is None and not self.strict_min_version: strict_min_version = amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID else: strict_min_version = ( self.strict_min_version or default_min_version) strict_max_version = ( self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION) try: min_appver, max_appver = get_appversions( app, strict_min_version, strict_max_version) yield Extractor.App( appdata=app, id=app.id, min=min_appver, max=max_appver) except AppVersion.DoesNotExist: pass
def test_appver_long(self): too_big = vnum(vint(MAXVERSION + 1)) just_right = vnum(vint(MAXVERSION)) assert self.check_appver_filters(too_big, floor_version(just_right)), ( 'All I ask is do not crash') assert self.check_appver_filters('9999999', '9999999.0') == ( [{'text': u'Firefox 9999999.0', 'selected': True, 'urlparams': {'appver': '9999999.0'}, 'children': []}, {'text': u'Firefox 5.0', 'selected': False, 'urlparams': {'appver': '5.0'}, 'children': []}]) assert self.check_appver_filters('99999999', '99999999.0') == ( [{'text': u'Firefox 99999999.0', 'selected': True, 'urlparams': {'appver': '99999999.0'}, 'children': []}, {'text': u'Firefox 5.0', 'selected': False, 'urlparams': {'appver': '5.0'}, 'children': []}])
def find_jetpacks(minver, maxver): """ Find all jetpack files that aren't disabled. Files that should be upgraded will have needs_upgrade=True. """ from .models import File statuses = amo.VALID_ADDON_STATUSES files = (File.objects.filter( jetpack_version__isnull=False, version__addon__auto_repackage=True, version__addon__status__in=statuses, version__addon__disabled_by_user=False).exclude( status=amo.STATUS_DISABLED).no_cache().select_related('version')) files = sorted(files, key=lambda f: (f.version.addon_id, f.version.id)) # Figure out which files need to be upgraded. for file_ in files: file_.needs_upgrade = False # If any files for this add-on are reviewed, take the last reviewed file # plus all newer files. Otherwise, only upgrade the latest file. for _group, fs in groupby(files, key=lambda f: f.version.addon_id): fs = list(fs) if any(f.status in amo.REVIEWED_STATUSES for f in fs): for file_ in reversed(fs): file_.needs_upgrade = True if file_.status in amo.REVIEWED_STATUSES: break else: fs[-1].needs_upgrade = True # Make sure only old files are marked. for file_ in [f for f in files if f.needs_upgrade]: if not (vint(minver) <= vint(file_.jetpack_version) < vint(maxver)): file_.needs_upgrade = False return files
def apps(self): """Get `AppVersion`s for the application.""" apps = ( (amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION), (amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID) ) doesnt_support_no_id = ( self.strict_min_version and (vint(self.strict_min_version) < vint(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID)) ) if self.guid is None and doesnt_support_no_id: raise forms.ValidationError( _('GUID is required for Firefox 47 and below.') ) couldnt_find_version = False for app, default_min_version in apps: if self.guid is None and not self.strict_min_version: strict_min_version = amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID else: strict_min_version = ( self.strict_min_version or default_min_version) strict_max_version = ( self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION) skip_app = ( self.strict_min_version and vint(self.strict_min_version) < vint(default_min_version) ) # Don't attempt to add support for this app to the WebExtension # if the `strict_min_version` is below the default minimum version # that is required to run WebExtensions (48.* for Android and 42.* # for Firefox). if skip_app: continue try: min_appver, max_appver = get_appversions( app, strict_min_version, strict_max_version) yield Extractor.App( appdata=app, id=app.id, min=min_appver, max=max_appver) except AppVersion.DoesNotExist: couldnt_find_version = True specified_versions = self.strict_min_version or self.strict_max_version if couldnt_find_version and specified_versions: msg = _( 'Cannot find min/max version. Maybe ' '"strict_min_version" or "strict_max_version" ' 'contains an unsupported version?') raise forms.ValidationError(msg)
def index(request, version=None): template = "compat/index.html" COMPAT = [v for v in amo.COMPAT if v["app"] == request.APP.id] compat_dict = dict((v["main"], v) for v in COMPAT) if not COMPAT: return render(request, template, {"results": False}) if version not in compat_dict: return http.HttpResponseRedirect(reverse("compat.index", args=[COMPAT[0]["main"]])) qs = AppCompat.search() binary = None initial = {"appver": "%s-%s" % (request.APP.id, version), "type": "all"} initial.update(request.GET.items()) form = CompatForm(initial) if request.GET and form.is_valid(): if form.cleaned_data["appver"]: app, ver = form.cleaned_data["appver"].split("-") if int(app) != request.APP.id or ver != version: new = reverse("compat.index", args=[ver], add_prefix=False) url = "/%s%s" % (amo.APP_IDS[int(app)].short, new) type_ = form.cleaned_data["type"] or None return http.HttpResponseRedirect(urlparams(url, type=type_)) if form.cleaned_data["type"] != "all": binary = form.cleaned_data["type"] == "binary" compat, app = compat_dict[version], str(request.APP.id) compat_queries = ( ( "prev", qs.query( **{ "top_95.%s.%s" % (app, vint(compat["previous"])): True, "support.%s.max__gte" % app: vint(compat["previous"]), } ), ), ("top_95", qs.query(**{"top_95_all.%s" % app: True})), ("all", qs), ) compat_levels = [(key, version_compat(queryset, compat, app, binary)) for key, queryset in compat_queries] usage_addons, usage_total = usage_stats(request, compat, app, binary) return render( request, template, { "version": version, "usage_addons": usage_addons, "usage_total": usage_total, "compat_levels": compat_levels, "form": form, "results": True, "show_previous": request.GET.get("previous"), }, )
def index(request, version=None): template = 'compat/index.html' COMPAT = [v for v in amo.COMPAT if v['app'] == request.APP.id] compat_dict = dict((v['main'], v) for v in COMPAT) if not COMPAT: return render(request, template, {'results': False}) if version not in compat_dict: return http.HttpResponseRedirect( reverse('compat.index', args=[COMPAT[0]['main']])) qs = AppCompat.search() binary = None initial = {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'} initial.update(request.GET.items()) form = CompatForm(initial) if request.GET and form.is_valid(): if form.cleaned_data['appver']: app, ver = form.cleaned_data['appver'].split('-') if int(app) != request.APP.id or ver != version: new = reverse('compat.index', args=[ver], add_prefix=False) url = '/%s%s' % (amo.APP_IDS[int(app)].short, new) type_ = form.cleaned_data['type'] or None return http.HttpResponseRedirect(urlparams(url, type=type_)) if form.cleaned_data['type'] != 'all': binary = form.cleaned_data['type'] == 'binary' compat, app = compat_dict[version], str(request.APP.id) compat_queries = ( ('prev', qs.query( **{ 'top_95.%s.%s' % (app, vint(compat['previous'])): True, 'support.%s.max__gte' % app: vint(compat['previous']) })), ('top_95', qs.query(**{'top_95_all.%s' % app: True})), ('all', qs), ) compat_levels = [(key, version_compat(queryset, compat, app, binary)) for key, queryset in compat_queries] usage_addons, usage_total = usage_stats(request, compat, app, binary) return render( request, template, { 'version': version, 'usage_addons': usage_addons, 'usage_total': usage_total, 'compat_levels': compat_levels, 'form': form, 'results': True, 'show_previous': request.GET.get('previous') })
def compat_stats(request, app, ver, minimum, ratio, binary): # Get the list of add-ons for usage stats. # Show add-ons marked as incompatible with this current version having # greater than 10 incompatible reports and whose average exceeds 80%. ver_int = str(vint(ver)) prefix = 'works.%s.%s' % (app, ver_int) qs = (AppCompat.search() .filter(**{'%s.failure__gt' % prefix: minimum, '%s.failure_ratio__gt' % prefix: ratio, 'support.%s.max__gte' % app: 0}) .order_by('-%s.failure_ratio' % prefix, '-%s.total' % prefix) .values_dict()) if binary is not None: qs = qs.filter(binary=binary) addons = amo.utils.paginate(request, qs) for obj in addons.object_list: obj['usage'] = obj['usage'][app] obj['max_version'] = obj['max_version'][app] obj['works'] = obj['works'][app].get(ver_int, {}) # Get all overrides for this add-on. obj['overrides'] = CompatOverride.objects.filter(addon__id=obj['id']) # Determine if there is an override for this current app version. obj['has_override'] = obj['overrides'].filter( _compat_ranges__min_app_version=ver + 'a1').exists() return addons, CompatTotals.objects.get(app=app).total
def compat_stats(request, version, minimum, ratio, binary): # Get the list of add-ons for usage stats. # Show add-ons marked as incompatible with this current version having # greater than 10 incompatible reports and whose average exceeds 80%. version_int = str(vint(version)) prefix = 'works.%s' % version_int qs = (AppCompat.search().filter( **{ '%s.failure__gt' % prefix: minimum, '%s.failure_ratio__gt' % prefix: ratio, 'support.max__gte': 0 }).order_by('-%s.failure_ratio' % prefix, '-%s.total' % prefix).values_dict()) if binary is not None: qs = qs.filter(binary=binary) addons = amo.utils.paginate(request, qs) for obj in addons.object_list: obj['usage'] = obj['usage'] obj['max_version'] = obj['max_version'] obj['works'] = obj['works'].get(version_int, {}) # Get all overrides for this add-on. obj['overrides'] = CompatOverride.objects.filter(addon__id=obj['id']) # Determine if there is an override for this current app version. obj['has_override'] = obj['overrides'].filter( _compat_ranges__min_app_version=version + 'a1').exists() return addons, CompatTotals.objects.get().total
def version_compat(qs, compat, app, binary): facets = [] for v, prev in zip(compat["versions"], (None,) + compat["versions"]): d = {"from": vint(v)} if prev: d["to"] = vint(prev) facets.append(d) # Pick up everything else for an Other count. facets.append({"to": vint(compat["versions"][-1])}) facet = {"range": {"support.%s.max" % app: facets}} if binary is not None: qs = qs.query(binary=binary) qs = qs.facet(by_status=facet) result = qs[:0].raw() total_addons = result["hits"]["total"] ranges = result["facets"]["by_status"]["ranges"] titles = compat["versions"] + (_("Other"),) faceted = [(v, r["count"]) for v, r in zip(titles, ranges)] return total_addons, faceted
def index(request, version=None): template = 'compat/index.html' COMPAT = [v for v in amo.COMPAT if v['app'] == request.APP.id] compat_dict = dict((v['main'], v) for v in COMPAT) if not COMPAT: return render(request, template, {'results': False}) if version not in compat_dict: return http.HttpResponseRedirect(reverse('compat.index', args=[COMPAT[0]['main']])) qs = AppCompat.search() binary = None initial = {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'} initial.update(request.GET.items()) form = CompatForm(initial) if request.GET and form.is_valid(): if form.cleaned_data['appver']: app, ver = form.cleaned_data['appver'].split('-') if int(app) != request.APP.id or ver != version: new = reverse('compat.index', args=[ver], add_prefix=False) url = '/%s%s' % (amo.APP_IDS[int(app)].short, new) type_ = form.cleaned_data['type'] or None return http.HttpResponseRedirect(urlparams(url, type=type_)) if form.cleaned_data['type'] != 'all': binary = form.cleaned_data['type'] == 'binary' compat, app = compat_dict[version], str(request.APP.id) compat_queries = ( ('prev', qs.query(**{ 'top_95.%s.%s' % (app, vint(compat['previous'])): True, 'support.%s.max__gte' % app: vint(compat['previous'])})), ('top_95', qs.query(**{'top_95_all.%s' % app: True})), ('all', qs), ) compat_levels = [(key, version_compat(queryset, compat, app, binary)) for key, queryset in compat_queries] usage_addons, usage_total = usage_stats(request, compat, app, binary) return render(request, template, {'version': version, 'usage_addons': usage_addons, 'usage_total': usage_total, 'compat_levels': compat_levels, 'form': form, 'results': True, 'show_previous': request.GET.get('previous')})
def version_compat(qs, compat, app, binary): facets = [] for v, prev in zip(compat['versions'], (None,) + compat['versions']): d = {'from': vint(v)} if prev: d['to'] = vint(prev) facets.append(d) # Pick up everything else for an Other count. facets.append({'to': vint(compat['versions'][-1])}) facet = {'range': {'support.%s.max' % app: facets}} if binary is not None: qs = qs.query(binary=binary) qs = qs.facet(by_status=facet) result = qs[:0].raw() total_addons = result['hits']['total'] ranges = result['facets']['by_status']['ranges'] titles = compat['versions'] + (_('Other'),) faceted = [(v, r['count']) for v, r in zip(titles, ranges)] return total_addons, faceted
def version_compat(qs, compat, app, binary): facets = [] for v, prev in zip(compat['versions'], (None, ) + compat['versions']): d = {'from': vint(v)} if prev: d['to'] = vint(prev) facets.append(d) # Pick up everything else for an Other count. facets.append({'to': vint(compat['versions'][-1])}) facet = {'range': {'support.%s.max' % app: facets}} if binary is not None: qs = qs.query(binary=binary) qs = qs.facet(by_status=facet) result = qs[:0].raw() total_addons = result['hits']['total'] ranges = result['facets']['by_status']['ranges'] titles = compat['versions'] + (_('Other'), ) faceted = [(v, r['count']) for v, r in zip(titles, ranges)] return total_addons, faceted
def usage_stats(request, compat, app, binary=None): # Get the list of add-ons for usage stats. qs = AppCompat.search().order_by("-usage.%s" % app).values_dict() if request.GET.get("previous"): qs = qs.filter(**{"support.%s.max__gte" % app: vint(compat["previous"])}) else: qs = qs.filter(**{"support.%s.max__gte" % app: 0}) if binary is not None: qs = qs.filter(binary=binary) addons = amo_utils.paginate(request, qs) for obj in addons.object_list: obj["usage"] = obj["usage"][app] obj["max_version"] = obj["max_version"][app] return addons, CompatTotals.objects.get(app=app).total
def usage_stats(request, compat, app, binary=None): # Get the list of add-ons for usage stats. qs = AppCompat.search().order_by('-usage.%s' % app).values_dict() if request.GET.get('previous'): qs = qs.filter( **{'support.%s.max__gte' % app: vint(compat['previous'])}) else: qs = qs.filter(**{'support.%s.max__gte' % app: 0}) if binary is not None: qs = qs.filter(binary=binary) addons = amo_utils.paginate(request, qs) for obj in addons.object_list: obj['usage'] = obj['usage'][app] obj['max_version'] = obj['max_version'][app] return addons, CompatTotals.objects.get(app=app).total
def matches_user_agent(cls, user_agent): for user_agent_re in cls.user_agent_re: match = user_agent_re.search(user_agent) if match: v = match.groups()[0] return vint(cls.min_display_version) <= vint(v)
def compatibility_report(index=None): docs = defaultdict(dict) indices = get_indices(index) # Gather all the data for the index. log.info(u'Generating Firefox compat report.') latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=amo.FIREFOX.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_ADDON_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): if (amo.FIREFOX not in addon.compatible_apps or addon.compatible_apps[amo.FIREFOX] is None): # Ignore this add-on if it does not have compat information # for Firefox. continue current_version = { 'id': addon.current_version.pk, 'version': addon.current_version.version, } doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, guid=addon.guid, binary=addon.binary_components, name=unicode(addon.name), created=addon.created, current_version=current_version) doc['count'] = chunk[addon.id] doc['usage'] = updates[addon.id] doc['top_95'] = {} # Populate with default counts for all versions. doc['works'] = {vint(version['main']): { 'success': 0, 'failure': 0, 'total': 0, 'failure_ratio': 0.0, } for version in FIREFOX_COMPAT} # Group reports by `major`.`minor` app version. reports = (CompatReport.objects .filter(guid=addon.guid, app_guid=amo.FIREFOX.guid) .values_list('app_version', 'works_properly') .annotate(Count('id'))) for ver, works_properly, cnt in reports: ver = vint(floor_version(ver)) major = [v['main'] for v in FIREFOX_COMPAT if vint(v['previous']) < ver <= vint(v['main'])] if major: w = doc['works'][vint(major[0])] # Tally number of success and failure reports. w['success' if works_properly else 'failure'] += cnt w['total'] += cnt # Calculate % of incompatibility reports. w['failure_ratio'] = w['failure'] / float(w['total']) compat = addon.compatible_apps[amo.FIREFOX] doc['support'] = {'min': compat.min.version_int, 'max': compat.max.version_int} doc['max_version'] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. compat_total, created = CompatTotals.objects.safer_get_or_create( defaults={'total': total}) if not created: compat_total.update(total=total) # Figure out which add-ons are in the top 95%. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): # Ignore the updates we skipped because of bad app compatibility. if addon in docs: running_total += count docs[addon]['top_95_all'] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # version. for compat in FIREFOX_COMPAT: version = vint(compat['previous']) # Find all the docs that have a max_version compatible with version. supported = [compat_doc for compat_doc in docs.values() if compat_doc['support']['max'] >= version] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][version] = running_total < (.95 * total) # Send it all to ES. bulk = [] for id_, doc in docs.items(): for index in set(indices): bulk.append({ "_source": doc, "_id": id_, "_type": AppCompat.get_mapping_type(), "_index": index or AppCompat._get_index(), }) es = amo_search.get_es() log.info('Bulk indexing %s compat docs on %s indices' % ( len(docs), len(indices))) elasticsearch.helpers.bulk(es, bulk, chunk_size=150) es.indices.refresh()
def apps(self): """Get `AppVersion`s for the application.""" apps = ( (amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION), (amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID) ) if self.type != amo.ADDON_STATICTHEME else ( (amo.FIREFOX, amo.DEFAULT_STATIC_THEME_MIN_VERSION_FIREFOX), ) doesnt_support_no_id = ( self.strict_min_version and (vint(self.strict_min_version) < vint(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID)) ) if self.guid is None and doesnt_support_no_id: raise forms.ValidationError( ugettext('GUID is required for Firefox 47 and below.') ) # If a minimum strict version is specified, it needs to be higher # than the version when Firefox started supporting WebExtensions # (We silently ignore apps that the add-on is not compatible with # below, but we need to be at least compatible with Firefox...) unsupported_no_matter_what = ( self.strict_min_version and vint(self.strict_min_version) < vint(amo.DEFAULT_WEBEXT_MIN_VERSION)) if unsupported_no_matter_what: msg = ugettext('Lowest supported "strict_min_version" is 42.0.') raise forms.ValidationError(msg) couldnt_find_version = False for app, default_min_version in apps: if self.guid is None and not self.strict_min_version: strict_min_version = max(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID, default_min_version) else: strict_min_version = ( self.strict_min_version or default_min_version) strict_max_version = ( self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION) # Don't attempt to add support for this app to the WebExtension # if the `strict_min_version` is below the default minimum version # that is required to run WebExtensions (48.* for Android and 42.* # for Firefox). skip_app = ( self.strict_min_version and vint(self.strict_min_version) < vint(default_min_version) ) if skip_app: continue try: min_appver, max_appver = get_appversions( app, strict_min_version, strict_max_version) yield Extractor.App( appdata=app, id=app.id, min=min_appver, max=max_appver) except AppVersion.DoesNotExist: couldnt_find_version = True specified_versions = self.strict_min_version or self.strict_max_version if couldnt_find_version and specified_versions: msg = ugettext( 'Cannot find min/max version. Maybe ' '"strict_min_version" or "strict_max_version" ' 'contains an unsupported version?') raise forms.ValidationError(msg)
def apps(self): """Get `AppVersion`s for the application.""" type_ = self.type if type_ == amo.ADDON_LPAPP: # Langpack are only compatible with Thunderbird desktop at the moment. # https://github.com/mozilla/addons-server/issues/8381 # They are all strictly compatible with a specific version, so # the default min version here doesn't matter much. apps = ((amo.THUNDERBIRD, amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD), ) elif type_ == amo.ADDON_STATICTHEME: # Static themes are only compatible with Thunderbird >= 60. apps = ((amo.THUNDERBIRD, amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD), ) elif type_ == amo.ADDON_DICT: # WebExt dicts are only compatible with Thunderbird >= 60.5. apps = ((amo.THUNDERBIRD, amo.DEFAULT_WEBEXT_DICT_MIN_VERSION_THUNDERBIRD), ) else: apps = ((amo.THUNDERBIRD, amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD), ) doesnt_support_no_id = (self.strict_min_version and (vint(self.strict_min_version) < vint( amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID))) if self.guid is None: raise forms.ValidationError( ugettext( 'GUID is required for Thunderbird Mail Extensions, including Themes.' )) # If a minimum strict version is specified, it needs to be higher # than the version when Firefox started supporting WebExtensions # (We silently ignore apps that the add-on is not compatible with # below, but we need to be at least compatible with Firefox...) unsupported_no_matter_what = ( self.strict_min_version and vint(self.strict_min_version) < vint( amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD)) if unsupported_no_matter_what: msg = ugettext('Lowest supported "strict_min_version" is 60.0.') raise forms.ValidationError(msg) couldnt_find_version = False for app, default_min_version in apps: if self.guid is None and not self.strict_min_version: strict_min_version = max(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID, default_min_version) else: strict_min_version = (self.strict_min_version or default_min_version) strict_max_version = (self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION) # Don't attempt to add support for this app to the WebExtension # if the `strict_min_version` is below the default minimum version # that is required to run WebExtensions (48.* for Android and 42.* # for Firefox). skip_app = ( self.strict_min_version and vint(self.strict_min_version) < vint(default_min_version)) if skip_app: continue try: min_appver, max_appver = get_appversions( app, strict_min_version, strict_max_version) yield Extractor.App(appdata=app, id=app.id, min=min_appver, max=max_appver) except AppVersion.DoesNotExist: couldnt_find_version = True specified_versions = self.strict_min_version or self.strict_max_version if couldnt_find_version and specified_versions: msg = ugettext('Cannot find min/max version. Maybe ' '"strict_min_version" or "strict_max_version" ' 'contains an unsupported version?') raise forms.ValidationError(msg)
def compatibility_report(index=None): docs = defaultdict(dict) indices = get_indices(index) # Gather all the data for the index. log.info(u'Generating Firefox compat report.') latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=amo.FIREFOX.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_ADDON_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): if (amo.FIREFOX not in addon.compatible_apps or addon.compatible_apps[amo.FIREFOX] is None): # Ignore this add-on if it does not have compat information # for Firefox. continue current_version = { 'id': addon.current_version.pk, 'version': addon.current_version.version, } doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, guid=addon.guid, binary=addon.binary_components, name=unicode(addon.name), created=addon.created, current_version=current_version) doc['count'] = chunk[addon.id] doc['usage'] = updates[addon.id] doc['top_95'] = {} # Populate with default counts for all versions. doc['works'] = { vint(version['main']): { 'success': 0, 'failure': 0, 'total': 0, 'failure_ratio': 0.0, } for version in FIREFOX_COMPAT } # Group reports by `major`.`minor` app version. reports = (CompatReport.objects.filter( guid=addon.guid, app_guid=amo.FIREFOX.guid).values_list( 'app_version', 'works_properly').annotate(Count('id'))) for ver, works_properly, cnt in reports: ver = vint(floor_version(ver)) major = [ v['main'] for v in FIREFOX_COMPAT if vint(v['previous']) < ver <= vint(v['main']) ] if major: w = doc['works'][vint(major[0])] # Tally number of success and failure reports. w['success' if works_properly else 'failure'] += cnt w['total'] += cnt # Calculate % of incompatibility reports. w['failure_ratio'] = w['failure'] / float(w['total']) compat = addon.compatible_apps[amo.FIREFOX] doc['support'] = { 'min': compat.min.version_int, 'max': compat.max.version_int } doc['max_version'] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. compat_total, created = CompatTotals.objects.safer_get_or_create( defaults={'total': total}) if not created: compat_total.update(total=total) # Figure out which add-ons are in the top 95%. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): # Ignore the updates we skipped because of bad app compatibility. if addon in docs: running_total += count docs[addon]['top_95_all'] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # version. for compat in FIREFOX_COMPAT: version = vint(compat['previous']) # Find all the docs that have a max_version compatible with version. supported = [ compat_doc for compat_doc in docs.values() if compat_doc['support']['max'] >= version ] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][version] = running_total < (.95 * total) # Send it all to ES. bulk = [] for id_, doc in docs.items(): for index in set(indices): bulk.append({ "_source": doc, "_id": id_, "_type": AppCompat.get_mapping_type(), "_index": index or AppCompat._get_index(), }) es = amo_search.get_es() log.info('Bulk indexing %s compat docs on %s indices' % (len(docs), len(indices))) elasticsearch.helpers.bulk(es, bulk, chunk_size=150) es.indices.refresh()
def compatibility_report(index=None): docs = defaultdict(dict) indices = get_indices(index) # Gather all the data for the index. for app in amo.APP_USAGE: versions = [c for c in amo.COMPAT if c['app'] == app.id] log.info(u'Making compat report for %s.' % app.pretty) latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=app.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): current_version = { 'id': addon.current_version.pk, 'version': addon.current_version.version, } doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, guid=addon.guid, binary=addon.binary_components, name=unicode(addon.name), created=addon.created, current_version=current_version) doc['count'] = chunk[addon.id] doc.setdefault('top_95', defaultdict(lambda: defaultdict(dict))) doc.setdefault('top_95_all', {}) doc.setdefault('usage', {})[app.id] = updates[addon.id] doc.setdefault('works', {}).setdefault(app.id, {}) # Populate with default counts for all app versions. for ver in versions: doc['works'][app.id][vint(ver['main'])] = { 'success': 0, 'failure': 0, 'total': 0, 'failure_ratio': 0.0, } # Group reports by `major`.`minor` app version. reports = (CompatReport.objects .filter(guid=addon.guid, app_guid=app.guid) .values_list('app_version', 'works_properly') .annotate(Count('id'))) for ver, works_properly, cnt in reports: ver = vint(floor_version(ver)) major = [v['main'] for v in versions if vint(v['previous']) < ver <= vint(v['main'])] if major: w = doc['works'][app.id][vint(major[0])] # Tally number of success and failure reports. w['success' if works_properly else 'failure'] += cnt w['total'] += cnt # Calculate % of incompatibility reports. w['failure_ratio'] = w['failure'] / float(w['total']) if app not in addon.compatible_apps: continue compat = addon.compatible_apps[app] d = {'min': compat.min.version_int, 'max': compat.max.version_int} doc.setdefault('support', {})[app.id] = d doc.setdefault('max_version', {})[app.id] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. compat_total, created = CompatTotals.objects.safer_get_or_create( app=app.id, defaults={'total': total}) if not created: compat_total.update(total=total) # Figure out which add-ons are in the top 95% for this app. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): running_total += count docs[addon]['top_95_all'][app.id] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # app + version combo. for compat in amo.COMPAT: app, ver = compat['app'], vint(compat['previous']) # Find all the docs that have a max_version compatible with ver. supported = [compat_doc for compat_doc in docs.values() if (app in compat_doc.get('support', {}) and compat_doc['support'][app]['max'] >= ver)] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][app][ver] = running_total < (.95 * total) # Send it all to the index. for chunk in chunked(docs.values(), 150): for doc in chunk: for index in indices: AppCompat.index(doc, id=doc['id'], refresh=False, index=index) es = amo_search.get_es() es.indices.refresh()
def apps(self): """Get `AppVersion`s for the application.""" apps = ((amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION), (amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID )) if self.type != amo.ADDON_STATICTHEME else ( (amo.FIREFOX, amo.DEFAULT_STATIC_THEME_MIN_VERSION_FIREFOX), ) doesnt_support_no_id = (self.strict_min_version and (vint(self.strict_min_version) < vint( amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID))) if self.guid is None and doesnt_support_no_id: raise forms.ValidationError( ugettext('GUID is required for Firefox 47 and below.')) # If a minimum strict version is specified, it needs to be higher # than the version when Firefox started supporting WebExtensions # (We silently ignore apps that the add-on is not compatible with # below, but we need to be at least compatible with Firefox...) unsupported_no_matter_what = (self.strict_min_version and vint( self.strict_min_version) < vint(amo.DEFAULT_WEBEXT_MIN_VERSION)) if unsupported_no_matter_what: msg = ugettext('Lowest supported "strict_min_version" is 42.0.') raise forms.ValidationError(msg) couldnt_find_version = False for app, default_min_version in apps: if self.guid is None and not self.strict_min_version: strict_min_version = max(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID, default_min_version) else: strict_min_version = (self.strict_min_version or default_min_version) strict_max_version = (self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION) # Don't attempt to add support for this app to the WebExtension # if the `strict_min_version` is below the default minimum version # that is required to run WebExtensions (48.* for Android and 42.* # for Firefox). skip_app = ( self.strict_min_version and vint(self.strict_min_version) < vint(default_min_version)) if skip_app: continue try: min_appver, max_appver = get_appversions( app, strict_min_version, strict_max_version) yield Extractor.App(appdata=app, id=app.id, min=min_appver, max=max_appver) except AppVersion.DoesNotExist: couldnt_find_version = True specified_versions = self.strict_min_version or self.strict_max_version if couldnt_find_version and specified_versions: msg = ugettext('Cannot find min/max version. Maybe ' '"strict_min_version" or "strict_max_version" ' 'contains an unsupported version?') raise forms.ValidationError(msg)
def compatibility_report(index=None): docs = defaultdict(dict) indices = get_indices(index) # Gather all the data for the index. for app in amo.APP_USAGE: versions = [c for c in amo.COMPAT if c['app'] == app.id] log.info(u'Making compat report for %s.' % app.pretty) latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=app.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): current_version = { 'id': addon.current_version.pk, 'version': addon.current_version.version, } doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, guid=addon.guid, binary=addon.binary_components, name=unicode(addon.name), created=addon.created, current_version=current_version) doc['count'] = chunk[addon.id] doc.setdefault('top_95', defaultdict(lambda: defaultdict(dict))) doc.setdefault('top_95_all', {}) doc.setdefault('usage', {})[app.id] = updates[addon.id] doc.setdefault('works', {}).setdefault(app.id, {}) # Populate with default counts for all app versions. for ver in versions: doc['works'][app.id][vint(ver['main'])] = { 'success': 0, 'failure': 0, 'total': 0, 'failure_ratio': 0.0, } # Group reports by `major`.`minor` app version. reports = (CompatReport.objects.filter( guid=addon.guid, app_guid=app.guid).values_list( 'app_version', 'works_properly').annotate(Count('id'))) for ver, works_properly, cnt in reports: ver = vint(floor_version(ver)) major = [ v['main'] for v in versions if vint(v['previous']) < ver <= vint(v['main']) ] if major: w = doc['works'][app.id][vint(major[0])] # Tally number of success and failure reports. w['success' if works_properly else 'failure'] += cnt w['total'] += cnt # Calculate % of incompatibility reports. w['failure_ratio'] = w['failure'] / float(w['total']) if app not in addon.compatible_apps: continue compat = addon.compatible_apps[app] d = { 'min': compat.min.version_int, 'max': compat.max.version_int } doc.setdefault('support', {})[app.id] = d doc.setdefault('max_version', {})[app.id] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. compat_total, created = CompatTotals.objects.safer_get_or_create( app=app.id, defaults={'total': total}) if not created: compat_total.update(total=total) # Figure out which add-ons are in the top 95% for this app. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): running_total += count docs[addon]['top_95_all'][app.id] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # app + version combo. for compat in amo.COMPAT: app, ver = compat['app'], vint(compat['previous']) # Find all the docs that have a max_version compatible with ver. supported = [ compat_doc for compat_doc in docs.values() if (app in compat_doc.get('support', {}) and compat_doc['support'][app]['max'] >= ver) ] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][app][ver] = running_total < (.95 * total) # Send it all to the index. for chunk in chunked(docs.values(), 150): for doc in chunk: for index in indices: AppCompat.index(doc, id=doc['id'], refresh=False, index=index) es = amo_search.get_es() es.indices.refresh()