def find_jetpacks(minver, maxver):
"""
Find all jetpack files that aren't disabled.
Files that should be upgraded will have needs_upgrade=True.
"""
from .models import File
statuses = amo.VALID_STATUSES
files = (File.objects.filter(jetpack_version__isnull=False,
version__addon__auto_repackage=True,
version__addon__status__in=statuses,
version__addon__disabled_by_user=False)
.exclude(status=amo.STATUS_DISABLED).no_cache()
.select_related('version'))
files = sorted(files, key=lambda f: (f.version.addon_id, f.version.id))
# Figure out which files need to be upgraded.
for file_ in files:
file_.needs_upgrade = False
# If any files for this add-on are reviewed, take the last reviewed file
# plus all newer files. Otherwise, only upgrade the latest file.
for _group, fs in groupby(files, key=lambda f: f.version.addon_id):
fs = list(fs)
if any(f.status in amo.REVIEWED_STATUSES for f in fs):
for file_ in reversed(fs):
file_.needs_upgrade = True
if file_.status in amo.REVIEWED_STATUSES:
break
else:
fs[-1].needs_upgrade = True
# Make sure only old files are marked.
for file_ in [f for f in files if f.needs_upgrade]:
if not (vint(minver) <= vint(file_.jetpack_version) < vint(maxver)):
file_.needs_upgrade = False
return files
def apps(self):
"""Get `AppVersion`s for the application."""
apps = (
(amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION),
(amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID)
)
doesnt_support_no_id = (
self.strict_min_version and
(vint(self.strict_min_version) <
vint(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID))
)
if self.guid is None and doesnt_support_no_id:
raise forms.ValidationError(
_('GUID is required for Firefox 47 and below.')
)
for app, default_min_version in apps:
if self.guid is None and not self.strict_min_version:
strict_min_version = amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID
else:
strict_min_version = (
self.strict_min_version or default_min_version)
strict_max_version = (
self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION)
try:
min_appver, max_appver = get_appversions(
app, strict_min_version, strict_max_version)
yield Extractor.App(
appdata=app, id=app.id, min=min_appver, max=max_appver)
except AppVersion.DoesNotExist:
pass
def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
assert self.check_appver_filters(too_big, floor_version(just_right)), (
'All I ask is do not crash')
assert self.check_appver_filters('9999999', '9999999.0') == (
[{'text': u'Firefox 9999999.0',
'selected': True,
'urlparams': {'appver': '9999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
assert self.check_appver_filters('99999999', '99999999.0') == (
[{'text': u'Firefox 99999999.0',
'selected': True,
'urlparams': {'appver': '99999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
def find_jetpacks(minver, maxver):
"""
Find all jetpack files that aren't disabled.
Files that should be upgraded will have needs_upgrade=True.
"""
from .models import File
statuses = amo.VALID_ADDON_STATUSES
files = (File.objects.filter(
jetpack_version__isnull=False,
version__addon__auto_repackage=True,
version__addon__status__in=statuses,
version__addon__disabled_by_user=False).exclude(
status=amo.STATUS_DISABLED).no_cache().select_related('version'))
files = sorted(files, key=lambda f: (f.version.addon_id, f.version.id))
# Figure out which files need to be upgraded.
for file_ in files:
file_.needs_upgrade = False
# If any files for this add-on are reviewed, take the last reviewed file
# plus all newer files. Otherwise, only upgrade the latest file.
for _group, fs in groupby(files, key=lambda f: f.version.addon_id):
fs = list(fs)
if any(f.status in amo.REVIEWED_STATUSES for f in fs):
for file_ in reversed(fs):
file_.needs_upgrade = True
if file_.status in amo.REVIEWED_STATUSES:
break
else:
fs[-1].needs_upgrade = True
# Make sure only old files are marked.
for file_ in [f for f in files if f.needs_upgrade]:
if not (vint(minver) <= vint(file_.jetpack_version) < vint(maxver)):
file_.needs_upgrade = False
return files
def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
assert self.check_appver_filters(too_big, floor_version(just_right)), (
'All I ask is do not crash')
assert self.check_appver_filters('9999999', '9999999.0') == (
[{'text': u'Firefox 9999999.0',
'selected': True,
'urlparams': {'appver': '9999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
assert self.check_appver_filters('99999999', '99999999.0') == (
[{'text': u'Firefox 99999999.0',
'selected': True,
'urlparams': {'appver': '99999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
def apps(self):
"""Get `AppVersion`s for the application."""
apps = (
(amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION),
(amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID)
)
doesnt_support_no_id = (
self.strict_min_version and
(vint(self.strict_min_version) <
vint(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID))
)
if self.guid is None and doesnt_support_no_id:
raise forms.ValidationError(
_('GUID is required for Firefox 47 and below.')
)
couldnt_find_version = False
for app, default_min_version in apps:
if self.guid is None and not self.strict_min_version:
strict_min_version = amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID
else:
strict_min_version = (
self.strict_min_version or default_min_version)
strict_max_version = (
self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION)
skip_app = (
self.strict_min_version and vint(self.strict_min_version) <
vint(default_min_version)
)
# Don't attempt to add support for this app to the WebExtension
# if the `strict_min_version` is below the default minimum version
# that is required to run WebExtensions (48.* for Android and 42.*
# for Firefox).
if skip_app:
continue
try:
min_appver, max_appver = get_appversions(
app, strict_min_version, strict_max_version)
yield Extractor.App(
appdata=app, id=app.id, min=min_appver, max=max_appver)
except AppVersion.DoesNotExist:
couldnt_find_version = True
specified_versions = self.strict_min_version or self.strict_max_version
if couldnt_find_version and specified_versions:
msg = _(
'Cannot find min/max version. Maybe '
'"strict_min_version" or "strict_max_version" '
'contains an unsupported version?')
raise forms.ValidationError(msg)
def apps(self):
"""Get `AppVersion`s for the application."""
apps = (
(amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION),
(amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID)
)
doesnt_support_no_id = (
self.strict_min_version and
(vint(self.strict_min_version) <
vint(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID))
)
if self.guid is None and doesnt_support_no_id:
raise forms.ValidationError(
_('GUID is required for Firefox 47 and below.')
)
couldnt_find_version = False
for app, default_min_version in apps:
if self.guid is None and not self.strict_min_version:
strict_min_version = amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID
else:
strict_min_version = (
self.strict_min_version or default_min_version)
strict_max_version = (
self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION)
skip_app = (
self.strict_min_version and vint(self.strict_min_version) <
vint(default_min_version)
)
# Don't attempt to add support for this app to the WebExtension
# if the `strict_min_version` is below the default minimum version
# that is required to run WebExtensions (48.* for Android and 42.*
# for Firefox).
if skip_app:
continue
try:
min_appver, max_appver = get_appversions(
app, strict_min_version, strict_max_version)
yield Extractor.App(
appdata=app, id=app.id, min=min_appver, max=max_appver)
except AppVersion.DoesNotExist:
couldnt_find_version = True
specified_versions = self.strict_min_version or self.strict_max_version
if couldnt_find_version and specified_versions:
msg = _(
'Cannot find min/max version. Maybe '
'"strict_min_version" or "strict_max_version" '
'contains an unsupported version?')
raise forms.ValidationError(msg)
def index(request, version=None):
template = "compat/index.html"
COMPAT = [v for v in amo.COMPAT if v["app"] == request.APP.id]
compat_dict = dict((v["main"], v) for v in COMPAT)
if not COMPAT:
return render(request, template, {"results": False})
if version not in compat_dict:
return http.HttpResponseRedirect(reverse("compat.index", args=[COMPAT[0]["main"]]))
qs = AppCompat.search()
binary = None
initial = {"appver": "%s-%s" % (request.APP.id, version), "type": "all"}
initial.update(request.GET.items())
form = CompatForm(initial)
if request.GET and form.is_valid():
if form.cleaned_data["appver"]:
app, ver = form.cleaned_data["appver"].split("-")
if int(app) != request.APP.id or ver != version:
new = reverse("compat.index", args=[ver], add_prefix=False)
url = "/%s%s" % (amo.APP_IDS[int(app)].short, new)
type_ = form.cleaned_data["type"] or None
return http.HttpResponseRedirect(urlparams(url, type=type_))
if form.cleaned_data["type"] != "all":
binary = form.cleaned_data["type"] == "binary"
compat, app = compat_dict[version], str(request.APP.id)
compat_queries = (
(
"prev",
qs.query(
**{
"top_95.%s.%s" % (app, vint(compat["previous"])): True,
"support.%s.max__gte" % app: vint(compat["previous"]),
}
),
),
("top_95", qs.query(**{"top_95_all.%s" % app: True})),
("all", qs),
)
compat_levels = [(key, version_compat(queryset, compat, app, binary)) for key, queryset in compat_queries]
usage_addons, usage_total = usage_stats(request, compat, app, binary)
return render(
request,
template,
{
"version": version,
"usage_addons": usage_addons,
"usage_total": usage_total,
"compat_levels": compat_levels,
"form": form,
"results": True,
"show_previous": request.GET.get("previous"),
},
)
def index(request, version=None):
template = 'compat/index.html'
COMPAT = [v for v in amo.COMPAT if v['app'] == request.APP.id]
compat_dict = dict((v['main'], v) for v in COMPAT)
if not COMPAT:
return render(request, template, {'results': False})
if version not in compat_dict:
return http.HttpResponseRedirect(
reverse('compat.index', args=[COMPAT[0]['main']]))
qs = AppCompat.search()
binary = None
initial = {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'}
initial.update(request.GET.items())
form = CompatForm(initial)
if request.GET and form.is_valid():
if form.cleaned_data['appver']:
app, ver = form.cleaned_data['appver'].split('-')
if int(app) != request.APP.id or ver != version:
new = reverse('compat.index', args=[ver], add_prefix=False)
url = '/%s%s' % (amo.APP_IDS[int(app)].short, new)
type_ = form.cleaned_data['type'] or None
return http.HttpResponseRedirect(urlparams(url, type=type_))
if form.cleaned_data['type'] != 'all':
binary = form.cleaned_data['type'] == 'binary'
compat, app = compat_dict[version], str(request.APP.id)
compat_queries = (
('prev',
qs.query(
**{
'top_95.%s.%s' % (app, vint(compat['previous'])): True,
'support.%s.max__gte' % app: vint(compat['previous'])
})),
('top_95', qs.query(**{'top_95_all.%s' % app: True})),
('all', qs),
)
compat_levels = [(key, version_compat(queryset, compat, app, binary))
for key, queryset in compat_queries]
usage_addons, usage_total = usage_stats(request, compat, app, binary)
return render(
request, template, {
'version': version,
'usage_addons': usage_addons,
'usage_total': usage_total,
'compat_levels': compat_levels,
'form': form,
'results': True,
'show_previous': request.GET.get('previous')
})
def compat_stats(request, app, ver, minimum, ratio, binary):
# Get the list of add-ons for usage stats.
# Show add-ons marked as incompatible with this current version having
# greater than 10 incompatible reports and whose average exceeds 80%.
ver_int = str(vint(ver))
prefix = 'works.%s.%s' % (app, ver_int)
qs = (AppCompat.search()
.filter(**{'%s.failure__gt' % prefix: minimum,
'%s.failure_ratio__gt' % prefix: ratio,
'support.%s.max__gte' % app: 0})
.order_by('-%s.failure_ratio' % prefix,
'-%s.total' % prefix)
.values_dict())
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo.utils.paginate(request, qs)
for obj in addons.object_list:
obj['usage'] = obj['usage'][app]
obj['max_version'] = obj['max_version'][app]
obj['works'] = obj['works'][app].get(ver_int, {})
# Get all overrides for this add-on.
obj['overrides'] = CompatOverride.objects.filter(addon__id=obj['id'])
# Determine if there is an override for this current app version.
obj['has_override'] = obj['overrides'].filter(
_compat_ranges__min_app_version=ver + 'a1').exists()
return addons, CompatTotals.objects.get(app=app).total
def compat_stats(request, version, minimum, ratio, binary):
# Get the list of add-ons for usage stats.
# Show add-ons marked as incompatible with this current version having
# greater than 10 incompatible reports and whose average exceeds 80%.
version_int = str(vint(version))
prefix = 'works.%s' % version_int
qs = (AppCompat.search().filter(
**{
'%s.failure__gt' % prefix: minimum,
'%s.failure_ratio__gt' % prefix: ratio,
'support.max__gte': 0
}).order_by('-%s.failure_ratio' % prefix,
'-%s.total' % prefix).values_dict())
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo.utils.paginate(request, qs)
for obj in addons.object_list:
obj['usage'] = obj['usage']
obj['max_version'] = obj['max_version']
obj['works'] = obj['works'].get(version_int, {})
# Get all overrides for this add-on.
obj['overrides'] = CompatOverride.objects.filter(addon__id=obj['id'])
# Determine if there is an override for this current app version.
obj['has_override'] = obj['overrides'].filter(
_compat_ranges__min_app_version=version + 'a1').exists()
return addons, CompatTotals.objects.get().total
def version_compat(qs, compat, app, binary):
facets = []
for v, prev in zip(compat["versions"], (None,) + compat["versions"]):
d = {"from": vint(v)}
if prev:
d["to"] = vint(prev)
facets.append(d)
# Pick up everything else for an Other count.
facets.append({"to": vint(compat["versions"][-1])})
facet = {"range": {"support.%s.max" % app: facets}}
if binary is not None:
qs = qs.query(binary=binary)
qs = qs.facet(by_status=facet)
result = qs[:0].raw()
total_addons = result["hits"]["total"]
ranges = result["facets"]["by_status"]["ranges"]
titles = compat["versions"] + (_("Other"),)
faceted = [(v, r["count"]) for v, r in zip(titles, ranges)]
return total_addons, faceted
def index(request, version=None):
template = 'compat/index.html'
COMPAT = [v for v in amo.COMPAT if v['app'] == request.APP.id]
compat_dict = dict((v['main'], v) for v in COMPAT)
if not COMPAT:
return render(request, template, {'results': False})
if version not in compat_dict:
return http.HttpResponseRedirect(reverse('compat.index',
args=[COMPAT[0]['main']]))
qs = AppCompat.search()
binary = None
initial = {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'}
initial.update(request.GET.items())
form = CompatForm(initial)
if request.GET and form.is_valid():
if form.cleaned_data['appver']:
app, ver = form.cleaned_data['appver'].split('-')
if int(app) != request.APP.id or ver != version:
new = reverse('compat.index', args=[ver], add_prefix=False)
url = '/%s%s' % (amo.APP_IDS[int(app)].short, new)
type_ = form.cleaned_data['type'] or None
return http.HttpResponseRedirect(urlparams(url, type=type_))
if form.cleaned_data['type'] != 'all':
binary = form.cleaned_data['type'] == 'binary'
compat, app = compat_dict[version], str(request.APP.id)
compat_queries = (
('prev', qs.query(**{
'top_95.%s.%s' % (app, vint(compat['previous'])): True,
'support.%s.max__gte' % app: vint(compat['previous'])})),
('top_95', qs.query(**{'top_95_all.%s' % app: True})),
('all', qs),
)
compat_levels = [(key, version_compat(queryset, compat, app, binary))
for key, queryset in compat_queries]
usage_addons, usage_total = usage_stats(request, compat, app, binary)
return render(request, template,
{'version': version, 'usage_addons': usage_addons,
'usage_total': usage_total, 'compat_levels': compat_levels,
'form': form, 'results': True,
'show_previous': request.GET.get('previous')})
def version_compat(qs, compat, app, binary):
facets = []
for v, prev in zip(compat['versions'], (None,) + compat['versions']):
d = {'from': vint(v)}
if prev:
d['to'] = vint(prev)
facets.append(d)
# Pick up everything else for an Other count.
facets.append({'to': vint(compat['versions'][-1])})
facet = {'range': {'support.%s.max' % app: facets}}
if binary is not None:
qs = qs.query(binary=binary)
qs = qs.facet(by_status=facet)
result = qs[:0].raw()
total_addons = result['hits']['total']
ranges = result['facets']['by_status']['ranges']
titles = compat['versions'] + (_('Other'),)
faceted = [(v, r['count']) for v, r in zip(titles, ranges)]
return total_addons, faceted
def version_compat(qs, compat, app, binary):
facets = []
for v, prev in zip(compat['versions'], (None, ) + compat['versions']):
d = {'from': vint(v)}
if prev:
d['to'] = vint(prev)
facets.append(d)
# Pick up everything else for an Other count.
facets.append({'to': vint(compat['versions'][-1])})
facet = {'range': {'support.%s.max' % app: facets}}
if binary is not None:
qs = qs.query(binary=binary)
qs = qs.facet(by_status=facet)
result = qs[:0].raw()
total_addons = result['hits']['total']
ranges = result['facets']['by_status']['ranges']
titles = compat['versions'] + (_('Other'), )
faceted = [(v, r['count']) for v, r in zip(titles, ranges)]
return total_addons, faceted
def usage_stats(request, compat, app, binary=None):
# Get the list of add-ons for usage stats.
qs = AppCompat.search().order_by("-usage.%s" % app).values_dict()
if request.GET.get("previous"):
qs = qs.filter(**{"support.%s.max__gte" % app: vint(compat["previous"])})
else:
qs = qs.filter(**{"support.%s.max__gte" % app: 0})
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo_utils.paginate(request, qs)
for obj in addons.object_list:
obj["usage"] = obj["usage"][app]
obj["max_version"] = obj["max_version"][app]
return addons, CompatTotals.objects.get(app=app).total
def usage_stats(request, compat, app, binary=None):
# Get the list of add-ons for usage stats.
qs = AppCompat.search().order_by('-usage.%s' % app).values_dict()
if request.GET.get('previous'):
qs = qs.filter(
**{'support.%s.max__gte' % app: vint(compat['previous'])})
else:
qs = qs.filter(**{'support.%s.max__gte' % app: 0})
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo_utils.paginate(request, qs)
for obj in addons.object_list:
obj['usage'] = obj['usage'][app]
obj['max_version'] = obj['max_version'][app]
return addons, CompatTotals.objects.get(app=app).total
def matches_user_agent(cls, user_agent):
for user_agent_re in cls.user_agent_re:
match = user_agent_re.search(user_agent)
if match:
v = match.groups()[0]
return vint(cls.min_display_version) <= vint(v)
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
log.info(u'Generating Firefox compat report.')
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=amo.FIREFOX.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_ADDON_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
if (amo.FIREFOX not in addon.compatible_apps or
addon.compatible_apps[amo.FIREFOX] is None):
# Ignore this add-on if it does not have compat information
# for Firefox.
continue
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id, slug=addon.slug, guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name), created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc['usage'] = updates[addon.id]
doc['top_95'] = {}
# Populate with default counts for all versions.
doc['works'] = {vint(version['main']): {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
} for version in FIREFOX_COMPAT}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects
.filter(guid=addon.guid, app_guid=amo.FIREFOX.guid)
.values_list('app_version', 'works_properly')
.annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [v['main'] for v in FIREFOX_COMPAT
if vint(v['previous']) < ver <= vint(v['main'])]
if major:
w = doc['works'][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
compat = addon.compatible_apps[amo.FIREFOX]
doc['support'] = {'min': compat.min.version_int,
'max': compat.max.version_int}
doc['max_version'] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95%.
running_total = 0
for addon, count in sorted(updates.items(), key=lambda x: x[1],
reverse=True):
# Ignore the updates we skipped because of bad app compatibility.
if addon in docs:
running_total += count
docs[addon]['top_95_all'] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# version.
for compat in FIREFOX_COMPAT:
version = vint(compat['previous'])
# Find all the docs that have a max_version compatible with version.
supported = [compat_doc for compat_doc in docs.values()
if compat_doc['support']['max'] >= version]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][version] = running_total < (.95 * total)
# Send it all to ES.
bulk = []
for id_, doc in docs.items():
for index in set(indices):
bulk.append({
"_source": doc,
"_id": id_,
"_type": AppCompat.get_mapping_type(),
"_index": index or AppCompat._get_index(),
})
es = amo_search.get_es()
log.info('Bulk indexing %s compat docs on %s indices' % (
len(docs), len(indices)))
elasticsearch.helpers.bulk(es, bulk, chunk_size=150)
es.indices.refresh()
def apps(self):
"""Get `AppVersion`s for the application."""
apps = (
(amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION),
(amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID)
) if self.type != amo.ADDON_STATICTHEME else (
(amo.FIREFOX, amo.DEFAULT_STATIC_THEME_MIN_VERSION_FIREFOX),
)
doesnt_support_no_id = (
self.strict_min_version and
(vint(self.strict_min_version) <
vint(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID))
)
if self.guid is None and doesnt_support_no_id:
raise forms.ValidationError(
ugettext('GUID is required for Firefox 47 and below.')
)
# If a minimum strict version is specified, it needs to be higher
# than the version when Firefox started supporting WebExtensions
# (We silently ignore apps that the add-on is not compatible with
# below, but we need to be at least compatible with Firefox...)
unsupported_no_matter_what = (
self.strict_min_version and vint(self.strict_min_version) <
vint(amo.DEFAULT_WEBEXT_MIN_VERSION))
if unsupported_no_matter_what:
msg = ugettext('Lowest supported "strict_min_version" is 42.0.')
raise forms.ValidationError(msg)
couldnt_find_version = False
for app, default_min_version in apps:
if self.guid is None and not self.strict_min_version:
strict_min_version = max(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID,
default_min_version)
else:
strict_min_version = (
self.strict_min_version or default_min_version)
strict_max_version = (
self.strict_max_version or amo.DEFAULT_WEBEXT_MAX_VERSION)
# Don't attempt to add support for this app to the WebExtension
# if the `strict_min_version` is below the default minimum version
# that is required to run WebExtensions (48.* for Android and 42.*
# for Firefox).
skip_app = (
self.strict_min_version and vint(self.strict_min_version) <
vint(default_min_version)
)
if skip_app:
continue
try:
min_appver, max_appver = get_appversions(
app, strict_min_version, strict_max_version)
yield Extractor.App(
appdata=app, id=app.id, min=min_appver, max=max_appver)
except AppVersion.DoesNotExist:
couldnt_find_version = True
specified_versions = self.strict_min_version or self.strict_max_version
if couldnt_find_version and specified_versions:
msg = ugettext(
'Cannot find min/max version. Maybe '
'"strict_min_version" or "strict_max_version" '
'contains an unsupported version?')
raise forms.ValidationError(msg)
def apps(self):
"""Get `AppVersion`s for the application."""
type_ = self.type
if type_ == amo.ADDON_LPAPP:
# Langpack are only compatible with Thunderbird desktop at the moment.
# https://github.com/mozilla/addons-server/issues/8381
# They are all strictly compatible with a specific version, so
# the default min version here doesn't matter much.
apps = ((amo.THUNDERBIRD,
amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD), )
elif type_ == amo.ADDON_STATICTHEME:
# Static themes are only compatible with Thunderbird >= 60.
apps = ((amo.THUNDERBIRD,
amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD), )
elif type_ == amo.ADDON_DICT:
# WebExt dicts are only compatible with Thunderbird >= 60.5.
apps = ((amo.THUNDERBIRD,
amo.DEFAULT_WEBEXT_DICT_MIN_VERSION_THUNDERBIRD), )
else:
apps = ((amo.THUNDERBIRD,
amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD), )
doesnt_support_no_id = (self.strict_min_version
and (vint(self.strict_min_version) < vint(
amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID)))
if self.guid is None:
raise forms.ValidationError(
ugettext(
'GUID is required for Thunderbird Mail Extensions, including Themes.'
))
# If a minimum strict version is specified, it needs to be higher
# than the version when Firefox started supporting WebExtensions
# (We silently ignore apps that the add-on is not compatible with
# below, but we need to be at least compatible with Firefox...)
unsupported_no_matter_what = (
self.strict_min_version and vint(self.strict_min_version) < vint(
amo.DEFAULT_WEBEXT_MIN_VERSION_THUNDERBIRD))
if unsupported_no_matter_what:
msg = ugettext('Lowest supported "strict_min_version" is 60.0.')
raise forms.ValidationError(msg)
couldnt_find_version = False
for app, default_min_version in apps:
if self.guid is None and not self.strict_min_version:
strict_min_version = max(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID,
default_min_version)
else:
strict_min_version = (self.strict_min_version
or default_min_version)
strict_max_version = (self.strict_max_version
or amo.DEFAULT_WEBEXT_MAX_VERSION)
# Don't attempt to add support for this app to the WebExtension
# if the `strict_min_version` is below the default minimum version
# that is required to run WebExtensions (48.* for Android and 42.*
# for Firefox).
skip_app = (
self.strict_min_version
and vint(self.strict_min_version) < vint(default_min_version))
if skip_app:
continue
try:
min_appver, max_appver = get_appversions(
app, strict_min_version, strict_max_version)
yield Extractor.App(appdata=app,
id=app.id,
min=min_appver,
max=max_appver)
except AppVersion.DoesNotExist:
couldnt_find_version = True
specified_versions = self.strict_min_version or self.strict_max_version
if couldnt_find_version and specified_versions:
msg = ugettext('Cannot find min/max version. Maybe '
'"strict_min_version" or "strict_max_version" '
'contains an unsupported version?')
raise forms.ValidationError(msg)
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
log.info(u'Generating Firefox compat report.')
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=amo.FIREFOX.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_ADDON_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
if (amo.FIREFOX not in addon.compatible_apps
or addon.compatible_apps[amo.FIREFOX] is None):
# Ignore this add-on if it does not have compat information
# for Firefox.
continue
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id,
slug=addon.slug,
guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name),
created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc['usage'] = updates[addon.id]
doc['top_95'] = {}
# Populate with default counts for all versions.
doc['works'] = {
vint(version['main']): {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
for version in FIREFOX_COMPAT
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects.filter(
guid=addon.guid, app_guid=amo.FIREFOX.guid).values_list(
'app_version', 'works_properly').annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [
v['main'] for v in FIREFOX_COMPAT
if vint(v['previous']) < ver <= vint(v['main'])
]
if major:
w = doc['works'][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
compat = addon.compatible_apps[amo.FIREFOX]
doc['support'] = {
'min': compat.min.version_int,
'max': compat.max.version_int
}
doc['max_version'] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95%.
running_total = 0
for addon, count in sorted(updates.items(),
key=lambda x: x[1],
reverse=True):
# Ignore the updates we skipped because of bad app compatibility.
if addon in docs:
running_total += count
docs[addon]['top_95_all'] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# version.
for compat in FIREFOX_COMPAT:
version = vint(compat['previous'])
# Find all the docs that have a max_version compatible with version.
supported = [
compat_doc for compat_doc in docs.values()
if compat_doc['support']['max'] >= version
]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][version] = running_total < (.95 * total)
# Send it all to ES.
bulk = []
for id_, doc in docs.items():
for index in set(indices):
bulk.append({
"_source": doc,
"_id": id_,
"_type": AppCompat.get_mapping_type(),
"_index": index or AppCompat._get_index(),
})
es = amo_search.get_es()
log.info('Bulk indexing %s compat docs on %s indices' %
(len(docs), len(indices)))
elasticsearch.helpers.bulk(es, bulk, chunk_size=150)
es.indices.refresh()
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
for app in amo.APP_USAGE:
versions = [c for c in amo.COMPAT if c['app'] == app.id]
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id, slug=addon.slug, guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name), created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
doc.setdefault('works', {}).setdefault(app.id, {})
# Populate with default counts for all app versions.
for ver in versions:
doc['works'][app.id][vint(ver['main'])] = {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects
.filter(guid=addon.guid, app_guid=app.guid)
.values_list('app_version', 'works_properly')
.annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [v['main'] for v in versions
if vint(v['previous']) < ver <= vint(v['main'])]
if major:
w = doc['works'][app.id][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {'min': compat.min.version_int,
'max': compat.max.version_int}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
app=app.id,
defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(), key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in amo.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [compat_doc for compat_doc in docs.values()
if (app in compat_doc.get('support', {}) and
compat_doc['support'][app]['max'] >= ver)]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in chunked(docs.values(), 150):
for doc in chunk:
for index in indices:
AppCompat.index(doc, id=doc['id'], refresh=False, index=index)
es = amo_search.get_es()
es.indices.refresh()
def apps(self):
"""Get `AppVersion`s for the application."""
apps = ((amo.FIREFOX, amo.DEFAULT_WEBEXT_MIN_VERSION),
(amo.ANDROID, amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID
)) if self.type != amo.ADDON_STATICTHEME else (
(amo.FIREFOX,
amo.DEFAULT_STATIC_THEME_MIN_VERSION_FIREFOX), )
doesnt_support_no_id = (self.strict_min_version
and (vint(self.strict_min_version) < vint(
amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID)))
if self.guid is None and doesnt_support_no_id:
raise forms.ValidationError(
ugettext('GUID is required for Firefox 47 and below.'))
# If a minimum strict version is specified, it needs to be higher
# than the version when Firefox started supporting WebExtensions
# (We silently ignore apps that the add-on is not compatible with
# below, but we need to be at least compatible with Firefox...)
unsupported_no_matter_what = (self.strict_min_version and vint(
self.strict_min_version) < vint(amo.DEFAULT_WEBEXT_MIN_VERSION))
if unsupported_no_matter_what:
msg = ugettext('Lowest supported "strict_min_version" is 42.0.')
raise forms.ValidationError(msg)
couldnt_find_version = False
for app, default_min_version in apps:
if self.guid is None and not self.strict_min_version:
strict_min_version = max(amo.DEFAULT_WEBEXT_MIN_VERSION_NO_ID,
default_min_version)
else:
strict_min_version = (self.strict_min_version
or default_min_version)
strict_max_version = (self.strict_max_version
or amo.DEFAULT_WEBEXT_MAX_VERSION)
# Don't attempt to add support for this app to the WebExtension
# if the `strict_min_version` is below the default minimum version
# that is required to run WebExtensions (48.* for Android and 42.*
# for Firefox).
skip_app = (
self.strict_min_version
and vint(self.strict_min_version) < vint(default_min_version))
if skip_app:
continue
try:
min_appver, max_appver = get_appversions(
app, strict_min_version, strict_max_version)
yield Extractor.App(appdata=app,
id=app.id,
min=min_appver,
max=max_appver)
except AppVersion.DoesNotExist:
couldnt_find_version = True
specified_versions = self.strict_min_version or self.strict_max_version
if couldnt_find_version and specified_versions:
msg = ugettext('Cannot find min/max version. Maybe '
'"strict_min_version" or "strict_max_version" '
'contains an unsupported version?')
raise forms.ValidationError(msg)
def matches_user_agent(cls, user_agent):
for user_agent_re in cls.user_agent_re:
match = user_agent_re.search(user_agent)
if match:
v = match.groups()[0]
return vint(cls.min_display_version) <= vint(v)
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
for app in amo.APP_USAGE:
versions = [c for c in amo.COMPAT if c['app'] == app.id]
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id,
slug=addon.slug,
guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name),
created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
doc.setdefault('works', {}).setdefault(app.id, {})
# Populate with default counts for all app versions.
for ver in versions:
doc['works'][app.id][vint(ver['main'])] = {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects.filter(
guid=addon.guid, app_guid=app.guid).values_list(
'app_version', 'works_properly').annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [
v['main'] for v in versions
if vint(v['previous']) < ver <= vint(v['main'])
]
if major:
w = doc['works'][app.id][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {
'min': compat.min.version_int,
'max': compat.max.version_int
}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
app=app.id, defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(),
key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in amo.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [
compat_doc for compat_doc in docs.values()
if (app in compat_doc.get('support', {})
and compat_doc['support'][app]['max'] >= ver)
]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in chunked(docs.values(), 150):
for doc in chunk:
for index in indices:
AppCompat.index(doc, id=doc['id'], refresh=False, index=index)
es = amo_search.get_es()
es.indices.refresh()
