def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
assert self.check_appver_filters(too_big, floor_version(just_right)), (
'All I ask is do not crash')
assert self.check_appver_filters('9999999', '9999999.0') == (
[{'text': u'Firefox 9999999.0',
'selected': True,
'urlparams': {'appver': '9999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
assert self.check_appver_filters('99999999', '99999999.0') == (
[{'text': u'Firefox 99999999.0',
'selected': True,
'urlparams': {'appver': '99999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
assert self.check_appver_filters(too_big, floor_version(just_right)), (
'All I ask is do not crash')
assert self.check_appver_filters('9999999', '9999999.0') == (
[{'text': u'Firefox 9999999.0',
'selected': True,
'urlparams': {'appver': '9999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
assert self.check_appver_filters('99999999', '99999999.0') == (
[{'text': u'Firefox 99999999.0',
'selected': True,
'urlparams': {'appver': '99999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
def check_appver_filters(self, appver, expected):
request = RequestFactory()
request.GET = {}
request.APP = amo.FIREFOX
facets = {
u'platforms': [{
u'doc_count': 58,
u'key': 1
}],
u'appversions': [{
u'doc_count': 58,
u'key': 5000000200100
}],
u'categories': [{
u'doc_count': 55,
u'key': 1
}],
u'tags': []
}
versions = version_sidebar(request, {'appver': floor_version(appver)},
facets)
all_ = versions.pop(0)
assert all_.text == 'Any %s' % unicode(request.APP.pretty)
assert not all_.selected == expected
return [v.__dict__ for v in versions]
def reporter_detail(request, guid):
try:
addon = Addon.objects.get(guid=guid)
except Addon.DoesNotExist:
addon = None
name = addon.name if addon else guid
qs = CompatReport.objects.filter(guid=guid)
show_listed_only = addon and not owner_or_unlisted_reviewer(request, addon)
if (addon and not addon.has_listed_versions() and show_listed_only):
# Not authorized? Let's pretend this addon simply doesn't exist.
name = guid
qs = CompatReport.objects.none()
elif show_listed_only:
unlisted_versions = addon.versions.filter(
channel=amo.RELEASE_CHANNEL_UNLISTED).values_list('version',
flat=True)
qs = qs.exclude(version__in=unlisted_versions)
form = AppVerForm(request.GET)
if request.GET and form.is_valid() and form.cleaned_data['appver']:
# Apply filters only if we have a good app/version combination.
version = form.cleaned_data['appver']
ver = vdict(floor_version(version))['major'] # 3.6 => 3
# Ideally we'd have a `version_int` column to do strict version
# comparing, but that's overkill for basic version filtering here.
qs = qs.filter(app_guid=amo.FIREFOX.guid,
app_version__startswith=str(ver) + '.')
works_ = dict(qs.values_list('works_properly').annotate(Count('id')))
works = {'success': works_.get(True, 0), 'failure': works_.get(False, 0)}
works_properly = request.GET.get('works_properly')
if works_properly:
qs = qs.filter(works_properly=works_properly)
reports = paginate(request, qs.order_by('-created'), 100)
return render(
request, 'compat/reporter_detail.html',
dict(reports=reports,
works=works,
works_properly=works_properly,
name=name,
guid=guid,
form=form))
def reporter_detail(request, guid):
try:
addon = Addon.objects.get(guid=guid)
except Addon.DoesNotExist:
addon = None
name = addon.name if addon else guid
qs = CompatReport.objects.filter(guid=guid)
show_listed_only = addon and not owner_or_unlisted_reviewer(request, addon)
if (addon and not addon.has_listed_versions() and show_listed_only):
# Not authorized? Let's pretend this addon simply doesn't exist.
name = guid
qs = CompatReport.objects.none()
elif show_listed_only:
unlisted_versions = addon.versions.filter(
channel=amo.RELEASE_CHANNEL_UNLISTED).values_list(
'version', flat=True)
qs = qs.exclude(version__in=unlisted_versions)
form = AppVerForm(request.GET)
if request.GET and form.is_valid() and form.cleaned_data['appver']:
# Apply filters only if we have a good app/version combination.
version = form.cleaned_data['appver']
ver = vdict(floor_version(version))['major'] # 3.6 => 3
# Ideally we'd have a `version_int` column to do strict version
# comparing, but that's overkill for basic version filtering here.
qs = qs.filter(app_guid=amo.FIREFOX.guid,
app_version__startswith=str(ver) + '.')
works_ = dict(qs.values_list('works_properly').annotate(Count('id')))
works = {'success': works_.get(True, 0), 'failure': works_.get(False, 0)}
works_properly = request.GET.get('works_properly')
if works_properly:
qs = qs.filter(works_properly=works_properly)
reports = paginate(request, qs.order_by('-created'), 100)
return render(request, 'compat/reporter_detail.html',
dict(reports=reports, works=works,
works_properly=works_properly,
name=name, guid=guid, form=form))
def reporter_detail(request, guid):
try:
addon = Addon.with_unlisted.get(guid=guid)
except Addon.DoesNotExist:
addon = None
name = addon.name if addon else guid
qs = CompatReport.objects.filter(guid=guid)
if (addon and not addon.is_listed
and not owner_or_unlisted_reviewer(request, addon)):
# Not authorized? Let's pretend this addon simply doesn't exist.
name = guid
qs = CompatReport.objects.none()
form = AppVerForm(request.GET)
if request.GET and form.is_valid() and form.cleaned_data['appver']:
# Apply filters only if we have a good app/version combination.
app, ver = form.cleaned_data['appver'].split('-')
app = amo.APP_IDS[int(app)]
ver = vdict(floor_version(ver))['major'] # 3.6 => 3
# Ideally we'd have a `version_int` column to do strict version
# comparing, but that's overkill for basic version filtering here.
qs = qs.filter(app_guid=app.guid,
app_version__startswith=str(ver) + '.')
works_ = dict(qs.values_list('works_properly').annotate(Count('id')))
works = {'success': works_.get(True, 0), 'failure': works_.get(False, 0)}
works_properly = request.GET.get('works_properly')
if works_properly:
qs = qs.filter(works_properly=works_properly)
reports = amo_utils.paginate(request, qs.order_by('-created'), 100)
return render(
request, 'compat/reporter_detail.html',
dict(reports=reports,
works=works,
works_properly=works_properly,
name=name,
guid=guid,
form=form))
def check_appver_filters(self, appver, expected):
request = RequestFactory()
request.GET = {}
request.APP = amo.FIREFOX
facets = {
u'platforms': [{u'doc_count': 58, u'key': 1}],
u'appversions': [{u'doc_count': 58, u'key': 5000000200100}],
u'categories': [{u'doc_count': 55, u'key': 1}],
u'tags': []
}
versions = version_sidebar(request,
{'appver': floor_version(appver)}, facets)
all_ = versions.pop(0)
assert all_.text == 'Any %s' % unicode(request.APP.pretty)
assert not all_.selected == expected
return [v.__dict__ for v in versions]
def reporter_detail(request, guid):
try:
addon = Addon.with_unlisted.get(guid=guid)
except Addon.DoesNotExist:
addon = None
name = addon.name if addon else guid
qs = CompatReport.objects.filter(guid=guid)
if (addon and not addon.is_listed and
not owner_or_unlisted_reviewer(request, addon)):
# Not authorized? Let's pretend this addon simply doesn't exist.
name = guid
qs = CompatReport.objects.none()
form = AppVerForm(request.GET)
if request.GET and form.is_valid() and form.cleaned_data['appver']:
# Apply filters only if we have a good app/version combination.
app, ver = form.cleaned_data['appver'].split('-')
app = amo.APP_IDS[int(app)]
ver = vdict(floor_version(ver))['major'] # 3.6 => 3
# Ideally we'd have a `version_int` column to do strict version
# comparing, but that's overkill for basic version filtering here.
qs = qs.filter(app_guid=app.guid,
app_version__startswith=str(ver) + '.')
works_ = dict(qs.values_list('works_properly').annotate(Count('id')))
works = {'success': works_.get(True, 0), 'failure': works_.get(False, 0)}
works_properly = request.GET.get('works_properly')
if works_properly:
qs = qs.filter(works_properly=works_properly)
reports = amo_utils.paginate(request, qs.order_by('-created'), 100)
return render(request, 'compat/reporter_detail.html',
dict(reports=reports, works=works,
works_properly=works_properly,
name=name, guid=guid, form=form))
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
for app in amo.APP_USAGE:
versions = [c for c in amo.COMPAT if c['app'] == app.id]
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id,
slug=addon.slug,
guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name),
created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
doc.setdefault('works', {}).setdefault(app.id, {})
# Populate with default counts for all app versions.
for ver in versions:
doc['works'][app.id][vint(ver['main'])] = {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects.filter(
guid=addon.guid, app_guid=app.guid).values_list(
'app_version', 'works_properly').annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [
v['main'] for v in versions
if vint(v['previous']) < ver <= vint(v['main'])
]
if major:
w = doc['works'][app.id][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {
'min': compat.min.version_int,
'max': compat.max.version_int
}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
app=app.id, defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(),
key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in amo.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [
compat_doc for compat_doc in docs.values()
if (app in compat_doc.get('support', {})
and compat_doc['support'][app]['max'] >= ver)
]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in chunked(docs.values(), 150):
for doc in chunk:
for index in indices:
AppCompat.index(doc, id=doc['id'], refresh=False, index=index)
es = amo_search.get_es()
es.indices.refresh()
'LATEST_FIREFOX_VERSION']
THUNDERBIRD.latest_version = product_details.thunderbird_versions[
'LATEST_THUNDERBIRD_VERSION']
MOBILE.latest_version = FIREFOX.latest_version
# This is a list of dictionaries that we should generate compat info for.
# app: should match FIREFOX.id.
# main: the app version we're generating compat info for.
# versions: version numbers to show in comparisons.
# previous: the major version before :main.
if FIREFOX.latest_version:
COMPAT = {FIREFOX.id: (), THUNDERBIRD.id: (), SEAMONKEY.id: ()}
for app in (FIREFOX, THUNDERBIRD):
for v in range(int(float(floor_version(app.latest_version))), 5, -1):
v_str = floor_version(str(v))
COMPAT[app.id] += ({
'app': app.id,
'main': v_str,
'versions': (v_str, v_str + 'a2', v_str + 'a1'),
'previous': floor_version(str(v - 1))
}, )
# This is because the oldest Thunderbird version is 6.0, and
# we need to include these older Firefox versions.
COMPAT[FIREFOX.id] += (
{
'app': FIREFOX.id,
'main': '5.0',
'versions': ('5.0', '5.0a2', '5.0a1'),
def c(x, y):
assert floor_version(x) == y
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
log.info(u'Generating Firefox compat report.')
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=amo.FIREFOX.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_ADDON_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
if (amo.FIREFOX not in addon.compatible_apps
or addon.compatible_apps[amo.FIREFOX] is None):
# Ignore this add-on if it does not have compat information
# for Firefox.
continue
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id,
slug=addon.slug,
guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name),
created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc['usage'] = updates[addon.id]
doc['top_95'] = {}
# Populate with default counts for all versions.
doc['works'] = {
vint(version['main']): {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
for version in FIREFOX_COMPAT
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects.filter(
guid=addon.guid, app_guid=amo.FIREFOX.guid).values_list(
'app_version', 'works_properly').annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [
v['main'] for v in FIREFOX_COMPAT
if vint(v['previous']) < ver <= vint(v['main'])
]
if major:
w = doc['works'][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
compat = addon.compatible_apps[amo.FIREFOX]
doc['support'] = {
'min': compat.min.version_int,
'max': compat.max.version_int
}
doc['max_version'] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95%.
running_total = 0
for addon, count in sorted(updates.items(),
key=lambda x: x[1],
reverse=True):
# Ignore the updates we skipped because of bad app compatibility.
if addon in docs:
running_total += count
docs[addon]['top_95_all'] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# version.
for compat in FIREFOX_COMPAT:
version = vint(compat['previous'])
# Find all the docs that have a max_version compatible with version.
supported = [
compat_doc for compat_doc in docs.values()
if compat_doc['support']['max'] >= version
]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][version] = running_total < (.95 * total)
# Send it all to ES.
bulk = []
for id_, doc in docs.items():
for index in set(indices):
bulk.append({
"_source": doc,
"_id": id_,
"_type": AppCompat.get_mapping_type(),
"_index": index or AppCompat._get_index(),
})
es = amo_search.get_es()
log.info('Bulk indexing %s compat docs on %s indices' %
(len(docs), len(indices)))
elasticsearch.helpers.bulk(es, bulk, chunk_size=150)
es.indices.refresh()
def clean_appver(self):
return floor_version(self.cleaned_data.get('appver'))
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
for app in amo.APP_USAGE:
versions = [c for c in amo.COMPAT if c['app'] == app.id]
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id, slug=addon.slug, guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name), created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
doc.setdefault('works', {}).setdefault(app.id, {})
# Populate with default counts for all app versions.
for ver in versions:
doc['works'][app.id][vint(ver['main'])] = {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects
.filter(guid=addon.guid, app_guid=app.guid)
.values_list('app_version', 'works_properly')
.annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [v['main'] for v in versions
if vint(v['previous']) < ver <= vint(v['main'])]
if major:
w = doc['works'][app.id][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {'min': compat.min.version_int,
'max': compat.max.version_int}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
app=app.id,
defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(), key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in amo.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [compat_doc for compat_doc in docs.values()
if (app in compat_doc.get('support', {}) and
compat_doc['support'][app]['max'] >= ver)]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in chunked(docs.values(), 150):
for doc in chunk:
for index in indices:
AppCompat.index(doc, id=doc['id'], refresh=False, index=index)
es = amo_search.get_es()
es.indices.refresh()
import olympia.core.logger
from olympia import amo
from olympia.search.utils import floor_version
log = olympia.core.logger.getLogger('z.compat')
# This is a list of dictionaries that we should generate compat info for.
# main: the app version we're generating compat info for.
# versions: version numbers to show in comparisons.
# previous: the major version before :main.
if amo.FIREFOX.latest_version:
# We only generate compatibility info for the last 8 major versions.
latest_version = int(float(floor_version(amo.FIREFOX.latest_version)))
FIREFOX_COMPAT = [{
'main': floor_version(v),
'versions': (floor_version(v),
floor_version(v) + 'a2',
floor_version(v) + 'a1'),
'previous': floor_version(v - 1)
} for v in range(latest_version, latest_version - 9, -1)]
else:
# Why don't you have `product_details` like the rest of us?
log.warning('You are missing `product_details`. '
'Run `python manage.py update_product_details` now.')
FIREFOX_COMPAT = {}
import commonware.log
from olympia import amo
from olympia.search.utils import floor_version
log = commonware.log.getLogger('z.compat')
# This is a list of dictionaries that we should generate compat info for.
# main: the app version we're generating compat info for.
# versions: version numbers to show in comparisons.
# previous: the major version before :main.
if amo.FIREFOX.latest_version:
# We only generate compatibility info for the last 8 major versions.
latest_version = int(float(floor_version(amo.FIREFOX.latest_version)))
FIREFOX_COMPAT = [{
'main':
floor_version(v),
'versions':
(floor_version(v), floor_version(v) + 'a2', floor_version(v) + 'a1'),
'previous':
floor_version(v - 1)
} for v in range(latest_version, latest_version - 9, -1)]
else:
# Why don't you have `product_details` like the rest of us?
log.warning('You are missing `product_details`. '
'Run `python manage.py update_product_details` now.')
FIREFOX_COMPAT = {}
def c(x, y):
eq_(floor_version(x), y)
def c(x, y):
eq_(floor_version(x), y)
def clean_appver(self):
return floor_version(self.cleaned_data.get('appver'))
THUNDERBIRD.latest_version = product_details.thunderbird_versions[
'LATEST_THUNDERBIRD_VERSION']
MOBILE.latest_version = FIREFOX.latest_version
# This is a list of dictionaries that we should generate compat info for.
# app: should match FIREFOX.id.
# main: the app version we're generating compat info for.
# versions: version numbers to show in comparisons.
# previous: the major version before :main.
if FIREFOX.latest_version:
COMPAT = {FIREFOX.id: (), THUNDERBIRD.id: (), SEAMONKEY.id: ()}
for app in (FIREFOX, THUNDERBIRD):
for v in range(int(float(floor_version(app.latest_version))), 5, -1):
v_str = floor_version(str(v))
COMPAT[app.id] += ({
'app': app.id,
'main': v_str,
'versions': (v_str, v_str + 'a2', v_str + 'a1'),
'previous': floor_version(str(v - 1))
},)
# This is because the oldest Thunderbird version is 6.0, and
# we need to include these older Firefox versions.
COMPAT[FIREFOX.id] += (
{'app': FIREFOX.id, 'main': '5.0',
'versions': ('5.0', '5.0a2', '5.0a1'),
'previous': '4.0'},
{'app': FIREFOX.id, 'main': '4.0',
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
log.info(u'Generating Firefox compat report.')
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=amo.FIREFOX.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_ADDON_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
if (amo.FIREFOX not in addon.compatible_apps or
addon.compatible_apps[amo.FIREFOX] is None):
# Ignore this add-on if it does not have compat information
# for Firefox.
continue
current_version = {
'id': addon.current_version.pk,
'version': addon.current_version.version,
}
doc = docs[addon.id]
doc.update(id=addon.id, slug=addon.slug, guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name), created=addon.created,
current_version=current_version)
doc['count'] = chunk[addon.id]
doc['usage'] = updates[addon.id]
doc['top_95'] = {}
# Populate with default counts for all versions.
doc['works'] = {vint(version['main']): {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
} for version in FIREFOX_COMPAT}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects
.filter(guid=addon.guid, app_guid=amo.FIREFOX.guid)
.values_list('app_version', 'works_properly')
.annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [v['main'] for v in FIREFOX_COMPAT
if vint(v['previous']) < ver <= vint(v['main'])]
if major:
w = doc['works'][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
compat = addon.compatible_apps[amo.FIREFOX]
doc['support'] = {'min': compat.min.version_int,
'max': compat.max.version_int}
doc['max_version'] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95%.
running_total = 0
for addon, count in sorted(updates.items(), key=lambda x: x[1],
reverse=True):
# Ignore the updates we skipped because of bad app compatibility.
if addon in docs:
running_total += count
docs[addon]['top_95_all'] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# version.
for compat in FIREFOX_COMPAT:
version = vint(compat['previous'])
# Find all the docs that have a max_version compatible with version.
supported = [compat_doc for compat_doc in docs.values()
if compat_doc['support']['max'] >= version]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][version] = running_total < (.95 * total)
# Send it all to ES.
bulk = []
for id_, doc in docs.items():
for index in set(indices):
bulk.append({
"_source": doc,
"_id": id_,
"_type": AppCompat.get_mapping_type(),
"_index": index or AppCompat._get_index(),
})
es = amo_search.get_es()
log.info('Bulk indexing %s compat docs on %s indices' % (
len(docs), len(indices)))
elasticsearch.helpers.bulk(es, bulk, chunk_size=150)
es.indices.refresh()
def c(x, y):
assert floor_version(x) == y
