def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
assert self.check_appver_filters(too_big, floor_version(just_right)), (
'All I ask is do not crash')
eq_(self.check_appver_filters('9999999', '9999999.0'),
[{'text': u'Firefox 9999999.0',
'selected': True,
'urlparams': {'appver': '9999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
eq_(self.check_appver_filters('99999999', '99999999.0'),
[{'text': u'Firefox 99999999.0',
'selected': True,
'urlparams': {'appver': '99999999.0'},
'children': []},
{'text': u'Firefox 5.0',
'selected': False,
'urlparams': {'appver': '5.0'},
'children': []}])
def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
assert self.check_appver_filters(too_big, floor_version(just_right)), "All I ask is do not crash"
eq_(
self.check_appver_filters("9999999", "9999999.0"),
[
{"text": u"Firefox 9999999.0", "selected": True, "urlparams": {"appver": "9999999.0"}, "children": []},
{"text": u"Firefox 5.0", "selected": False, "urlparams": {"appver": "5.0"}, "children": []},
],
)
eq_(
self.check_appver_filters("99999999", "99999999.0"),
[
{
"text": u"Firefox 99999999.0",
"selected": True,
"urlparams": {"appver": "99999999.0"},
"children": [],
},
{"text": u"Firefox 5.0", "selected": False, "urlparams": {"appver": "5.0"}, "children": []},
],
)
def find_jetpacks(minver, maxver):
"""
Find all jetpack files that aren't disabled.
Files that should be upgraded will have needs_upgrade=True.
"""
from .models import File
statuses = amo.VALID_STATUSES
files = (File.objects.filter(jetpack_version__isnull=False,
version__addon__auto_repackage=True,
version__addon__status__in=statuses,
version__addon__disabled_by_user=False)
.exclude(status=amo.STATUS_DISABLED).no_cache()
.select_related('version'))
files = sorted(files, key=lambda f: (f.version.addon_id, f.version.id))
# Figure out which files need to be upgraded.
for file_ in files:
file_.needs_upgrade = False
# If any files for this add-on are reviewed, take the last reviewed file
# plus all newer files. Otherwise, only upgrade the latest file.
for _group, fs in groupby(files, key=lambda f: f.version.addon_id):
fs = list(fs)
if any(f.status in amo.REVIEWED_STATUSES for f in fs):
for file_ in reversed(fs):
file_.needs_upgrade = True
if file_.status in amo.REVIEWED_STATUSES:
break
else:
fs[-1].needs_upgrade = True
# Make sure only old files are marked.
for file_ in [f for f in files if f.needs_upgrade]:
if not (vint(minver) <= vint(file_.jetpack_version) < vint(maxver)):
file_.needs_upgrade = False
return files
def check_jetpack_version(sender, **kw):
import files.tasks
from files.utils import JetpackUpgrader
minver, maxver = JetpackUpgrader().jetpack_versions()
qs = File.objects.filter(version__addon=sender, jetpack_version__isnull=False)
ids = [f.id for f in qs if vint(minver) <= vint(f.jetpack_version) < vint(maxver)]
if ids:
files.tasks.start_upgrade.delay(ids, priority="high")
def version_compat(qs, compat, app):
facets = []
for v, prev in zip(compat['versions'], (None,) + compat['versions']):
d = {'from': vint(v)}
if prev:
d['to'] = vint(prev)
facets.append(d)
qs = qs.facet(by_status={'range': {'support.%s.max' % app: facets}})
result = qs[:0].raw()
total_addons = result['hits']['total']
ranges = result['facets']['by_status']['ranges']
faceted = [(v, r['count']) for v, r in zip(compat['versions'], ranges)]
other = total_addons - sum(r[1] for r in faceted)
return total_addons, faceted + [(_('Other'), other)]
def compat_stats(request, app, ver, ratio, binary):
# Get the list of add-ons for usage stats.
redis = redisutils.connections["master"]
# Show add-ons marked as incompatible with this current version having
# greater than 10 incompatible reports and whose average exceeds 80%.
ver_int = str(vint(ver))
prefix = "works.%s.%s" % (app, ver_int)
qs = (
AppCompat.search()
.filter(
**{"%s.failure__gt" % prefix: 10, "%s.failure_ratio__gt" % prefix: ratio, "support.%s.max__gte" % app: 0}
)
.order_by("-%s.failure_ratio" % prefix, "-%s.total" % prefix)
.values_dict()
)
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo.utils.paginate(request, qs)
for obj in addons.object_list:
obj["usage"] = obj["usage"][app]
obj["max_version"] = obj["max_version"][app]
obj["works"] = obj["works"][app].get(ver_int, {})
# Get all overrides for this add-on.
obj["overrides"] = CompatOverride.objects.filter(addon__id=obj["id"])
# Determine if there is an override for this current app version.
obj["has_override"] = obj["overrides"].filter(_compat_ranges__min_app_version=ver + "a1").exists()
total = int(redis.hget("compat:%s" % app, "total"))
return addons, total
def compat_stats(request, app, ver, minimum, ratio, binary):
# Get the list of add-ons for usage stats.
# Show add-ons marked as incompatible with this current version having
# greater than 10 incompatible reports and whose average exceeds 80%.
ver_int = str(vint(ver))
prefix = 'works.%s.%s' % (app, ver_int)
qs = (AppCompat.search()
.filter(**{'%s.failure__gt' % prefix: minimum,
'%s.failure_ratio__gt' % prefix: ratio,
'support.%s.max__gte' % app: 0})
.order_by('-%s.failure_ratio' % prefix,
'-%s.total' % prefix)
.values_dict())
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo.utils.paginate(request, qs)
for obj in addons.object_list:
obj['usage'] = obj['usage'][app]
obj['max_version'] = obj['max_version'][app]
obj['works'] = obj['works'][app].get(ver_int, {})
# Get all overrides for this add-on.
obj['overrides'] = CompatOverride.objects.filter(addon__id=obj['id'])
# Determine if there is an override for this current app version.
obj['has_override'] = obj['overrides'].filter(
_compat_ranges__min_app_version=ver + 'a1').exists()
return addons, CompatTotals.objects.get(app=app).total
def start_upgrade(file_ids, sdk_version=None, priority='low', **kw):
upgrader = JetpackUpgrader()
minver, maxver = upgrader.jetpack_versions()
files = File.objects.filter(id__in=file_ids).select_related('version')
now = datetime.now()
filedata = {}
for file_ in files:
if not (file_.jetpack_version and
vint(minver) <= vint(file_.jetpack_version) < vint(maxver)):
continue
jp_log.info('Sending %s to builder for jetpack version %s.'
% (file_.id, maxver))
# Data stored locally so we can figure out job details and if it should
# be cancelled.
data = {'file': file_.id,
'version': maxver,
'time': now,
'uuid': uuid.uuid4().hex,
'status': 'Sent to builder',
'owner': 'bulk'}
# Data POSTed to the builder.
post = {'addon': file_.version.addon_id,
'file_id': file_.id,
'priority': priority,
'secret': settings.BUILDER_SECRET_KEY,
'uuid': data['uuid'],
'pingback': absolutify(reverse('amo.builder-pingback'))}
if file_.builder_version:
post['package_key'] = file_.builder_version
else:
# Older jetpacks might not have builderVersion in their harness.
post['location'] = file_.get_url_path('builder')
if sdk_version:
post['sdk_version'] = sdk_version
try:
jp_log.info(urllib.urlencode(post))
response = urllib2.urlopen(settings.BUILDER_UPGRADE_URL,
urllib.urlencode(post))
jp_log.info('Response from builder for %s: [%s] %s' %
(file_.id, response.code, response.read()))
except Exception:
jp_log.error('Could not talk to builder for %s.' % file_.id,
exc_info=True)
filedata[file_.id] = data
upgrader.files(filedata)
def index(request, version=None):
template = 'compat/index.html'
COMPAT = [v for v in amo.COMPAT if v['app'] == request.APP.id]
compat_dict = dict((v['main'], v) for v in COMPAT)
if not COMPAT:
return jingo.render(request, template, {'results': False})
if version not in compat_dict:
return http.HttpResponseRedirect(reverse('compat.index',
args=[COMPAT[0]['main']]))
qs = AppCompat.search()
binary = None
initial = {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'}
initial.update(request.GET.items())
form = CompatForm(initial)
if request.GET and form.is_valid():
if form.cleaned_data['appver']:
app, ver = form.cleaned_data['appver'].split('-')
if int(app) != request.APP.id or ver != version:
new = reverse('compat.index', args=[ver], add_prefix=False)
url = '/%s%s' % (amo.APP_IDS[int(app)].short, new)
type_ = form.cleaned_data['type'] or None
return http.HttpResponseRedirect(urlparams(url, type=type_))
if form.cleaned_data['type'] != 'all':
binary = form.cleaned_data['type'] == 'binary'
compat, app = compat_dict[version], str(request.APP.id)
compat_queries = (
('prev', qs.query(**{
'top_95.%s.%s' % (app, vint(compat['previous'])): True,
'support.%s.max__gte' % app: vint(compat['previous'])})),
('top_95', qs.query(**{'top_95_all.%s' % app: True})),
('all', qs),
)
compat_levels = [(key, version_compat(qs, compat, app, binary))
for key, qs in compat_queries]
usage_addons, usage_total = usage_stats(request, compat, app, binary)
return jingo.render(request, template,
{'version': version,
'usage_addons': usage_addons,
'usage_total': usage_total,
'compat_levels': compat_levels,
'form': form,
'results': True,
'show_previous': request.GET.get('previous')})
def start_upgrade(file_ids, sdk_version=None, priority="low", **kw):
upgrader = JetpackUpgrader()
minver, maxver = upgrader.jetpack_versions()
files = File.objects.filter(id__in=file_ids).select_related("version")
now = datetime.now()
filedata = {}
for file_ in files:
if not (file_.jetpack_version and vint(minver) <= vint(file_.jetpack_version) < vint(maxver)):
continue
jp_log.info("Sending %s to builder for jetpack version %s." % (file_.id, maxver))
# Data stored locally so we can figure out job details and if it should
# be cancelled.
data = {
"file": file_.id,
"version": maxver,
"time": now,
"uuid": uuid.uuid4().hex,
"status": "Sent to builder",
"owner": "bulk",
}
# Data POSTed to the builder.
post = {
"addon": file_.version.addon_id,
"file_id": file_.id,
"priority": priority,
"secret": settings.BUILDER_SECRET_KEY,
"uuid": data["uuid"],
"pingback": absolutify(reverse("amo.builder-pingback")),
}
if file_.builder_version:
post["package_key"] = file_.builder_version
else:
# Older jetpacks might not have builderVersion in their harness.
post["location"] = file_.get_url_path("builder")
if sdk_version:
post["sdk_version"] = sdk_version
try:
jp_log.info(urllib.urlencode(post))
response = urllib2.urlopen(settings.BUILDER_UPGRADE_URL, urllib.urlencode(post))
jp_log.info("Response from builder for %s: [%s] %s" % (file_.id, response.code, response.read()))
except Exception:
jp_log.error("Could not talk to builder for %s." % file_.id, exc_info=True)
filedata[file_.id] = data
upgrader.files(filedata)
def version_compat(qs, compat, app, binary):
facets = []
for v, prev in zip(compat['versions'], (None,) + compat['versions']):
d = {'from': vint(v)}
if prev:
d['to'] = vint(prev)
facets.append(d)
# Pick up everything else for an Other count.
facets.append({'to': vint(compat['versions'][-1])})
facet = {'range': {'support.%s.max' % app: facets}}
if binary is not None:
qs = qs.query(binary=binary)
qs = qs.facet(by_status=facet)
result = qs[:0].raw()
total_addons = result['hits']['total']
ranges = result['facets']['by_status']['ranges']
titles = compat['versions'] + (_('Other'),)
faceted = [(v, r['count']) for v, r in zip(titles, ranges)]
return total_addons, faceted
def usage_stats(request, compat, app):
# Get the list of add-ons for usage stats.
redis = redisutils.connections['master']
qs = (AppCompat.search().order_by('-usage.%s' % app).values_dict()
.filter(**{'support.%s.max__gte' % app: vint(compat['previous'])}))
addons = amo.utils.paginate(request, qs)
for obj in addons.object_list:
obj['usage'] = obj['usage'][app]
obj['max_version'] = obj['max_version'][app]
total = int(redis.hget('compat:%s' % app, 'total'))
return addons, total
def usage_stats(request, compat, app, binary=None):
# Get the list of add-ons for usage stats.
qs = AppCompat.search().order_by('-usage.%s' % app).values_dict()
if request.GET.get('previous'):
qs = qs.filter(**{
'support.%s.max__gte' % app: vint(compat['previous'])})
else:
qs = qs.filter(**{'support.%s.max__gte' % app: 0})
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo.utils.paginate(request, qs)
for obj in addons.object_list:
obj['usage'] = obj['usage'][app]
obj['max_version'] = obj['max_version'][app]
return addons, CompatTotals.objects.get(app=app).total
def usage_stats(request, compat, app, binary=None):
# Get the list of add-ons for usage stats.
qs = AppCompat.search().order_by('-usage.%s' % app).values_dict()
if request.GET.get('previous'):
qs = qs.filter(
**{'support.%s.max__gte' % app: vint(compat['previous'])})
else:
qs = qs.filter(**{'support.%s.max__gte' % app: 0})
if binary is not None:
qs = qs.filter(binary=binary)
addons = amo.utils.paginate(request, qs)
for obj in addons.object_list:
obj['usage'] = obj['usage'][app]
obj['max_version'] = obj['max_version'][app]
return addons, CompatTotals.objects.get(app=app).total
def _check_firefox(ua):
need_firefox, need_upgrade = True, True
for ua_res, min_version in mkt.platforms.APP_PLATFORMS:
for ua_re in ua_res:
match = ua_re.search(ua)
if match:
v = match.groups()[0]
# If we found a version at all, then this is Firefox.
need_firefox = False
# If we found a matching version, then we can install apps!
need_upgrade = vint(v) < min_version
return need_firefox, need_upgrade
def _check_firefox(ua):
need_firefox, need_upgrade = True, True
for ua_res, min_version in mkt.platforms.APP_PLATFORMS:
for ua_re in ua_res:
match = ua_re.search(ua)
if match:
v = match.groups()[0]
# If we found a version at all, then this is Firefox.
need_firefox = False
# If we found a matching version, then we can install apps!
need_upgrade = vint(v) < min_version
return need_firefox, need_upgrade
class FIREFOX(App):
id = 1
shortername = 'fx'
short = 'firefox'
pretty = _(u'Firefox')
browser = True
types = [ADDON_EXTENSION, ADDON_THEME, ADDON_DICT, ADDON_SEARCH,
ADDON_LPAPP, ADDON_PLUGIN, ADDON_PERSONA, ADDON_WEBAPP]
guid = '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}'
min_display_version = 3.0
# These versions were relabeled and should not be displayed.
exclude_versions = (3.1, 3.7, 4.2)
backup_version = vint('3.7.*')
user_agent_string = 'Firefox'
platforms = 'desktop' # DESKTOP_PLATFORMS (set in constants.platforms)
@classmethod
def matches_user_agent(cls, user_agent):
matches = cls.user_agent_string in user_agent
if 'Android' in user_agent or 'Mobile' in user_agent:
matches = False
return matches
def index(request, version=None):
COMPAT = [v for v in settings.COMPAT if v['app'] == request.APP.id]
compat_dict = dict((v['main'], v) for v in COMPAT)
if not COMPAT:
raise http.Http404()
if version not in compat_dict:
return redirect('compat.index', COMPAT[0]['main'])
compat, app = compat_dict[version], str(request.APP.id)
qs = AppCompat.search()
compat_queries = (
('prev', qs.query(top_95=True, **{
'support.%s.max__gte' % app: vint(compat['previous'])})),
('top_95', qs.query(top_95=True)),
('all', qs),
)
compat_levels = [(key, version_compat(qs, compat, app))
for key, qs in compat_queries]
usage_addons, usage_total = usage_stats(request, compat, app)
return jingo.render(request, 'compat/index.html',
{'version': version,
'usage_addons': usage_addons,
'usage_total': usage_total,
'compat_levels': compat_levels})
def matches_user_agent(cls, user_agent):
match = cls.user_agent_re.search(user_agent)
if match:
return vint(cls.min_display_version) <= vint(match.groups()[0])
def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
self.check_appver_filters(too_big, floor_version(just_right))
self.check_appver_filters("9999999", "9999999.0")
self.check_appver_filters("99999999", "99999999.0")
def compatibility_report(index=None, aliased=True):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
for app in amo.APP_USAGE:
versions = [c for c in settings.COMPAT if c['app'] == app.id]
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in amo.utils.chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
doc = docs[addon.id]
doc.update(id=addon.id, slug=addon.slug, guid=addon.guid,
self_hosted=addon.is_selfhosted(),
binary=addon.binary_components,
name=unicode(addon.name), created=addon.created,
current_version=addon.current_version.version,
current_version_id=addon.current_version.pk)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
doc.setdefault('works', {}).setdefault(app.id, {})
# Populate with default counts for all app versions.
for ver in versions:
doc['works'][app.id][vint(ver['main'])] = {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects
.filter(guid=addon.guid, app_guid=app.guid)
.values_list('app_version', 'works_properly')
.annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [v['main'] for v in versions
if vint(v['previous']) < ver <= vint(v['main'])]
if major:
w = doc['works'][app.id][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {'min': compat.min.version_int,
'max': compat.max.version_int}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
app=app.id,
defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(), key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in settings.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [doc for doc in docs.values()
if app in doc.get('support', {})
and doc['support'][app]['max'] >= ver]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in amo.utils.chunked(docs.values(), 150):
for doc in chunk:
for index in indices:
AppCompat.index(doc, id=doc['id'], bulk=True, index=index)
elasticutils.get_es().flush_bulk(forced=True)
import re
from tower import ugettext_lazy as _
from versions.compare import version_int as vint
# These are the minimum versions required for `navigator.mozApps` support.
APP_PLATFORMS = [
# Firefox for Desktop.
(
[
re.compile('Firefox/([\d.]+)')
],
vint('16.0')
),
# Firefox for Android.
(
[
re.compile('Fennec/([\d.]+)'),
re.compile('Android; Mobile; rv:([\d.]+)'),
re.compile('Mobile; rv:([\d.]+)')
],
vint('17.0')
)
]
def FREE_PLATFORMS():
return (
('free-firefoxos', _('Firefox OS')),
def test_appver_long(self):
too_big = vnum(vint(MAXVERSION + 1))
just_right = vnum(vint(MAXVERSION))
self.check_appver_filters(too_big, floor_version(just_right))
self.check_appver_filters('9999999', '9999999.0')
self.check_appver_filters('99999999', '99999999.0')
def compatibility_report():
redis = redisutils.connections['master']
docs = defaultdict(dict)
# Gather all the data for the index.
for app in amo.APP_USAGE:
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in amo.utils.chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
doc = docs[addon.id]
doc.update(id=addon.id, slug=addon.slug, binary=addon.binary,
name=unicode(addon.name), created=addon.created)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {'min': compat.min.version_int,
'max': compat.max.version_int}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
redis.hset('compat:%s' % app.id, 'total', total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(), key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in settings.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [doc for doc in docs.values()
if app in doc.get('support', {})
and doc['support'][app]['max'] >= ver]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in amo.utils.chunked(docs.values(), 150):
for doc in chunk:
AppCompat.index(doc, id=doc['id'], bulk=True)
elasticutils.get_es().flush_bulk(forced=True)
def matches_user_agent(cls, user_agent):
for user_agent_re in cls.user_agent_re:
match = user_agent_re.search(user_agent)
if match:
v = match.groups()[0]
return vint(cls.min_display_version) <= vint(v)
import re
from tower import ugettext_lazy as _
from versions.compare import version_int as vint
# These are the minimum versions required for `navigator.mozApps` support.
APP_PLATFORMS = [
# Firefox for Desktop.
([re.compile('Firefox/([\d.]+)')], vint('16.0')),
# Firefox for Android.
([
re.compile('Fennec/([\d.]+)'),
re.compile('Android; Mobile; rv:([\d.]+)'),
re.compile('Mobile; rv:([\d.]+)')
], vint('17.0'))
]
def FREE_PLATFORMS():
return (
('free-firefoxos', _('Firefox OS')),
('free-desktop', _('Firefox')),
('free-android-mobile', _('Firefox Mobile')),
('free-android-tablet', _('Firefox Tablet')),
)
def PAID_PLATFORMS(request=None):
import waffle
platforms = (('paid-firefoxos', _('Firefox OS')), )
def matches_user_agent(cls, user_agent):
for user_agent_re in cls.user_agent_re:
match = user_agent_re.search(user_agent)
if match:
v = match.groups()[0]
return vint(cls.min_display_version) <= vint(v)
def compatibility_report():
redis = redisutils.connections['master']
docs = defaultdict(dict)
# Gather all the data for the index.
for app in amo.APP_USAGE:
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in amo.utils.chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
doc = docs[addon.id]
doc.update(id=addon.id,
slug=addon.slug,
binary=addon.binary,
name=unicode(addon.name),
created=addon.created)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {
'min': compat.min.version_int,
'max': compat.max.version_int
}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
redis.hset('compat:%s' % app.id, 'total', total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(),
key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in settings.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [
doc for doc in docs.values() if app in doc.get('support', {})
and doc['support'][app]['max'] >= ver
]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in amo.utils.chunked(docs.values(), 150):
for doc in chunk:
AppCompat.index(doc, id=doc['id'], bulk=True)
elasticutils.get_es().flush_bulk(forced=True)
def compatibility_report(index=None):
docs = defaultdict(dict)
indices = get_indices(index)
# Gather all the data for the index.
for app in amo.APP_USAGE:
versions = [c for c in amo.COMPAT if c['app'] == app.id]
log.info(u'Making compat report for %s.' % app.pretty)
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
addon__disabled_by_user=False,
addon__status__in=amo.VALID_STATUSES,
addon___current_version__isnull=False,
date=latest)
updates = dict(qs.values_list('addon', 'count'))
for chunk in amo.utils.chunked(updates.items(), 50):
chunk = dict(chunk)
for addon in Addon.objects.filter(id__in=chunk):
doc = docs[addon.id]
doc.update(id=addon.id,
slug=addon.slug,
guid=addon.guid,
binary=addon.binary_components,
name=unicode(addon.name),
created=addon.created,
current_version=addon.current_version.version,
current_version_id=addon.current_version.pk)
doc['count'] = chunk[addon.id]
doc.setdefault('top_95',
defaultdict(lambda: defaultdict(dict)))
doc.setdefault('top_95_all', {})
doc.setdefault('usage', {})[app.id] = updates[addon.id]
doc.setdefault('works', {}).setdefault(app.id, {})
# Populate with default counts for all app versions.
for ver in versions:
doc['works'][app.id][vint(ver['main'])] = {
'success': 0,
'failure': 0,
'total': 0,
'failure_ratio': 0.0,
}
# Group reports by `major`.`minor` app version.
reports = (CompatReport.objects.filter(
guid=addon.guid, app_guid=app.guid).values_list(
'app_version', 'works_properly').annotate(Count('id')))
for ver, works_properly, cnt in reports:
ver = vint(floor_version(ver))
major = [
v['main'] for v in versions
if vint(v['previous']) < ver <= vint(v['main'])
]
if major:
w = doc['works'][app.id][vint(major[0])]
# Tally number of success and failure reports.
w['success' if works_properly else 'failure'] += cnt
w['total'] += cnt
# Calculate % of incompatibility reports.
w['failure_ratio'] = w['failure'] / float(w['total'])
if app not in addon.compatible_apps:
continue
compat = addon.compatible_apps[app]
d = {
'min': compat.min.version_int,
'max': compat.max.version_int
}
doc.setdefault('support', {})[app.id] = d
doc.setdefault('max_version', {})[app.id] = compat.max.version
total = sum(updates.values())
# Remember the total so we can show % of usage later.
compat_total, created = CompatTotals.objects.safer_get_or_create(
app=app.id, defaults={'total': total})
if not created:
compat_total.update(total=total)
# Figure out which add-ons are in the top 95% for this app.
running_total = 0
for addon, count in sorted(updates.items(),
key=lambda x: x[1],
reverse=True):
running_total += count
docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
# Mark the top 95% of add-ons compatible with the previous version for each
# app + version combo.
for compat in amo.COMPAT:
app, ver = compat['app'], vint(compat['previous'])
# Find all the docs that have a max_version compatible with ver.
supported = [
doc for doc in docs.values() if app in doc.get('support', {})
and doc['support'][app]['max'] >= ver
]
# Sort by count so we can get the top 95% most-used add-ons.
supported = sorted(supported, key=lambda d: d['count'], reverse=True)
total = sum(doc['count'] for doc in supported)
# Figure out which add-ons are in the top 95% for this app + version.
running_total = 0
for doc in supported:
running_total += doc['count']
doc['top_95'][app][ver] = running_total < (.95 * total)
# Send it all to the index.
for chunk in amo.utils.chunked(docs.values(), 150):
for doc in chunk:
for index in indices:
AppCompat.index(doc, id=doc['id'], refresh=False, index=index)
es = amo.search.get_es()
es.indices.refresh()
def test_appver_long(self):
too_big = vnum(vint(sys.maxint + 1))
just_right = vnum(vint(sys.maxint))
self.check_appver_filters(too_big, floor_version(just_right))
self.check_appver_filters('9999999', '9999999.0')
self.check_appver_filters('99999999', '99999999.0')
import re
from versions.compare import version_int as vint
# These are the minimum versions required for `navigator.mozApps` support.
APP_PLATFORMS = [
# Firefox for Desktop.
([re.compile("Firefox/([\d.]+)")], vint("16.0")),
# Firefox for Android.
(
[re.compile("Fennec/([\d.]+)"), re.compile("Android; Mobile; rv:([\d.]+)"), re.compile("Mobile; rv:([\d.]+)")],
vint("17.0"),
),
]
