def test_appver_long(self): too_big = vnum(vint(MAXVERSION + 1)) just_right = vnum(vint(MAXVERSION)) assert self.check_appver_filters(too_big, floor_version(just_right)), ( 'All I ask is do not crash') eq_(self.check_appver_filters('9999999', '9999999.0'), [{'text': u'Firefox 9999999.0', 'selected': True, 'urlparams': {'appver': '9999999.0'}, 'children': []}, {'text': u'Firefox 5.0', 'selected': False, 'urlparams': {'appver': '5.0'}, 'children': []}]) eq_(self.check_appver_filters('99999999', '99999999.0'), [{'text': u'Firefox 99999999.0', 'selected': True, 'urlparams': {'appver': '99999999.0'}, 'children': []}, {'text': u'Firefox 5.0', 'selected': False, 'urlparams': {'appver': '5.0'}, 'children': []}])
def test_appver_long(self): too_big = vnum(vint(MAXVERSION + 1)) just_right = vnum(vint(MAXVERSION)) assert self.check_appver_filters(too_big, floor_version(just_right)), "All I ask is do not crash" eq_( self.check_appver_filters("9999999", "9999999.0"), [ {"text": u"Firefox 9999999.0", "selected": True, "urlparams": {"appver": "9999999.0"}, "children": []}, {"text": u"Firefox 5.0", "selected": False, "urlparams": {"appver": "5.0"}, "children": []}, ], ) eq_( self.check_appver_filters("99999999", "99999999.0"), [ { "text": u"Firefox 99999999.0", "selected": True, "urlparams": {"appver": "99999999.0"}, "children": [], }, {"text": u"Firefox 5.0", "selected": False, "urlparams": {"appver": "5.0"}, "children": []}, ], )
def find_jetpacks(minver, maxver): """ Find all jetpack files that aren't disabled. Files that should be upgraded will have needs_upgrade=True. """ from .models import File statuses = amo.VALID_STATUSES files = (File.objects.filter(jetpack_version__isnull=False, version__addon__auto_repackage=True, version__addon__status__in=statuses, version__addon__disabled_by_user=False) .exclude(status=amo.STATUS_DISABLED).no_cache() .select_related('version')) files = sorted(files, key=lambda f: (f.version.addon_id, f.version.id)) # Figure out which files need to be upgraded. for file_ in files: file_.needs_upgrade = False # If any files for this add-on are reviewed, take the last reviewed file # plus all newer files. Otherwise, only upgrade the latest file. for _group, fs in groupby(files, key=lambda f: f.version.addon_id): fs = list(fs) if any(f.status in amo.REVIEWED_STATUSES for f in fs): for file_ in reversed(fs): file_.needs_upgrade = True if file_.status in amo.REVIEWED_STATUSES: break else: fs[-1].needs_upgrade = True # Make sure only old files are marked. for file_ in [f for f in files if f.needs_upgrade]: if not (vint(minver) <= vint(file_.jetpack_version) < vint(maxver)): file_.needs_upgrade = False return files
def check_jetpack_version(sender, **kw): import files.tasks from files.utils import JetpackUpgrader minver, maxver = JetpackUpgrader().jetpack_versions() qs = File.objects.filter(version__addon=sender, jetpack_version__isnull=False) ids = [f.id for f in qs if vint(minver) <= vint(f.jetpack_version) < vint(maxver)] if ids: files.tasks.start_upgrade.delay(ids, priority="high")
def version_compat(qs, compat, app): facets = [] for v, prev in zip(compat['versions'], (None,) + compat['versions']): d = {'from': vint(v)} if prev: d['to'] = vint(prev) facets.append(d) qs = qs.facet(by_status={'range': {'support.%s.max' % app: facets}}) result = qs[:0].raw() total_addons = result['hits']['total'] ranges = result['facets']['by_status']['ranges'] faceted = [(v, r['count']) for v, r in zip(compat['versions'], ranges)] other = total_addons - sum(r[1] for r in faceted) return total_addons, faceted + [(_('Other'), other)]
def compat_stats(request, app, ver, ratio, binary): # Get the list of add-ons for usage stats. redis = redisutils.connections["master"] # Show add-ons marked as incompatible with this current version having # greater than 10 incompatible reports and whose average exceeds 80%. ver_int = str(vint(ver)) prefix = "works.%s.%s" % (app, ver_int) qs = ( AppCompat.search() .filter( **{"%s.failure__gt" % prefix: 10, "%s.failure_ratio__gt" % prefix: ratio, "support.%s.max__gte" % app: 0} ) .order_by("-%s.failure_ratio" % prefix, "-%s.total" % prefix) .values_dict() ) if binary is not None: qs = qs.filter(binary=binary) addons = amo.utils.paginate(request, qs) for obj in addons.object_list: obj["usage"] = obj["usage"][app] obj["max_version"] = obj["max_version"][app] obj["works"] = obj["works"][app].get(ver_int, {}) # Get all overrides for this add-on. obj["overrides"] = CompatOverride.objects.filter(addon__id=obj["id"]) # Determine if there is an override for this current app version. obj["has_override"] = obj["overrides"].filter(_compat_ranges__min_app_version=ver + "a1").exists() total = int(redis.hget("compat:%s" % app, "total")) return addons, total
def compat_stats(request, app, ver, minimum, ratio, binary): # Get the list of add-ons for usage stats. # Show add-ons marked as incompatible with this current version having # greater than 10 incompatible reports and whose average exceeds 80%. ver_int = str(vint(ver)) prefix = 'works.%s.%s' % (app, ver_int) qs = (AppCompat.search() .filter(**{'%s.failure__gt' % prefix: minimum, '%s.failure_ratio__gt' % prefix: ratio, 'support.%s.max__gte' % app: 0}) .order_by('-%s.failure_ratio' % prefix, '-%s.total' % prefix) .values_dict()) if binary is not None: qs = qs.filter(binary=binary) addons = amo.utils.paginate(request, qs) for obj in addons.object_list: obj['usage'] = obj['usage'][app] obj['max_version'] = obj['max_version'][app] obj['works'] = obj['works'][app].get(ver_int, {}) # Get all overrides for this add-on. obj['overrides'] = CompatOverride.objects.filter(addon__id=obj['id']) # Determine if there is an override for this current app version. obj['has_override'] = obj['overrides'].filter( _compat_ranges__min_app_version=ver + 'a1').exists() return addons, CompatTotals.objects.get(app=app).total
def start_upgrade(file_ids, sdk_version=None, priority='low', **kw): upgrader = JetpackUpgrader() minver, maxver = upgrader.jetpack_versions() files = File.objects.filter(id__in=file_ids).select_related('version') now = datetime.now() filedata = {} for file_ in files: if not (file_.jetpack_version and vint(minver) <= vint(file_.jetpack_version) < vint(maxver)): continue jp_log.info('Sending %s to builder for jetpack version %s.' % (file_.id, maxver)) # Data stored locally so we can figure out job details and if it should # be cancelled. data = {'file': file_.id, 'version': maxver, 'time': now, 'uuid': uuid.uuid4().hex, 'status': 'Sent to builder', 'owner': 'bulk'} # Data POSTed to the builder. post = {'addon': file_.version.addon_id, 'file_id': file_.id, 'priority': priority, 'secret': settings.BUILDER_SECRET_KEY, 'uuid': data['uuid'], 'pingback': absolutify(reverse('amo.builder-pingback'))} if file_.builder_version: post['package_key'] = file_.builder_version else: # Older jetpacks might not have builderVersion in their harness. post['location'] = file_.get_url_path('builder') if sdk_version: post['sdk_version'] = sdk_version try: jp_log.info(urllib.urlencode(post)) response = urllib2.urlopen(settings.BUILDER_UPGRADE_URL, urllib.urlencode(post)) jp_log.info('Response from builder for %s: [%s] %s' % (file_.id, response.code, response.read())) except Exception: jp_log.error('Could not talk to builder for %s.' % file_.id, exc_info=True) filedata[file_.id] = data upgrader.files(filedata)
def index(request, version=None): template = 'compat/index.html' COMPAT = [v for v in amo.COMPAT if v['app'] == request.APP.id] compat_dict = dict((v['main'], v) for v in COMPAT) if not COMPAT: return jingo.render(request, template, {'results': False}) if version not in compat_dict: return http.HttpResponseRedirect(reverse('compat.index', args=[COMPAT[0]['main']])) qs = AppCompat.search() binary = None initial = {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'} initial.update(request.GET.items()) form = CompatForm(initial) if request.GET and form.is_valid(): if form.cleaned_data['appver']: app, ver = form.cleaned_data['appver'].split('-') if int(app) != request.APP.id or ver != version: new = reverse('compat.index', args=[ver], add_prefix=False) url = '/%s%s' % (amo.APP_IDS[int(app)].short, new) type_ = form.cleaned_data['type'] or None return http.HttpResponseRedirect(urlparams(url, type=type_)) if form.cleaned_data['type'] != 'all': binary = form.cleaned_data['type'] == 'binary' compat, app = compat_dict[version], str(request.APP.id) compat_queries = ( ('prev', qs.query(**{ 'top_95.%s.%s' % (app, vint(compat['previous'])): True, 'support.%s.max__gte' % app: vint(compat['previous'])})), ('top_95', qs.query(**{'top_95_all.%s' % app: True})), ('all', qs), ) compat_levels = [(key, version_compat(qs, compat, app, binary)) for key, qs in compat_queries] usage_addons, usage_total = usage_stats(request, compat, app, binary) return jingo.render(request, template, {'version': version, 'usage_addons': usage_addons, 'usage_total': usage_total, 'compat_levels': compat_levels, 'form': form, 'results': True, 'show_previous': request.GET.get('previous')})
def start_upgrade(file_ids, sdk_version=None, priority="low", **kw): upgrader = JetpackUpgrader() minver, maxver = upgrader.jetpack_versions() files = File.objects.filter(id__in=file_ids).select_related("version") now = datetime.now() filedata = {} for file_ in files: if not (file_.jetpack_version and vint(minver) <= vint(file_.jetpack_version) < vint(maxver)): continue jp_log.info("Sending %s to builder for jetpack version %s." % (file_.id, maxver)) # Data stored locally so we can figure out job details and if it should # be cancelled. data = { "file": file_.id, "version": maxver, "time": now, "uuid": uuid.uuid4().hex, "status": "Sent to builder", "owner": "bulk", } # Data POSTed to the builder. post = { "addon": file_.version.addon_id, "file_id": file_.id, "priority": priority, "secret": settings.BUILDER_SECRET_KEY, "uuid": data["uuid"], "pingback": absolutify(reverse("amo.builder-pingback")), } if file_.builder_version: post["package_key"] = file_.builder_version else: # Older jetpacks might not have builderVersion in their harness. post["location"] = file_.get_url_path("builder") if sdk_version: post["sdk_version"] = sdk_version try: jp_log.info(urllib.urlencode(post)) response = urllib2.urlopen(settings.BUILDER_UPGRADE_URL, urllib.urlencode(post)) jp_log.info("Response from builder for %s: [%s] %s" % (file_.id, response.code, response.read())) except Exception: jp_log.error("Could not talk to builder for %s." % file_.id, exc_info=True) filedata[file_.id] = data upgrader.files(filedata)
def version_compat(qs, compat, app, binary): facets = [] for v, prev in zip(compat['versions'], (None,) + compat['versions']): d = {'from': vint(v)} if prev: d['to'] = vint(prev) facets.append(d) # Pick up everything else for an Other count. facets.append({'to': vint(compat['versions'][-1])}) facet = {'range': {'support.%s.max' % app: facets}} if binary is not None: qs = qs.query(binary=binary) qs = qs.facet(by_status=facet) result = qs[:0].raw() total_addons = result['hits']['total'] ranges = result['facets']['by_status']['ranges'] titles = compat['versions'] + (_('Other'),) faceted = [(v, r['count']) for v, r in zip(titles, ranges)] return total_addons, faceted
def usage_stats(request, compat, app): # Get the list of add-ons for usage stats. redis = redisutils.connections['master'] qs = (AppCompat.search().order_by('-usage.%s' % app).values_dict() .filter(**{'support.%s.max__gte' % app: vint(compat['previous'])})) addons = amo.utils.paginate(request, qs) for obj in addons.object_list: obj['usage'] = obj['usage'][app] obj['max_version'] = obj['max_version'][app] total = int(redis.hget('compat:%s' % app, 'total')) return addons, total
def usage_stats(request, compat, app, binary=None): # Get the list of add-ons for usage stats. qs = AppCompat.search().order_by('-usage.%s' % app).values_dict() if request.GET.get('previous'): qs = qs.filter(**{ 'support.%s.max__gte' % app: vint(compat['previous'])}) else: qs = qs.filter(**{'support.%s.max__gte' % app: 0}) if binary is not None: qs = qs.filter(binary=binary) addons = amo.utils.paginate(request, qs) for obj in addons.object_list: obj['usage'] = obj['usage'][app] obj['max_version'] = obj['max_version'][app] return addons, CompatTotals.objects.get(app=app).total
def usage_stats(request, compat, app, binary=None): # Get the list of add-ons for usage stats. qs = AppCompat.search().order_by('-usage.%s' % app).values_dict() if request.GET.get('previous'): qs = qs.filter( **{'support.%s.max__gte' % app: vint(compat['previous'])}) else: qs = qs.filter(**{'support.%s.max__gte' % app: 0}) if binary is not None: qs = qs.filter(binary=binary) addons = amo.utils.paginate(request, qs) for obj in addons.object_list: obj['usage'] = obj['usage'][app] obj['max_version'] = obj['max_version'][app] return addons, CompatTotals.objects.get(app=app).total
def _check_firefox(ua): need_firefox, need_upgrade = True, True for ua_res, min_version in mkt.platforms.APP_PLATFORMS: for ua_re in ua_res: match = ua_re.search(ua) if match: v = match.groups()[0] # If we found a version at all, then this is Firefox. need_firefox = False # If we found a matching version, then we can install apps! need_upgrade = vint(v) < min_version return need_firefox, need_upgrade
class FIREFOX(App): id = 1 shortername = 'fx' short = 'firefox' pretty = _(u'Firefox') browser = True types = [ADDON_EXTENSION, ADDON_THEME, ADDON_DICT, ADDON_SEARCH, ADDON_LPAPP, ADDON_PLUGIN, ADDON_PERSONA, ADDON_WEBAPP] guid = '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}' min_display_version = 3.0 # These versions were relabeled and should not be displayed. exclude_versions = (3.1, 3.7, 4.2) backup_version = vint('3.7.*') user_agent_string = 'Firefox' platforms = 'desktop' # DESKTOP_PLATFORMS (set in constants.platforms) @classmethod def matches_user_agent(cls, user_agent): matches = cls.user_agent_string in user_agent if 'Android' in user_agent or 'Mobile' in user_agent: matches = False return matches
def index(request, version=None): COMPAT = [v for v in settings.COMPAT if v['app'] == request.APP.id] compat_dict = dict((v['main'], v) for v in COMPAT) if not COMPAT: raise http.Http404() if version not in compat_dict: return redirect('compat.index', COMPAT[0]['main']) compat, app = compat_dict[version], str(request.APP.id) qs = AppCompat.search() compat_queries = ( ('prev', qs.query(top_95=True, **{ 'support.%s.max__gte' % app: vint(compat['previous'])})), ('top_95', qs.query(top_95=True)), ('all', qs), ) compat_levels = [(key, version_compat(qs, compat, app)) for key, qs in compat_queries] usage_addons, usage_total = usage_stats(request, compat, app) return jingo.render(request, 'compat/index.html', {'version': version, 'usage_addons': usage_addons, 'usage_total': usage_total, 'compat_levels': compat_levels})
def matches_user_agent(cls, user_agent): match = cls.user_agent_re.search(user_agent) if match: return vint(cls.min_display_version) <= vint(match.groups()[0])
def test_appver_long(self): too_big = vnum(vint(MAXVERSION + 1)) just_right = vnum(vint(MAXVERSION)) self.check_appver_filters(too_big, floor_version(just_right)) self.check_appver_filters("9999999", "9999999.0") self.check_appver_filters("99999999", "99999999.0")
def compatibility_report(index=None, aliased=True): docs = defaultdict(dict) indices = get_indices(index) # Gather all the data for the index. for app in amo.APP_USAGE: versions = [c for c in settings.COMPAT if c['app'] == app.id] log.info(u'Making compat report for %s.' % app.pretty) latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=app.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in amo.utils.chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, guid=addon.guid, self_hosted=addon.is_selfhosted(), binary=addon.binary_components, name=unicode(addon.name), created=addon.created, current_version=addon.current_version.version, current_version_id=addon.current_version.pk) doc['count'] = chunk[addon.id] doc.setdefault('top_95', defaultdict(lambda: defaultdict(dict))) doc.setdefault('top_95_all', {}) doc.setdefault('usage', {})[app.id] = updates[addon.id] doc.setdefault('works', {}).setdefault(app.id, {}) # Populate with default counts for all app versions. for ver in versions: doc['works'][app.id][vint(ver['main'])] = { 'success': 0, 'failure': 0, 'total': 0, 'failure_ratio': 0.0, } # Group reports by `major`.`minor` app version. reports = (CompatReport.objects .filter(guid=addon.guid, app_guid=app.guid) .values_list('app_version', 'works_properly') .annotate(Count('id'))) for ver, works_properly, cnt in reports: ver = vint(floor_version(ver)) major = [v['main'] for v in versions if vint(v['previous']) < ver <= vint(v['main'])] if major: w = doc['works'][app.id][vint(major[0])] # Tally number of success and failure reports. w['success' if works_properly else 'failure'] += cnt w['total'] += cnt # Calculate % of incompatibility reports. w['failure_ratio'] = w['failure'] / float(w['total']) if app not in addon.compatible_apps: continue compat = addon.compatible_apps[app] d = {'min': compat.min.version_int, 'max': compat.max.version_int} doc.setdefault('support', {})[app.id] = d doc.setdefault('max_version', {})[app.id] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. compat_total, created = CompatTotals.objects.safer_get_or_create( app=app.id, defaults={'total': total}) if not created: compat_total.update(total=total) # Figure out which add-ons are in the top 95% for this app. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): running_total += count docs[addon]['top_95_all'][app.id] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # app + version combo. for compat in settings.COMPAT: app, ver = compat['app'], vint(compat['previous']) # Find all the docs that have a max_version compatible with ver. supported = [doc for doc in docs.values() if app in doc.get('support', {}) and doc['support'][app]['max'] >= ver] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][app][ver] = running_total < (.95 * total) # Send it all to the index. for chunk in amo.utils.chunked(docs.values(), 150): for doc in chunk: for index in indices: AppCompat.index(doc, id=doc['id'], bulk=True, index=index) elasticutils.get_es().flush_bulk(forced=True)
import re from tower import ugettext_lazy as _ from versions.compare import version_int as vint # These are the minimum versions required for `navigator.mozApps` support. APP_PLATFORMS = [ # Firefox for Desktop. ( [ re.compile('Firefox/([\d.]+)') ], vint('16.0') ), # Firefox for Android. ( [ re.compile('Fennec/([\d.]+)'), re.compile('Android; Mobile; rv:([\d.]+)'), re.compile('Mobile; rv:([\d.]+)') ], vint('17.0') ) ] def FREE_PLATFORMS(): return ( ('free-firefoxos', _('Firefox OS')),
def test_appver_long(self): too_big = vnum(vint(MAXVERSION + 1)) just_right = vnum(vint(MAXVERSION)) self.check_appver_filters(too_big, floor_version(just_right)) self.check_appver_filters('9999999', '9999999.0') self.check_appver_filters('99999999', '99999999.0')
def compatibility_report(): redis = redisutils.connections['master'] docs = defaultdict(dict) # Gather all the data for the index. for app in amo.APP_USAGE: log.info(u'Making compat report for %s.' % app.pretty) latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=app.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in amo.utils.chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, binary=addon.binary, name=unicode(addon.name), created=addon.created) doc['count'] = chunk[addon.id] doc.setdefault('top_95', defaultdict(lambda: defaultdict(dict))) doc.setdefault('top_95_all', {}) doc.setdefault('usage', {})[app.id] = updates[addon.id] if app not in addon.compatible_apps: continue compat = addon.compatible_apps[app] d = {'min': compat.min.version_int, 'max': compat.max.version_int} doc.setdefault('support', {})[app.id] = d doc.setdefault('max_version', {})[app.id] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. redis.hset('compat:%s' % app.id, 'total', total) # Figure out which add-ons are in the top 95% for this app. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): running_total += count docs[addon]['top_95_all'][app.id] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # app + version combo. for compat in settings.COMPAT: app, ver = compat['app'], vint(compat['previous']) # Find all the docs that have a max_version compatible with ver. supported = [doc for doc in docs.values() if app in doc.get('support', {}) and doc['support'][app]['max'] >= ver] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][app][ver] = running_total < (.95 * total) # Send it all to the index. for chunk in amo.utils.chunked(docs.values(), 150): for doc in chunk: AppCompat.index(doc, id=doc['id'], bulk=True) elasticutils.get_es().flush_bulk(forced=True)
def matches_user_agent(cls, user_agent): for user_agent_re in cls.user_agent_re: match = user_agent_re.search(user_agent) if match: v = match.groups()[0] return vint(cls.min_display_version) <= vint(v)
import re from tower import ugettext_lazy as _ from versions.compare import version_int as vint # These are the minimum versions required for `navigator.mozApps` support. APP_PLATFORMS = [ # Firefox for Desktop. ([re.compile('Firefox/([\d.]+)')], vint('16.0')), # Firefox for Android. ([ re.compile('Fennec/([\d.]+)'), re.compile('Android; Mobile; rv:([\d.]+)'), re.compile('Mobile; rv:([\d.]+)') ], vint('17.0')) ] def FREE_PLATFORMS(): return ( ('free-firefoxos', _('Firefox OS')), ('free-desktop', _('Firefox')), ('free-android-mobile', _('Firefox Mobile')), ('free-android-tablet', _('Firefox Tablet')), ) def PAID_PLATFORMS(request=None): import waffle platforms = (('paid-firefoxos', _('Firefox OS')), )
def compatibility_report(): redis = redisutils.connections['master'] docs = defaultdict(dict) # Gather all the data for the index. for app in amo.APP_USAGE: log.info(u'Making compat report for %s.' % app.pretty) latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=app.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in amo.utils.chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, binary=addon.binary, name=unicode(addon.name), created=addon.created) doc['count'] = chunk[addon.id] doc.setdefault('top_95', defaultdict(lambda: defaultdict(dict))) doc.setdefault('top_95_all', {}) doc.setdefault('usage', {})[app.id] = updates[addon.id] if app not in addon.compatible_apps: continue compat = addon.compatible_apps[app] d = { 'min': compat.min.version_int, 'max': compat.max.version_int } doc.setdefault('support', {})[app.id] = d doc.setdefault('max_version', {})[app.id] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. redis.hset('compat:%s' % app.id, 'total', total) # Figure out which add-ons are in the top 95% for this app. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): running_total += count docs[addon]['top_95_all'][app.id] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # app + version combo. for compat in settings.COMPAT: app, ver = compat['app'], vint(compat['previous']) # Find all the docs that have a max_version compatible with ver. supported = [ doc for doc in docs.values() if app in doc.get('support', {}) and doc['support'][app]['max'] >= ver ] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][app][ver] = running_total < (.95 * total) # Send it all to the index. for chunk in amo.utils.chunked(docs.values(), 150): for doc in chunk: AppCompat.index(doc, id=doc['id'], bulk=True) elasticutils.get_es().flush_bulk(forced=True)
def compatibility_report(index=None): docs = defaultdict(dict) indices = get_indices(index) # Gather all the data for the index. for app in amo.APP_USAGE: versions = [c for c in amo.COMPAT if c['app'] == app.id] log.info(u'Making compat report for %s.' % app.pretty) latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=app.id, addon__disabled_by_user=False, addon__status__in=amo.VALID_STATUSES, addon___current_version__isnull=False, date=latest) updates = dict(qs.values_list('addon', 'count')) for chunk in amo.utils.chunked(updates.items(), 50): chunk = dict(chunk) for addon in Addon.objects.filter(id__in=chunk): doc = docs[addon.id] doc.update(id=addon.id, slug=addon.slug, guid=addon.guid, binary=addon.binary_components, name=unicode(addon.name), created=addon.created, current_version=addon.current_version.version, current_version_id=addon.current_version.pk) doc['count'] = chunk[addon.id] doc.setdefault('top_95', defaultdict(lambda: defaultdict(dict))) doc.setdefault('top_95_all', {}) doc.setdefault('usage', {})[app.id] = updates[addon.id] doc.setdefault('works', {}).setdefault(app.id, {}) # Populate with default counts for all app versions. for ver in versions: doc['works'][app.id][vint(ver['main'])] = { 'success': 0, 'failure': 0, 'total': 0, 'failure_ratio': 0.0, } # Group reports by `major`.`minor` app version. reports = (CompatReport.objects.filter( guid=addon.guid, app_guid=app.guid).values_list( 'app_version', 'works_properly').annotate(Count('id'))) for ver, works_properly, cnt in reports: ver = vint(floor_version(ver)) major = [ v['main'] for v in versions if vint(v['previous']) < ver <= vint(v['main']) ] if major: w = doc['works'][app.id][vint(major[0])] # Tally number of success and failure reports. w['success' if works_properly else 'failure'] += cnt w['total'] += cnt # Calculate % of incompatibility reports. w['failure_ratio'] = w['failure'] / float(w['total']) if app not in addon.compatible_apps: continue compat = addon.compatible_apps[app] d = { 'min': compat.min.version_int, 'max': compat.max.version_int } doc.setdefault('support', {})[app.id] = d doc.setdefault('max_version', {})[app.id] = compat.max.version total = sum(updates.values()) # Remember the total so we can show % of usage later. compat_total, created = CompatTotals.objects.safer_get_or_create( app=app.id, defaults={'total': total}) if not created: compat_total.update(total=total) # Figure out which add-ons are in the top 95% for this app. running_total = 0 for addon, count in sorted(updates.items(), key=lambda x: x[1], reverse=True): running_total += count docs[addon]['top_95_all'][app.id] = running_total < (.95 * total) # Mark the top 95% of add-ons compatible with the previous version for each # app + version combo. for compat in amo.COMPAT: app, ver = compat['app'], vint(compat['previous']) # Find all the docs that have a max_version compatible with ver. supported = [ doc for doc in docs.values() if app in doc.get('support', {}) and doc['support'][app]['max'] >= ver ] # Sort by count so we can get the top 95% most-used add-ons. supported = sorted(supported, key=lambda d: d['count'], reverse=True) total = sum(doc['count'] for doc in supported) # Figure out which add-ons are in the top 95% for this app + version. running_total = 0 for doc in supported: running_total += doc['count'] doc['top_95'][app][ver] = running_total < (.95 * total) # Send it all to the index. for chunk in amo.utils.chunked(docs.values(), 150): for doc in chunk: for index in indices: AppCompat.index(doc, id=doc['id'], refresh=False, index=index) es = amo.search.get_es() es.indices.refresh()
def test_appver_long(self): too_big = vnum(vint(sys.maxint + 1)) just_right = vnum(vint(sys.maxint)) self.check_appver_filters(too_big, floor_version(just_right)) self.check_appver_filters('9999999', '9999999.0') self.check_appver_filters('99999999', '99999999.0')
import re from versions.compare import version_int as vint # These are the minimum versions required for `navigator.mozApps` support. APP_PLATFORMS = [ # Firefox for Desktop. ([re.compile("Firefox/([\d.]+)")], vint("16.0")), # Firefox for Android. ( [re.compile("Fennec/([\d.]+)"), re.compile("Android; Mobile; rv:([\d.]+)"), re.compile("Mobile; rv:([\d.]+)")], vint("17.0"), ), ]