def incompatible_latest_apps(self):
"""Returns a list of applications with which this add-on is
incompatible (based on the latest version).
"""
return [a for a, v in self.compatible_apps.items() if v and
version_int(v.max.version) < version_int(a.latest_version)]
def test_bump_version_in_model(self, mock_sign_file):
# We want to make sure each file has been signed.
self.file2 = amo.tests.file_factory(version=self.version)
self.file2.update(filename='jetpack-b.xpi')
backup_file2_path = u'{0}.backup_signature'.format(
self.file2.file_path)
try:
with amo.tests.copy_file('apps/files/fixtures/files/jetpack.xpi',
self.file_.file_path):
with amo.tests.copy_file(
'apps/files/fixtures/files/jetpack.xpi',
self.file2.file_path):
file_hash = self.file_.generate_hash()
file2_hash = self.file2.generate_hash()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
tasks.sign_addons([self.addon.pk])
assert mock_sign_file.call_count == 2
self.version.reload()
assert self.version.version == '1.3.1-signed'
assert self.version.version_int == version_int(
'1.3.1-signed')
assert file_hash != self.file_.generate_hash()
assert file2_hash != self.file2.generate_hash()
self.assert_backup()
assert os.path.exists(backup_file2_path)
finally:
if os.path.exists(backup_file2_path):
os.unlink(backup_file2_path)
def test_version_int():
"""Tests that version_int. Corrects our versions."""
eq_(version_int('3.5.0a1pre2'), 3050000001002)
eq_(version_int(''), 200100)
eq_(version_int(sys.maxint), sys.maxint)
eq_(version_int(sys.maxint + 1), sys.maxint)
eq_(version_int('9999999'), sys.maxint)
def __init__(self, request, platform, version):
self.request = request
self.platform = platform
self.version = version
self.compat_mode = 'strict'
if version_int(self.version) >= version_int('10.0'):
self.compat_mode = 'ignore'
def test_version_int():
"""Tests that version_int. Corrects our versions."""
eq_(version_int('3.5.0a1pre2'), 3050000001002)
eq_(version_int(''), 200100)
eq_(version_int(MAXVERSION), MAXVERSION)
eq_(version_int(MAXVERSION + 1), MAXVERSION)
eq_(version_int('9999999'), MAXVERSION)
def find_jetpacks(jp_version):
"""
Find all jetpack files that aren't disabled.
Files that should be upgraded will have needs_upgrade=True.
"""
statuses = amo.VALID_STATUSES
files = (File.objects.filter(jetpack_version__isnull=False,
version__addon__status__in=statuses,
version__addon__disabled_by_user=False)
.exclude(status=amo.STATUS_DISABLED).no_cache()
.select_related('version'))
files = sorted(files, key=lambda f: (f.version.addon_id, f.version.id))
# Figure out which files need to be upgraded.
for file_ in files:
file_.needs_upgrade = False
# If any files for this add-on are reviewed, take the last reviewed file
# plus all newer files. Otherwise, only upgrade the latest file.
for _, fs in groupby(files, key=lambda f: f.version.addon_id):
fs = list(fs)
if any(f.status in amo.REVIEWED_STATUSES for f in fs):
for file_ in reversed(fs):
file_.needs_upgrade = True
if file_.status in amo.REVIEWED_STATUSES:
break
else:
fs[-1].needs_upgrade = True
# Make sure only old files are marked.
for file_ in [f for f in files if f.needs_upgrade]:
if version_int(file_.jetpack_version) >= version_int(jp_version):
file_.needs_upgrade = False
return files
def __init__(self, request, platform, version):
self.request = request
self.platform = platform
self.version = version
self.compat_mode = 'strict'
if (waffle.switch_is_active('d2c-at-the-disco') and
version_int(self.version) >= version_int('10.0')):
self.compat_mode = 'ignore'
def get_compat_mode(version):
# Returns appropriate compat mode based on app version.
# Replace when we are ready to deal with bug 711698.
vint = version_int(version)
if waffle.switch_is_active("d2c-at-the-disco"):
return "ignore" if vint >= version_int("10.0") else "strict"
else:
return "strict"
def get_compat_mode(version):
# Returns appropriate compat mode based on app version.
# Replace when we are ready to deal with bug 711698.
vint = version_int(version)
if waffle.switch_is_active('d2c-at-the-disco'):
return 'ignore' if vint >= version_int('10.0') else 'strict'
else:
return 'strict'
def check_jetpack_version(sender, **kw):
import files.tasks
from zadmin.models import get_config
jetpack_version = get_config('jetpack_version')
qs = File.objects.filter(version__addon=sender,
jetpack_version__isnull=False)
ids = [f.id for f in qs
if version_int(f.jetpack_version) < version_int(jetpack_version)]
if ids:
files.tasks.start_upgrade.delay(jetpack_version, ids, priority='high')
def _filter_search(request,
qs,
query,
filters,
sorting,
sorting_default='-weekly_downloads',
types=[]):
"""Filter an ES queryset based on a list of filters."""
APP = request.APP
# Intersection of the form fields present and the filters we want to apply.
show = [f for f in filters if query.get(f)]
if query.get('q'):
qs = qs.query(or_=name_query(query['q']))
if 'platform' in show and query['platform'] in amo.PLATFORM_DICT:
ps = (amo.PLATFORM_DICT[query['platform']].id, amo.PLATFORM_ALL.id)
# If we've selected "All Systems" don't filter by platform.
if ps[0] != ps[1]:
qs = qs.filter(platform__in=ps)
if 'appver' in show:
# Get a min version less than X.0.
low = version_int(query['appver'])
# Get a max version greater than X.0a.
high = version_int(query['appver'] + 'a')
# If we're not using D2C then fall back to appversion checking.
extensions_shown = (not query.get('atype')
or query['atype'] == amo.ADDON_EXTENSION)
if not extensions_shown or low < version_int('10.0'):
qs = qs.filter(
**{
'appversion.%s.max__gte' % APP.id: high,
'appversion.%s.min__lte' % APP.id: low
})
if 'atype' in show and query['atype'] in amo.ADDON_TYPES:
qs = qs.filter(type=query['atype'])
else:
qs = qs.filter(type__in=types)
if 'cat' in show:
cat = (Category.objects.filter(id=query['cat']).filter(
Q(application=APP.id) | Q(type=amo.ADDON_SEARCH)))
if not cat.exists():
show.remove('cat')
if 'cat' in show:
qs = qs.filter(category=query['cat'])
if 'tag' in show:
qs = qs.filter(tag=query['tag'])
if 'sort' in show:
qs = qs.order_by(sorting[query['sort']])
elif not query.get('q'):
# Sort by a default if there was no query so results are predictable.
qs = qs.order_by(sorting_default)
return qs
def test_dont_sign_dont_bump_version_bad_zipfile(self, mock_sign_file):
with amo.tests.copy_file(__file__, self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
tasks.sign_addons([self.addon.pk])
assert not mock_sign_file.called
self.version.reload()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
assert file_hash == self.file_.generate_hash()
self.assert_no_backup()
def test_dont_sign_dont_bump_sign_error(self, mock_sign_file):
mock_sign_file.side_effect = IOError()
with amo.tests.copy_file("apps/files/fixtures/files/jetpack.xpi", self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
tasks.sign_addons([self.addon.pk])
assert mock_sign_file.called
self.version.reload()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
assert file_hash == self.file_.generate_hash()
self.assert_no_backup()
def test_dont_bump_not_signed(self, mock_sign_file):
mock_sign_file.return_value = None # Pretend we didn't sign.
with amo.tests.copy_file("apps/files/fixtures/files/jetpack.xpi", self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
tasks.sign_addons([self.addon.pk])
assert mock_sign_file.called
self.version.reload()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
assert file_hash == self.file_.generate_hash()
self.assert_no_backup()
def test_resign_bump_version_in_model_if_force(self, mock_sign_file):
with amo.tests.copy_file("apps/files/fixtures/files/new-addon-signature.xpi", self.file_.file_path):
self.file_.update(is_signed=True)
file_hash = self.file_.generate_hash()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
tasks.sign_addons([self.addon.pk], force=True)
assert mock_sign_file.called
self.version.reload()
assert self.version.version == "1.3.1-signed"
assert self.version.version_int == version_int("1.3.1-signed")
assert file_hash != self.file_.generate_hash()
self.assert_backup()
def test_sign_bump_non_ascii_version(self, mock_sign_file):
"""Sign versions which have non-ascii version numbers."""
self.version.update(version=u"é1.3")
with amo.tests.copy_file("apps/files/fixtures/files/jetpack.xpi", self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == u"é1.3"
assert self.version.version_int == version_int("1.3")
tasks.sign_addons([self.addon.pk])
assert mock_sign_file.called
self.version.reload()
assert self.version.version == u"é1.3.1-signed"
assert self.version.version_int == version_int(u"é1.3.1-signed")
assert file_hash != self.file_.generate_hash()
self.assert_backup()
def test_sign_bump_old_versions_default_compat(self, mock_sign_file):
"""Sign files which are old, but default to compatible."""
with amo.tests.copy_file("apps/files/fixtures/files/jetpack.xpi", self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
self.set_max_appversion(settings.MIN_D2C_VERSION)
tasks.sign_addons([self.addon.pk])
assert mock_sign_file.called
self.version.reload()
assert self.version.version == "1.3.1-signed"
assert self.version.version_int == version_int("1.3.1-signed")
assert file_hash != self.file_.generate_hash()
self.assert_backup()
def test_sign_bump_new_versions_not_default_compat(self, mock_sign_file):
"""Sign files which are recent, event if not default to compatible."""
with amo.tests.copy_file("apps/files/fixtures/files/jetpack.xpi", self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
self.file_.update(binary_components=True, strict_compatibility=True)
tasks.sign_addons([self.addon.pk])
assert mock_sign_file.called
self.version.reload()
assert self.version.version == "1.3.1-signed"
assert self.version.version_int == version_int("1.3.1-signed")
assert file_hash != self.file_.generate_hash()
self.assert_backup()
def test_no_bump_unreviewed(self, mock_sign_file):
"""Don't bump nor sign unreviewed files."""
for status in amo.UNREVIEWED_STATUSES + (amo.STATUS_BETA,):
self.file_.update(status=amo.STATUS_UNREVIEWED)
with amo.tests.copy_file("apps/files/fixtures/files/jetpack.xpi", self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
tasks.sign_addons([self.addon.pk])
assert not mock_sign_file.called
self.version.reload()
assert self.version.version == "1.3"
assert self.version.version_int == version_int("1.3")
assert file_hash == self.file_.generate_hash()
self.assert_no_backup()
def test_dont_resign_dont_bump_version_in_model(self, mock_sign_file):
with amo.tests.copy_file(
'apps/files/fixtures/files/new-addon-signature.xpi',
self.file_.file_path):
self.file_.update(is_signed=True)
file_hash = self.file_.generate_hash()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
tasks.sign_addons([self.addon.pk])
assert not mock_sign_file.called
self.version.reload()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
assert file_hash == self.file_.generate_hash()
self.assert_no_backup()
def get_recs_from_ids(cls, addons, app, version, compat_mode="strict"):
vint = compare.version_int(version)
recs = RecommendedCollection.build_recs(addons)
qs = Addon.objects.public().filter(id__in=recs, appsupport__app=app.id, appsupport__min__lte=vint)
if compat_mode == "strict":
qs = qs.filter(appsupport__max__gte=vint)
return recs, qs
def make_langpack(version):
versions = (version, '%s.*' % version)
for version in versions:
AppVersion.objects.get_or_create(application=amo.FIREFOX.id,
version=version,
version_int=version_int(version))
return make_xpi({
'install.rdf': """<?xml version="1.0"?>
<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:em="http://www.mozilla.org/2004/em-rdf#">
<Description about="urn:mozilla:install-manifest"
em:id="*****@*****.**"
em:name="Foo Language Pack"
em:version="{0}"
em:type="8"
em:creator="mozilla.org">
<em:targetApplication>
<Description>
<em:id>{{ec8030f7-c20a-464f-9b0e-13a3a9e97384}}</em:id>
<em:minVersion>{0}</em:minVersion>
<em:maxVersion>{1}</em:maxVersion>
</Description>
</em:targetApplication>
</Description>
</RDF>
""".format(*versions)
}).read()
def supports_firefox(file_obj):
"""Return True if the file support a high enough version of Firefox.
We only sign files that are at least compatible with Firefox
MIN_NOT_D2C_VERSION, or Firefox MIN_NOT_D2C_VERSION if they are not default
to compatible.
"""
apps = file_obj.version.apps.all()
if not file_obj.binary_components and not file_obj.strict_compatibility:
# Version is "default to compatible".
return apps.filter(max__application=amo.FIREFOX.id, max__version_int__gte=version_int(settings.MIN_D2C_VERSION))
else:
# Version isn't "default to compatible".
return apps.filter(
max__application=amo.FIREFOX.id, max__version_int__gte=version_int(settings.MIN_NOT_D2C_VERSION)
)
def __init__(self, request, platform, version):
self.request = request
self.platform = platform
self.version = version
self.compat_mode = "strict"
if waffle.switch_is_active("d2c-at-the-disco") and version_int(self.version) >= version_int("10.0"):
self.compat_mode = "ignore"
def get_recs_from_ids(cls, addons, app, version):
vint = compare.version_int(version)
recs = RecommendedCollection.build_recs(addons)
qs = (Addon.objects.public()
.filter(id__in=recs, appsupport__app=app.id,
appsupport__min__lte=vint, appsupport__max__gte=vint))
return recs, qs[:Collection.RECOMMENDATION_LIMIT]
def test_sign_bump_non_ascii_filename(self, mock_sign_file):
"""Sign files which have non-ascii filenames."""
self.file_.update(filename=u'jétpack.xpi')
with amo.tests.copy_file(
'apps/files/fixtures/files/jetpack.xpi',
self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
tasks.sign_addons([self.addon.pk])
assert mock_sign_file.called
self.version.reload()
assert self.version.version == '1.3.1-signed'
assert self.version.version_int == version_int('1.3.1-signed')
assert file_hash != self.file_.generate_hash()
self.assert_backup()
def extract(addon):
"""Extract indexable attributes from an add-on."""
attrs = ('id', 'slug', 'app_slug', 'created', 'last_updated',
'weekly_downloads', 'bayesian_rating', 'average_daily_users',
'status', 'type', 'hotness', 'is_disabled', 'premium_type')
d = dict(zip(attrs, attrgetter(*attrs)(addon)))
# Coerce the Translation into a string.
d['name_sort'] = unicode(addon.name).lower()
translations = addon.translations
d['name'] = list(set(string for _, string in translations[addon.name_id]))
d['description'] = list(set(string for _, string
in translations[addon.description_id]))
d['summary'] = list(set(string for _, string
in translations[addon.summary_id]))
d['authors'] = [a.name for a in addon.listed_authors]
d['device'] = getattr(addon, 'device_ids', [])
# This is an extra query, not good for perf.
d['category'] = getattr(addon, 'category_ids', [])
d['tags'] = getattr(addon, 'tag_list', [])
d['price'] = getattr(addon, 'price', 0.0)
if addon.current_version:
d['platforms'] = [p.id for p in
addon.current_version.supported_platforms]
d['appversion'] = {}
for app, appver in addon.compatible_apps.items():
if appver:
min_, max_ = appver.min.version_int, appver.max.version_int
else:
# Fake wide compatibility for search tools and personas.
min_, max_ = 0, version_int('9999')
d['appversion'][app.id] = dict(min=min_, max=max_)
d['has_version'] = addon._current_version != None
d['app'] = [app.id for app in addon.compatible_apps.keys()]
if addon.type == amo.ADDON_PERSONA:
# This would otherwise get attached when by the transformer.
d['weekly_downloads'] = addon.persona.popularity
# Boost on popularity.
d['_boost'] = addon.persona.popularity ** .2
else:
# Boost by the number of users on a logarithmic scale. The maximum
# boost (11,000,000 users for adblock) is about 5x.
d['_boost'] = addon.average_daily_users ** .2
# Double the boost if the add-on is public.
if addon.status == amo.STATUS_PUBLIC:
d['_boost'] = max(d['_boost'], 1) * 4
# Indices for each language. languages is a list of locales we want to
# index with analyzer if the string's locale matches.
for analyzer, languages in amo.SEARCH_ANALYZER_MAP.iteritems():
d['name_' + analyzer] = list(
set(string for locale, string in translations[addon.name_id]
if locale.lower() in languages))
d['summary_' + analyzer] = list(
set(string for locale, string in translations[addon.summary_id]
if locale.lower() in languages))
d['description_' + analyzer] = list(
set(string for locale, string in translations[addon.description_id]
if locale.lower() in languages))
return d
def not_signed():
assert not mock_sign_file.called
self.version.reload()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
assert file_hash == self.file_.generate_hash()
self.assert_no_backup()
def test_dont_sign_dont_bump_old_versions(self, mock_sign_file):
"""Don't sign files which are too old, or not default to compatible."""
def not_signed():
assert not mock_sign_file.called
self.version.reload()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
assert file_hash == self.file_.generate_hash()
self.assert_no_backup()
with amo.tests.copy_file('apps/files/fixtures/files/jetpack.xpi',
self.file_.file_path):
file_hash = self.file_.generate_hash()
assert self.version.version == '1.3'
assert self.version.version_int == version_int('1.3')
# Too old, don't sign.
self.set_max_appversion('1') # Very very old.
tasks.sign_addons([self.addon.pk])
not_signed()
# MIN_D2C_VERSION, but strict compat: don't sign.
self.set_max_appversion(settings.MIN_D2C_VERSION)
self.file_.update(strict_compatibility=True)
tasks.sign_addons([self.addon.pk])
not_signed()
# MIN_D2C_VERSION, but binary component: don't sign.
self.file_.update(strict_compatibility=False,
binary_components=True)
tasks.sign_addons([self.addon.pk])
not_signed()
def compatibility_report():
redis = redisutils.connections['master']
# for app in amo.APP_USAGE:
for compat in settings.COMPAT:
app = amo.APPS_ALL[compat['app']]
version = compat['version']
log.info(u'Making compat report for %s %s.' % (app.pretty, version))
versions = (('latest', version), ('beta', version + 'b'),
('alpha', compat['alpha']))
rv = dict((k, 0) for k in dict(versions))
rv['other'] = 0
ignore = (amo.STATUS_NULL, amo.STATUS_DISABLED)
qs = (Addon.objects.exclude(type=amo.ADDON_PERSONA, status__in=ignore)
.filter(appsupport__app=app.id, name__locale='en-us'))
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
date=latest)
total = qs.aggregate(Sum('count'))['count__sum']
addons = list(qs.values_list('addon', 'count', 'addon__appsupport__min',
'addon__appsupport__max'))
# Count up the top 95% of addons by ADU.
adus = 0
for addon, count, minver, maxver in addons:
# Don't count add-ons that weren't compatible with the previous
# release
if maxver < vc.version_int(compat['previous']):
continue
if adus < .95 * total:
adus += count
else:
break
for key, version in versions:
if minver <= vc.version_int(version) <= maxver:
rv[key] += 1
break
else:
rv['other'] += 1
log.info(u'Compat for %s %s: %s' % (app.pretty, version, rv))
key = '%s:%s' % (app.id, version)
redis.hmset('compat:' + key, rv)
def test_version_asterix_compare():
eq_(version_int('*'), version_int('99'))
assert version_int('98.*') < version_int('*')
eq_(version_int('5.*'), version_int('5.99'))
assert version_int('5.*') > version_int('5.0.*')
def test_version_int_compare():
eq_(version_int('3.6.*'), version_int('3.6.99'))
assert version_int('3.6.*') > version_int('3.6.8')
def search(request, tag_name=None, template=None):
APP = request.APP
types = (amo.ADDON_EXTENSION, amo.ADDON_THEME, amo.ADDON_DICT,
amo.ADDON_SEARCH, amo.ADDON_LPAPP)
fixed = fix_search_query(request.GET)
if fixed is not request.GET:
return redirect(urlparams(request.path, **fixed), permanent=True)
form = ESSearchForm(request.GET or {})
form.is_valid() # Let the form try to clean data.
category = request.GET.get('cat')
query = form.cleaned_data
if category == 'collections':
return _collections(request)
elif category == 'personas' or query.get('atype') == amo.ADDON_PERSONA:
return _personas(request)
sort, extra_sort = split_choices(form.fields['sort'].choices, 'created')
qs = (Addon.search().filter(
status__in=amo.REVIEWED_STATUSES, is_disabled=False, app=APP.id).facet(
tags={'terms': {
'field': 'tag'
}},
platforms={'terms': {
'field': 'platform'
}},
appversions={'terms': {
'field': 'appversion.%s.max' % APP.id
}},
categories={'terms': {
'field': 'category',
'size': 100
}}))
if query.get('q'):
qs = qs.query(or_=name_query(query['q']))
if tag_name or query.get('tag'):
qs = qs.filter(tag=tag_name or query['tag'])
if query.get('platform') and query['platform'] in amo.PLATFORM_DICT:
ps = (amo.PLATFORM_DICT[query['platform']].id, amo.PLATFORM_ALL.id)
qs = qs.filter(platform__in=ps)
if query.get('appver'):
# Get a min version less than X.0.
low = version_int(query['appver'])
# Get a max version greater than X.0a.
high = version_int(query['appver'] + 'a')
qs = qs.filter(
**{
'appversion.%s.max__gte' % APP.id: high,
'appversion.%s.min__lte' % APP.id: low
})
if query.get('atype') and query['atype'] in amo.ADDON_TYPES:
qs = qs.filter(type=query['atype'])
if query['atype'] == amo.ADDON_SEARCH:
# Search add-ons should not be searched by ADU, so replace 'Users'
# sort with 'Weekly Downloads'.
sort[1] = extra_sort[1]
del extra_sort[1]
else:
qs = qs.filter(type__in=types)
if query.get('cat'):
qs = qs.filter(category=query['cat'])
if query.get('sort'):
mapping = {
'users': '-average_daily_users',
'rating': '-bayesian_rating',
'created': '-created',
'name': 'name_sort',
'downloads': '-weekly_downloads',
'updated': '-last_updated',
'hotness': '-hotness'
}
qs = qs.order_by(mapping[query['sort']])
elif not query.get('q'):
# Sort by weekly downloads if there was no query so we get predictable
# results.
qs = qs.order_by('-weekly_downloads')
pager = amo.utils.paginate(request, qs)
ctx = {
'is_pjax': request.META.get('HTTP_X_PJAX'),
'pager': pager,
'query': query,
'form': form,
'sort_opts': sort,
'extra_sort_opts': extra_sort,
'sorting': sort_sidebar(request, query, form),
}
if not ctx['is_pjax']:
facets = pager.object_list.facets
ctx.update({
'categories': category_sidebar(request, query, facets),
'platforms': platform_sidebar(request, query, facets),
'versions': version_sidebar(request, query, facets),
'tags': tag_sidebar(request, query, facets),
})
return jingo.render(request, template, ctx)
def sign_addons(addon_ids, force=False, **kw):
"""Used to sign all the versions of an addon.
This is used in the 'sign_addons' and 'process_addons --task sign_addons'
management commands.
It also bumps the version number of the file and the Version, so the
Firefox extension update mecanism picks this new signed version and
installs it.
"""
log.info(u'[{0}] Signing addons.'.format(len(addon_ids)))
def file_supports_firefox(version):
"""Return a Q object: files supporting at least a firefox version."""
return Q(version__apps__max__application=amo.FIREFOX.id,
version__apps__max__version_int__gte=version_int(version))
is_default_compatible = Q(binary_components=False,
strict_compatibility=False)
# We only want to sign files that are at least compatible with Firefox
# MIN_D2C_VERSION, or Firefox MIN_NOT_D2C_VERSION if they are not default
# to compatible.
# The signing feature should be supported from Firefox 40 and above, but
# we're still signing some files that are a bit older just in case.
ff_version_filter = ((is_default_compatible
& file_supports_firefox(settings.MIN_D2C_VERSION)) |
(~is_default_compatible & file_supports_firefox(
settings.MIN_NOT_D2C_VERSION)))
addons_emailed = []
# We only care about extensions and (complete) themes. The latter is
# because they may have multi-package XPIs, containing extensions.
for version in Version.objects.filter(
addon_id__in=addon_ids,
addon__type__in=[amo.ADDON_EXTENSION, amo.ADDON_THEME]):
# We only sign files that have been reviewed and are compatible with
# versions of Firefox that are recent enough.
to_sign = version.files.filter(ff_version_filter,
status__in=amo.REVIEWED_STATUSES)
# We only care about multi-package XPIs for themes, because they may
# have extensions inside.
if version.addon.type == amo.ADDON_THEME:
to_sign = version.files.filter(is_multi_package=True)
if force:
to_sign = to_sign.all()
else:
to_sign = to_sign.filter(is_signed=False)
if not to_sign:
log.info(u'Not signing addon {0}, version {1} (no files or already'
u' signed)'.format(version.addon, version))
continue
log.info(u'Signing addon {0}, version {1}'.format(
version.addon, version))
bump_version = False # Did we sign at least one file?
for file_obj in to_sign:
if not os.path.isfile(file_obj.file_path):
log.info(u'File {0} does not exist, skip'.format(file_obj.pk))
continue
# Save the original file, before bumping the version.
backup_path = u'{0}.backup_signature'.format(file_obj.file_path)
shutil.copy(file_obj.file_path, backup_path)
try:
# Need to bump the version (modify install.rdf or package.json)
# before the file is signed.
bump_version_number(file_obj)
if file_obj.status == amo.STATUS_PUBLIC:
server = settings.SIGNING_SERVER
else:
server = settings.PRELIMINARY_SIGNING_SERVER
signed = bool(sign_file(file_obj, server))
if signed: # Bump the version number if at least one signed.
bump_version = True
else: # We didn't sign, so revert the version bump.
shutil.move(backup_path, file_obj.file_path)
except:
log.error(u'Failed signing file {0}'.format(file_obj.pk),
exc_info=True)
# Revert the version bump, restore the backup.
shutil.move(backup_path, file_obj.file_path)
# Now update the Version model, if we signed at least one file.
if bump_version:
bumped_version = _dot_one(version.version)
version.update(version=bumped_version,
version_int=version_int(bumped_version))
addon = version.addon
if addon.pk not in addons_emailed:
# Send a mail to the owners/devs warning them we've
# automatically signed their addon.
qs = (AddonUser.objects.filter(
role=amo.AUTHOR_ROLE_OWNER,
addon=addon).exclude(user__email=None))
emails = qs.values_list('user__email', flat=True)
subject = MAIL_SUBJECT.format(addon=addon.name)
message = MAIL_MESSAGE.format(
addon=addon.name,
addon_url=amo.helpers.absolutify(
addon.get_dev_url(action='versions')))
amo.utils.send_mail(
subject,
message,
recipient_list=emails,
fail_silently=True,
headers={'Reply-To': '*****@*****.**'})
addons_emailed.append(addon.pk)
def save(self, *args, **kw):
if not self.version_int:
self.version_int = compare.version_int(self.version)
return super(AppVersion, self).save(*args, **kw)
def addon_filter(addons, addon_type, limit, app, platform, version,
compat_mode='strict', shuffle=True):
"""
Filter addons by type, application, app version, and platform.
Add-ons that support the current locale will be sorted to front of list.
Shuffling will be applied to the add-ons supporting the locale and the
others separately.
Doing this in the database takes too long, so we in code and wrap it in
generous caching.
"""
APP = app
if addon_type.upper() != 'ALL':
try:
addon_type = int(addon_type)
if addon_type:
addons = [a for a in addons if a.type == addon_type]
except ValueError:
# `addon_type` is ALL or a type id. Otherwise we ignore it.
pass
# Take out personas since they don't have versions.
groups = dict(partition(addons,
lambda x: x.type == amo.ADDON_PERSONA))
personas, addons = groups.get(True, []), groups.get(False, [])
platform = platform.lower()
if platform != 'all' and platform in amo.PLATFORM_DICT:
pid = amo.PLATFORM_DICT[platform]
f = lambda ps: pid in ps or amo.PLATFORM_ALL in ps
addons = [a for a in addons
if f(a.current_version.supported_platforms)]
if version is not None:
vint = version_int(version)
f_strict = lambda app: (app.min.version_int <= vint
<= app.max.version_int)
f_ignore = lambda app: app.min.version_int <= vint
xs = [(a, a.compatible_apps) for a in addons]
# Iterate over addons, checking compatibility depending on compat_mode.
addons = []
for addon, apps in xs:
app = apps.get(APP)
if compat_mode == 'strict':
if app and f_strict(app):
addons.append(addon)
elif compat_mode == 'ignore':
if app and f_ignore(app):
addons.append(addon)
elif compat_mode == 'normal':
# This does a db hit but it's cached. This handles the cases
# for strict opt-in, binary components, and compat overrides.
v = addon.compatible_version(APP.id, version, platform,
compat_mode)
if v: # There's a compatible version.
addons.append(addon)
# Put personas back in.
addons.extend(personas)
# We prefer add-ons that support the current locale.
lang = translation.get_language()
partitioner = lambda x: (x.description is not None and
(x.description.locale == lang))
groups = dict(partition(addons, partitioner))
good, others = groups.get(True, []), groups.get(False, [])
if shuffle:
random.shuffle(good)
random.shuffle(others)
# If limit=0, we return all addons with `good` coming before `others`.
# Otherwise pad `good` if less than the limit and return the limit.
if limit > 0:
if len(good) < limit:
good.extend(others[:limit - len(good)])
return good[:limit]
else:
good.extend(others)
return good
def extract(addon):
"""Extract indexable attributes from an add-on."""
attrs = ('id', 'slug', 'created', 'last_updated', 'weekly_downloads',
'bayesian_rating', 'average_daily_users', 'status', 'type',
'hotness', 'is_disabled', 'premium_type')
d = dict(zip(attrs, attrgetter(*attrs)(addon)))
# Coerce the Translation into a string.
d['name_sort'] = unicode(addon.name).lower()
translations = addon.translations
d['name'] = list(set(string for _, string in translations[addon.name_id]))
d['description'] = list(
set(string for _, string in translations[addon.description_id]))
d['summary'] = list(
set(string for _, string in translations[addon.summary_id]))
d['authors'] = [a.name for a in addon.listed_authors]
d['device'] = getattr(addon, 'device_ids', [])
# This is an extra query, not good for perf.
d['category'] = getattr(addon, 'category_ids', [])
d['tags'] = getattr(addon, 'tag_list', [])
d['price'] = getattr(addon, 'price', 0.0)
if addon.current_version:
d['platforms'] = [
p.id for p in addon.current_version.supported_platforms
]
d['appversion'] = {}
for app, appver in addon.compatible_apps.items():
if appver:
min_, max_ = appver.min.version_int, appver.max.version_int
else:
# Fake wide compatibility for search tools and personas.
min_, max_ = 0, version_int('9999')
d['appversion'][app.id] = dict(min=min_, max=max_)
try:
d['has_version'] = addon._current_version is not None
except ObjectDoesNotExist:
d['has_version'] = None
d['app'] = [app.id for app in addon.compatible_apps.keys()]
if addon.type == amo.ADDON_PERSONA:
try:
# This would otherwise get attached when by the transformer.
d['weekly_downloads'] = addon.persona.popularity
# Boost on popularity.
d['boost'] = addon.persona.popularity**.2
d['has_theme_rereview'] = (
addon.persona.rereviewqueuetheme_set.exists())
except Persona.DoesNotExist:
# The addon won't have a persona while it's being created.
pass
else:
# Boost by the number of users on a logarithmic scale. The maximum
# boost (11,000,000 users for adblock) is about 5x.
d['boost'] = addon.average_daily_users**.2
# Double the boost if the add-on is public.
if addon.status == amo.STATUS_PUBLIC and 'boost' in d:
d['boost'] = max(d['boost'], 1) * 4
# Indices for each language. languages is a list of locales we want to
# index with analyzer if the string's locale matches.
for analyzer, languages in amo.SEARCH_ANALYZER_MAP.iteritems():
if (not settings.ES_USE_PLUGINS
and analyzer in amo.SEARCH_ANALYZER_PLUGINS):
continue
d['name_' + analyzer] = list(
set(string for locale, string in translations[addon.name_id]
if locale.lower() in languages))
d['summary_' + analyzer] = list(
set(string for locale, string in translations[addon.summary_id]
if locale.lower() in languages))
d['description_' + analyzer] = list(
set(string for locale, string in translations[addon.description_id]
if locale.lower() in languages))
return d
def set_max_appversion(self, version):
"""Set self.max_appversion to the given version."""
self.max_appversion.update(version=version,
version_int=version_int(version))
def between(ver, min, max):
if not (min and max):
return True
return version_int(min) < ver < version_int(max)
def unsign_addons(addon_ids, force=False, **kw):
"""Used to unsign all the versions of an addon that were previously signed.
This is used to revert the signing in case we need to.
It first moves the backup of the signed file back over its original one,
then un-bump the version, and finally re-hash the file.
"""
log.info(u'[{0}] Unsigning addons.'.format(len(addon_ids)))
bumped_suffix = u'.1-signed'
def file_supports_firefox(version):
"""Return a Q object: files supporting at least a firefox version."""
return Q(version__apps__max__application=amo.FIREFOX.id,
version__apps__max__version_int__gte=version_int(version))
is_default_compatible = Q(binary_components=False,
strict_compatibility=False)
# We only want to unsign files that are at least compatible with Firefox
# MIN_D2C_VERSION, or Firefox MIN_NOT_D2C_VERSION if they are not default
# to compatible.
# The signing feature should be supported from Firefox 40 and above, but
# we're still signing some files that are a bit older just in case.
ff_version_filter = ((is_default_compatible
& file_supports_firefox(settings.MIN_D2C_VERSION)) |
(~is_default_compatible & file_supports_firefox(
settings.MIN_NOT_D2C_VERSION)))
addons_emailed = []
# We only care about extensions.
for version in Version.objects.filter(addon_id__in=addon_ids,
addon__type=amo.ADDON_EXTENSION):
# We only unsign files that have been reviewed and are compatible with
# versions of Firefox that are recent enough.
if not version.version.endswith(bumped_suffix):
log.info(u'Version {0} was not bumped, skip.'.format(version.pk))
continue
to_unsign = version.files.filter(ff_version_filter,
status__in=amo.REVIEWED_STATUSES)
if force:
to_unsign = to_unsign.all()
else:
to_unsign = to_unsign.filter(is_signed=False)
if not to_unsign:
log.info(u'Not unsigning addon {0}, version {1} (no files or not '
u'signed)'.format(version.addon, version))
continue
log.info(u'Unsigning addon {0}, version {1}'.format(
version.addon, version))
for file_obj in to_unsign:
if not os.path.isfile(file_obj.file_path):
log.info(u'File {0} does not exist, skip'.format(file_obj.pk))
continue
backup_path = u'{0}.backup_signature'.format(file_obj.file_path)
if not os.path.isfile(backup_path):
log.info(
u'Backup {0} does not exist, skip'.format(backup_path))
continue
# Restore the backup.
shutil.move(backup_path, file_obj.file_path)
file_obj.update(cert_serial_num='', hash=file_obj.generate_hash())
# Now update the Version model, to unbump its version.
unbumped_version = version.version[:-len(bumped_suffix)]
version.update(version=unbumped_version,
version_int=version_int(unbumped_version))
# Warn addon owners that we restored backups.
addon = version.addon
if addon.pk not in addons_emailed:
# Send a mail to the owners/devs warning them we've
# unsigned their addon and restored backups.
qs = (AddonUser.objects.filter(
role=amo.AUTHOR_ROLE_OWNER,
addon=addon).exclude(user__email=None))
emails = qs.values_list('user__email', flat=True)
subject = MAIL_UNSIGN_SUBJECT.format(addon=addon.name)
message = MAIL_UNSIGN_MESSAGE.format(
addon=addon.name,
addon_url=amo.helpers.absolutify(
addon.get_dev_url(action='versions')))
amo.utils.send_mail(
subject,
message,
recipient_list=emails,
fail_silently=True,
headers={'Reply-To': '*****@*****.**'})
addons_emailed.append(addon.pk)
def test_version_int_unicode():
eq_(version_int(u'\u2322 ugh stephend'), 200100)
def test_version_int():
"""Tests that version_int. Corrects our versions."""
eq_(version_int("3.5.0a1pre2"), 3050000001002)
eq_(version_int(""), 200100)
search_opts['tag'] = tag
client = SearchClient()
try:
results = client.query(query, **search_opts)
except SearchError, e:
log.error('Sphinx Error: %s' % e)
return jingo.render(request, 'search/down.html', locals(), status=503)
version_filters = client.meta['versions']
# If we are filtering by a version, make sure we explicitly list it.
if search_opts['version']:
try:
version_filters += (version_int(search_opts['version']),)
except UnicodeEncodeError:
pass # We didn't want to list you anyway.
versions = _get_versions(request, client.meta['versions'],
search_opts['version'])
categories = _get_categories(request, client.meta['categories'],
addon_type, category)
tags = _get_tags(request, client.meta['tags'], tag)
platforms = _get_platforms(request, client.meta['platforms'],
search_opts['platform'])
sort_tabs = _get_sorts(request, sort)
sort_opts = _get_sort_menu(request, sort)
pager = amo.utils.paginate(request, results, search_opts['limit'])
def file_supports_firefox(version):
"""Return a Q object: files supporting at least a firefox version."""
return Q(version__apps__max__application=amo.FIREFOX.id,
version__apps__max__version_int__gte=version_int(version))
def extract(addon):
"""Extract indexable attributes from an add-on."""
attrs = ('id', 'slug', 'app_slug', 'created', 'last_updated',
'weekly_downloads', 'bayesian_rating', 'average_daily_users',
'status', 'type', 'hotness', 'is_disabled', 'premium_type',
'uses_flash')
d = dict(zip(attrs, attrgetter(*attrs)(addon)))
# Coerce the Translation into a string.
d['name_sort'] = unicode(addon.name).lower()
translations = addon.translations
d['name'] = list(set(string for _, string in translations[addon.name_id]))
d['description'] = list(
set(string for _, string in translations[addon.description_id]))
d['summary'] = list(
set(string for _, string in translations[addon.summary_id]))
d['authors'] = [a.name for a in addon.listed_authors]
d['device'] = getattr(addon, 'device_ids', [])
# This is an extra query, not good for perf.
d['category'] = getattr(addon, 'category_ids', [])
d['tags'] = getattr(addon, 'tag_list', [])
d['price'] = getattr(addon, 'price', 0.0)
if addon.current_version:
d['platforms'] = [
p.id for p in addon.current_version.supported_platforms
]
d['appversion'] = {}
for app, appver in addon.compatible_apps.items():
if appver:
min_, max_ = appver.min.version_int, appver.max.version_int
else:
# Fake wide compatibility for search tools and personas.
min_, max_ = 0, version_int('9999')
d['appversion'][app.id] = dict(min=min_, max=max_)
try:
d['has_version'] = addon._current_version is not None
except ObjectDoesNotExist:
d['has_version'] = None
d['app'] = [app.id for app in addon.compatible_apps.keys()]
if addon.type == amo.ADDON_PERSONA:
try:
# This would otherwise get attached when by the transformer.
d['weekly_downloads'] = addon.persona.popularity
# Boost on popularity.
d['_boost'] = addon.persona.popularity**.2
d['has_theme_rereview'] = (
addon.persona.rereviewqueuetheme_set.exists())
except Persona.DoesNotExist:
# The addon won't have a persona while it's being created.
pass
elif addon.type == amo.ADDON_WEBAPP:
installed_ids = list(
Installed.objects.filter(addon=addon).values_list('id', flat=True))
d['popularity'] = d['_boost'] = len(installed_ids)
# Calculate regional popularity for "mature regions"
# (installs + reviews/installs from that region).
installs = dict(
ClientData.objects.filter(installed__in=installed_ids).annotate(
region_counts=Count('region')).values_list(
'region', 'region_counts').distinct())
for region in mkt.regions.ALL_REGION_IDS:
cnt = installs.get(region, 0)
if cnt:
# Magic number (like all other scores up in this piece).
d['popularity_%s' % region] = d['popularity'] + cnt * 10
else:
d['popularity_%s' % region] = len(installed_ids)
d['_boost'] += cnt * 10
d['app_type'] = (amo.ADDON_WEBAPP_PACKAGED
if addon.is_packaged else amo.ADDON_WEBAPP_HOSTED)
else:
# Boost by the number of users on a logarithmic scale. The maximum
# boost (11,000,000 users for adblock) is about 5x.
d['_boost'] = addon.average_daily_users**.2
# Double the boost if the add-on is public.
if addon.status == amo.STATUS_PUBLIC and 'boost' in d:
d['_boost'] = max(d['_boost'], 1) * 4
# Indices for each language. languages is a list of locales we want to
# index with analyzer if the string's locale matches.
for analyzer, languages in amo.SEARCH_ANALYZER_MAP.iteritems():
if (not settings.ES_USE_PLUGINS
and analyzer in amo.SEARCH_ANALYZER_PLUGINS):
continue
d['name_' + analyzer] = list(
set(string for locale, string in translations[addon.name_id]
if locale.lower() in languages))
d['summary_' + analyzer] = list(
set(string for locale, string in translations[addon.summary_id]
if locale.lower() in languages))
d['description_' + analyzer] = list(
set(string for locale, string in translations[addon.description_id]
if locale.lower() in languages))
return d
def get_compat_mode(version):
# Returns appropriate compat mode based on app version.
# Replace when we are ready to deal with bug 711698.
vint = version_int(version)
return 'ignore' if vint >= version_int('10.0') else 'strict'
def fix_let_scope_bustage_in_addons(addon_ids):
"""Used to fix the "let scope bustage" (bug 1224686) in the last version of
the provided add-ons.
This is used in the 'fix_let_scope_bustage' management commands.
It also bumps the version number of the file and the Version, so the
Firefox extension update mecanism picks this new fixed version and installs
it.
"""
log.info(u'[{0}] Fixing addons.'.format(len(addon_ids)))
addons_emailed = []
for addon in Addon.objects.filter(id__in=addon_ids):
# We only care about the latest added version for each add-on.
version = addon.versions.first()
log.info(u'Fixing addon {0}, version {1}'.format(addon, version))
bumped_version_number = u'{0}.1-let-fixed'.format(version.version)
for file_obj in version.files.all():
if not os.path.isfile(file_obj.file_path):
log.info(u'File {0} does not exist, skip'.format(file_obj.pk))
continue
# Save the original file, before bumping the version.
backup_path = u'{0}.backup_let_fix'.format(file_obj.file_path)
shutil.copy(file_obj.file_path, backup_path)
try:
# Apply the fix itself.
fix_let_scope_bustage_in_xpi(file_obj.file_path)
except:
log.error(u'Failed fixing file {0}'.format(file_obj.pk),
exc_info=True)
# Revert the fix by restoring the backup.
shutil.move(backup_path, file_obj.file_path)
continue # We move to the next file.
# Need to bump the version (modify install.rdf or package.json)
# before the file is signed.
update_version_number(file_obj, bumped_version_number)
if file_obj.is_signed: # Only sign if it was already signed.
if file_obj.status == amo.STATUS_PUBLIC:
server = settings.SIGNING_SERVER
else:
server = settings.PRELIMINARY_SIGNING_SERVER
sign_file(file_obj, server)
# Now update the Version model.
version.update(version=bumped_version_number,
version_int=version_int(bumped_version_number))
addon = version.addon
if addon.pk not in addons_emailed:
# Send a mail to the owners/devs warning them we've
# automatically fixed their addon.
qs = (AddonUser.objects.filter(
role=amo.AUTHOR_ROLE_OWNER,
addon=addon).exclude(user__email__isnull=True))
emails = qs.values_list('user__email', flat=True)
subject = MAIL_SUBJECT.format(addon=addon.name)
message = MAIL_MESSAGE.format(
addon=addon.name,
addon_url=amo.helpers.absolutify(
addon.get_dev_url(action='versions')))
amo.utils.send_mail(
subject,
message,
recipient_list=emails,
fail_silently=True,
headers={'Reply-To': '*****@*****.**'})
addons_emailed.append(addon.pk)