def setUp(self):
super(TestModelBase, self).setUp()
self.saved_cb = amo_models._on_change_callbacks.copy()
amo_models._on_change_callbacks.clear()
self.cb = Mock()
self.cb.__name__ = 'testing_mock_callback'
Addon.on_change(self.cb)
def fake_object(self, data):
"""Create a fake instance of Addon and related models from ES data."""
obj = Addon(id=data['id'], slug=data['slug'], is_listed=True)
if data['current_version'] and data['current_version']['files']:
data_version = data['current_version']
obj._current_version = Version(
id=data_version['id'],
reviewed=self.handle_date(data_version['reviewed']),
version=data_version['version'])
obj._current_version.all_files = [
File(
id=file_['id'], created=self.handle_date(file_['created']),
hash=file_['hash'], filename=file_['filename'],
size=file_['size'], status=file_['status'])
for file_ in data_version['files']
]
# Attach base attributes that have the same name/format in ES and in
# the model.
self._attach_fields(
obj, data,
('average_daily_users', 'bayesian_rating', 'created',
'default_locale', 'guid', 'hotness', 'is_listed', 'last_updated',
'public_stats', 'slug', 'status', 'type', 'weekly_downloads'))
# Attach attributes that do not have the same name/format in ES.
obj.tag_list = data['tags']
obj.disabled_by_user = data['is_disabled'] # Not accurate, but enough.
# Attach translations (they require special treatment).
self._attach_translations(obj, data, self.translated_fields)
return obj
def test_filter_or(self):
qs = Addon.search().filter(type=1).filter(or_=dict(status=1, app=2))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [
# {'term': {'type': 1}},
# {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]},
# ]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
or_clause = sorted(filters['and'])[0]
assert or_clause.keys() == ['or']
assert {'term': {'status': 1}} in or_clause['or']
assert {'term': {'app': 2}} in or_clause['or']
qs = Addon.search().filter(type=1, or_=dict(status=1, app=2))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [
# {'term': {'type': 1}},
# {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]},
# ]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
or_clause = sorted(filters['and'])[0]
assert or_clause.keys() == ['or']
assert {'term': {'status': 1}} in or_clause['or']
assert {'term': {'app': 2}} in or_clause['or']
def test_extra_order_by(self):
qs = Addon.search().extra(order_by=['-rating'])
assert qs._build_query()['sort'] == [{'rating': 'desc'}]
qs = Addon.search().order_by('-id').extra(order_by=['-rating'])
assert qs._build_query()['sort'] == [
{'id': 'desc'}, {'rating': 'desc'}]
def test_count_non_dsl_mode(self):
p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False)
assert p._count is None
p.page(1)
assert p.count == Addon.search().count()
def test_multiple_ignored(self):
cb = Mock()
cb.__name__ = 'something'
old = len(amo_models._on_change_callbacks[Addon])
Addon.on_change(cb)
assert len(amo_models._on_change_callbacks[Addon]) == old + 1
Addon.on_change(cb)
assert len(amo_models._on_change_callbacks[Addon]) == old + 1
def test_extra_query(self):
qs = Addon.search().extra(query={'type': 1})
eq_(qs._build_query()['query']['function_score']['query'],
{'term': {'type': 1}})
qs = Addon.search().filter(status=1).extra(query={'type': 1})
filtered = qs._build_query()['query']['filtered']
eq_(filtered['query']['function_score']['query'],
{'term': {'type': 1}})
eq_(filtered['filter'], [{'term': {'status': 1}}])
def test_extra_query(self):
qs = Addon.search().extra(query={'type': 1})
assert qs._build_query()['query'] == (
{'term': {'type': 1}})
qs = Addon.search().filter(status=1).extra(query={'type': 1})
filtered = qs._build_query()['query']['bool']
assert filtered['must'] == (
[{'term': {'type': 1}}])
assert filtered['filter'] == [{'term': {'status': 1}}]
def create_file(self, **kwargs):
addon = Addon()
addon.save()
ver = Version(version='0.1')
ver.addon = addon
ver.save()
f = File(**kwargs)
f.version = ver
f.save()
return f
def test_extra_filter(self):
qs = Addon.search().extra(filter={'category__in': [1, 2]})
eq_(qs._build_query()['query']['filtered']['filter'],
[{'in': {'category': [1, 2]}}])
qs = (Addon.search().filter(type=1)
.extra(filter={'category__in': [1, 2]}))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]}
eq_(filters.keys(), ['and'])
ok_({'term': {'type': 1}} in filters['and'])
ok_({'in': {'category': [1, 2]}} in filters['and'])
def test_extra_filter(self):
qs = Addon.search().extra(filter={'category__in': [1, 2]})
assert qs._build_query()['query']['filtered']['filter'] == (
[{'in': {'category': [1, 2]}}])
qs = (Addon.search().filter(type=1)
.extra(filter={'category__in': [1, 2]}))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
assert {'in': {'category': [1, 2]}} in filters['and']
def test_extra_filter(self):
qs = Addon.search().extra(filter={'category__in': [1, 2]})
assert qs._build_query()['query']['bool']['filter'] == (
[{'terms': {'category': [1, 2]}}])
qs = (Addon.search().filter(type=1)
.extra(filter={'category__in': [1, 2]}))
filters = qs._build_query()['query']['bool']['filter']
# Filters:
# [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}]
assert len(filters) == 2
assert {'term': {'type': 1}} in filters
assert {'terms': {'category': [1, 2]}} in filters
def test_count_non_dsl_mode(self):
addon_factory()
addon_factory()
addon_factory()
self.refresh()
p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False)
assert p.count == 3
p.page(1)
assert p.count == 3
assert p.count == Addon.search().count()
def wrapper(request, addon_id=None, *args, **kw):
"""Provides an addon instance to the view given addon_id, which can be
an Addon pk, guid or a slug."""
assert addon_id, 'Must provide addon id, guid or slug'
lookup_field = Addon.get_lookup_field(addon_id)
if lookup_field == 'slug':
addon = get_object_or_404(qs(), slug=addon_id)
else:
try:
if lookup_field == 'pk':
addon = qs().get(id=addon_id)
elif lookup_field == 'guid':
addon = qs().get(guid=addon_id)
except Addon.DoesNotExist:
raise http.Http404
# Don't get in an infinite loop if addon.slug.isdigit().
if addon.slug and addon.slug != addon_id:
url = request.path.replace(addon_id, addon.slug, 1)
if request.GET:
url += '?' + request.GET.urlencode()
return http.HttpResponsePermanentRedirect(url)
# If the addon is unlisted it needs either an owner/viewer/dev/support,
# or an unlisted addon reviewer.
if not (addon.has_listed_versions() or
owner_or_unlisted_reviewer(request, addon)):
raise http.Http404
return f(request, addon, *args, **kw)
def es_extensions(request, category=None, template=None):
TYPE = amo.ADDON_EXTENSION
if category is not None:
q = Category.objects.filter(application=request.APP.id, type=TYPE)
category = get_object_or_404(q, slug=category)
if ('sort' not in request.GET and not request.MOBILE
and category and category.count > 4):
return category_landing(request, category)
qs = (Addon.search().filter(type=TYPE, app=request.APP.id,
is_disabled=False,
status__in=amo.REVIEWED_STATUSES))
filter = ESAddonFilter(request, qs, key='sort', default='popular')
qs, sorting = filter.qs, filter.field
src = 'cb-btn-%s' % sorting
dl_src = 'cb-dl-%s' % sorting
if category:
qs = qs.filter(category=category.id)
addons = amo.utils.paginate(request, qs)
return render(request, template,
{'section': 'extensions', 'addon_type': TYPE,
'category': category, 'addons': addons,
'filter': filter, 'sorting': sorting,
'sort_opts': filter.opts, 'src': src,
'dl_src': dl_src, 'search_cat': '%s,0' % TYPE})
def test_facet_range(self):
facet = {'range': {'status': [{'lte': 3}, {'gte': 5}]}}
# Pass a copy so edits aren't propagated back here.
qs = Addon.search().filter(app=1).facet(by_status=dict(facet))
assert qs._build_query()['query']['filtered']['filter'] == (
[{'term': {'app': 1}}])
assert qs._build_query()['facets'] == {'by_status': facet}
def test_indexed_count(self):
# Did all the right addons get indexed?
count = Addon.search().filter(type=1, is_disabled=False).count()
# Created in the setUpClass.
assert count == 4 == (
Addon.objects.filter(disabled_by_user=False,
status__in=amo.VALID_ADDON_STATUSES).count())
def handle_upload(self, request, addon, version_string):
if "upload" in request.FILES:
filedata = request.FILES["upload"]
else:
raise forms.ValidationError(_(u'Missing "upload" key in multipart file data.'), status.HTTP_400_BAD_REQUEST)
# Parse the file to get and validate package data with the addon.
pkg = parse_addon(filedata, addon)
if not acl.submission_allowed(request.user, pkg):
raise forms.ValidationError(_(u"You cannot submit this type of add-on"), status.HTTP_400_BAD_REQUEST)
version_string = version_string or pkg["version"]
if version_string and pkg["version"] != version_string:
raise forms.ValidationError(_("Version does not match the manifest file."), status.HTTP_400_BAD_REQUEST)
if addon is not None and addon.versions.filter(version=version_string).exists():
raise forms.ValidationError(_("Version already exists."), status.HTTP_409_CONFLICT)
dont_allow_no_guid = not addon and not pkg.get("guid", None) and not pkg.get("is_webextension", False)
if dont_allow_no_guid:
raise forms.ValidationError(
_("Only WebExtensions are allowed to omit the GUID"), status.HTTP_400_BAD_REQUEST
)
if addon is None:
addon = Addon.create_addon_from_upload_data(data=pkg, user=request.user, upload=filedata, is_listed=False)
created = True
else:
created = False
file_upload = handle_upload(filedata=filedata, user=request.user, addon=addon, submit=True)
return file_upload, created
def handle_upload(self, request, addon, version_string):
if 'upload' in request.FILES:
filedata = request.FILES['upload']
else:
raise forms.ValidationError(
_(u'Missing "upload" key in multipart file data.'),
status.HTTP_400_BAD_REQUEST)
# Parse the file to get and validate package data with the addon.
pkg = parse_addon(filedata, addon)
if not acl.submission_allowed(request.user, pkg):
raise forms.ValidationError(
_(u'You cannot submit this type of add-on'),
status.HTTP_400_BAD_REQUEST)
version_string = version_string or pkg['version']
if version_string and pkg['version'] != version_string:
raise forms.ValidationError(
_('Version does not match the manifest file.'),
status.HTTP_400_BAD_REQUEST)
if (addon is not None and
addon.versions.filter(version=version_string).exists()):
raise forms.ValidationError(
_('Version already exists.'),
status.HTTP_409_CONFLICT)
dont_allow_no_guid = (
not addon and not pkg.get('guid', None) and
not pkg.get('is_webextension', False))
if dont_allow_no_guid:
raise forms.ValidationError(
_('Only WebExtensions are allowed to omit the GUID'),
status.HTTP_400_BAD_REQUEST)
if addon is None:
addon = Addon.create_addon_from_upload_data(
data=pkg, user=request.user, upload=filedata, is_listed=False)
created = True
channel = amo.RELEASE_CHANNEL_UNLISTED
else:
created = False
last_version = addon.find_latest_version_including_rejected()
if last_version:
channel = last_version.channel
else:
# TODO: we need to properly handle channels here and fail if
# no previous version to guess with. Also need to allow the
# channel to be selected for versions.
channel = (amo.RELEASE_CHANNEL_LISTED if addon.is_listed else
amo.RELEASE_CHANNEL_UNLISTED)
file_upload = handle_upload(
filedata=filedata, user=request.user, addon=addon, submit=True,
channel=channel)
return file_upload, created
def test_and(self):
qs = Addon.search().filter(type=1, category__in=[1, 2])
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
assert {'in': {'category': [1, 2]}} in filters['and']
def test_and(self):
qs = Addon.search().filter(type=1, category__in=[1, 2])
filters = qs._build_query()['query']['bool']['filter']
# Filters:
# [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}]
assert len(filters) == 2
assert {'term': {'type': 1}} in filters
assert {'terms': {'category': [1, 2]}} in filters
def test_change_is_not_recursive(self):
class fn:
called = False
def callback(old_attr=None, new_attr=None, instance=None,
sender=None, **kw):
fn.called = True
# Both save and update should be protected:
instance.update(site_specific=False)
instance.save()
Addon.on_change(callback)
addon = Addon.objects.get(pk=3615)
addon.save()
assert fn.called
def test_validate_number(self):
p = amo.utils.ESPaginator(Addon.search(), 20)
# A bad number raises an exception.
with self.assertRaises(paginator.PageNotAnInteger):
p.page('a')
# A large number is ignored.
p.page(99)
def create_addon(self, license=None):
data = self.cleaned_data
a = Addon(guid=data['guid'],
name=data['name'],
type=data['type'],
status=amo.STATUS_UNREVIEWED,
homepage=data['homepage'],
summary=data['summary'])
a.save()
AddonUser(addon=a, user=self.request.user).save()
self.addon = a
# Save Version, attach License
self.create_version(license=license)
amo.log(amo.LOG.CREATE_ADDON, a)
log.info('Addon %d saved' % a.id)
return a
def test_paginate_returns_this_paginator(self):
request = MagicMock()
request.GET.get.return_value = 1
request.GET.urlencode.return_value = ''
request.path = ''
qs = Addon.search()
pager = paginate(request, qs)
assert isinstance(pager.paginator, ESPaginator)
def create_personas(self, number, persona_extras=None):
persona_extras = persona_extras or {}
addon = Addon.objects.get(id=15679)
for i in range(number):
a = Addon(type=amo.ADDON_PERSONA)
a.name = 'persona-%s' % i
a.all_categories = []
a.save()
v = Version.objects.get(addon=addon)
v.addon = a
v.pk = None
v.save()
p = Persona(addon_id=a.id, persona_id=i, **persona_extras)
p.save()
a.persona = p
a._current_version = v
a.status = amo.STATUS_PUBLIC
a.save()
def test_query_multiple_and_range(self):
qs = Addon.search().query(type=1, status__gte=1)
query = qs._build_query()['query']
# Query:
# {'bool': {'must': [{'term': {'type': 1}},
# {'range': {'status': {'gte': 1}}}, ]}}
assert query.keys() == ['bool']
assert query['bool'].keys() == ['must']
assert {'term': {'type': 1}} in query['bool']['must']
assert {'range': {'status': {'gte': 1}}} in query['bool']['must']
def test_featured_ids(self):
FeaturedCollection.objects.filter(collection__addons=3615)[0].delete()
another = Addon.objects.get(id=1003)
self.change_addon(another, 'en-US')
items = Addon.featured_random(amo.FIREFOX, 'en-US')
# The order should be random within those boundaries.
assert [1003, 3481] == sorted(items[0:2])
assert [1001, 2464, 7661, 15679] == sorted(items[2:])
def test_query_or(self):
qs = Addon.search().query(or_=dict(type=1, status__gte=2))
query = qs._build_query()['query']['function_score']['query']
# Query:
# {'bool': {'should': [{'term': {'type': 1}},
# {'range': {'status': {'gte': 2}}}, ]}}
assert query.keys() == ['bool']
assert query['bool'].keys() == ['should']
assert {'term': {'type': 1}} in query['bool']['should']
assert {'range': {'status': {'gte': 2}}} in query['bool']['should']
def test_query_multiple_and_range(self):
qs = Addon.search().query(type=1, status__gte=1)
query = qs._build_query()['query']['function_score']['query']
# Query:
# {'bool': {'must': [{'term': {'type': 1}},
# {'range': {'status': {'gte': 1}}}, ]}}
eq_(query.keys(), ['bool'])
eq_(query['bool'].keys(), ['must'])
ok_({'term': {'type': 1}} in query['bool']['must'])
ok_({'range': {'status': {'gte': 1}}} in query['bool']['must'])
def test_es_paginator(self):
qs = Addon.search()
pager = amo.utils.paginate(self.request, qs)
assert isinstance(pager.paginator, amo.utils.ESPaginator)
def add_static_theme_from_lwt(lwt):
from olympia.activity.models import AddonLog
timer = StopWatch(
'addons.tasks.migrate_lwts_to_static_theme.add_from_lwt.')
timer.start()
olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID))
# Try to handle LWT with no authors
author = (lwt.listed_authors or [_get_lwt_default_author()])[0]
# Wrap zip in FileUpload for Addon/Version from_upload to consume.
upload = FileUpload.objects.create(
user=author, valid=True)
filename = uuid.uuid4().hex + '.xpi'
destination = os.path.join(user_media_path('addons'), 'temp', filename)
build_static_theme_xpi_from_lwt(lwt, destination)
upload.update(path=destination, name=filename)
timer.log_interval('1.build_xpi')
# Create addon + version
parsed_data = parse_addon(upload, user=author)
timer.log_interval('2a.parse_addon')
addon = Addon.initialize_addon_from_upload(
parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author)
addon_updates = {}
timer.log_interval('2b.initialize_addon')
# static themes are only compatible with Firefox at the moment,
# not Android
version = Version.from_upload(
upload, addon, selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
timer.log_interval('3.initialize_version')
# Set category
lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat.
lwt_category_slug = lwt_category.slug if lwt_category else 'other'
for app, type_dict in CATEGORIES.items():
static_theme_categories = type_dict.get(amo.ADDON_STATICTHEME, [])
static_category = static_theme_categories.get(
lwt_category_slug, static_theme_categories.get('other'))
AddonCategory.objects.create(
addon=addon,
category=Category.from_static_category(static_category, True))
timer.log_interval('4.set_categories')
# Set license
lwt_license = PERSONA_LICENSES_IDS.get(
lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright
static_license = License.objects.get(builtin=lwt_license.builtin)
version.update(license=static_license)
timer.log_interval('5.set_license')
# Set tags
for addon_tag in AddonTag.objects.filter(addon=lwt):
AddonTag.objects.create(addon=addon, tag=addon_tag.tag)
timer.log_interval('6.set_tags')
# Steal the ratings (even with soft delete they'll be deleted anyway)
addon_updates.update(
average_rating=lwt.average_rating,
bayesian_rating=lwt.bayesian_rating,
total_ratings=lwt.total_ratings,
text_ratings_count=lwt.text_ratings_count)
Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version)
timer.log_interval('7.move_ratings')
# Replace the lwt in collections
CollectionAddon.objects.filter(addon=lwt).update(addon=addon)
# Modify the activity log entry too.
rating_activity_log_ids = [
l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review']
addonlog_qs = AddonLog.objects.filter(
addon=lwt, activity_log__action__in=rating_activity_log_ids)
[alog.transfer(addon) for alog in addonlog_qs.iterator()]
timer.log_interval('8.move_activity_logs')
# Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats
# dashboard and future update counts, and copy the average_daily_users.
# hotness will be recalculated by the deliver_hotness() cron in a more
# reliable way that we could do, so skip it entirely.
migrate_theme_update_count(lwt, addon)
addon_updates.update(
average_daily_users=lwt.persona.popularity or 0,
hotness=0)
timer.log_interval('9.copy_statistics')
# Logging
activity.log_create(
amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author)
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(
datestatuschanged=lwt.last_updated,
reviewed=datetime.now(),
status=amo.STATUS_PUBLIC)
timer.log_interval('10.sign_files')
addon_updates['status'] = amo.STATUS_PUBLIC
# set the modified and creation dates to match the original.
addon_updates['created'] = lwt.created
addon_updates['modified'] = lwt.modified
addon_updates['last_updated'] = lwt.last_updated
addon.update(**addon_updates)
return addon
def check_xpi_info(xpi_info, addon=None, xpi_file=None, user=None):
from olympia.addons.models import Addon, DeniedGuid
guid = xpi_info['guid']
# If we allow the guid to be omitted we assume that one was generated
# or existed before and use that one.
# An example are WebExtensions that don't require a guid but we generate
# one once they're uploaded. Now, if you update that WebExtension we
# just use the original guid.
if addon and not guid:
xpi_info['guid'] = guid = addon.guid
if guid:
if user:
deleted_guid_clashes = Addon.unfiltered.exclude(
authors__id=user.id).filter(guid=guid)
else:
deleted_guid_clashes = Addon.unfiltered.filter(guid=guid)
if addon and addon.guid != guid:
msg = gettext('The add-on ID in your manifest.json (%s) '
'does not match the ID of your add-on on AMO (%s)')
raise forms.ValidationError(msg % (guid, addon.guid))
if (not addon
# Non-deleted add-ons.
and
(Addon.objects.filter(guid=guid).exists()
# DeniedGuid objects for deletions for Mozilla disabled add-ons
or DeniedGuid.objects.filter(guid=guid).exists()
# Deleted add-ons that don't belong to the uploader.
or deleted_guid_clashes.exists())):
raise forms.ValidationError(gettext('Duplicate add-on ID found.'))
if len(xpi_info['version']) > 32:
raise forms.ValidationError(
gettext('Version numbers should have fewer than 32 characters.'))
if not VERSION_RE.match(xpi_info['version']):
raise forms.ValidationError(
gettext('Version numbers should only contain letters, numbers, '
'and these punctuation characters: +*.-_.'))
if xpi_info.get('type') == amo.ADDON_STATICTHEME:
max_size = settings.MAX_STATICTHEME_SIZE
if xpi_file and xpi_file.size > max_size:
raise forms.ValidationError(
gettext('Maximum size for WebExtension themes is {0}.').format(
filesizeformat(max_size)))
if xpi_file:
# Make sure we pass in a copy of `xpi_info` since
# `resolve_webext_translations` modifies data in-place
translations = Addon.resolve_webext_translations(
xpi_info.copy(), xpi_file)
verify_mozilla_trademark(translations['name'], user)
# Parse the file to get and validate package data with the addon.
if not acl.experiments_submission_allowed(user, xpi_info):
raise forms.ValidationError(
gettext('You cannot submit this type of add-on'))
if not addon and not acl.reserved_guid_addon_submission_allowed(
user, xpi_info):
raise forms.ValidationError(
gettext(
'You cannot submit an add-on using an ID ending with this suffix'
))
if not acl.mozilla_signed_extension_submission_allowed(user, xpi_info):
raise forms.ValidationError(
gettext('You cannot submit a Mozilla Signed Extension'))
if (not addon and guid and guid.lower().endswith(amo.RESERVED_ADDON_GUIDS)
and not xpi_info.get('is_mozilla_signed_extension')):
raise forms.ValidationError(
gettext(
'Add-ons using an ID ending with this suffix need to be signed with '
'privileged certificate before being submitted'))
if not acl.langpack_submission_allowed(user, xpi_info):
raise forms.ValidationError(
gettext('You cannot submit a language pack'))
if not acl.site_permission_addons_submission_allowed(user, xpi_info):
raise forms.ValidationError(
gettext('You cannot submit this type of add-on'))
return xpi_info
def test_cache_key():
# Test that we are not taking the db into account when building our
# cache keys for django-cache-machine. See bug 928881.
assert Addon._cache_key(1, 'default') == Addon._cache_key(1, 'slave')
def handle_upload(self, request, addon, version_string, guid=None):
if 'upload' in request.FILES:
filedata = request.FILES['upload']
else:
raise forms.ValidationError(
_(u'Missing "upload" key in multipart file data.'),
status.HTTP_400_BAD_REQUEST)
# Parse the file to get and validate package data with the addon.
pkg = parse_addon(filedata, addon)
if not acl.submission_allowed(request.user, pkg):
raise forms.ValidationError(
_(u'You cannot submit this type of add-on'),
status.HTTP_400_BAD_REQUEST)
if addon is not None and addon.status == amo.STATUS_DISABLED:
raise forms.ValidationError(
_('You cannot add versions to an addon that has status: %s.' %
amo.STATUS_CHOICES_ADDON[amo.STATUS_DISABLED]),
status.HTTP_400_BAD_REQUEST)
version_string = version_string or pkg['version']
if version_string and pkg['version'] != version_string:
raise forms.ValidationError(
_('Version does not match the manifest file.'),
status.HTTP_400_BAD_REQUEST)
if (addon is not None
and addon.versions.filter(version=version_string).exists()):
raise forms.ValidationError(_('Version already exists.'),
status.HTTP_409_CONFLICT)
package_guid = pkg.get('guid', None)
dont_allow_no_guid = (not addon and not package_guid
and not pkg.get('is_webextension', False))
if dont_allow_no_guid:
raise forms.ValidationError(
_('Only WebExtensions are allowed to omit the GUID'),
status.HTTP_400_BAD_REQUEST)
if guid is not None and not addon and not package_guid:
# No guid was present in the package, but one was provided in the
# URL, so we take it instead of generating one ourselves. But
# first, validate it properly.
if not amo.ADDON_GUID_PATTERN.match(guid):
raise forms.ValidationError(_('Invalid GUID in URL'),
status.HTTP_400_BAD_REQUEST)
pkg['guid'] = guid
# channel will be ignored for new addons.
if addon is None:
channel = amo.RELEASE_CHANNEL_UNLISTED # New is always unlisted.
addon = Addon.create_addon_from_upload_data(data=pkg,
user=request.user,
upload=filedata,
channel=channel)
created = True
else:
created = False
channel_param = request.POST.get('channel')
channel = amo.CHANNEL_CHOICES_LOOKUP.get(channel_param)
if not channel:
last_version = (addon.find_latest_version(None, exclude=()))
if last_version:
channel = last_version.channel
else:
channel = amo.RELEASE_CHANNEL_UNLISTED # Treat as new.
will_have_listed = channel == amo.RELEASE_CHANNEL_LISTED
if not addon.has_complete_metadata(
has_listed_versions=will_have_listed):
raise forms.ValidationError(
_('You cannot add a listed version to this addon '
'via the API due to missing metadata. '
'Please submit via the website'),
status.HTTP_400_BAD_REQUEST)
file_upload = handle_upload(filedata=filedata,
user=request.user,
addon=addon,
submit=True,
channel=channel)
return file_upload, created
def check_xpi_info(xpi_info, addon=None, xpi_file=None, user=None):
from olympia.addons.models import Addon, DeniedGuid
guid = xpi_info['guid']
is_webextension = xpi_info.get('is_webextension', False)
# If we allow the guid to be omitted we assume that one was generated
# or existed before and use that one.
# An example are WebExtensions that don't require a guid but we generate
# one once they're uploaded. Now, if you update that WebExtension we
# just use the original guid.
if addon and not guid and is_webextension:
xpi_info['guid'] = guid = addon.guid
if not guid and not is_webextension:
raise forms.ValidationError(ugettext('Could not find an add-on ID.'))
if guid:
current_user = core.get_user()
if current_user:
deleted_guid_clashes = Addon.unfiltered.exclude(
authors__id=current_user.id).filter(guid=guid)
else:
deleted_guid_clashes = Addon.unfiltered.filter(guid=guid)
if addon and addon.guid != guid:
msg = ugettext(
'The add-on ID in your manifest.json or install.rdf (%s) '
'does not match the ID of your add-on on AMO (%s)')
raise forms.ValidationError(msg % (guid, addon.guid))
if (not addon and
# Non-deleted add-ons.
(
Addon.objects.filter(guid=guid).exists() or
# DeniedGuid objects for deletions for Mozilla disabled add-ons
DeniedGuid.objects.filter(guid=guid).exists() or
# Deleted add-ons that don't belong to the uploader.
deleted_guid_clashes.exists())):
raise forms.ValidationError(ugettext('Duplicate add-on ID found.'))
if len(xpi_info['version']) > 32:
raise forms.ValidationError(
ugettext('Version numbers should have fewer than 32 characters.'))
if not VERSION_RE.match(xpi_info['version']):
raise forms.ValidationError(
ugettext('Version numbers should only contain letters, numbers, '
'and these punctuation characters: +*.-_.'))
if is_webextension and xpi_info.get('type') == amo.ADDON_STATICTHEME:
max_size = settings.MAX_STATICTHEME_SIZE
if xpi_file and os.path.getsize(xpi_file.name) > max_size:
raise forms.ValidationError(
ugettext(
u'Maximum size for WebExtension themes is {0}.').format(
filesizeformat(max_size)))
if xpi_file:
# Make sure we pass in a copy of `xpi_info` since
# `resolve_webext_translations` modifies data in-place
translations = Addon.resolve_webext_translations(
xpi_info.copy(), xpi_file)
verify_mozilla_trademark(translations['name'], core.get_user())
# Parse the file to get and validate package data with the addon.
if not acl.submission_allowed(user, xpi_info):
raise forms.ValidationError(
ugettext(u'You cannot submit this type of add-on'))
if not addon and not system_addon_submission_allowed(user, xpi_info):
guids = ' or '.join('"' + guid + '"'
for guid in amo.SYSTEM_ADDON_GUIDS)
raise forms.ValidationError(
ugettext(u'You cannot submit an add-on with a guid ending '
u'%s' % guids))
if not mozilla_signed_extension_submission_allowed(user, xpi_info):
raise forms.ValidationError(
ugettext(u'You cannot submit a Mozilla Signed Extension'))
return xpi_info
def test_getitem(self):
addons = list(Addon.search())
assert addons[0] == Addon.search()[0]
def test_iter(self):
qs = Addon.search().filter(type=1, is_disabled=False)
assert len(qs) == len(list(qs))
def test_count(self):
assert Addon.search().count() == 6
def test_object_result_slice(self):
addon = self._addons[0]
qs = Addon.search().filter(id=addon.id)
assert addon == qs[0]
def test_values(self):
qs = Addon.search().values('name')
assert qs._build_query()['fields'] == ['id', 'name']
def test_extra_bad_key(self):
with self.assertRaises(AssertionError):
Addon.search().extra(x=1)
def test_len(self):
qs = Addon.search()
qs._results_cache = [1]
assert len(qs) == 1
def test_slice_stop_zero(self):
qs = Addon.search()[:0]
assert qs._build_query()['size'] == 0
def unindex_addons(ids, **kw):
for addon in ids:
log.info('Removing addon [%s] from search index.' % addon)
Addon.unindex(addon)
def test_slice(self):
qs = Addon.search()[5:12]
assert qs._build_query()['from'] == 5
assert qs._build_query()['size'] == 7
def process_request(self, query, addon_type='ALL', limit=10,
platform='ALL', version=None, compat_mode='strict'):
"""
Query the search backend and serve up the XML.
"""
limit = min(MAX_LIMIT, int(limit))
app_id = self.request.APP.id
# We currently filter for status=PUBLIC for all versions. If
# that changes, the contract for API version 1.5 requires
# that we continue filtering for it there.
filters = {
'app': app_id,
'status': amo.STATUS_PUBLIC,
'is_listed': True,
'is_experimental': False,
'is_disabled': False,
'has_version': True,
}
# Opts may get overridden by query string filters.
opts = {
'addon_type': addon_type,
'version': version,
}
# Specific case for Personas (bug 990768): if we search providing the
# Persona addon type (9), don't filter on the platform as Personas
# don't have compatible platforms to filter on.
if addon_type != '9':
opts['platform'] = platform
if self.version < 1.5:
# Fix doubly encoded query strings.
try:
query = urllib.unquote(query.encode('ascii'))
except UnicodeEncodeError:
# This fails if the string is already UTF-8.
pass
query, qs_filters, params = extract_filters(query, opts)
qs = Addon.search().query(or_=name_query(query))
filters.update(qs_filters)
if 'type' not in filters:
# Filter by ALL types, which is really all types except for apps.
filters['type__in'] = list(amo.ADDON_SEARCH_TYPES)
qs = qs.filter(**filters)
qs = qs[:limit]
total = qs.count()
results = []
for addon in qs:
compat_version = addon.compatible_version(app_id,
params['version'],
params['platform'],
compat_mode)
# Specific case for Personas (bug 990768): if we search providing
# the Persona addon type (9), then don't look for a compatible
# version.
if compat_version or addon_type == '9':
addon.compat_version = compat_version
results.append(addon)
if len(results) == limit:
break
else:
# We're excluding this addon because there are no
# compatible versions. Decrement the total.
total -= 1
return self.render('legacy_api/search.xml', {
'results': results,
'total': total,
# For caching
'version': version,
'compat_mode': compat_mode,
})
def test_order_by_multiple(self):
qs = Addon.search().order_by('-rating', 'id')
assert qs._build_query()['sort'] == [{'rating': 'desc'}, 'id']
def test_generate_filename_ja(self):
f = File()
f.version = Version(version='0.1.7')
f.version.compatible_apps = (amo.FIREFOX, )
f.version.addon = Addon(name=u' フォクすけ といっしょ')
eq_(f.generate_filename(), 'addon-0.1.7-fx.xpi')
def test_order_by_asc(self):
qs = Addon.search().order_by('rating')
assert qs._build_query()['sort'] == ['rating']
def test_get_unfiltered_manager(self):
Addon.get_unfiltered_manager() == Addon.unfiltered
UserProfile.get_unfiltered_manager() == UserProfile.objects
def test_order_by_desc(self):
qs = Addon.search().order_by('-rating')
assert qs._build_query()['sort'] == [{'rating': 'desc'}]
def fake_object(self, data):
"""Create a fake instance of Addon and related models from ES data."""
obj = Addon(id=data['id'], slug=data['slug'])
# Attach base attributes that have the same name/format in ES and in
# the model.
self._attach_fields(
obj, data,
('average_daily_users', 'bayesian_rating', 'created',
'default_locale', 'guid', 'has_eula', 'has_privacy_policy',
'hotness', 'icon_type', 'is_experimental', 'last_updated',
'modified', 'public_stats', 'slug', 'status', 'type',
'view_source', 'weekly_downloads'))
# Attach attributes that do not have the same name/format in ES.
obj.tag_list = data['tags']
obj.disabled_by_user = data['is_disabled'] # Not accurate, but enough.
obj.all_categories = [
CATEGORIES_BY_ID[cat_id] for cat_id in data.get('category', [])
]
# Attach translations (they require special treatment).
self._attach_translations(obj, data, self.translated_fields)
# Attach related models (also faking them). `current_version` is a
# property we can't write to, so we use the underlying field which
# begins with an underscore. `current_beta_version` and
# `latest_unlisted_version` are writeable cached_property so we can
# directly write to them.
obj.current_beta_version = self.fake_version_object(
obj, data.get('current_beta_version'), amo.RELEASE_CHANNEL_LISTED)
obj._current_version = self.fake_version_object(
obj, data.get('current_version'), amo.RELEASE_CHANNEL_LISTED)
obj.latest_unlisted_version = self.fake_version_object(
obj, data.get('latest_unlisted_version'),
amo.RELEASE_CHANNEL_UNLISTED)
data_authors = data.get('listed_authors', [])
obj.listed_authors = [
UserProfile(id=data_author['id'],
display_name=data_author['name'],
username=data_author['username'])
for data_author in data_authors
]
# We set obj.all_previews to the raw preview data because
# ESPreviewSerializer will handle creating the fake Preview object
# for us when its to_representation() method is called.
obj.all_previews = data.get('previews', [])
obj.average_rating = data.get('ratings', {}).get('average')
obj.total_reviews = data.get('ratings', {}).get('count')
if data['type'] == amo.ADDON_PERSONA:
persona_data = data.get('persona')
if persona_data:
obj.persona = Persona(
addon=obj,
accentcolor=persona_data['accentcolor'],
display_username=persona_data['author'],
header=persona_data['header'],
footer=persona_data['footer'],
persona_id=1 if persona_data['is_new'] else None,
textcolor=persona_data['textcolor'])
else:
# Sadly, https://code.djangoproject.com/ticket/14368 prevents
# us from setting obj.persona = None. This is fixed in
# Django 1.9, but in the meantime, work around it by creating
# a Persona instance with a custom attribute indicating that
# it should not be used.
obj.persona = Persona()
obj.persona._broken = True
return obj
if is_beta:
log.error('[@None] Not creating beta version {0} for new '
'"{1}" language pack'.format(data['version'], xpi))
return
if (Addon.objects.filter(name__localized_string=data['name'])
.exists()):
data['old_name'] = data['name']
data['name'] = u'{0} ({1})'.format(
data['old_name'], data['apps'][0].appdata.pretty)
log.warning(u'[@None] Creating langpack {guid}: Add-on with '
u'name {old_name!r} already exists, trying '
u'{name!r}.'.format(**data))
addon = Addon.from_upload(
upload, [amo.PLATFORM_ALL.id], parsed_data=data)
AddonUser(addon=addon, user=owner).save()
version = addon.versions.get()
if addon.default_locale.lower() == lang.lower():
addon.target_locale = addon.default_locale
addon.save()
log.info('[@None] Created new "{0}" language pack, version {1}'
.format(xpi, data['version']))
# Set the category
for app in version.compatible_apps:
static_category = (
CATEGORIES.get(app.id, []).get(amo.ADDON_LPAPP, [])
def test_count(self):
p = amo.utils.ESPaginator(Addon.search(), 20)
assert p._count is None
p.page(1)
assert p.count == Addon.search().count()
def get_bayesian_rating(self):
q = Addon.search().filter(id=self.addon.id)
return list(q.values_dict('bayesian_rating'))[0]['bayesian_rating'][0]
def test_clone(self):
# Doing a filter creates a new ES object.
qs = Addon.search()
qs2 = qs.filter(type=1)
assert 'filtered' not in qs._build_query()['query']
assert 'filtered' in qs2._build_query()['query']
def search(request, tag_name=None, template=None):
APP = request.APP
types = (amo.ADDON_EXTENSION, amo.ADDON_THEME, amo.ADDON_DICT,
amo.ADDON_SEARCH, amo.ADDON_LPAPP)
category = request.GET.get('cat')
if category == 'collections':
extra_params = {'sort': {'newest': 'created'}}
else:
extra_params = None
fixed = fix_search_query(request.GET, extra_params=extra_params)
if fixed is not request.GET:
return http.HttpResponsePermanentRedirect(urlparams(request.path,
**fixed))
facets = request.GET.copy()
# In order to differentiate between "all versions" and an undefined value,
# we use "any" instead of "" in the frontend.
if 'appver' in facets and facets['appver'] == 'any':
facets['appver'] = ''
form = ESSearchForm(facets or {})
form.is_valid() # Let the form try to clean data.
form_data = form.cleaned_data
if tag_name:
form_data['tag'] = tag_name
if category == 'collections':
return _collections(request)
elif category == 'themes' or form_data.get('atype') == amo.ADDON_PERSONA:
return _personas(request)
sort, extra_sort = split_choices(form.sort_choices, 'created')
if form_data.get('atype') == amo.ADDON_SEARCH:
# Search add-ons should not be searched by ADU, so replace 'Users'
# sort with 'Weekly Downloads'.
sort, extra_sort = list(sort), list(extra_sort)
sort[1] = extra_sort[1]
del extra_sort[1]
# Perform search, using aggregation so that we can build the facets UI.
# Note that we don't need to aggregate on platforms, that facet it built
# from our constants directly, using the current application for this
# request (request.APP).
appversion_field = 'current_version.compatible_apps.%s.max' % APP.id
qs = (Addon.search_public().filter(app=APP.id)
.aggregate(tags={'terms': {'field': 'tags'}},
appversions={'terms': {'field': appversion_field}},
categories={'terms': {'field': 'category', 'size': 200}})
)
filters = ['atype', 'appver', 'cat', 'sort', 'tag', 'platform']
mapping = {'users': '-average_daily_users',
'rating': '-bayesian_rating',
'created': '-created',
'name': 'name_sort',
'downloads': '-weekly_downloads',
'updated': '-last_updated',
'hotness': '-hotness'}
qs = _filter_search(request, qs, form_data, filters, mapping, types=types)
pager = amo.utils.paginate(request, qs)
ctx = {
'is_pjax': request.META.get('HTTP_X_PJAX'),
'pager': pager,
'query': form_data,
'form': form,
'sort_opts': sort,
'extra_sort_opts': extra_sort,
'sorting': sort_sidebar(request, form_data, form),
'sort': form_data.get('sort'),
}
if not ctx['is_pjax']:
aggregations = pager.object_list.aggregations
ctx.update({
'tag': tag_name,
'categories': category_sidebar(request, form_data, aggregations),
'platforms': platform_sidebar(request, form_data),
'versions': version_sidebar(request, form_data, aggregations),
'tags': tag_sidebar(request, form_data, aggregations),
})
return render(request, template, ctx)
def test_object_result(self):
qs = Addon.search().filter(id=self._addons[0].id)[:1]
assert self._addons[:1] == list(qs)
def test_count_uses_cached_results(self):
qs = Addon.search()
qs._results_cache = mock.Mock()
qs._results_cache.count = mock.sentinel.count
assert qs.count() == mock.sentinel.count
