def setUp(self): super(TestModelBase, self).setUp() self.saved_cb = amo_models._on_change_callbacks.copy() amo_models._on_change_callbacks.clear() self.cb = Mock() self.cb.__name__ = 'testing_mock_callback' Addon.on_change(self.cb)
def fake_object(self, data): """Create a fake instance of Addon and related models from ES data.""" obj = Addon(id=data['id'], slug=data['slug'], is_listed=True) if data['current_version'] and data['current_version']['files']: data_version = data['current_version'] obj._current_version = Version( id=data_version['id'], reviewed=self.handle_date(data_version['reviewed']), version=data_version['version']) obj._current_version.all_files = [ File( id=file_['id'], created=self.handle_date(file_['created']), hash=file_['hash'], filename=file_['filename'], size=file_['size'], status=file_['status']) for file_ in data_version['files'] ] # Attach base attributes that have the same name/format in ES and in # the model. self._attach_fields( obj, data, ('average_daily_users', 'bayesian_rating', 'created', 'default_locale', 'guid', 'hotness', 'is_listed', 'last_updated', 'public_stats', 'slug', 'status', 'type', 'weekly_downloads')) # Attach attributes that do not have the same name/format in ES. obj.tag_list = data['tags'] obj.disabled_by_user = data['is_disabled'] # Not accurate, but enough. # Attach translations (they require special treatment). self._attach_translations(obj, data, self.translated_fields) return obj
def test_filter_or(self): qs = Addon.search().filter(type=1).filter(or_=dict(status=1, app=2)) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [ # {'term': {'type': 1}}, # {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}, # ]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] or_clause = sorted(filters['and'])[0] assert or_clause.keys() == ['or'] assert {'term': {'status': 1}} in or_clause['or'] assert {'term': {'app': 2}} in or_clause['or'] qs = Addon.search().filter(type=1, or_=dict(status=1, app=2)) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [ # {'term': {'type': 1}}, # {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}, # ]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] or_clause = sorted(filters['and'])[0] assert or_clause.keys() == ['or'] assert {'term': {'status': 1}} in or_clause['or'] assert {'term': {'app': 2}} in or_clause['or']
def test_extra_order_by(self): qs = Addon.search().extra(order_by=['-rating']) assert qs._build_query()['sort'] == [{'rating': 'desc'}] qs = Addon.search().order_by('-id').extra(order_by=['-rating']) assert qs._build_query()['sort'] == [ {'id': 'desc'}, {'rating': 'desc'}]
def test_count_non_dsl_mode(self): p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False) assert p._count is None p.page(1) assert p.count == Addon.search().count()
def test_multiple_ignored(self): cb = Mock() cb.__name__ = 'something' old = len(amo_models._on_change_callbacks[Addon]) Addon.on_change(cb) assert len(amo_models._on_change_callbacks[Addon]) == old + 1 Addon.on_change(cb) assert len(amo_models._on_change_callbacks[Addon]) == old + 1
def test_extra_query(self): qs = Addon.search().extra(query={'type': 1}) eq_(qs._build_query()['query']['function_score']['query'], {'term': {'type': 1}}) qs = Addon.search().filter(status=1).extra(query={'type': 1}) filtered = qs._build_query()['query']['filtered'] eq_(filtered['query']['function_score']['query'], {'term': {'type': 1}}) eq_(filtered['filter'], [{'term': {'status': 1}}])
def test_extra_query(self): qs = Addon.search().extra(query={'type': 1}) assert qs._build_query()['query'] == ( {'term': {'type': 1}}) qs = Addon.search().filter(status=1).extra(query={'type': 1}) filtered = qs._build_query()['query']['bool'] assert filtered['must'] == ( [{'term': {'type': 1}}]) assert filtered['filter'] == [{'term': {'status': 1}}]
def create_file(self, **kwargs): addon = Addon() addon.save() ver = Version(version='0.1') ver.addon = addon ver.save() f = File(**kwargs) f.version = ver f.save() return f
def test_extra_filter(self): qs = Addon.search().extra(filter={'category__in': [1, 2]}) eq_(qs._build_query()['query']['filtered']['filter'], [{'in': {'category': [1, 2]}}]) qs = (Addon.search().filter(type=1) .extra(filter={'category__in': [1, 2]})) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]} eq_(filters.keys(), ['and']) ok_({'term': {'type': 1}} in filters['and']) ok_({'in': {'category': [1, 2]}} in filters['and'])
def test_extra_filter(self): qs = Addon.search().extra(filter={'category__in': [1, 2]}) assert qs._build_query()['query']['filtered']['filter'] == ( [{'in': {'category': [1, 2]}}]) qs = (Addon.search().filter(type=1) .extra(filter={'category__in': [1, 2]})) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] assert {'in': {'category': [1, 2]}} in filters['and']
def test_extra_filter(self): qs = Addon.search().extra(filter={'category__in': [1, 2]}) assert qs._build_query()['query']['bool']['filter'] == ( [{'terms': {'category': [1, 2]}}]) qs = (Addon.search().filter(type=1) .extra(filter={'category__in': [1, 2]})) filters = qs._build_query()['query']['bool']['filter'] # Filters: # [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}] assert len(filters) == 2 assert {'term': {'type': 1}} in filters assert {'terms': {'category': [1, 2]}} in filters
def test_count_non_dsl_mode(self): addon_factory() addon_factory() addon_factory() self.refresh() p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False) assert p.count == 3 p.page(1) assert p.count == 3 assert p.count == Addon.search().count()
def wrapper(request, addon_id=None, *args, **kw): """Provides an addon instance to the view given addon_id, which can be an Addon pk, guid or a slug.""" assert addon_id, 'Must provide addon id, guid or slug' lookup_field = Addon.get_lookup_field(addon_id) if lookup_field == 'slug': addon = get_object_or_404(qs(), slug=addon_id) else: try: if lookup_field == 'pk': addon = qs().get(id=addon_id) elif lookup_field == 'guid': addon = qs().get(guid=addon_id) except Addon.DoesNotExist: raise http.Http404 # Don't get in an infinite loop if addon.slug.isdigit(). if addon.slug and addon.slug != addon_id: url = request.path.replace(addon_id, addon.slug, 1) if request.GET: url += '?' + request.GET.urlencode() return http.HttpResponsePermanentRedirect(url) # If the addon is unlisted it needs either an owner/viewer/dev/support, # or an unlisted addon reviewer. if not (addon.has_listed_versions() or owner_or_unlisted_reviewer(request, addon)): raise http.Http404 return f(request, addon, *args, **kw)
def es_extensions(request, category=None, template=None): TYPE = amo.ADDON_EXTENSION if category is not None: q = Category.objects.filter(application=request.APP.id, type=TYPE) category = get_object_or_404(q, slug=category) if ('sort' not in request.GET and not request.MOBILE and category and category.count > 4): return category_landing(request, category) qs = (Addon.search().filter(type=TYPE, app=request.APP.id, is_disabled=False, status__in=amo.REVIEWED_STATUSES)) filter = ESAddonFilter(request, qs, key='sort', default='popular') qs, sorting = filter.qs, filter.field src = 'cb-btn-%s' % sorting dl_src = 'cb-dl-%s' % sorting if category: qs = qs.filter(category=category.id) addons = amo.utils.paginate(request, qs) return render(request, template, {'section': 'extensions', 'addon_type': TYPE, 'category': category, 'addons': addons, 'filter': filter, 'sorting': sorting, 'sort_opts': filter.opts, 'src': src, 'dl_src': dl_src, 'search_cat': '%s,0' % TYPE})
def test_facet_range(self): facet = {'range': {'status': [{'lte': 3}, {'gte': 5}]}} # Pass a copy so edits aren't propagated back here. qs = Addon.search().filter(app=1).facet(by_status=dict(facet)) assert qs._build_query()['query']['filtered']['filter'] == ( [{'term': {'app': 1}}]) assert qs._build_query()['facets'] == {'by_status': facet}
def test_indexed_count(self): # Did all the right addons get indexed? count = Addon.search().filter(type=1, is_disabled=False).count() # Created in the setUpClass. assert count == 4 == ( Addon.objects.filter(disabled_by_user=False, status__in=amo.VALID_ADDON_STATUSES).count())
def handle_upload(self, request, addon, version_string): if "upload" in request.FILES: filedata = request.FILES["upload"] else: raise forms.ValidationError(_(u'Missing "upload" key in multipart file data.'), status.HTTP_400_BAD_REQUEST) # Parse the file to get and validate package data with the addon. pkg = parse_addon(filedata, addon) if not acl.submission_allowed(request.user, pkg): raise forms.ValidationError(_(u"You cannot submit this type of add-on"), status.HTTP_400_BAD_REQUEST) version_string = version_string or pkg["version"] if version_string and pkg["version"] != version_string: raise forms.ValidationError(_("Version does not match the manifest file."), status.HTTP_400_BAD_REQUEST) if addon is not None and addon.versions.filter(version=version_string).exists(): raise forms.ValidationError(_("Version already exists."), status.HTTP_409_CONFLICT) dont_allow_no_guid = not addon and not pkg.get("guid", None) and not pkg.get("is_webextension", False) if dont_allow_no_guid: raise forms.ValidationError( _("Only WebExtensions are allowed to omit the GUID"), status.HTTP_400_BAD_REQUEST ) if addon is None: addon = Addon.create_addon_from_upload_data(data=pkg, user=request.user, upload=filedata, is_listed=False) created = True else: created = False file_upload = handle_upload(filedata=filedata, user=request.user, addon=addon, submit=True) return file_upload, created
def handle_upload(self, request, addon, version_string): if 'upload' in request.FILES: filedata = request.FILES['upload'] else: raise forms.ValidationError( _(u'Missing "upload" key in multipart file data.'), status.HTTP_400_BAD_REQUEST) # Parse the file to get and validate package data with the addon. pkg = parse_addon(filedata, addon) if not acl.submission_allowed(request.user, pkg): raise forms.ValidationError( _(u'You cannot submit this type of add-on'), status.HTTP_400_BAD_REQUEST) version_string = version_string or pkg['version'] if version_string and pkg['version'] != version_string: raise forms.ValidationError( _('Version does not match the manifest file.'), status.HTTP_400_BAD_REQUEST) if (addon is not None and addon.versions.filter(version=version_string).exists()): raise forms.ValidationError( _('Version already exists.'), status.HTTP_409_CONFLICT) dont_allow_no_guid = ( not addon and not pkg.get('guid', None) and not pkg.get('is_webextension', False)) if dont_allow_no_guid: raise forms.ValidationError( _('Only WebExtensions are allowed to omit the GUID'), status.HTTP_400_BAD_REQUEST) if addon is None: addon = Addon.create_addon_from_upload_data( data=pkg, user=request.user, upload=filedata, is_listed=False) created = True channel = amo.RELEASE_CHANNEL_UNLISTED else: created = False last_version = addon.find_latest_version_including_rejected() if last_version: channel = last_version.channel else: # TODO: we need to properly handle channels here and fail if # no previous version to guess with. Also need to allow the # channel to be selected for versions. channel = (amo.RELEASE_CHANNEL_LISTED if addon.is_listed else amo.RELEASE_CHANNEL_UNLISTED) file_upload = handle_upload( filedata=filedata, user=request.user, addon=addon, submit=True, channel=channel) return file_upload, created
def test_and(self): qs = Addon.search().filter(type=1, category__in=[1, 2]) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] assert {'in': {'category': [1, 2]}} in filters['and']
def test_and(self): qs = Addon.search().filter(type=1, category__in=[1, 2]) filters = qs._build_query()['query']['bool']['filter'] # Filters: # [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}] assert len(filters) == 2 assert {'term': {'type': 1}} in filters assert {'terms': {'category': [1, 2]}} in filters
def test_change_is_not_recursive(self): class fn: called = False def callback(old_attr=None, new_attr=None, instance=None, sender=None, **kw): fn.called = True # Both save and update should be protected: instance.update(site_specific=False) instance.save() Addon.on_change(callback) addon = Addon.objects.get(pk=3615) addon.save() assert fn.called
def test_validate_number(self): p = amo.utils.ESPaginator(Addon.search(), 20) # A bad number raises an exception. with self.assertRaises(paginator.PageNotAnInteger): p.page('a') # A large number is ignored. p.page(99)
def create_addon(self, license=None): data = self.cleaned_data a = Addon(guid=data['guid'], name=data['name'], type=data['type'], status=amo.STATUS_UNREVIEWED, homepage=data['homepage'], summary=data['summary']) a.save() AddonUser(addon=a, user=self.request.user).save() self.addon = a # Save Version, attach License self.create_version(license=license) amo.log(amo.LOG.CREATE_ADDON, a) log.info('Addon %d saved' % a.id) return a
def test_paginate_returns_this_paginator(self): request = MagicMock() request.GET.get.return_value = 1 request.GET.urlencode.return_value = '' request.path = '' qs = Addon.search() pager = paginate(request, qs) assert isinstance(pager.paginator, ESPaginator)
def create_personas(self, number, persona_extras=None): persona_extras = persona_extras or {} addon = Addon.objects.get(id=15679) for i in range(number): a = Addon(type=amo.ADDON_PERSONA) a.name = 'persona-%s' % i a.all_categories = [] a.save() v = Version.objects.get(addon=addon) v.addon = a v.pk = None v.save() p = Persona(addon_id=a.id, persona_id=i, **persona_extras) p.save() a.persona = p a._current_version = v a.status = amo.STATUS_PUBLIC a.save()
def test_query_multiple_and_range(self): qs = Addon.search().query(type=1, status__gte=1) query = qs._build_query()['query'] # Query: # {'bool': {'must': [{'term': {'type': 1}}, # {'range': {'status': {'gte': 1}}}, ]}} assert query.keys() == ['bool'] assert query['bool'].keys() == ['must'] assert {'term': {'type': 1}} in query['bool']['must'] assert {'range': {'status': {'gte': 1}}} in query['bool']['must']
def test_featured_ids(self): FeaturedCollection.objects.filter(collection__addons=3615)[0].delete() another = Addon.objects.get(id=1003) self.change_addon(another, 'en-US') items = Addon.featured_random(amo.FIREFOX, 'en-US') # The order should be random within those boundaries. assert [1003, 3481] == sorted(items[0:2]) assert [1001, 2464, 7661, 15679] == sorted(items[2:])
def test_query_or(self): qs = Addon.search().query(or_=dict(type=1, status__gte=2)) query = qs._build_query()['query']['function_score']['query'] # Query: # {'bool': {'should': [{'term': {'type': 1}}, # {'range': {'status': {'gte': 2}}}, ]}} assert query.keys() == ['bool'] assert query['bool'].keys() == ['should'] assert {'term': {'type': 1}} in query['bool']['should'] assert {'range': {'status': {'gte': 2}}} in query['bool']['should']
def test_query_multiple_and_range(self): qs = Addon.search().query(type=1, status__gte=1) query = qs._build_query()['query']['function_score']['query'] # Query: # {'bool': {'must': [{'term': {'type': 1}}, # {'range': {'status': {'gte': 1}}}, ]}} eq_(query.keys(), ['bool']) eq_(query['bool'].keys(), ['must']) ok_({'term': {'type': 1}} in query['bool']['must']) ok_({'range': {'status': {'gte': 1}}} in query['bool']['must'])
def test_es_paginator(self): qs = Addon.search() pager = amo.utils.paginate(self.request, qs) assert isinstance(pager.paginator, amo.utils.ESPaginator)
def add_static_theme_from_lwt(lwt): from olympia.activity.models import AddonLog timer = StopWatch( 'addons.tasks.migrate_lwts_to_static_theme.add_from_lwt.') timer.start() olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID)) # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create( user=author, valid=True) filename = uuid.uuid4().hex + '.xpi' destination = os.path.join(user_media_path('addons'), 'temp', filename) build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination, name=filename) timer.log_interval('1.build_xpi') # Create addon + version parsed_data = parse_addon(upload, user=author) timer.log_interval('2a.parse_addon') addon = Addon.initialize_addon_from_upload( parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) addon_updates = {} timer.log_interval('2b.initialize_addon') # static themes are only compatible with Firefox at the moment, # not Android version = Version.from_upload( upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) timer.log_interval('3.initialize_version') # Set category lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' for app, type_dict in CATEGORIES.items(): static_theme_categories = type_dict.get(amo.ADDON_STATICTHEME, []) static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create( addon=addon, category=Category.from_static_category(static_category, True)) timer.log_interval('4.set_categories') # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) timer.log_interval('5.set_license') # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) timer.log_interval('6.set_tags') # Steal the ratings (even with soft delete they'll be deleted anyway) addon_updates.update( average_rating=lwt.average_rating, bayesian_rating=lwt.bayesian_rating, total_ratings=lwt.total_ratings, text_ratings_count=lwt.text_ratings_count) Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version) timer.log_interval('7.move_ratings') # Replace the lwt in collections CollectionAddon.objects.filter(addon=lwt).update(addon=addon) # Modify the activity log entry too. rating_activity_log_ids = [ l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review'] addonlog_qs = AddonLog.objects.filter( addon=lwt, activity_log__action__in=rating_activity_log_ids) [alog.transfer(addon) for alog in addonlog_qs.iterator()] timer.log_interval('8.move_activity_logs') # Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats # dashboard and future update counts, and copy the average_daily_users. # hotness will be recalculated by the deliver_hotness() cron in a more # reliable way that we could do, so skip it entirely. migrate_theme_update_count(lwt, addon) addon_updates.update( average_daily_users=lwt.persona.popularity or 0, hotness=0) timer.log_interval('9.copy_statistics') # Logging activity.log_create( amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update( datestatuschanged=lwt.last_updated, reviewed=datetime.now(), status=amo.STATUS_PUBLIC) timer.log_interval('10.sign_files') addon_updates['status'] = amo.STATUS_PUBLIC # set the modified and creation dates to match the original. addon_updates['created'] = lwt.created addon_updates['modified'] = lwt.modified addon_updates['last_updated'] = lwt.last_updated addon.update(**addon_updates) return addon
def check_xpi_info(xpi_info, addon=None, xpi_file=None, user=None): from olympia.addons.models import Addon, DeniedGuid guid = xpi_info['guid'] # If we allow the guid to be omitted we assume that one was generated # or existed before and use that one. # An example are WebExtensions that don't require a guid but we generate # one once they're uploaded. Now, if you update that WebExtension we # just use the original guid. if addon and not guid: xpi_info['guid'] = guid = addon.guid if guid: if user: deleted_guid_clashes = Addon.unfiltered.exclude( authors__id=user.id).filter(guid=guid) else: deleted_guid_clashes = Addon.unfiltered.filter(guid=guid) if addon and addon.guid != guid: msg = gettext('The add-on ID in your manifest.json (%s) ' 'does not match the ID of your add-on on AMO (%s)') raise forms.ValidationError(msg % (guid, addon.guid)) if (not addon # Non-deleted add-ons. and (Addon.objects.filter(guid=guid).exists() # DeniedGuid objects for deletions for Mozilla disabled add-ons or DeniedGuid.objects.filter(guid=guid).exists() # Deleted add-ons that don't belong to the uploader. or deleted_guid_clashes.exists())): raise forms.ValidationError(gettext('Duplicate add-on ID found.')) if len(xpi_info['version']) > 32: raise forms.ValidationError( gettext('Version numbers should have fewer than 32 characters.')) if not VERSION_RE.match(xpi_info['version']): raise forms.ValidationError( gettext('Version numbers should only contain letters, numbers, ' 'and these punctuation characters: +*.-_.')) if xpi_info.get('type') == amo.ADDON_STATICTHEME: max_size = settings.MAX_STATICTHEME_SIZE if xpi_file and xpi_file.size > max_size: raise forms.ValidationError( gettext('Maximum size for WebExtension themes is {0}.').format( filesizeformat(max_size))) if xpi_file: # Make sure we pass in a copy of `xpi_info` since # `resolve_webext_translations` modifies data in-place translations = Addon.resolve_webext_translations( xpi_info.copy(), xpi_file) verify_mozilla_trademark(translations['name'], user) # Parse the file to get and validate package data with the addon. if not acl.experiments_submission_allowed(user, xpi_info): raise forms.ValidationError( gettext('You cannot submit this type of add-on')) if not addon and not acl.reserved_guid_addon_submission_allowed( user, xpi_info): raise forms.ValidationError( gettext( 'You cannot submit an add-on using an ID ending with this suffix' )) if not acl.mozilla_signed_extension_submission_allowed(user, xpi_info): raise forms.ValidationError( gettext('You cannot submit a Mozilla Signed Extension')) if (not addon and guid and guid.lower().endswith(amo.RESERVED_ADDON_GUIDS) and not xpi_info.get('is_mozilla_signed_extension')): raise forms.ValidationError( gettext( 'Add-ons using an ID ending with this suffix need to be signed with ' 'privileged certificate before being submitted')) if not acl.langpack_submission_allowed(user, xpi_info): raise forms.ValidationError( gettext('You cannot submit a language pack')) if not acl.site_permission_addons_submission_allowed(user, xpi_info): raise forms.ValidationError( gettext('You cannot submit this type of add-on')) return xpi_info
def test_cache_key(): # Test that we are not taking the db into account when building our # cache keys for django-cache-machine. See bug 928881. assert Addon._cache_key(1, 'default') == Addon._cache_key(1, 'slave')
def handle_upload(self, request, addon, version_string, guid=None): if 'upload' in request.FILES: filedata = request.FILES['upload'] else: raise forms.ValidationError( _(u'Missing "upload" key in multipart file data.'), status.HTTP_400_BAD_REQUEST) # Parse the file to get and validate package data with the addon. pkg = parse_addon(filedata, addon) if not acl.submission_allowed(request.user, pkg): raise forms.ValidationError( _(u'You cannot submit this type of add-on'), status.HTTP_400_BAD_REQUEST) if addon is not None and addon.status == amo.STATUS_DISABLED: raise forms.ValidationError( _('You cannot add versions to an addon that has status: %s.' % amo.STATUS_CHOICES_ADDON[amo.STATUS_DISABLED]), status.HTTP_400_BAD_REQUEST) version_string = version_string or pkg['version'] if version_string and pkg['version'] != version_string: raise forms.ValidationError( _('Version does not match the manifest file.'), status.HTTP_400_BAD_REQUEST) if (addon is not None and addon.versions.filter(version=version_string).exists()): raise forms.ValidationError(_('Version already exists.'), status.HTTP_409_CONFLICT) package_guid = pkg.get('guid', None) dont_allow_no_guid = (not addon and not package_guid and not pkg.get('is_webextension', False)) if dont_allow_no_guid: raise forms.ValidationError( _('Only WebExtensions are allowed to omit the GUID'), status.HTTP_400_BAD_REQUEST) if guid is not None and not addon and not package_guid: # No guid was present in the package, but one was provided in the # URL, so we take it instead of generating one ourselves. But # first, validate it properly. if not amo.ADDON_GUID_PATTERN.match(guid): raise forms.ValidationError(_('Invalid GUID in URL'), status.HTTP_400_BAD_REQUEST) pkg['guid'] = guid # channel will be ignored for new addons. if addon is None: channel = amo.RELEASE_CHANNEL_UNLISTED # New is always unlisted. addon = Addon.create_addon_from_upload_data(data=pkg, user=request.user, upload=filedata, channel=channel) created = True else: created = False channel_param = request.POST.get('channel') channel = amo.CHANNEL_CHOICES_LOOKUP.get(channel_param) if not channel: last_version = (addon.find_latest_version(None, exclude=())) if last_version: channel = last_version.channel else: channel = amo.RELEASE_CHANNEL_UNLISTED # Treat as new. will_have_listed = channel == amo.RELEASE_CHANNEL_LISTED if not addon.has_complete_metadata( has_listed_versions=will_have_listed): raise forms.ValidationError( _('You cannot add a listed version to this addon ' 'via the API due to missing metadata. ' 'Please submit via the website'), status.HTTP_400_BAD_REQUEST) file_upload = handle_upload(filedata=filedata, user=request.user, addon=addon, submit=True, channel=channel) return file_upload, created
def check_xpi_info(xpi_info, addon=None, xpi_file=None, user=None): from olympia.addons.models import Addon, DeniedGuid guid = xpi_info['guid'] is_webextension = xpi_info.get('is_webextension', False) # If we allow the guid to be omitted we assume that one was generated # or existed before and use that one. # An example are WebExtensions that don't require a guid but we generate # one once they're uploaded. Now, if you update that WebExtension we # just use the original guid. if addon and not guid and is_webextension: xpi_info['guid'] = guid = addon.guid if not guid and not is_webextension: raise forms.ValidationError(ugettext('Could not find an add-on ID.')) if guid: current_user = core.get_user() if current_user: deleted_guid_clashes = Addon.unfiltered.exclude( authors__id=current_user.id).filter(guid=guid) else: deleted_guid_clashes = Addon.unfiltered.filter(guid=guid) if addon and addon.guid != guid: msg = ugettext( 'The add-on ID in your manifest.json or install.rdf (%s) ' 'does not match the ID of your add-on on AMO (%s)') raise forms.ValidationError(msg % (guid, addon.guid)) if (not addon and # Non-deleted add-ons. ( Addon.objects.filter(guid=guid).exists() or # DeniedGuid objects for deletions for Mozilla disabled add-ons DeniedGuid.objects.filter(guid=guid).exists() or # Deleted add-ons that don't belong to the uploader. deleted_guid_clashes.exists())): raise forms.ValidationError(ugettext('Duplicate add-on ID found.')) if len(xpi_info['version']) > 32: raise forms.ValidationError( ugettext('Version numbers should have fewer than 32 characters.')) if not VERSION_RE.match(xpi_info['version']): raise forms.ValidationError( ugettext('Version numbers should only contain letters, numbers, ' 'and these punctuation characters: +*.-_.')) if is_webextension and xpi_info.get('type') == amo.ADDON_STATICTHEME: max_size = settings.MAX_STATICTHEME_SIZE if xpi_file and os.path.getsize(xpi_file.name) > max_size: raise forms.ValidationError( ugettext( u'Maximum size for WebExtension themes is {0}.').format( filesizeformat(max_size))) if xpi_file: # Make sure we pass in a copy of `xpi_info` since # `resolve_webext_translations` modifies data in-place translations = Addon.resolve_webext_translations( xpi_info.copy(), xpi_file) verify_mozilla_trademark(translations['name'], core.get_user()) # Parse the file to get and validate package data with the addon. if not acl.submission_allowed(user, xpi_info): raise forms.ValidationError( ugettext(u'You cannot submit this type of add-on')) if not addon and not system_addon_submission_allowed(user, xpi_info): guids = ' or '.join('"' + guid + '"' for guid in amo.SYSTEM_ADDON_GUIDS) raise forms.ValidationError( ugettext(u'You cannot submit an add-on with a guid ending ' u'%s' % guids)) if not mozilla_signed_extension_submission_allowed(user, xpi_info): raise forms.ValidationError( ugettext(u'You cannot submit a Mozilla Signed Extension')) return xpi_info
def test_getitem(self): addons = list(Addon.search()) assert addons[0] == Addon.search()[0]
def test_iter(self): qs = Addon.search().filter(type=1, is_disabled=False) assert len(qs) == len(list(qs))
def test_count(self): assert Addon.search().count() == 6
def test_object_result_slice(self): addon = self._addons[0] qs = Addon.search().filter(id=addon.id) assert addon == qs[0]
def test_values(self): qs = Addon.search().values('name') assert qs._build_query()['fields'] == ['id', 'name']
def test_extra_bad_key(self): with self.assertRaises(AssertionError): Addon.search().extra(x=1)
def test_len(self): qs = Addon.search() qs._results_cache = [1] assert len(qs) == 1
def test_slice_stop_zero(self): qs = Addon.search()[:0] assert qs._build_query()['size'] == 0
def unindex_addons(ids, **kw): for addon in ids: log.info('Removing addon [%s] from search index.' % addon) Addon.unindex(addon)
def test_slice(self): qs = Addon.search()[5:12] assert qs._build_query()['from'] == 5 assert qs._build_query()['size'] == 7
def process_request(self, query, addon_type='ALL', limit=10, platform='ALL', version=None, compat_mode='strict'): """ Query the search backend and serve up the XML. """ limit = min(MAX_LIMIT, int(limit)) app_id = self.request.APP.id # We currently filter for status=PUBLIC for all versions. If # that changes, the contract for API version 1.5 requires # that we continue filtering for it there. filters = { 'app': app_id, 'status': amo.STATUS_PUBLIC, 'is_listed': True, 'is_experimental': False, 'is_disabled': False, 'has_version': True, } # Opts may get overridden by query string filters. opts = { 'addon_type': addon_type, 'version': version, } # Specific case for Personas (bug 990768): if we search providing the # Persona addon type (9), don't filter on the platform as Personas # don't have compatible platforms to filter on. if addon_type != '9': opts['platform'] = platform if self.version < 1.5: # Fix doubly encoded query strings. try: query = urllib.unquote(query.encode('ascii')) except UnicodeEncodeError: # This fails if the string is already UTF-8. pass query, qs_filters, params = extract_filters(query, opts) qs = Addon.search().query(or_=name_query(query)) filters.update(qs_filters) if 'type' not in filters: # Filter by ALL types, which is really all types except for apps. filters['type__in'] = list(amo.ADDON_SEARCH_TYPES) qs = qs.filter(**filters) qs = qs[:limit] total = qs.count() results = [] for addon in qs: compat_version = addon.compatible_version(app_id, params['version'], params['platform'], compat_mode) # Specific case for Personas (bug 990768): if we search providing # the Persona addon type (9), then don't look for a compatible # version. if compat_version or addon_type == '9': addon.compat_version = compat_version results.append(addon) if len(results) == limit: break else: # We're excluding this addon because there are no # compatible versions. Decrement the total. total -= 1 return self.render('legacy_api/search.xml', { 'results': results, 'total': total, # For caching 'version': version, 'compat_mode': compat_mode, })
def test_order_by_multiple(self): qs = Addon.search().order_by('-rating', 'id') assert qs._build_query()['sort'] == [{'rating': 'desc'}, 'id']
def test_generate_filename_ja(self): f = File() f.version = Version(version='0.1.7') f.version.compatible_apps = (amo.FIREFOX, ) f.version.addon = Addon(name=u' フォクすけ といっしょ') eq_(f.generate_filename(), 'addon-0.1.7-fx.xpi')
def test_order_by_asc(self): qs = Addon.search().order_by('rating') assert qs._build_query()['sort'] == ['rating']
def test_get_unfiltered_manager(self): Addon.get_unfiltered_manager() == Addon.unfiltered UserProfile.get_unfiltered_manager() == UserProfile.objects
def test_order_by_desc(self): qs = Addon.search().order_by('-rating') assert qs._build_query()['sort'] == [{'rating': 'desc'}]
def fake_object(self, data): """Create a fake instance of Addon and related models from ES data.""" obj = Addon(id=data['id'], slug=data['slug']) # Attach base attributes that have the same name/format in ES and in # the model. self._attach_fields( obj, data, ('average_daily_users', 'bayesian_rating', 'created', 'default_locale', 'guid', 'has_eula', 'has_privacy_policy', 'hotness', 'icon_type', 'is_experimental', 'last_updated', 'modified', 'public_stats', 'slug', 'status', 'type', 'view_source', 'weekly_downloads')) # Attach attributes that do not have the same name/format in ES. obj.tag_list = data['tags'] obj.disabled_by_user = data['is_disabled'] # Not accurate, but enough. obj.all_categories = [ CATEGORIES_BY_ID[cat_id] for cat_id in data.get('category', []) ] # Attach translations (they require special treatment). self._attach_translations(obj, data, self.translated_fields) # Attach related models (also faking them). `current_version` is a # property we can't write to, so we use the underlying field which # begins with an underscore. `current_beta_version` and # `latest_unlisted_version` are writeable cached_property so we can # directly write to them. obj.current_beta_version = self.fake_version_object( obj, data.get('current_beta_version'), amo.RELEASE_CHANNEL_LISTED) obj._current_version = self.fake_version_object( obj, data.get('current_version'), amo.RELEASE_CHANNEL_LISTED) obj.latest_unlisted_version = self.fake_version_object( obj, data.get('latest_unlisted_version'), amo.RELEASE_CHANNEL_UNLISTED) data_authors = data.get('listed_authors', []) obj.listed_authors = [ UserProfile(id=data_author['id'], display_name=data_author['name'], username=data_author['username']) for data_author in data_authors ] # We set obj.all_previews to the raw preview data because # ESPreviewSerializer will handle creating the fake Preview object # for us when its to_representation() method is called. obj.all_previews = data.get('previews', []) obj.average_rating = data.get('ratings', {}).get('average') obj.total_reviews = data.get('ratings', {}).get('count') if data['type'] == amo.ADDON_PERSONA: persona_data = data.get('persona') if persona_data: obj.persona = Persona( addon=obj, accentcolor=persona_data['accentcolor'], display_username=persona_data['author'], header=persona_data['header'], footer=persona_data['footer'], persona_id=1 if persona_data['is_new'] else None, textcolor=persona_data['textcolor']) else: # Sadly, https://code.djangoproject.com/ticket/14368 prevents # us from setting obj.persona = None. This is fixed in # Django 1.9, but in the meantime, work around it by creating # a Persona instance with a custom attribute indicating that # it should not be used. obj.persona = Persona() obj.persona._broken = True return obj
if is_beta: log.error('[@None] Not creating beta version {0} for new ' '"{1}" language pack'.format(data['version'], xpi)) return if (Addon.objects.filter(name__localized_string=data['name']) .exists()): data['old_name'] = data['name'] data['name'] = u'{0} ({1})'.format( data['old_name'], data['apps'][0].appdata.pretty) log.warning(u'[@None] Creating langpack {guid}: Add-on with ' u'name {old_name!r} already exists, trying ' u'{name!r}.'.format(**data)) addon = Addon.from_upload( upload, [amo.PLATFORM_ALL.id], parsed_data=data) AddonUser(addon=addon, user=owner).save() version = addon.versions.get() if addon.default_locale.lower() == lang.lower(): addon.target_locale = addon.default_locale addon.save() log.info('[@None] Created new "{0}" language pack, version {1}' .format(xpi, data['version'])) # Set the category for app in version.compatible_apps: static_category = ( CATEGORIES.get(app.id, []).get(amo.ADDON_LPAPP, [])
def test_count(self): p = amo.utils.ESPaginator(Addon.search(), 20) assert p._count is None p.page(1) assert p.count == Addon.search().count()
def get_bayesian_rating(self): q = Addon.search().filter(id=self.addon.id) return list(q.values_dict('bayesian_rating'))[0]['bayesian_rating'][0]
def test_clone(self): # Doing a filter creates a new ES object. qs = Addon.search() qs2 = qs.filter(type=1) assert 'filtered' not in qs._build_query()['query'] assert 'filtered' in qs2._build_query()['query']
def search(request, tag_name=None, template=None): APP = request.APP types = (amo.ADDON_EXTENSION, amo.ADDON_THEME, amo.ADDON_DICT, amo.ADDON_SEARCH, amo.ADDON_LPAPP) category = request.GET.get('cat') if category == 'collections': extra_params = {'sort': {'newest': 'created'}} else: extra_params = None fixed = fix_search_query(request.GET, extra_params=extra_params) if fixed is not request.GET: return http.HttpResponsePermanentRedirect(urlparams(request.path, **fixed)) facets = request.GET.copy() # In order to differentiate between "all versions" and an undefined value, # we use "any" instead of "" in the frontend. if 'appver' in facets and facets['appver'] == 'any': facets['appver'] = '' form = ESSearchForm(facets or {}) form.is_valid() # Let the form try to clean data. form_data = form.cleaned_data if tag_name: form_data['tag'] = tag_name if category == 'collections': return _collections(request) elif category == 'themes' or form_data.get('atype') == amo.ADDON_PERSONA: return _personas(request) sort, extra_sort = split_choices(form.sort_choices, 'created') if form_data.get('atype') == amo.ADDON_SEARCH: # Search add-ons should not be searched by ADU, so replace 'Users' # sort with 'Weekly Downloads'. sort, extra_sort = list(sort), list(extra_sort) sort[1] = extra_sort[1] del extra_sort[1] # Perform search, using aggregation so that we can build the facets UI. # Note that we don't need to aggregate on platforms, that facet it built # from our constants directly, using the current application for this # request (request.APP). appversion_field = 'current_version.compatible_apps.%s.max' % APP.id qs = (Addon.search_public().filter(app=APP.id) .aggregate(tags={'terms': {'field': 'tags'}}, appversions={'terms': {'field': appversion_field}}, categories={'terms': {'field': 'category', 'size': 200}}) ) filters = ['atype', 'appver', 'cat', 'sort', 'tag', 'platform'] mapping = {'users': '-average_daily_users', 'rating': '-bayesian_rating', 'created': '-created', 'name': 'name_sort', 'downloads': '-weekly_downloads', 'updated': '-last_updated', 'hotness': '-hotness'} qs = _filter_search(request, qs, form_data, filters, mapping, types=types) pager = amo.utils.paginate(request, qs) ctx = { 'is_pjax': request.META.get('HTTP_X_PJAX'), 'pager': pager, 'query': form_data, 'form': form, 'sort_opts': sort, 'extra_sort_opts': extra_sort, 'sorting': sort_sidebar(request, form_data, form), 'sort': form_data.get('sort'), } if not ctx['is_pjax']: aggregations = pager.object_list.aggregations ctx.update({ 'tag': tag_name, 'categories': category_sidebar(request, form_data, aggregations), 'platforms': platform_sidebar(request, form_data), 'versions': version_sidebar(request, form_data, aggregations), 'tags': tag_sidebar(request, form_data, aggregations), }) return render(request, template, ctx)
def test_object_result(self): qs = Addon.search().filter(id=self._addons[0].id)[:1] assert self._addons[:1] == list(qs)
def test_count_uses_cached_results(self): qs = Addon.search() qs._results_cache = mock.Mock() qs._results_cache.count = mock.sentinel.count assert qs.count() == mock.sentinel.count