def app_search(request):
results = []
q = request.GET.get('q', u'').lower().strip()
limit = (lkp.MAX_RESULTS if request.GET.get('all_results')
else lkp.SEARCH_LIMIT)
fields = ('name', 'app_slug')
non_es_fields = ['id', 'name__localized_string'] + list(fields)
if q.isnumeric():
qs = Webapp.objects.filter(pk=q).values(*non_es_fields)[:limit]
else:
# Try to load by GUID:
qs = Webapp.objects.filter(guid=q).values(*non_es_fields)[:limit]
if not qs.count():
# TODO: Update to `.fields(...)` when the DSL supports it.
qs = (WebappIndexer.search()
.query(_expand_query(q, fields))[:limit])
qs = qs.execute()
for app in qs:
if isinstance(app, dict):
# This is a result from the database.
app['url'] = reverse('lookup.app_summary', args=[app['id']])
app['name'] = app['name__localized_string']
results.append(app)
else:
# This is a result from elasticsearch which returns `Result`
# objects and name as a list, one for each locale.
for name in app.name:
results.append({
'id': app.id,
'url': reverse('lookup.app_summary', args=[app.id]),
'app_slug': app.get('app_slug'),
'name': name,
})
return {'results': results}
def app_search(request):
results = []
q = request.GET.get('q', u'').lower().strip()
limit = (lkp.MAX_RESULTS if request.GET.get('all_results')
else lkp.SEARCH_LIMIT)
fields = ('name', 'app_slug')
non_es_fields = ['id', 'name__localized_string'] + list(fields)
if q.isnumeric():
qs = Webapp.objects.filter(pk=q).values(*non_es_fields)[:limit]
else:
# Try to load by GUID:
qs = Webapp.objects.filter(guid=q).values(*non_es_fields)[:limit]
if not qs.count():
qs = (WebappIndexer.search()
.query(_expand_query(q, fields))[:limit])
# TODO: Update to `.fields(...)` when the DSL supports it.
qs = qs.execute()
for app in qs:
if isinstance(app, dict):
# This is a result from the database.
app['url'] = reverse('lookup.app_summary', args=[app['id']])
app['name'] = app['name__localized_string']
results.append(app)
else:
# This is a result from elasticsearch which returns `Result`
# objects and name as a list, one for each locale.
for name in app.name:
results.append({
'id': app.id,
'url': reverse('lookup.app_summary', args=[app.id]),
'app_slug': app.get('app_slug'),
'name': name,
})
return {'results': results}
def get_updates_queue(self):
# Updated apps, i.e. apps that have been published but have new
# unreviewed versions, go in this queue.
if self.use_es:
must = [
es_filter.Terms(status=mkt.WEBAPPS_APPROVED_STATUSES),
es_filter.Term(**{'latest_version.status':
mkt.STATUS_PENDING}),
es_filter.Terms(app_type=[mkt.ADDON_WEBAPP_PACKAGED,
mkt.ADDON_WEBAPP_PRIVILEGED]),
es_filter.Term(is_disabled=False),
es_filter.Term(is_escalated=False),
]
return WebappIndexer.search().filter('bool', must=must)
return (Version.objects.filter(
# Note: this will work as long as we disable files of existing
# unreviewed versions when a new version is uploaded.
files__status=mkt.STATUS_PENDING,
addon__disabled_by_user=False,
addon__is_packaged=True,
addon__status__in=mkt.WEBAPPS_APPROVED_STATUSES)
.exclude(addon__id__in=self.excluded_ids)
.exclude(addon__tags__tag_text='homescreen')
.order_by('nomination', 'created')
.select_related('addon', 'files').no_transforms())
def get_updates_queue(self):
if self.use_es:
must = [
es_filter.Terms(status=amo.WEBAPPS_APPROVED_STATUSES),
es_filter.Term(
**{'latest_version.status': amo.STATUS_PENDING}),
es_filter.Terms(app_type=[
amo.ADDON_WEBAPP_PACKAGED, amo.ADDON_WEBAPP_PRIVILEGED
]),
es_filter.Term(is_disabled=False),
es_filter.Term(is_escalated=False),
]
return WebappIndexer.search().filter('bool', must=must)
return (Version.objects.no_cache().filter(
# Note: this will work as long as we disable files of existing
# unreviewed versions when a new version is uploaded.
files__status=amo.STATUS_PENDING,
addon__disabled_by_user=False,
addon__is_packaged=True,
addon__status__in=amo.WEBAPPS_APPROVED_STATUSES).exclude(
addon__id__in=self.excluded_ids).order_by(
'nomination',
'created').select_related('addon',
'files').no_transforms())
def get_updates_queue(self):
# Updated apps, i.e. apps that have been published but have new
# unreviewed versions, go in this queue.
if self.use_es:
must = [
es_filter.Terms(status=mkt.WEBAPPS_APPROVED_STATUSES),
es_filter.Term(
**{'latest_version.status': mkt.STATUS_PENDING}),
es_filter.Terms(app_type=[
mkt.ADDON_WEBAPP_PACKAGED, mkt.ADDON_WEBAPP_PRIVILEGED
]),
es_filter.Term(is_disabled=False),
es_filter.Term(is_escalated=False),
]
return WebappIndexer.search().filter('bool', must=must)
return (Version.objects.filter(
# Note: this will work as long as we disable files of existing
# unreviewed versions when a new version is uploaded.
files__status=mkt.STATUS_PENDING,
addon__disabled_by_user=False,
addon__is_packaged=True,
addon__status__in=mkt.WEBAPPS_APPROVED_STATUSES).exclude(
addon__id__in=self.excluded_ids).exclude(
addon__tags__tag_text='homescreen').order_by(
'nomination',
'created').select_related('addon',
'files').no_transforms())
def app_search(request):
results = []
q = request.GET.get("q", u"").lower().strip()
limit = lkp.MAX_RESULTS if request.GET.get("all_results") else lkp.SEARCH_LIMIT
fields = ("name", "app_slug")
non_es_fields = ["id", "name__localized_string"] + list(fields)
if q.isnumeric():
qs = Webapp.objects.filter(pk=q).values(*non_es_fields)[:limit]
else:
# Try to load by GUID:
qs = Webapp.objects.filter(guid=q).values(*non_es_fields)[:limit]
if not qs.count():
qs = WebappIndexer.search().query(_expand_query(q, fields))[:limit]
# TODO: Update to `.fields(...)` when the DSL supports it.
qs = qs.execute()
for app in qs:
if isinstance(app, dict):
# This is a result from the database.
app["url"] = reverse("lookup.app_summary", args=[app["id"]])
app["name"] = app["name__localized_string"]
results.append(app)
else:
# This is a result from elasticsearch which returns `Result`
# objects and name as a list, one for each locale.
for name in app.name:
results.append(
{
"id": app.id,
"url": reverse("lookup.app_summary", args=[app.id]),
"app_slug": app.get("app_slug"),
"name": name,
}
)
return {"results": results}
def _filter(self, req=None, data=None):
req = req or RequestFactory().get('/', data=data or {})
req.user = AnonymousUser()
queryset = WebappIndexer.search()
for filter_class in self.filter_classes:
queryset = filter_class().filter_queryset(req, queryset,
self.view_class)
return queryset.to_dict()
def _filter(self, req=None, data=None):
req = req or RequestFactory().get('/', data=data or {})
req.user = AnonymousUser()
queryset = WebappIndexer.search()
for filter_class in self.filter_classes:
queryset = filter_class().filter_queryset(req, queryset,
self.view_class)
return queryset.to_dict()
def search(self, request):
form_data = self.get_search_data(request)
query = form_data.get('q', '')
qs = WebappIndexer.search()
if form_data.get('status') != 'any':
qs = qs.filter('term', status=form_data.get('status'))
qs = self.apply_filters(request, qs, data=form_data)
qs = apply_reviewer_filters(request, qs, data=form_data)
page = self.paginate_queryset(qs)
return self.get_pagination_serializer(page), query
def setUp(self):
self.webapp = Webapp.objects.get(pk=337141)
self.request = RequestFactory().get('/')
self.request.user = AnonymousUser()
RegionMiddleware().process_request(self.request)
self.reindex(Webapp, 'webapp')
self.indexer = WebappIndexer.search().filter(
'term', id=self.webapp.id).execute().hits[0]
self.serializer = SimpleESAppSerializer(self.indexer,
context={'request': self.request})
def get_escalated_queue(self):
if self.use_es:
must = [
es_filter.Term(is_disabled=False),
es_filter.Term(is_escalated=True),
]
return WebappIndexer.search().filter('bool', must=must)
return EscalationQueue.objects.filter(
addon__disabled_by_user=False)
def setUp(self):
self.webapp = Webapp.objects.get(pk=337141)
self.request = RequestFactory().get('/')
self.request.user = AnonymousUser()
RegionMiddleware().process_request(self.request)
self.reindex(Webapp)
self.indexer = WebappIndexer.search().filter(
'term', id=self.webapp.id).execute().hits[0]
self.serializer = SimpleESAppSerializer(
self.indexer, context={'request': self.request})
def get_escalated_queue(self):
# Apps and homescreens flagged for escalation go in this queue.
if self.use_es:
must = [
es_filter.Term(is_disabled=False),
es_filter.Term(is_escalated=True),
]
return WebappIndexer.search().filter('bool', must=must)
return EscalationQueue.objects.filter(
addon__disabled_by_user=False)
def get_rereview_queue(self):
if self.use_es:
must = [
es_filter.Term(is_rereviewed=True),
es_filter.Term(is_disabled=False),
es_filter.Term(is_escalated=False),
]
return WebappIndexer.search().filter('bool', must=must)
return (RereviewQueue.objects.no_cache().filter(
addon__disabled_by_user=False).exclude(
addon__in=self.excluded_ids))
def get_rereview_queue(self):
if self.use_es:
must = [
es_filter.Term(is_rereviewed=True),
es_filter.Term(is_disabled=False),
es_filter.Term(is_escalated=False),
]
return WebappIndexer.search().filter('bool', must=must)
return (RereviewQueue.objects.
filter(addon__disabled_by_user=False).
exclude(addon__in=self.excluded_ids))
def test_excluded_fields(self):
ok_(WebappIndexer.hidden_fields)
data = WebappIndexer.search().execute().hits
eq_(len(data), 1)
obj = data[0]
ok_('trending_2' not in obj)
ok_('popularity_2' not in obj)
ok_('name_translations' in obj)
ok_('name' not in obj)
ok_('name_l10n_english' not in obj)
ok_('name_sort' not in obj)
ok_('name.raw' not in obj)
def test_excluded_fields(self):
ok_(WebappIndexer.hidden_fields)
data = WebappIndexer.search().execute().hits
eq_(len(data), 1)
obj = data[0]
ok_('trending_2' not in obj)
ok_('popularity_2' not in obj)
ok_('name_translations' in obj)
ok_('name' not in obj)
ok_('name_l10n_english' not in obj)
ok_('name_sort' not in obj)
ok_('name.raw' not in obj)
def get_pending_queue(self):
if self.use_es:
sq = WebappIndexer.search()
sq = sq.filter('term', status=amo.STATUS_PENDING)
sq = sq.filter('term', **{'latest_version.status': amo.STATUS_PENDING})
sq = sq.filter('term', is_escalated=False)
sq = sq.filter('term', is_disabled=False)
return sq
return (Version.objects.no_cache().filter(
files__status=amo.STATUS_PENDING,
addon__disabled_by_user=False,
addon__status=amo.STATUS_PENDING)
.exclude(addon__id__in=self.excluded_ids)
.order_by('nomination', 'created')
.select_related('addon', 'files').no_transforms())
def test_single_hit(self):
"""Test the ESPaginator only queries ES one time."""
es = WebappIndexer.get_es()
orig_search = es.search
es.counter = 0
def monkey_search(*args, **kwargs):
es.counter += 1
return orig_search(*args, **kwargs)
es.search = monkey_search
ESPaginator(WebappIndexer.search(), 5).object_list.execute()
eq_(es.counter, 1)
es.search = orig_search
def get_apps(self, request, app_ids):
"""
Takes a list of app_ids. Gets the apps, including filters.
Returns an app_map for serializer context.
"""
sq = WebappIndexer.search()
if request.QUERY_PARAMS.get('filtering', '1') == '1':
# With filtering (default).
for backend in self.filter_backends:
sq = backend().filter_queryset(request, sq, self)
sq = WebappIndexer.filter_by_apps(app_ids, sq)
# Store the apps to attach to feed elements later.
with statsd.timer('mkt.feed.views.apps_query'):
apps = sq.execute().hits
return dict((app.id, app) for app in apps)
def get_apps(self, request, app_ids):
"""
Takes a list of app_ids. Gets the apps, including filters.
Returns an app_map for serializer context.
"""
sq = WebappIndexer.search()
if request.query_params.get('filtering', '1') == '1':
# With filtering (default).
for backend in self.filter_backends:
sq = backend().filter_queryset(request, sq, self)
sq = WebappIndexer.filter_by_apps(app_ids, sq)
# Store the apps to attach to feed elements later.
with statsd.timer('mkt.feed.views.apps_query'):
apps = sq.execute().hits
return dict((app.id, app) for app in apps)
def get_pending_queue(self):
if self.use_es:
must = [
es_filter.Term(status=mkt.STATUS_PENDING),
es_filter.Term(**{'latest_version.status':
mkt.STATUS_PENDING}),
es_filter.Term(is_escalated=False),
es_filter.Term(is_disabled=False),
]
return WebappIndexer.search().filter('bool', must=must)
return (Version.objects.filter(
files__status=mkt.STATUS_PENDING,
addon__disabled_by_user=False,
addon__status=mkt.STATUS_PENDING)
.exclude(addon__id__in=self.excluded_ids)
.order_by('nomination', 'created')
.select_related('addon', 'files').no_transforms())
def search(self, request):
# Parse form.
form = self.form_class(request.GET if request else None)
if not form.is_valid():
raise form_errors(form)
form_data = form.cleaned_data
# Status filter.
data = search_form_to_es_fields(form_data)
if form_data.get('status') != 'any':
data.update(status=form_data.get('status'))
# Do filter.
sq = apply_reviewer_filters(request, WebappIndexer.search(),
data=form_data)
sq = WebappIndexer.get_app_filter(request, data, sq=sq, no_filter=True)
page = self.paginate_queryset(sq)
return self.get_pagination_serializer(page), request.GET.get('q', '')
def get_apps(self, request, app_ids):
"""
Takes a list of app_ids. Gets the apps, including filters.
Returns an app_map for serializer context.
"""
if request.QUERY_PARAMS.get('filtering', '1') == '0':
# Without filtering.
sq = WebappIndexer.search().filter(es_filter.Bool(
should=[es_filter.Terms(id=app_ids)]
))[0:len(app_ids)]
else:
# With filtering.
sq = WebappIndexer.get_app_filter(request, {
'device': self._get_device(request)
}, app_ids=app_ids)
# Store the apps to attach to feed elements later.
apps = sq.execute().hits
return dict((app.id, app) for app in apps)
def search(self, request):
# Parse form.
form = self.form_class(request.GET if request else None)
if not form.is_valid():
raise form_errors(form)
form_data = form.cleaned_data
# Status filter.
data = search_form_to_es_fields(form_data)
if form_data.get('status') != 'any':
data.update(status=form_data.get('status'))
# Do filter.
sq = apply_reviewer_filters(request, WebappIndexer.search(),
data=form_data)
sq = WebappIndexer.get_app_filter(request, data, sq=sq, no_filter=True)
page = self.paginate_queryset(sq)
return self.get_pagination_serializer(page), request.GET.get('q', '')
def get_apps(self, request, app_ids):
"""
Takes a list of app_ids. Gets the apps, including filters.
Returns an app_map for serializer context.
"""
if request.QUERY_PARAMS.get('filtering', '1') == '0':
# Without filtering.
sq = WebappIndexer.search().filter(es_filter.Bool(
should=[es_filter.Terms(id=app_ids)]
))[0:len(app_ids)]
else:
# With filtering.
sq = WebappIndexer.get_app_filter(request, {
'device': self._get_device(request)
}, app_ids=app_ids)
# Store the apps to attach to feed elements later.
apps = sq.execute().hits
return dict((app.id, app) for app in apps)
def get_pending_queue(self):
if self.use_es:
must = [
es_filter.Term(status=amo.STATUS_PENDING),
es_filter.Term(
**{'latest_version.status': amo.STATUS_PENDING}),
es_filter.Term(is_escalated=False),
es_filter.Term(is_disabled=False),
]
return WebappIndexer.search().filter('bool', must=must)
return (Version.objects.no_cache().filter(
files__status=amo.STATUS_PENDING,
addon__disabled_by_user=False,
addon__status=amo.STATUS_PENDING).exclude(
addon__id__in=self.excluded_ids).order_by(
'nomination',
'created').select_related('addon',
'files').no_transforms())
def get_homescreen_queue(self):
# Both unreviewed homescreens and published homescreens with new
# unreviewed versions go in this queue.
if self.use_es:
must = [
es_filter.Term(**{'latest_version.status':
mkt.STATUS_PENDING}),
es_filter.Term(is_escalated=False),
es_filter.Term(is_disabled=False),
es_filter.Term(is_homescreen=True),
]
return WebappIndexer.search().filter('bool', must=must)
return (Version.objects.filter(
files__status=mkt.STATUS_PENDING,
addon__disabled_by_user=False,
addon__tags__tag_text='homescreen')
.exclude(addon__id__in=self.excluded_ids)
.order_by('nomination', 'created')
.select_related('addon', 'files').no_transforms())
def get_pending_queue(self):
# Unreviewed apps go in this queue.
if self.use_es:
must = [
es_filter.Term(status=mkt.STATUS_PENDING),
es_filter.Term(**{"latest_version.status": mkt.STATUS_PENDING}),
es_filter.Term(is_escalated=False),
es_filter.Term(is_disabled=False),
]
return WebappIndexer.search().filter("bool", must=must)
return (
Version.objects.filter(
files__status=mkt.STATUS_PENDING, addon__disabled_by_user=False, addon__status=mkt.STATUS_PENDING
)
.exclude(addon__id__in=self.excluded_ids)
.exclude(addon__tags__tag_text="homescreen")
.order_by("nomination", "created")
.select_related("addon", "files")
.no_transforms()
)
def get_updates_queue(self):
if self.use_es:
sq = WebappIndexer.search()
sq = sq.filter('terms', status=amo.WEBAPPS_APPROVED_STATUSES)
sq = sq.filter('term', **{'latest_version.status': amo.STATUS_PENDING})
sq = sq.filter('terms', app_type=[amo.ADDON_WEBAPP_PACKAGED,
amo.ADDON_WEBAPP_PRIVILEGED])
sq = sq.filter('term', is_disabled=False)
sq = sq.filter('term', is_escalated=False)
return sq
return (Version.objects.no_cache().filter(
# Note: this will work as long as we disable files of existing
# unreviewed versions when a new version is uploaded.
files__status=amo.STATUS_PENDING,
addon__disabled_by_user=False,
addon__is_packaged=True,
addon__status__in=amo.WEBAPPS_APPROVED_STATUSES)
.exclude(addon__id__in=self.excluded_ids)
.order_by('nomination', 'created')
.select_related('addon', 'files').no_transforms())
def get_obj(self):
return WebappIndexer.search().filter('term',
id=self.app.pk).execute().hits[0]
def get_queryset(self):
return WebappIndexer.search()
def serialize(self, app, profile=None):
data = WebappIndexer.search().filter('term',
id=app.pk).execute().hits[0]
a = TVESAppSerializer(instance=data, context={'request': self.request})
return a.data
def test_feed_collection_group(self):
app = WebappIndexer.search().filter(
'term', id=self.app.pk).execute().hits[0]
app['group_translations'] = [{'lang': 'en-US', 'string': 'My Group'}]
res = ESAppSerializer(app, context={'request': self.request})
eq_(res.data['group'], {'en-US': 'My Group'})
def get_obj(self):
return WebappIndexer.search().filter(
'term', id=self.app.pk).execute().hits[0]
def serialize(self, app, profile=None):
data = WebappIndexer.search().filter("term", id=app.pk).execute().hits[0]
a = TVESAppSerializer(instance=data, context={"request": self.request})
return a.data
def get_queryset(self):
return WebappIndexer.search()
def test_feed_collection_group(self):
app = WebappIndexer.search().filter('term',
id=self.app.pk).execute().hits[0]
app['group_translations'] = [{'lang': 'en-US', 'string': 'My Group'}]
res = ESAppSerializer(app, context={'request': self.request})
eq_(res.data['group'], {'en-US': 'My Group'})
