def test_sphinx_indexer(self):
"""
This tests that sphinx will properly index an addon.
"""
# we have to specify to sphinx to look at test_ dbs
c = SearchClient()
results = c.query('Delicious')
assert results[0].id == 3615, results
def test_sphinx_indexer(self):
"""
This tests that sphinx will properly index an addon.
"""
# we have to specify to sphinx to look at test_ dbs
c = SearchClient()
results = c.query('Delicious')
assert results[0].id == 3615, results
def process_request(self,
query,
addon_type='ALL',
limit=10,
platform='ALL',
version=None):
"""
This queries sphinx with `query` and serves the results in xml.
"""
if query.startswith('guid:'):
return self.guid_search(query, limit)
sc = SearchClient()
limit = min(MAX_LIMIT, int(limit))
opts = {'app': self.request.APP.id}
if addon_type.upper() != 'ALL':
try:
opts['type'] = int(addon_type)
except ValueError:
# `addon_type` is ALL or a type id. Otherwise we ignore it.
pass
if version:
opts['version'] = version
if platform.upper() != 'ALL':
opts['platform'] = platform.lower()
if self.version < 1.5:
# By default we show public addons only for api_version < 1.5
opts['status'] = [amo.STATUS_PUBLIC]
# Fix doubly encoded query strings
try:
query = urllib.unquote(query.encode('ascii'))
except UnicodeEncodeError:
# This fails if the string is already UTF-8.
pass
try:
results = sc.query(query, limit=limit, **opts)
except SearchError:
return self.render_msg('Could not connect to Sphinx search.',
ERROR,
status=503,
mimetype=self.mimetype)
return self.render('api/search.xml', {
'results': results,
'total': sc.total_found
})
def search(request, tag_name=None, template=None):
# If the form is invalid we still want to have a query.
query = request.REQUEST.get('q', '')
search_opts = {
'meta': ('versions', 'categories', 'tags', 'platforms'),
'version': None,
}
form = SearchForm(request)
form.is_valid() # Let the form try to clean data.
category = form.cleaned_data.get('cat')
if category == 'collections':
return _collections(request)
elif category == 'personas':
return _personas(request)
# TODO: Let's change the form values to something less gross when
# Remora dies in a fire.
query = form.cleaned_data['q']
addon_type = form.cleaned_data.get('atype', 0)
tag = tag_name if tag_name is not None else form.cleaned_data.get('tag')
if tag_name:
search_opts['show_personas'] = True
page = form.cleaned_data['page']
sort = form.cleaned_data.get('sort')
search_opts['version'] = form.cleaned_data.get('lver')
search_opts['limit'] = form.cleaned_data.get('pp', DEFAULT_NUM_RESULTS)
search_opts['platform'] = form.cleaned_data.get('pid', amo.PLATFORM_ALL)
search_opts['sort'] = sort
search_opts['app'] = request.APP.id
search_opts['offset'] = (page - 1) * search_opts['limit']
if category:
search_opts['category'] = category
elif addon_type:
search_opts['type'] = addon_type
search_opts['tag'] = tag
client = SearchClient()
try:
results = client.query(query, **search_opts)
except SearchError, e:
log.error('Sphinx Error: %s' % e)
return jingo.render(request, 'search/down.html', locals(), status=503)
def process_request(self, query, addon_type='ALL', limit=10,
platform='ALL', version=None):
"""
This queries sphinx with `query` and serves the results in xml.
"""
sc = SearchClient()
limit = min(MAX_LIMIT, int(limit))
opts = {'app': self.request.APP.id}
if addon_type.upper() != 'ALL':
try:
opts['type'] = int(addon_type)
except ValueError:
# `addon_type` is ALL or a type id. Otherwise we ignore it.
pass
if version:
opts['version'] = version
if platform.upper() != 'ALL':
opts['platform'] = platform.lower()
# By default we show public addons only for api_version < 1.5
statuses = [amo.STATUS_PUBLIC]
if (self.version >= 1.5
and not self.request.REQUEST.get('hide_sandbox')):
statuses.append(amo.STATUS_UNREVIEWED)
opts['status'] = statuses
# Fix doubly encoded query strings
if self.version < 1.5:
try:
query = urllib.unquote(query.encode('ascii'))
except UnicodeEncodeError:
# This fails if the string is already UTF-8.
pass
try:
results = sc.query(query, limit=limit, **opts)
except SearchError:
return self.render_msg('Could not connect to Sphinx search.',
ERROR, status=503, mimetype=self.mimetype)
return self.render('api/search.xml',
{'results': results, 'total': sc.total_found})
def ajax_search(request):
""" Returns a json feed of ten results for auto-complete used in
collections.
[
{"id": 123, "name": "best addon", "icon": "http://path/to/icon"},
...
]
"""
q = request.GET.get('q', '')
client = SearchClient()
try:
results = client.query(q, limit=10)
items = [dict(id=result.id, label=unicode(result.name),
icon=result.icon_url, value=unicode(result.name).lower())
for result in results]
except SearchError:
items = []
return HttpResponse(json.dumps(items), mimetype='application/json')
def ajax_search(request):
""" Returns a json feed of ten results for auto-complete used in
collections.
[
{"id": 123, "name": "best addon", "icon": "http://path/to/icon"},
...
]
"""
q = request.GET.get('q', '')
client = SearchClient()
try:
results = client.query('@name ' + q, limit=10,
match=sphinx.SPH_MATCH_EXTENDED2)
return [dict(id=result.id, label=unicode(result.name),
icon=result.icon_url, value=unicode(result.name).lower())
for result in results]
except SearchError:
return []
def ajax_search(request):
""" Returns a json feed of ten results for auto-complete used in
collections.
[
{"id": 123, "name": "best addon", "icon": "http://path/to/icon"},
...
]
"""
q = request.GET.get('q', '')
client = SearchClient()
try:
results = client.query('@name ' + q, limit=10,
match=sphinx.SPH_MATCH_EXTENDED2)
return [dict(id=result.id, label=unicode(result.name),
icon=result.icon_url, value=unicode(result.name).lower())
for result in results]
except SearchError:
return []
def process_request(self, query, addon_type="ALL", limit=10, platform="ALL", version=None):
"""
This queries sphinx with `query` and serves the results in xml.
"""
sc = SearchClient()
opts = {"app": self.request.APP.id}
if addon_type.upper() != "ALL":
try:
opts["type"] = int(addon_type)
except ValueError:
# `addon_type` is ALL or a type id. Otherwise we ignore it.
pass
if version:
opts["version"] = version
if platform.upper() != "ALL":
opts["platform"] = platform.lower()
# By default we show public addons only for api_version < 1.5
statuses = [amo.STATUS_PUBLIC]
if self.version >= 1.5 and not self.request.REQUEST.get("hide_sandbox"):
statuses.append(amo.STATUS_SANDBOX)
opts["status"] = statuses
# Fix doubly encoded query strings
if self.version < 1.5:
query = urllib.unquote(query.encode("ascii"))
try:
results = sc.query(query, limit=int(limit), **opts)
except SearchError:
return self.render_msg("Could not connect to Sphinx search.", ERROR, status=503, mimetype=self.mimetype)
if self.format == "xml":
return self.render("api/search.xml", {"results": results, "total": sc.total_found})
def _sphinx_api_search(self, query, addon_type="ALL", limit=10, platform="ALL", version=None, compat_mode="strict"):
"""
This queries sphinx with `query` and serves the results in xml.
"""
sc = SearchClient()
limit = min(MAX_LIMIT, int(limit))
opts = {"app": self.request.APP.id}
if addon_type.upper() != "ALL":
try:
opts["type"] = int(addon_type)
except ValueError:
# `addon_type` is ALL or a type id. Otherwise we ignore it.
pass
if version:
opts["version"] = version
if platform.upper() != "ALL":
opts["platform"] = platform.lower()
if self.version < 1.5:
# By default we show public addons only for api_version < 1.5
opts["status"] = [amo.STATUS_PUBLIC]
# Fix doubly encoded query strings
try:
query = urllib.unquote(query.encode("ascii"))
except UnicodeEncodeError:
# This fails if the string is already UTF-8.
pass
try:
results = sc.query(query, limit=limit, **opts)
except SearchError:
return self.render_msg("Could not connect to Sphinx search.", ERROR, status=503, mimetype=self.mimetype)
return self.render("api/search.xml", {"results": results, "total": sc.total_found})
def _get_results(request, meta=[], client=None):
form = ReporterSearchForm(request.GET)
if form.is_valid():
data = form.cleaned_data
query = data.get('q', '')
product = data.get('product') or FIREFOX.short
version = data.get('version')
search_opts = _get_results_opts(request, data, product, meta)
c = client or Client()
opinions = c.query(query, **search_opts)
metas = c.meta
else:
opinions = []
product = request.default_prod
query = ''
version = Version(LATEST_BETAS[product]).simplified
metas = {}
product = PRODUCTS.get(product, FIREFOX)
return (opinions, form, product, version, metas)
def _get_results(request, meta=[], client=None):
form = ReporterSearchForm(request.GET)
if form.is_valid():
data = form.cleaned_data
query = data.get('q', '')
product = data.get('product') or request.default_prod.short
version = data.get('version')
search_opts = _get_results_opts(request, data, product, meta)
type_filter = search_opts['type'] if 'type' in search_opts else None
c = client or Client()
opinions = c.query(query, **search_opts)
metas = c.meta
else:
opinions = []
type_filter = None
product = request.default_prod
query = ''
version = (getattr(product, 'default_version', None)
or Version(LATEST_BETAS[product]).simplified)
metas = {}
product = PRODUCTS.get(product, FIREFOX)
return (opinions, form, product, version, metas, type_filter)
def dashboard(request):
"""Front page view."""
# Defaults
app = request.default_app
version = simplify_version(LATEST_BETAS[app])
# Frequent terms
term_params = {
'product': app.id,
'version': version,
}
frequent_terms = Term.objects.frequent(
**term_params)[:settings.TRENDS_COUNT]
# opinions queryset for demographics
latest_opinions = Opinion.objects.browse(**term_params)
latest_beta = Opinion.objects.filter(version=version, product=app.id)
# Sites clusters
sites = SiteSummary.objects.filter(version__exact=version).filter(
positive__exact=None).filter(os__exact=None)[:settings.TRENDS_COUNT]
# search form to generate various form elements.
search_form = ReporterSearchForm()
try:
c = Client()
search_opts = dict(product=app.short, version=version)
c.query('', meta=('type', 'locale', 'os', 'day_sentiment'), **search_opts)
metas = c.meta
daily = c.meta.get('day_sentiment', {})
chart_data = dict(series=[
dict(name=_('Praise'), data=daily['praise']),
dict(name=_('Issues'), data=daily['issue']),
dict(name=_('Suggestion'), data=daily['suggestion']),
],
)
total = c.total_found
except SearchError:
metas = {}
total = latest_beta.count()
chart_data = None
data = {'opinions': latest_opinions.all()[:settings.MESSAGES_COUNT],
'opinion_count': total,
'opinion_types': OPINION_TYPES,
'OPINION_PRAISE': OPINION_PRAISE,
'OPINION_ISSUE': OPINION_ISSUE,
'OPINION_SUGGESTION': OPINION_SUGGESTION,
'product': app.short,
'products': PROD_CHOICES,
'sentiments': get_sentiment(metas.get('type', [])),
'terms': stats.frequent_terms(qs=frequent_terms),
'demo': dict(locale=metas.get('locale'), os=metas.get('os')),
'sites': sites,
'version': version,
'versions': VERSION_CHOICES[app],
'search_form': search_form,
'chart_data_json': json.dumps(chart_data),
}
if not request.mobile_site:
template = 'dashboard/dashboard.html'
else:
template = 'dashboard/mobile/dashboard.html'
return jingo.render(request, template, data)
def dashboard(request):
prod = request.default_prod
version = getattr(prod, "default_version", None) or Version(LATEST_BETAS[prod]).simplified
search_form = ReporterSearchForm()
# Frequent terms
term_params = {"product": prod.id, "version": version}
# # opinions queryset for demographics
latest_opinions = Opinion.objects.browse(**term_params)
# # Sites clusters
# sites = SiteSummary.objects.filter(version__exact=version).filter(
# positive__exact=None).filter(
# platform__exact=None)[:settings.TRENDS_COUNT]
# sites = SiteSummary.objects.all()
# Get the desktop site's absolute URL for use in the settings tab
desktop_site = Site.objects.get(id=settings.DESKTOP_SITE_ID)
try:
c = Client()
search_opts = dict(product=prod.short, version=version)
c.query("", meta=("type", "locale", "manufacturer", "device", "day_sentiment"), **search_opts)
metas = c.meta
daily = c.meta.get("day_sentiment", {})
chart_data = dict(
series=[
dict(name=_("Praise"), data=daily["praise"]),
dict(name=_("Issues"), data=daily["issue"]),
dict(name=_("Ideas"), data=daily["idea"]),
]
)
total = c.total_found
except SearchError:
metas = {}
total = latest_opinions.count()
chart_data = None
data = {
"opinions": latest_opinions.all()[: settings.MESSAGES_COUNT],
"opinion_count": total,
"product": prod,
"products": PROD_CHOICES,
"sent": get_sentiment(metas.get("type", [])),
"locales": metas.get("locale"),
"platforms": metas.get("platform"),
"devices": metas.get("device"),
"manufacturers": metas.get("manufacturer"),
# 'sites': sites,
"version": version,
"versions": VERSION_CHOICES[prod],
"chart_data_json": json.dumps(chart_data),
"defaults": get_defaults(search_form),
"search_form": search_form,
"desktop_url": "http://" + desktop_site.domain,
}
if not request.mobile_site:
template = "dashboard/beta.html"
else:
template = "dashboard/mobile/beta.html"
return jingo.render(request, template, data)
def dashboard(request):
"""Beta dashboard."""
# Defaults
prod = request.default_prod
version = Version(LATEST_BETAS[prod]).simplified
search_form = ReporterSearchForm()
# Frequent terms
term_params = {
'product': prod.id,
'version': version,
}
frequent_terms = Term.objects.frequent(
**term_params)[:settings.TRENDS_COUNT]
# opinions queryset for demographics
latest_opinions = Opinion.objects.browse(**term_params)
latest_beta = Opinion.objects.filter(version=version, product=prod.id)
# Sites clusters
sites = SiteSummary.objects.filter(version__exact=version).filter(
positive__exact=None).filter(
platform__exact=None)[:settings.TRENDS_COUNT]
try:
c = Client()
search_opts = dict(product=prod.short, version=version)
c.query('', meta=('type', 'locale', 'platform', 'day_sentiment'),
**search_opts)
metas = c.meta
daily = c.meta.get('day_sentiment', {})
chart_data = dict(series=[
dict(name=_('Praise'), data=daily['praise']),
dict(name=_('Issues'), data=daily['issue']),
dict(name=_('Ideas'), data=daily['idea']),
])
total = c.total_found
except SearchError:
metas = {}
total = latest_beta.count()
chart_data = None
data = {'opinions': latest_opinions.all()[:settings.MESSAGES_COUNT],
'opinion_count': total,
'product': prod,
'products': PROD_CHOICES,
'sentiments': get_sentiment(metas.get('type', [])),
'terms': stats.frequent_terms(qs=frequent_terms),
'locales': metas.get('locale'),
'platforms': metas.get('platform'),
'sites': sites,
'version': version,
'versions': VERSION_CHOICES[prod],
'chart_data_json': json.dumps(chart_data),
'defaults': get_defaults(search_form),
'search_form': search_form,
}
if not request.mobile_site:
template = 'dashboard/beta.html'
else:
template = 'dashboard/mobile/beta.html'
return jingo.render(request, template, data)
def search(request, tag_name=None):
# If the form is invalid we still want to have a query.
query = request.REQUEST.get('q', '')
search_opts = {
'meta': ('versions', 'categories', 'tags'),
'version': None,
}
form = SearchForm(request)
form.is_valid() # Let the form try to clean data.
# TODO(davedash): remove this feature when we remove Application for
# the search advanced form
# Redirect if appid != request.APP.id
appid = form.cleaned_data['appid']
if request.APP.id != appid:
new_app = amo.APP_IDS.get(appid)
return HttpResponseRedirect(
urlresolvers.get_app_redirect(new_app))
category = form.cleaned_data.get('cat')
if category == 'collections':
return _collections(request)
elif category == 'personas':
return _personas(request)
# TODO: Let's change the form values to something less gross when
# Remora dies in a fire.
query = form.cleaned_data['q']
addon_type = form.cleaned_data.get('atype', 0)
tag = tag_name if tag_name is not None else form.cleaned_data.get('tag')
page = form.cleaned_data['page']
last_updated = form.cleaned_data.get('lup')
sort = form.cleaned_data.get('sort')
search_opts['version'] = form.cleaned_data.get('lver')
search_opts['limit'] = form.cleaned_data.get('pp', DEFAULT_NUM_RESULTS)
search_opts['platform'] = form.cleaned_data.get('pid', amo.PLATFORM_ALL)
search_opts['sort'] = sort
search_opts['app'] = request.APP.id
search_opts['offset'] = (page - 1) * search_opts['limit']
delta_dict = {
'1 day ago': timedelta(days=1),
'1 week ago': timedelta(days=7),
'1 month ago': timedelta(days=30),
'3 months ago': timedelta(days=90),
'6 months ago': timedelta(days=180),
'1 year ago': timedelta(days=365),
}
delta = delta_dict.get(last_updated)
if delta:
search_opts['before'] = int(
time.mktime((datetime.now() - delta).timetuple()))
if category:
search_opts['category'] = category
elif addon_type:
search_opts['type'] = addon_type
search_opts['tag'] = tag
client = SearchClient()
try:
results = client.query(query, **search_opts)
except SearchError:
return jingo.render(request, 'search/down.html', locals(), status=503)
version_filters = client.meta['versions']
# If we are filtering by a version, make sure we explicitly list it.
if search_opts['version']:
try:
version_filters += (version_int(search_opts['version']),)
except UnicodeEncodeError:
pass # We didn't want to list you anyway.
versions = _get_versions(request, client.meta['versions'],
search_opts['version'])
categories = _get_categories(request, client.meta['categories'],
addon_type, category)
tags = _get_tags(request, client.meta['tags'], tag)
sort_tabs = _get_sorts(request, sort)
pager = amo.utils.paginate(request, results, search_opts['limit'])
return jingo.render(request, 'search/results.html', {
'pager': pager, 'query': query, 'tag': tag,
'versions': versions, 'categories': categories, 'tags': tags,
'sort_tabs': sort_tabs, 'sort': sort})
def test_alpha_sort(self):
"This verifies that alpha sort is case insensitive."
c = SearchClient()
results = c.query('', sort='name')
ordering = [unicode(a.name).lower() for a in results]
eq_(ordering, sorted(ordering))
import datetime
import socket
from mock import patch
from nose.tools import eq_
import input
from feedback.models import Opinion
from search.client import Client, SearchError, extract_filters
from search.tests import SphinxTestCase
query = lambda x='', **kwargs: Client().query(x, **kwargs)
num_results = lambda x='', **kwargs: len(query(x, **kwargs))
class SearchTest(SphinxTestCase):
def test_default_ordering(self):
"""Any query should return results in rev-chron order."""
r = query()
dates = [o.created for o in r]
eq_(dates, sorted(dates, reverse=True), "These aren't revchron.")
r = query('Firefox')
dates = [o.created for o in r]
eq_(dates, sorted(dates, reverse=True), "These aren't revchron.")
@patch('search.client.sphinx.SphinxClient.RunQueries')
def test_errors(self, sphinx):
for error in (socket.timeout(), Exception(),):
sphinx.side_effect = error
self.assertRaises(SearchError, query)
def test_alpha_sort(self):
"This verifies that alpha sort is case insensitive."
c = SearchClient()
results = c.query('', sort='name')
ordering = [unicode(a.name).lower() for a in results]
eq_(ordering, sorted(ordering))
def dashboard(request):
prod = request.default_prod
version = (getattr(prod, 'default_version', None)
or Version(LATEST_BETAS[prod]).simplified)
search_form = ReporterSearchForm()
# Frequent terms
term_params = {
'product': prod.id,
'version': version,
}
# # opinions queryset for demographics
latest_opinions = Opinion.objects.browse(**term_params)
# # Sites clusters
# sites = SiteSummary.objects.filter(version__exact=version).filter(
# positive__exact=None).filter(
# platform__exact=None)[:settings.TRENDS_COUNT]
# sites = SiteSummary.objects.all()
# Get the desktop site's absolute URL for use in the settings tab
desktop_site = Site.objects.get(id=settings.DESKTOP_SITE_ID)
try:
c = Client()
search_opts = dict(product=prod.short, version=version)
c.query('',
meta=('type', 'locale', 'manufacturer', 'device',
'day_sentiment'),
**search_opts)
metas = c.meta
daily = c.meta.get('day_sentiment', {})
chart_data = dict(series=[
dict(name=_('Praise'), data=daily['praise']),
dict(name=_('Issues'), data=daily['issue']),
dict(name=_('Ideas'), data=daily['idea']),
])
total = c.total_found
except SearchError:
metas = {}
total = latest_opinions.count()
chart_data = None
data = {
'opinions': latest_opinions.all()[:settings.MESSAGES_COUNT],
'opinion_count': total,
'product': prod,
'products': PROD_CHOICES,
'sent': get_sentiment(metas.get('type', [])),
'locales': metas.get('locale'),
'platforms': metas.get('platform'),
'devices': metas.get('device'),
'manufacturers': metas.get('manufacturer'),
# 'sites': sites,
'version': version,
'versions': VERSION_CHOICES[prod],
'chart_data_json': json.dumps(chart_data),
'defaults': get_defaults(search_form),
'search_form': search_form,
'desktop_url': 'http://' + desktop_site.domain,
}
if not request.mobile_site:
template = 'dashboard/beta.html'
else:
template = 'dashboard/mobile/beta.html'
return jingo.render(request, template, data)
def search(request, tag_name=None):
# If the form is invalid we still want to have a query.
query = request.REQUEST.get('q', '')
search_opts = {
'meta': ('versions', 'categories', 'tags'),
'version': None,
}
form = SearchForm(request)
form.is_valid() # Let the form try to clean data.
# TODO(davedash): remove this feature when we remove Application for
# the search advanced form
# Redirect if appid != request.APP.id
appid = form.cleaned_data['appid']
if request.APP.id != appid:
new_app = amo.APP_IDS.get(appid)
return HttpResponseRedirect(
urlresolvers.get_app_redirect(new_app))
category = form.cleaned_data.get('cat')
if category == 'collections':
return _collections(request)
elif category == 'personas':
return _personas(request)
# TODO: Let's change the form values to something less gross when
# Remora dies in a fire.
query = form.cleaned_data['q']
addon_type = form.cleaned_data.get('atype', 0)
tag = tag_name if tag_name is not None else form.cleaned_data.get('tag')
page = form.cleaned_data['page']
last_updated = form.cleaned_data.get('lup')
sort = form.cleaned_data.get('sort')
search_opts['version'] = form.cleaned_data.get('lver')
search_opts['limit'] = form.cleaned_data.get('pp', DEFAULT_NUM_RESULTS)
search_opts['platform'] = form.cleaned_data.get('pid', amo.PLATFORM_ALL)
search_opts['sort'] = sort
search_opts['app'] = request.APP.id
search_opts['offset'] = (page - 1) * search_opts['limit']
delta_dict = {
'1 day ago': timedelta(days=1),
'1 week ago': timedelta(days=7),
'1 month ago': timedelta(days=30),
'3 months ago': timedelta(days=90),
'6 months ago': timedelta(days=180),
'1 year ago': timedelta(days=365),
}
delta = delta_dict.get(last_updated)
if delta:
search_opts['before'] = int(
time.mktime((datetime.now() - delta).timetuple()))
if category:
search_opts['category'] = category
elif addon_type:
search_opts['type'] = addon_type
search_opts['tag'] = tag
client = SearchClient()
try:
results = client.query(query, **search_opts)
except SearchError, e:
log.error('Sphinx Error: %s' %e)
return jingo.render(request, 'search/down.html', locals(), status=503)
