def library_autocomplete(request):
"""
'Live' search by name
"""
from search.helpers import package_query
from elasticutils import F
q = request.GET.get('q')
limit = request.GET.get('limit')
try:
limit = int(limit)
except:
limit = settings.LIBRARY_AUTOCOMPLETE_LIMIT
ids = (settings.MINIMUM_PACKAGE_ID, settings.MINIMUM_PACKAGE_ID - 1)
notAddonKit = ~(F(id_number=ids[0]) | F(id_number=ids[1]))
onlyMyPrivateLibs = (F(active=True) | F(author=request.user.id))
try:
qs = (Package.search().query(or_=package_query(q)).filter(
type='l').filter(notAddonKit).filter(onlyMyPrivateLibs))
found = qs[:limit]
except Exception, ex:
log.exception('Library autocomplete error')
found = []
def test_filter_or_3(self):
eq_(len(self.get_s().filter(F(tag='awesome') | F(tag='boat') |
F(tag='boring'))), 5)
eq_(len(self.get_s().filter(or_={'foo': 'bar',
'or_': {'tag': 'boat',
'width': '5'}
})), 3)
def test_filter_or_3(self):
s = self.get_s().filter(F(tag='awesome') | F(tag='boat') |
F(tag='boring'))
eq_(s._build_query(), {
'filter': {
'or': [
{'term': {'tag': 'awesome'}},
{'term': {'tag': 'boat'}},
{'term': {'tag': 'boring'}}
]
}
})
eq_(s.count(), 5)
# This is kind of a crazy case.
s = self.get_s().filter(or_={'foo': 'bar',
'or_': {'tag': 'boat', 'width': '5'}})
eq_(s._build_query(), {
'filter': {
'or': [
{'or': [
{'term': {'width': '5'}},
{'term': {'tag': 'boat'}}
]},
{'term': {'foo': 'bar'}}
]
}
})
eq_(s.count(), 3)
def filter_username(self, value):
username_lower = value.lower()
username_filter = F(iusername__prefix=username_lower) | F(
idisplay_name__prefix=username_lower)
return self._filter_by_users(username_filter)
def package_search(searchq='', user=None, score_on=None, **filters):
"""This provides some sane defaults to filter on when searching Packages"""
# This is a filtered query, that says we want to do a query, but not have
# to deal with version_text='initial' or 'copy'
notInitialOrCopy = ~(F(version_name='initial') | F(version_name='copy'))
qs = Package.search().values_obj('copies_count','times_depended',
'activity','size').filter(notInitialOrCopy,
**filters).filter(F(active=True))
# Add type facet (minus any type filter)
facetFilter = dict((k, v) for k, v in filters.items() if k != 'type')
if facetFilter:
facetFilter = notInitialOrCopy & F(**facetFilter)
else:
facetFilter = notInitialOrCopy
qs = qs.facet(types={'terms': {'field': 'type'},
'facet_filter': facetFilter.filters})
if searchq:
qs = qs.query(or_=package_query(searchq))
if user and user.is_authenticated():
qs = qs.facet(author={'terms': {
'field': 'author',
'script':'term == %d ? true : false' % user.id}
})
return qs
def test_f_mutation_with_or(self):
"""Make sure OR doesn't mutate operands."""
f1 = F(fielda='tag', fieldb='boat')
f2 = F(fieldc='car')
f1 | f2
# Should only contain f1 filters.
eq_(sorted(f1.filters[0]['and']),
sorted([('fielda', 'tag'), ('fieldb', 'boat')]))
# Should only contain f2 filters.
eq_(f2.filters, [('fieldc', 'car')])
def test_f_mutation_with_or(self):
"""Make sure OR doesn't mutate operands."""
f1 = F(fielda='tag', fieldb='boat')
f2 = F(fieldc='car')
f1 | f2
# Should only contain f1 filters.
eq_(sorted(f1.filters['and']),
sorted([{'term': {'fielda': 'tag'}},
{'term': {'fieldb': 'boat'}}]))
# Should only contain f2 filters.
eq_(f2.filters, {'term': {'fieldc': 'car'}})
def _filter_by_users(self, users_filter, invert=False):
users = UserMappingType.reshape(
UserMappingType.search()
# Optimization: Filter out users that have never contributed.
.filter(~F(last_contribution_date=None)).filter(
users_filter).values_dict("id").everything())
user_ids = [u["id"] for u in users]
res = F(creator_id__in=user_ids)
if invert:
res = ~res
return res
def test_filter_f_and_ff(self):
s = self.get_s().filter(F(tag='awesome') & F(foo='car', width='7'))
eq_(s._build_query(),
{
'filter': {
'and': [
{'term': {'width': '7'}},
{'term': {'foo': 'car'}},
{'term': {'tag': 'awesome'}}
]
}
}
)
eq_(s.count(), 1)
def test_filter_not_not_f(self):
f = F(tag='awesome')
f = ~f
f = ~f
s = self.get_s().filter(f)
eq_(s._build_query(), {'filter': {'term': {'tag': 'awesome'}}})
eq_(s.count(), 3)
def get_facet_response_for_choice_fields(query_with_criteria, choice_fields,
form_model_id):
facet_results = []
for field in choice_fields:
field_name = es_questionnaire_field_name(field.code,
form_model_id) + "_exact"
facet_response = query_with_criteria.facet(
field_name, filtered=True).facet_counts()
facet_result_options = []
facet_result = {
"es_field_name": field_name,
"facets": facet_result_options,
# find total submissions containing specified answer
"total":
query_with_criteria.filter(~F(**{field_name: None})).count()
}
for option, facet_list in facet_response.iteritems():
for facet in facet_list:
facet_result_options.append({
"term": facet['term'],
"count": facet['count']
})
facet_results.append(facet_result)
return facet_results
def filter_page(self, value):
"""Validate the page numbers, but don't return any filters."""
try:
page = fields.IntegerField().to_internal_value(value)
except fields.ValidationError:
page = 1
if page < 1:
page = 1
self.query_values['page'] = page
return F()
def filter_page_size(self, value):
"""Validate the page sizes, but don't return any filters."""
try:
page_size = fields.IntegerField().to_internal_value(value)
except fields.ValidationError:
page_size = 20
if not (1 <= page_size < 100):
page_size = 100
self.query_values['page_size'] = page_size
return F()
def test_f_mutation_with_not(self):
"""Make sure NOT doesn't mutate operands."""
f1 = F(fielda='tag')
f2 = ~f1
# Change f2 to see if it changes f1.
f2.filters['not']['filter']['term']['fielda'] = 'boat'
# Should only contain f1 filters.
eq_(f1.filters, {'term': {'fielda': 'tag'}})
# Should only contain f2 tweaked filter.
eq_(f2.filters, {'not': {'filter': {'term': {'fielda': 'boat'}}}})
def test_f_mutation_with_not(self):
"""Make sure NOT doesn't mutate operands."""
f1 = F(fielda='tag')
f2 = ~f1
# Change f2 to see if it changes f1.
f2.filters[0]['not']['filter'] = [('fielda', 'boat')]
# Should only contain f1 filters.
eq_(f1.filters, [('fielda', 'tag')])
# Should only contain f2 tweaked filter.
eq_(f2.filters, [{'not': {'filter': [('fielda', 'boat')]}}])
def apply_reviewer_filters(request, qs, data=None):
for k in ('has_info_request', 'has_editor_comment'):
if data.get(k, None) is not None:
qs = qs.filter(**{
'latest_version.%s' % k: data[k]
})
if data.get('is_escalated', None) is not None:
qs = qs.filter(is_escalated=data['is_escalated'])
is_tarako = data.get('is_tarako')
if is_tarako is not None:
if is_tarako:
qs = qs.filter(tags='tarako')
else:
qs = qs.filter(~F(tags='tarako'))
return qs
def get_filter(self, f, term):
# Action must be either None or one that can be handled by Elasticutils
# According to Elasticutils
# Available actions are : startswith, prefix, in, range and distance
action = self.search_actions.get(f, self.default_action)
if not action and f in self.prefix_fields:
action = 'prefix'
field_action = '{0}__{1}'.format(f, action) if action else f
# If term is an empty string, we are looking for a missing relation
if term == '':
term = None
return F(**{field_action: term})
def get_filters(self):
self.query_values = self.get_default_query()
# request.GET is a multidict, so simple `.update(request.GET)` causes
# everything to be a list. This converts it into a plain single dict.
self.query_values.update(dict(self.request.GET.items()))
f = F()
for key, value in self.query_values.items():
filter_method = getattr(self, 'filter_' + key, None)
if filter_method is None:
self.warnings.append('Unknown filter {}'.format(key))
else:
f &= filter_method(value)
return f
def test_filter_not(self):
s = self.get_s().filter(~F(tag='awesome'))
eq_(s._build_query(), {
'filter': {
'not': {
'filter': {'term': {'tag': 'awesome'}}
}
}
})
eq_(s.count(), 3)
s = self.get_s().filter(~(F(tag='boring') | F(tag='boat')))
eq_(s._build_query(), {
'filter': {
'not': {
'filter': {
'or': [
{'term': {'tag': 'boring'}},
{'term': {'tag': 'boat'}}
]
}
}
}
})
eq_(s.count(), 4)
s = self.get_s().filter(~F(tag='boat')).filter(~F(foo='bar'))
eq_(s._build_query(), {
'filter': {
'and': [
{'not': {'filter': {'term': {'tag': 'boat'}}}},
{'not': {'filter': {'term': {'foo': 'bar'}}}}
]
}
})
eq_(s.count(), 4)
s = self.get_s().filter(~F(tag='boat', foo='barf'))
eq_(s._build_query(), {
'filter': {
'not': {
'filter': {
'and': [
{'term': {'foo': 'barf'}},
{'term': {'tag': 'boat'}}
]
}
}
}
})
eq_(s.count(), 6)
def issue_distribution(index):
q = S().indexes(index).doctypes('IssueData') \
.filter(state='open') \
.filter(~F(assignee=None)) \
.values_dict()
q = all(q)
# open, assigned
issues = list(q)
distribution = {}
for issue in issues:
login = issue['assignee']['login']
if login not in distribution:
distribution[login] = {
'person': issue['assignee'],
'issues': [issue]
}
else:
distribution[login]['issues'].append(issue)
return distribution
def filter_ordering(self, value):
"""Validate sort order, but don't return any filters."""
if value not in self.get_allowed_orderings():
self.warnings.append('Invalid sort order: {}'.format(value))
return F()
def get_filters(self):
f = super(TopContributorsQuestions, self).get_filters()
f &= F(by_asker=False)
return f
def get_data(self, request):
super(TopContributorsQuestions, self).get_data(request)
# This is the base of all the metrics. Each metric branches off from
# this to get a particular metric type, since we can't do Aggregates.
query = AnswerMetricsMappingType.search()
base_filters = self.get_filters()
# This branch is to get the total number of answers for each user.
answer_query = (
query
.filter(base_filters)
.facet('creator_id', filtered=True, size=BIG_NUMBER))
# This branch gets the number of answers that are solutions for each user.
solutions_filter = base_filters & F(is_solution=True)
solutions_query = (
query
.filter(solutions_filter)
.facet('creator_id', filtered=True, size=BIG_NUMBER))
# This branch gets the number of helpful votes across all answers for
# each user. It is a raw facet because elasticutils only supports the
# term facet type in non-raw facets. Because it is raw facet, we have
# to also put the filter in the facet ourselves.
helpful_query = (
query
.facet_raw(
creator_id={
'terms_stats': {
'key_field': 'creator_id',
'value_field': 'helpful_count',
},
'facet_filter': query._process_filters(base_filters.filters),
}))
# Collect three lists of objects that correlates users and the appropriate metric count
creator_answer_counts = answer_query.facet_counts()['creator_id']['terms']
creator_solutions_counts = solutions_query.facet_counts()['creator_id']['terms']
creator_helpful_counts = helpful_query.facet_counts()['creator_id']['terms']
# Combine all the metric types into one big list.
combined = defaultdict(lambda: {
'answer_count': 0,
'solution_count': 0,
'helpful_vote_count': 0,
})
for d in creator_answer_counts:
combined[d['term']]['user_id'] = d['term']
combined[d['term']]['answer_count'] = d['count']
for d in creator_solutions_counts:
combined[d['term']]['user_id'] = d['term']
combined[d['term']]['solution_count'] = d['count']
for d in creator_helpful_counts:
combined[d['term']]['user_id'] = d['term']
# Since this is a term_stats filter, not just a term filter, it is total, not count.
combined[d['term']]['helpful_vote_count'] = int(d['total'])
# Sort by answer count, and get just the ids into a list.
sort_key = self.query_values['ordering']
if sort_key[0] == '-':
sort_reverse = True
sort_key = sort_key[1:]
else:
sort_reverse = False
top_contributors = combined.values()
top_contributors.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
user_ids = [c['user_id'] for c in top_contributors]
full_count = len(user_ids)
# Paginate those user ids.
page_start = (self.query_values['page'] - 1) * self.query_values['page_size']
page_end = page_start + self.query_values['page_size']
user_ids = user_ids[page_start:page_end]
# Get full user objects for every id on this page.
users = UserMappingType.reshape(
UserMappingType
.search()
.filter(id__in=user_ids)
.values_dict('id', 'username', 'display_name', 'avatar', 'last_contribution_date')
[:self.query_values['page_size']])
# For ever user object found, mix in the metrics counts for that user,
# and then reshape the data to make more sense to clients.
data = []
for u in users:
d = combined[u['id']]
d['user'] = u
d['last_contribution_date'] = d['user'].get('last_contribution_date', None)
d.pop('user_id', None)
d['user'].pop('id', None)
d['user'].pop('last_contribution_date', None)
data.append(d)
# One last sort, since ES didn't return the users in any particular order.
data.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
# Add ranks to the objects.
for i, contributor in enumerate(data, 1):
contributor['rank'] = page_start + i
return {
'results': data,
'count': full_count,
'filters': self.query_values,
'allowed_orderings': self.get_allowed_orderings(),
'warnings': self.warnings,
}
def filter_last_contribution_date__lt(self, value):
# This query usually covers a lot of users, so inverting it makes it a lot faster.
date = fields.DateField().to_internal_value(value)
dt = datetime.combine(date, datetime.max.time())
return self._filter_by_users(~F(last_contribution_date__lt=dt), invert=True)
def filter_product(self, value):
return F(product=value)
def filter_last_contribution_date__gt(self, value):
date = fields.DateField().to_internal_value(value)
dt = datetime.combine(date, datetime.max.time())
return self._filter_by_users(F(last_contribution_date__gt=dt))
def filter_locale(self, value):
return F(locale=value)
def filter_enddate(self, value):
date = fields.DateField().to_internal_value(value)
dt = datetime.combine(date, datetime.max.time())
return F(created__lte=dt)
def get_data(self, request):
super(TopContributorsQuestions, self).get_data(request)
# This is the base of all the metrics. Each metric branches off from
# this to get a particular metric type, since we can't do Aggregates.
query = AnswerMetricsMappingType.search()
base_filters = self.get_filters()
# This branch is to get the total number of answers for each user.
answer_query = query.filter(base_filters).facet("creator_id",
filtered=True,
size=BIG_NUMBER)
# This branch gets the number of answers that are solutions for each user.
solutions_filter = base_filters & F(is_solution=True)
solutions_query = query.filter(solutions_filter).facet("creator_id",
filtered=True,
size=BIG_NUMBER)
# This branch gets the number of helpful votes across all answers for
# each user. It is a raw facet because elasticutils only supports the
# term facet type in non-raw facets. Because it is raw facet, we have
# to also put the filter in the facet ourselves.
helpful_query = query.facet_raw(
creator_id={
"terms_stats": {
"key_field": "creator_id",
"value_field": "helpful_count",
},
"facet_filter": query._process_filters(base_filters.filters),
})
# Collect three lists of objects that correlates users and the appropriate metric count
creator_answer_counts = answer_query.facet_counts(
)["creator_id"]["terms"]
creator_solutions_counts = solutions_query.facet_counts(
)["creator_id"]["terms"]
creator_helpful_counts = helpful_query.facet_counts(
)["creator_id"]["terms"]
# Combine all the metric types into one big list.
combined = defaultdict(lambda: {
"answer_count": 0,
"solution_count": 0,
"helpful_vote_count": 0,
})
for d in creator_answer_counts:
combined[d["term"]]["user_id"] = d["term"]
combined[d["term"]]["answer_count"] = d["count"]
for d in creator_solutions_counts:
combined[d["term"]]["user_id"] = d["term"]
combined[d["term"]]["solution_count"] = d["count"]
for d in creator_helpful_counts:
combined[d["term"]]["user_id"] = d["term"]
# Since this is a term_stats filter, not just a term filter, it is total, not count.
combined[d["term"]]["helpful_vote_count"] = int(d["total"])
# Sort by answer count, and get just the ids into a list.
sort_key = self.query_values["ordering"]
if sort_key[0] == "-":
sort_reverse = True
sort_key = sort_key[1:]
else:
sort_reverse = False
top_contributors = list(combined.values())
top_contributors.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
user_ids = [c["user_id"] for c in top_contributors]
full_count = len(user_ids)
# Paginate those user ids.
page_start = (self.query_values["page"] -
1) * self.query_values["page_size"]
page_end = page_start + self.query_values["page_size"]
user_ids = user_ids[page_start:page_end]
# Get full user objects for every id on this page.
users = UserMappingType.reshape(
UserMappingType.search().filter(id__in=user_ids).values_dict(
"id", "username", "display_name", "avatar",
"last_contribution_date")[:self.query_values["page_size"]])
# For ever user object found, mix in the metrics counts for that user,
# and then reshape the data to make more sense to clients.
data = []
for u in users:
d = combined[u["id"]]
d["user"] = u
d["last_contribution_date"] = d["user"].get(
"last_contribution_date", None)
d.pop("user_id", None)
d["user"].pop("id", None)
d["user"].pop("last_contribution_date", None)
data.append(d)
# One last sort, since ES didn't return the users in any particular order.
data.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
# Add ranks to the objects.
for i, contributor in enumerate(data, 1):
contributor["rank"] = page_start + i
return {
"results": data,
"count": full_count,
"filters": self.query_values,
"allowed_orderings": self.get_allowed_orderings(),
"warnings": self.warnings,
}
def test_filter_raw_overrides_everything(self):
s = self.get_s().filter_raw({'term': {'tag': 'awesome'}})
s = s.filter(tag='boring')
s = s.filter(F(tag='end'))
eq_(s._build_query(),
{'filter': {'term': {'tag': 'awesome'}}})
