def library_autocomplete(request): """ 'Live' search by name """ from search.helpers import package_query from elasticutils import F q = request.GET.get('q') limit = request.GET.get('limit') try: limit = int(limit) except: limit = settings.LIBRARY_AUTOCOMPLETE_LIMIT ids = (settings.MINIMUM_PACKAGE_ID, settings.MINIMUM_PACKAGE_ID - 1) notAddonKit = ~(F(id_number=ids[0]) | F(id_number=ids[1])) onlyMyPrivateLibs = (F(active=True) | F(author=request.user.id)) try: qs = (Package.search().query(or_=package_query(q)).filter( type='l').filter(notAddonKit).filter(onlyMyPrivateLibs)) found = qs[:limit] except Exception, ex: log.exception('Library autocomplete error') found = []
def test_filter_or_3(self): eq_(len(self.get_s().filter(F(tag='awesome') | F(tag='boat') | F(tag='boring'))), 5) eq_(len(self.get_s().filter(or_={'foo': 'bar', 'or_': {'tag': 'boat', 'width': '5'} })), 3)
def test_filter_or_3(self): s = self.get_s().filter(F(tag='awesome') | F(tag='boat') | F(tag='boring')) eq_(s._build_query(), { 'filter': { 'or': [ {'term': {'tag': 'awesome'}}, {'term': {'tag': 'boat'}}, {'term': {'tag': 'boring'}} ] } }) eq_(s.count(), 5) # This is kind of a crazy case. s = self.get_s().filter(or_={'foo': 'bar', 'or_': {'tag': 'boat', 'width': '5'}}) eq_(s._build_query(), { 'filter': { 'or': [ {'or': [ {'term': {'width': '5'}}, {'term': {'tag': 'boat'}} ]}, {'term': {'foo': 'bar'}} ] } }) eq_(s.count(), 3)
def filter_username(self, value): username_lower = value.lower() username_filter = F(iusername__prefix=username_lower) | F( idisplay_name__prefix=username_lower) return self._filter_by_users(username_filter)
def package_search(searchq='', user=None, score_on=None, **filters): """This provides some sane defaults to filter on when searching Packages""" # This is a filtered query, that says we want to do a query, but not have # to deal with version_text='initial' or 'copy' notInitialOrCopy = ~(F(version_name='initial') | F(version_name='copy')) qs = Package.search().values_obj('copies_count','times_depended', 'activity','size').filter(notInitialOrCopy, **filters).filter(F(active=True)) # Add type facet (minus any type filter) facetFilter = dict((k, v) for k, v in filters.items() if k != 'type') if facetFilter: facetFilter = notInitialOrCopy & F(**facetFilter) else: facetFilter = notInitialOrCopy qs = qs.facet(types={'terms': {'field': 'type'}, 'facet_filter': facetFilter.filters}) if searchq: qs = qs.query(or_=package_query(searchq)) if user and user.is_authenticated(): qs = qs.facet(author={'terms': { 'field': 'author', 'script':'term == %d ? true : false' % user.id} }) return qs
def test_f_mutation_with_or(self): """Make sure OR doesn't mutate operands.""" f1 = F(fielda='tag', fieldb='boat') f2 = F(fieldc='car') f1 | f2 # Should only contain f1 filters. eq_(sorted(f1.filters[0]['and']), sorted([('fielda', 'tag'), ('fieldb', 'boat')])) # Should only contain f2 filters. eq_(f2.filters, [('fieldc', 'car')])
def test_f_mutation_with_or(self): """Make sure OR doesn't mutate operands.""" f1 = F(fielda='tag', fieldb='boat') f2 = F(fieldc='car') f1 | f2 # Should only contain f1 filters. eq_(sorted(f1.filters['and']), sorted([{'term': {'fielda': 'tag'}}, {'term': {'fieldb': 'boat'}}])) # Should only contain f2 filters. eq_(f2.filters, {'term': {'fieldc': 'car'}})
def _filter_by_users(self, users_filter, invert=False): users = UserMappingType.reshape( UserMappingType.search() # Optimization: Filter out users that have never contributed. .filter(~F(last_contribution_date=None)).filter( users_filter).values_dict("id").everything()) user_ids = [u["id"] for u in users] res = F(creator_id__in=user_ids) if invert: res = ~res return res
def test_filter_f_and_ff(self): s = self.get_s().filter(F(tag='awesome') & F(foo='car', width='7')) eq_(s._build_query(), { 'filter': { 'and': [ {'term': {'width': '7'}}, {'term': {'foo': 'car'}}, {'term': {'tag': 'awesome'}} ] } } ) eq_(s.count(), 1)
def test_filter_not_not_f(self): f = F(tag='awesome') f = ~f f = ~f s = self.get_s().filter(f) eq_(s._build_query(), {'filter': {'term': {'tag': 'awesome'}}}) eq_(s.count(), 3)
def get_facet_response_for_choice_fields(query_with_criteria, choice_fields, form_model_id): facet_results = [] for field in choice_fields: field_name = es_questionnaire_field_name(field.code, form_model_id) + "_exact" facet_response = query_with_criteria.facet( field_name, filtered=True).facet_counts() facet_result_options = [] facet_result = { "es_field_name": field_name, "facets": facet_result_options, # find total submissions containing specified answer "total": query_with_criteria.filter(~F(**{field_name: None})).count() } for option, facet_list in facet_response.iteritems(): for facet in facet_list: facet_result_options.append({ "term": facet['term'], "count": facet['count'] }) facet_results.append(facet_result) return facet_results
def filter_page(self, value): """Validate the page numbers, but don't return any filters.""" try: page = fields.IntegerField().to_internal_value(value) except fields.ValidationError: page = 1 if page < 1: page = 1 self.query_values['page'] = page return F()
def filter_page_size(self, value): """Validate the page sizes, but don't return any filters.""" try: page_size = fields.IntegerField().to_internal_value(value) except fields.ValidationError: page_size = 20 if not (1 <= page_size < 100): page_size = 100 self.query_values['page_size'] = page_size return F()
def test_f_mutation_with_not(self): """Make sure NOT doesn't mutate operands.""" f1 = F(fielda='tag') f2 = ~f1 # Change f2 to see if it changes f1. f2.filters['not']['filter']['term']['fielda'] = 'boat' # Should only contain f1 filters. eq_(f1.filters, {'term': {'fielda': 'tag'}}) # Should only contain f2 tweaked filter. eq_(f2.filters, {'not': {'filter': {'term': {'fielda': 'boat'}}}})
def test_f_mutation_with_not(self): """Make sure NOT doesn't mutate operands.""" f1 = F(fielda='tag') f2 = ~f1 # Change f2 to see if it changes f1. f2.filters[0]['not']['filter'] = [('fielda', 'boat')] # Should only contain f1 filters. eq_(f1.filters, [('fielda', 'tag')]) # Should only contain f2 tweaked filter. eq_(f2.filters, [{'not': {'filter': [('fielda', 'boat')]}}])
def apply_reviewer_filters(request, qs, data=None): for k in ('has_info_request', 'has_editor_comment'): if data.get(k, None) is not None: qs = qs.filter(**{ 'latest_version.%s' % k: data[k] }) if data.get('is_escalated', None) is not None: qs = qs.filter(is_escalated=data['is_escalated']) is_tarako = data.get('is_tarako') if is_tarako is not None: if is_tarako: qs = qs.filter(tags='tarako') else: qs = qs.filter(~F(tags='tarako')) return qs
def get_filter(self, f, term): # Action must be either None or one that can be handled by Elasticutils # According to Elasticutils # Available actions are : startswith, prefix, in, range and distance action = self.search_actions.get(f, self.default_action) if not action and f in self.prefix_fields: action = 'prefix' field_action = '{0}__{1}'.format(f, action) if action else f # If term is an empty string, we are looking for a missing relation if term == '': term = None return F(**{field_action: term})
def get_filters(self): self.query_values = self.get_default_query() # request.GET is a multidict, so simple `.update(request.GET)` causes # everything to be a list. This converts it into a plain single dict. self.query_values.update(dict(self.request.GET.items())) f = F() for key, value in self.query_values.items(): filter_method = getattr(self, 'filter_' + key, None) if filter_method is None: self.warnings.append('Unknown filter {}'.format(key)) else: f &= filter_method(value) return f
def test_filter_not(self): s = self.get_s().filter(~F(tag='awesome')) eq_(s._build_query(), { 'filter': { 'not': { 'filter': {'term': {'tag': 'awesome'}} } } }) eq_(s.count(), 3) s = self.get_s().filter(~(F(tag='boring') | F(tag='boat'))) eq_(s._build_query(), { 'filter': { 'not': { 'filter': { 'or': [ {'term': {'tag': 'boring'}}, {'term': {'tag': 'boat'}} ] } } } }) eq_(s.count(), 4) s = self.get_s().filter(~F(tag='boat')).filter(~F(foo='bar')) eq_(s._build_query(), { 'filter': { 'and': [ {'not': {'filter': {'term': {'tag': 'boat'}}}}, {'not': {'filter': {'term': {'foo': 'bar'}}}} ] } }) eq_(s.count(), 4) s = self.get_s().filter(~F(tag='boat', foo='barf')) eq_(s._build_query(), { 'filter': { 'not': { 'filter': { 'and': [ {'term': {'foo': 'barf'}}, {'term': {'tag': 'boat'}} ] } } } }) eq_(s.count(), 6)
def issue_distribution(index): q = S().indexes(index).doctypes('IssueData') \ .filter(state='open') \ .filter(~F(assignee=None)) \ .values_dict() q = all(q) # open, assigned issues = list(q) distribution = {} for issue in issues: login = issue['assignee']['login'] if login not in distribution: distribution[login] = { 'person': issue['assignee'], 'issues': [issue] } else: distribution[login]['issues'].append(issue) return distribution
def filter_ordering(self, value): """Validate sort order, but don't return any filters.""" if value not in self.get_allowed_orderings(): self.warnings.append('Invalid sort order: {}'.format(value)) return F()
def get_filters(self): f = super(TopContributorsQuestions, self).get_filters() f &= F(by_asker=False) return f
def get_data(self, request): super(TopContributorsQuestions, self).get_data(request) # This is the base of all the metrics. Each metric branches off from # this to get a particular metric type, since we can't do Aggregates. query = AnswerMetricsMappingType.search() base_filters = self.get_filters() # This branch is to get the total number of answers for each user. answer_query = ( query .filter(base_filters) .facet('creator_id', filtered=True, size=BIG_NUMBER)) # This branch gets the number of answers that are solutions for each user. solutions_filter = base_filters & F(is_solution=True) solutions_query = ( query .filter(solutions_filter) .facet('creator_id', filtered=True, size=BIG_NUMBER)) # This branch gets the number of helpful votes across all answers for # each user. It is a raw facet because elasticutils only supports the # term facet type in non-raw facets. Because it is raw facet, we have # to also put the filter in the facet ourselves. helpful_query = ( query .facet_raw( creator_id={ 'terms_stats': { 'key_field': 'creator_id', 'value_field': 'helpful_count', }, 'facet_filter': query._process_filters(base_filters.filters), })) # Collect three lists of objects that correlates users and the appropriate metric count creator_answer_counts = answer_query.facet_counts()['creator_id']['terms'] creator_solutions_counts = solutions_query.facet_counts()['creator_id']['terms'] creator_helpful_counts = helpful_query.facet_counts()['creator_id']['terms'] # Combine all the metric types into one big list. combined = defaultdict(lambda: { 'answer_count': 0, 'solution_count': 0, 'helpful_vote_count': 0, }) for d in creator_answer_counts: combined[d['term']]['user_id'] = d['term'] combined[d['term']]['answer_count'] = d['count'] for d in creator_solutions_counts: combined[d['term']]['user_id'] = d['term'] combined[d['term']]['solution_count'] = d['count'] for d in creator_helpful_counts: combined[d['term']]['user_id'] = d['term'] # Since this is a term_stats filter, not just a term filter, it is total, not count. combined[d['term']]['helpful_vote_count'] = int(d['total']) # Sort by answer count, and get just the ids into a list. sort_key = self.query_values['ordering'] if sort_key[0] == '-': sort_reverse = True sort_key = sort_key[1:] else: sort_reverse = False top_contributors = combined.values() top_contributors.sort(key=lambda d: d[sort_key], reverse=sort_reverse) user_ids = [c['user_id'] for c in top_contributors] full_count = len(user_ids) # Paginate those user ids. page_start = (self.query_values['page'] - 1) * self.query_values['page_size'] page_end = page_start + self.query_values['page_size'] user_ids = user_ids[page_start:page_end] # Get full user objects for every id on this page. users = UserMappingType.reshape( UserMappingType .search() .filter(id__in=user_ids) .values_dict('id', 'username', 'display_name', 'avatar', 'last_contribution_date') [:self.query_values['page_size']]) # For ever user object found, mix in the metrics counts for that user, # and then reshape the data to make more sense to clients. data = [] for u in users: d = combined[u['id']] d['user'] = u d['last_contribution_date'] = d['user'].get('last_contribution_date', None) d.pop('user_id', None) d['user'].pop('id', None) d['user'].pop('last_contribution_date', None) data.append(d) # One last sort, since ES didn't return the users in any particular order. data.sort(key=lambda d: d[sort_key], reverse=sort_reverse) # Add ranks to the objects. for i, contributor in enumerate(data, 1): contributor['rank'] = page_start + i return { 'results': data, 'count': full_count, 'filters': self.query_values, 'allowed_orderings': self.get_allowed_orderings(), 'warnings': self.warnings, }
def filter_last_contribution_date__lt(self, value): # This query usually covers a lot of users, so inverting it makes it a lot faster. date = fields.DateField().to_internal_value(value) dt = datetime.combine(date, datetime.max.time()) return self._filter_by_users(~F(last_contribution_date__lt=dt), invert=True)
def filter_product(self, value): return F(product=value)
def filter_last_contribution_date__gt(self, value): date = fields.DateField().to_internal_value(value) dt = datetime.combine(date, datetime.max.time()) return self._filter_by_users(F(last_contribution_date__gt=dt))
def filter_locale(self, value): return F(locale=value)
def filter_enddate(self, value): date = fields.DateField().to_internal_value(value) dt = datetime.combine(date, datetime.max.time()) return F(created__lte=dt)
def get_data(self, request): super(TopContributorsQuestions, self).get_data(request) # This is the base of all the metrics. Each metric branches off from # this to get a particular metric type, since we can't do Aggregates. query = AnswerMetricsMappingType.search() base_filters = self.get_filters() # This branch is to get the total number of answers for each user. answer_query = query.filter(base_filters).facet("creator_id", filtered=True, size=BIG_NUMBER) # This branch gets the number of answers that are solutions for each user. solutions_filter = base_filters & F(is_solution=True) solutions_query = query.filter(solutions_filter).facet("creator_id", filtered=True, size=BIG_NUMBER) # This branch gets the number of helpful votes across all answers for # each user. It is a raw facet because elasticutils only supports the # term facet type in non-raw facets. Because it is raw facet, we have # to also put the filter in the facet ourselves. helpful_query = query.facet_raw( creator_id={ "terms_stats": { "key_field": "creator_id", "value_field": "helpful_count", }, "facet_filter": query._process_filters(base_filters.filters), }) # Collect three lists of objects that correlates users and the appropriate metric count creator_answer_counts = answer_query.facet_counts( )["creator_id"]["terms"] creator_solutions_counts = solutions_query.facet_counts( )["creator_id"]["terms"] creator_helpful_counts = helpful_query.facet_counts( )["creator_id"]["terms"] # Combine all the metric types into one big list. combined = defaultdict(lambda: { "answer_count": 0, "solution_count": 0, "helpful_vote_count": 0, }) for d in creator_answer_counts: combined[d["term"]]["user_id"] = d["term"] combined[d["term"]]["answer_count"] = d["count"] for d in creator_solutions_counts: combined[d["term"]]["user_id"] = d["term"] combined[d["term"]]["solution_count"] = d["count"] for d in creator_helpful_counts: combined[d["term"]]["user_id"] = d["term"] # Since this is a term_stats filter, not just a term filter, it is total, not count. combined[d["term"]]["helpful_vote_count"] = int(d["total"]) # Sort by answer count, and get just the ids into a list. sort_key = self.query_values["ordering"] if sort_key[0] == "-": sort_reverse = True sort_key = sort_key[1:] else: sort_reverse = False top_contributors = list(combined.values()) top_contributors.sort(key=lambda d: d[sort_key], reverse=sort_reverse) user_ids = [c["user_id"] for c in top_contributors] full_count = len(user_ids) # Paginate those user ids. page_start = (self.query_values["page"] - 1) * self.query_values["page_size"] page_end = page_start + self.query_values["page_size"] user_ids = user_ids[page_start:page_end] # Get full user objects for every id on this page. users = UserMappingType.reshape( UserMappingType.search().filter(id__in=user_ids).values_dict( "id", "username", "display_name", "avatar", "last_contribution_date")[:self.query_values["page_size"]]) # For ever user object found, mix in the metrics counts for that user, # and then reshape the data to make more sense to clients. data = [] for u in users: d = combined[u["id"]] d["user"] = u d["last_contribution_date"] = d["user"].get( "last_contribution_date", None) d.pop("user_id", None) d["user"].pop("id", None) d["user"].pop("last_contribution_date", None) data.append(d) # One last sort, since ES didn't return the users in any particular order. data.sort(key=lambda d: d[sort_key], reverse=sort_reverse) # Add ranks to the objects. for i, contributor in enumerate(data, 1): contributor["rank"] = page_start + i return { "results": data, "count": full_count, "filters": self.query_values, "allowed_orderings": self.get_allowed_orderings(), "warnings": self.warnings, }
def test_filter_raw_overrides_everything(self): s = self.get_s().filter_raw({'term': {'tag': 'awesome'}}) s = s.filter(tag='boring') s = s.filter(F(tag='end')) eq_(s._build_query(), {'filter': {'term': {'tag': 'awesome'}}})