def test_can_apply_multiple_sort_conditions_asc(self): self.whoosh_backend.add_doc(dict(id="2", type="ticket2")) self.whoosh_backend.add_doc(dict(id="3", type="ticket1")) self.whoosh_backend.add_doc(dict(id="4", type="ticket3")) self.whoosh_backend.add_doc(dict(id="1", type="ticket1")) result = self.whoosh_backend.query( query.Every(), sort=[SortInstruction("type", ASC), SortInstruction("id", ASC)], fields=("id", "type"), ) self.print_result(result) self.assertEqual([{ 'type': 'ticket1', 'id': '1' }, { 'type': 'ticket1', 'id': '3' }, { 'type': 'ticket2', 'id': '2' }, { 'type': 'ticket3', 'id': '4' }], result.docs)
def test_can_create_href_with_multiple_sort(self): href = RequestParameters(self.req).create_href(sort=[ SortInstruction("field1", ASC), SortInstruction("field2", DESC), ]) href = unquote(href) self.assertIn("sort=field1+asc,+field2+desc", href)
def test_can_return_empty_result(self): result = self.whoosh_backend.query( query.Every(), sort=[SortInstruction("type", ASC), SortInstruction("id", DESC)], fields=("id", "type"), facets=("type", "product")) self.print_result(result) self.assertEqual(0, result.hits)
def test_can_sort_by_score_and_date(self): the_first_date = datetime(2012, 12, 1) the_second_date = datetime(2012, 12, 2) the_third_date = datetime(2012, 12, 3) exact_match_string = "texttofind" not_exact_match_string = "texttofind bla" self.whoosh_backend.add_doc( dict( id="1", type="ticket", summary=not_exact_match_string, time=the_first_date, )) self.whoosh_backend.add_doc( dict( id="2", type="ticket", summary=exact_match_string, time=the_second_date, )) self.whoosh_backend.add_doc( dict( id="3", type="ticket", summary=not_exact_match_string, time=the_third_date, )) self.whoosh_backend.add_doc( dict( id="4", type="ticket", summary="some text out of search scope", time=the_third_date, )) parsed_query = self.parser.parse("summary:texttofind") result = self.whoosh_backend.query( parsed_query, sort=[SortInstruction(SCORE, ASC), SortInstruction("time", DESC)], ) self.print_result(result) self.assertEqual(3, result.hits) docs = result.docs #must be found first, because the highest score (of exact match) self.assertEqual("2", docs[0]["id"]) #must be found second, because the time order DESC self.assertEqual("3", docs[1]["id"]) #must be found third, because the time order DESC self.assertEqual("1", docs[2]["id"])
def test_can_parse_multiple_sort_terms(self): self.assertEqual(None, self._evaluate_sort(" ")) self.assertEqual(None, self._evaluate_sort(" , , ")) self.assertEqual([SortInstruction("f1", ASC)], self._evaluate_sort(" f1 ")) self.assertEqual([SortInstruction("f1", ASC)], self._evaluate_sort(" f1 asc")) self.assertEqual([SortInstruction("f1", DESC)], self._evaluate_sort("f1 desc")) self.assertEqual( [SortInstruction("f1", ASC), SortInstruction("f2", DESC)], self._evaluate_sort("f1, f2 desc"))
def test_can_do_facet_count(self): self.whoosh_backend.add_doc(dict(id="1", type="ticket", product="A")) self.whoosh_backend.add_doc(dict(id="2", type="ticket", product="B")) self.whoosh_backend.add_doc(dict(id="3", type="wiki", product="A")) result = self.whoosh_backend.query( query.Every(), sort=[SortInstruction("type", ASC), SortInstruction("id", DESC)], fields=("id", "type"), facets=("type", "product")) self.print_result(result) self.assertEqual(3, result.hits) facets = result.facets self.assertEqual({"ticket": 2, "wiki": 1}, facets["type"]) self.assertEqual({"A": 2, "B": 1}, facets["product"])
def test_can_apply_sorting(self): #arrange self.insert_ticket("T1", component="c1", status="new", milestone="A") self.insert_ticket("T2", component="c1", status="new", milestone="B") self.insert_ticket("T3", component="c3", status="new", milestone="C") #act self.req.args[RequestParameters.QUERY] = "*" self.req.args[RequestParameters.SORT] = "component, milestone desc" data = self.process_request() #assert api_sort = data["debug"]["api_parameters"]["sort"] self.assertEqual([ SortInstruction("component", ASC), SortInstruction("milestone", DESC), ], api_sort) ids = [item["summary"] for item in data["results"].items] self.assertEqual(["T2", "T1", "T3"], ids)
def _parse_sort(self, sort_string): if not sort_string: return None sort_terms = sort_string.split(",") sort = [] for term in sort_terms: term = term.strip() if not term: continue term_parts = term.split() parts_count = len(term_parts) if parts_count == 1: sort.append(SortInstruction(term_parts[0], ASC)) elif parts_count == 2: sort.append(SortInstruction(term_parts[0], term_parts[1])) else: raise TracError("Invalid sort term %s " % term) return sort if sort else None
def _create_headers_item(self, field): current_sort_direction = self._get_current_sort_direction_for_field( field) href_sort_direction = DESC if current_sort_direction == ASC else ASC return dict( name=field, href=self.parameters.create_href(skip_page=True, sort=SortInstruction( field, href_sort_direction)), #TODO:add translated column label. Now it is really temporary # workaround label=field, sort=current_sort_direction, )
def test_that_filter_queries_applied(self): #arrange self.insert_ticket("t1", status="closed", component="c1") self.insert_ticket("t2", status="closed", component="c1") self.insert_ticket("t3", status="closed", component="NotInFilterCriteria") #act results = self.search_api.query( "*", filter=['status:"closed"', 'component:"c1"'], sort=[SortInstruction("id", ASC)]) self.print_result(results) #assert self.assertEqual(2, results.hits) docs = results.docs self.assertEqual("t1", docs[0]["summary"]) self.assertEqual("t2", docs[1]["summary"])
def test_can_retrieve_docs(self): self.whoosh_backend.add_doc(dict(id="1", type="ticket")) self.whoosh_backend.add_doc(dict(id="2", type="ticket")) result = self.whoosh_backend.query( query.Every(), sort = [SortInstruction("id", ASC)], ) self.print_result(result) self.assertEqual(2, result.hits) docs = result.docs self.assertEqual( {'id': u'1', 'type': u'ticket', 'unique_id': u'empty:ticket:1', 'score': u'1'}, docs[0]) self.assertEqual( {'id': u'2', 'type': u'ticket', 'unique_id': u'empty:ticket:2', 'score': u'2'}, docs[1])
class RequestContext(object): DATA_ACTIVE_FILTER_QUERIES = 'active_filter_queries' DATA_ACTIVE_PRODUCT = 'active_product' DATA_ACTIVE_QUERY = 'active_query' DATA_BREADCRUMBS_TEMPLATE = 'resourcepath_template' DATA_HEADERS = "headers" DATA_ALL_VIEWS = "all_views" DATA_VIEW = "view" DATA_VIEW_GRID = "grid" DATA_FACET_COUNTS = 'facet_counts' DATA_DEBUG = 'debug' DATA_PAGE_HREF = 'page_href' DATA_RESULTS = 'results' DATA_PRODUCT_LIST = 'search_product_list' DATA_QUERY = 'query' DATA_QUICK_JUMP = "quickjump" DATA_QUERY_SUGGESTION = 'query_suggestion' DATA_SEARCH_EXTRAS = 'extra_search_fields' #bhsearch may support more pluggable views later VIEWS_SUPPORTED = ( (None, "Free text"), (DATA_VIEW_GRID, "Grid"), ) VIEWS_WITH_KNOWN_FIELDS = [DATA_VIEW_GRID] OBLIGATORY_FIELDS_TO_SELECT = [IndexFields.ID, IndexFields.TYPE] DEFAULT_SORT = [SortInstruction(SCORE, ASC), SortInstruction("time", DESC)] def __init__( self, env, req, search_participants, default_view, all_grid_fields, default_facets, global_quicksearch, query_suggestions, ): self.env = env self.req = req self.requires_redirect = False self._handle_multiproduct_parameters(req, global_quicksearch) self.parameters = RequestParameters(req, href=get_global_env(self.env).href) self.data = { self.DATA_QUERY: self.parameters.query, self.DATA_SEARCH_EXTRAS: [], } self.search_participants = search_participants self.default_view = default_view self.all_grid_fields = all_grid_fields self.default_facets = default_facets self.view = None self.page = self.parameters.page self.pagelen = self.parameters.pagelen self.query_suggestions = query_suggestions if self.parameters.sort: self.sort = self.parameters.sort else: self.sort = self.DEFAULT_SORT self.allowed_participants, self.sorted_participants = \ self._get_allowed_participants(req) if self.parameters.type in self.allowed_participants: self.active_type = self.parameters.type self.active_participant = self.allowed_participants[ self.active_type] else: self.active_type = None self.active_participant = None self.active_product = self.parameters.product self._prepare_active_type() self._prepare_hidden_search_fields() self._prepare_quick_jump() # Compatibility with trac search self._process_legacy_type_filters(req, search_participants) if not req.path_info.startswith(BHSEARCH_URL): self.requires_redirect = True self.fields = self._prepare_fields_and_view() self.query_filter = self._prepare_query_filter() self.facets = self._prepare_facets() def _handle_multiproduct_parameters(self, req, global_quicksearch): if not using_multiproduct(self.env): return if self.env.parent is not None: if not global_quicksearch: req.args[RequestParameters.PRODUCT] = \ self.env.product.prefix self.requires_redirect = True def _get_allowed_participants(self, req): allowed_participants = {} ordered_participants = [] for participant in self.search_participants: if participant.is_allowed(req): allowed_participants[ participant.get_participant_type()] = participant ordered_participants.append(participant) return allowed_participants, ordered_participants def _prepare_active_type(self): active_type = self.parameters.type if active_type and active_type not in self.allowed_participants: raise TracError( _("Unsupported resource type: '%(name)s'", name=active_type)) def _prepare_hidden_search_fields(self): if self.active_type: self.data[self.DATA_SEARCH_EXTRAS].append( (RequestParameters.TYPE, self.active_type)) if self.parameters.product: self.data[self.DATA_SEARCH_EXTRAS].append( (RequestParameters.PRODUCT, self.parameters.product)) if self.parameters.view: self.data[self.DATA_SEARCH_EXTRAS].append( (RequestParameters.VIEW, self.parameters.view)) if self.parameters.sort: self.data[self.DATA_SEARCH_EXTRAS].append( (RequestParameters.SORT, self.parameters.sort_string)) for filter_query in self.parameters.filter_queries: self.data[self.DATA_SEARCH_EXTRAS].append( (RequestParameters.FILTER_QUERY, filter_query)) def _prepare_quick_jump(self): if not self.parameters.query: return check_result = self._check_quickjump(self.req, self.parameters.query) if check_result: self.data[self.DATA_QUICK_JUMP] = check_result #the method below is "copy/paste" from trac search/web_ui.py def _check_quickjump(self, req, kwd): """Look for search shortcuts""" # pylint: disable=maybe-no-member noquickjump = int(req.args.get('noquickjump', '0')) # Source quickjump FIXME: delegate to ISearchSource.search_quickjump quickjump_href = None if kwd[0] == '/': quickjump_href = req.href.browser(kwd) name = kwd description = _('Browse repository path %(path)s', path=kwd) else: context = web_context(req, 'search') link = find_element(extract_link(self.env, context, kwd), 'href') if link is not None: quickjump_href = link.attrib.get('href') name = link.children description = link.attrib.get('title', '') if quickjump_href: # Only automatically redirect to local quickjump links base_path = req.base_path.replace('@', '%40') redirect_href = quickjump_href.replace('@', '%40') if not redirect_href.startswith(base_path or '/'): noquickjump = True if noquickjump: return { 'href': quickjump_href, 'name': tag.EM(name), 'description': description } else: req.redirect(quickjump_href) def _prepare_fields_and_view(self): self._add_views_selector() self.view = self._get_view() if self.view: self.data[self.DATA_VIEW] = self.view fields_to_select = None if self.view in self.VIEWS_WITH_KNOWN_FIELDS: if self.active_participant: fields_in_view = self.active_participant.\ get_default_view_fields(self.view) elif self.view == self.DATA_VIEW_GRID: fields_in_view = self.all_grid_fields else: raise TracError("Unsupported view: %s" % self.view) self.data[self.DATA_HEADERS] = [ self._create_headers_item(field) for field in fields_in_view ] fields_to_select = self._add_obligatory_fields(fields_in_view) return fields_to_select def _add_views_selector(self): active_view = self.parameters.view all_views = [] for view, label in self.VIEWS_SUPPORTED: all_views.append( dict(label=_(label), href=self.parameters.create_href( view=view, skip_view=(view is None)), is_active=(view == active_view))) self.data[self.DATA_ALL_VIEWS] = all_views def _get_view(self): view = self.parameters.view if view is None: if self.active_participant: view = self.active_participant.get_default_view() else: view = self.default_view if view is not None: view = view.strip().lower() if view == "": view = None return view def _add_obligatory_fields(self, fields_in_view): fields_to_select = list(fields_in_view) for obligatory_field in self.OBLIGATORY_FIELDS_TO_SELECT: if obligatory_field is not fields_to_select: fields_to_select.append(obligatory_field) return fields_to_select def _create_headers_item(self, field): current_sort_direction = self._get_current_sort_direction_for_field( field) href_sort_direction = DESC if current_sort_direction == ASC else ASC return dict( name=field, href=self.parameters.create_href(skip_page=True, sort=SortInstruction( field, href_sort_direction)), #TODO:add translated column label. Now it is really temporary # workaround label=field, sort=current_sort_direction, ) def _get_current_sort_direction_for_field(self, field): if self.sort and len(self.sort) == 1: single_sort = self.sort[0] if single_sort.field == field: return single_sort.order return None def _prepare_query_filter(self): query_filters = list(self.parameters.filter_queries) if self.active_type: query_filters.append( self._create_term_expression(IndexFields.TYPE, self.active_type)) if self.active_product is not None: query_filters.append( self._create_term_expression(IndexFields.PRODUCT, self.active_product or None)) return query_filters def _create_term_expression(self, field, field_value): if field_value is None: query = "NOT (%s:*)" % field elif isinstance(field_value, basestring): query = '%s:"%s"' % (field, field_value) else: query = '%s:%s' % (field, field_value) return query def _prepare_facets(self): #TODO: add possibility of specifying facets in query parameters if self.active_participant: facets = self.active_participant.get_default_facets() else: facets = self.default_facets return facets def _process_legacy_type_filters(self, req, search_participants): legacy_type_filters = [ sp.get_participant_type() for sp in search_participants if sp.get_participant_type() in req.args ] if legacy_type_filters: params = self.parameters.params if len(legacy_type_filters) == 1: self.parameters.type = params[RequestParameters.TYPE] = \ legacy_type_filters[0] else: filter_queries = self.parameters.filter_queries if params[ RequestParameters.FILTER_QUERY] is not filter_queries: params[RequestParameters.FILTER_QUERY] = filter_queries filter_queries.append('type:(%s)' % ' OR '.join(legacy_type_filters)) self.requires_redirect = True def _process_doc(self, doc): ui_doc = dict(doc) if doc.get('product'): env = ProductEnvironment(self.env, doc['product']) product_href = ProductEnvironment.resolve_href(env, self.env) # pylint: disable=too-many-function-args ui_doc["href"] = product_href(doc['type'], doc['id']) else: ui_doc["href"] = self.req.href(doc['type'], doc['id']) if doc['content']: ui_doc['content'] = shorten_result(doc['content']) if doc['time']: ui_doc['date'] = user_time(self.req, format_datetime, doc['time']) is_free_text_view = self.view is None if is_free_text_view: participant = self.allowed_participants[doc['type']] ui_doc['title'] = participant.format_search_results(doc) return ui_doc def _prepare_results(self, result_docs, hits): ui_docs = [self._process_doc(doc) for doc in result_docs] results = Paginator(ui_docs, self.page - 1, self.pagelen, hits) self._prepare_shown_pages(results) results.current_page = { 'href': None, 'class': 'current', 'string': str(results.page + 1), 'title': None } parameters = self.parameters if results.has_next_page: next_href = parameters.create_href(page=parameters.page + 1) add_link(self.req, 'next', next_href, _('Next Page')) if results.has_previous_page: prev_href = parameters.create_href(page=parameters.page - 1) add_link(self.req, 'prev', prev_href, _('Previous Page')) self.data[self.DATA_RESULTS] = results prevnext_nav(self.req, _('Previous'), _('Next')) def _prepare_shown_pages(self, results): shown_pages = results.get_shown_pages(self.pagelen) page_data = [] for shown_page in shown_pages: page_href = self.parameters.create_href(page=shown_page) page_data.append( [page_href, None, str(shown_page), 'page ' + str(shown_page)]) fields = ['href', 'class', 'string', 'title'] results.shown_pages = [dict(zip(fields, p)) for p in page_data] def process_results(self, query_result): docs = self._prepare_docs(query_result.docs, query_result.highlighting) self._prepare_results(docs, query_result.hits) self._prepare_result_facet_counts(query_result.facets) self._prepare_breadcrumbs() self._prepare_query_suggestion(query_result.query_suggestion) self.data[self.DATA_DEBUG] = query_result.debug if self.parameters.debug: self.data[self.DATA_DEBUG]['enabled'] = True self.data[self.DATA_SEARCH_EXTRAS].append(('debug', '1')) self.data[self.DATA_PAGE_HREF] = self.parameters.create_href() def _prepare_result_facet_counts(self, result_facets): """ Sample query_result.facets content returned by query { 'component': {None:2}, 'milestone': {None:1, 'm1':1}, } returned facet_count contains href parameters: { 'component': {None: {'count':2, href:'...'}, 'milestone': { None: {'count':1,, href:'...'}, 'm1':{'count':1, href:'...'} }, } """ facet_counts = [] if result_facets: for field in self.facets: if field == IndexFields.PRODUCT and \ not using_multiproduct(self.env): continue facets_dict = result_facets.get(field, {}) per_field_dict = dict() for field_value, count in facets_dict.iteritems(): if field == IndexFields.TYPE: href = self.parameters.create_href(skip_page=True, force_filters=[], type=field_value) elif field == IndexFields.PRODUCT: href = self.parameters.create_href( skip_page=True, product=field_value or u'', ) else: href = self.parameters.create_href( skip_page=True, additional_filter=self._create_term_expression( field, field_value)) per_field_dict[field_value] = dict(count=count, href=href) facet_counts.append((_(field), per_field_dict)) self.data[self.DATA_FACET_COUNTS] = facet_counts def _prepare_docs(self, docs, highlights): new_docs = [] for doc, highlight in zip(docs, highlights): doc = defaultdict(str, doc) for field in highlight.iterkeys(): highlighted_field = 'hilited_%s' % field if highlight[field]: fragment = self._create_genshi_fragment(highlight[field]) doc[highlighted_field] = fragment else: doc[highlighted_field] = '' new_docs.append(doc) return new_docs def _create_genshi_fragment(self, html_fragment): return tag(HTML(html_fragment)) def _prepare_breadcrumbs(self): self._prepare_breadcrumbs_template() self._prepare_product_breadcrumb() self._prepare_query_filter_breadcrumbs() def _prepare_breadcrumbs_template(self): self.data[self.DATA_BREADCRUMBS_TEMPLATE] = 'bhsearch_breadcrumbs.html' def _prepare_product_breadcrumb(self): if not using_multiproduct(self.env): return product_search = lambda x: self.parameters.create_href(product=x) all_products_search = self.parameters.create_href(skip_product=True) global_product = [(u'', _(u'Global product'), product_search(u''))] products = \ ProductModule.get_product_list(self.env, self.req, product_search) all_products = [(None, _(u'All products'), all_products_search)] search_product_list = global_product + products + all_products # pylint: disable=unused-variable for prefix, name, url in search_product_list: if prefix == self.active_product: self.data[self.DATA_ACTIVE_PRODUCT] = name break else: self.data[self.DATA_ACTIVE_PRODUCT] = self.active_product self.data[self.DATA_PRODUCT_LIST] = search_product_list def _prepare_query_filter_breadcrumbs(self): current_filters = self.parameters.filter_queries def remove_filter_from_list(filter_to_remove): new_filters = list(current_filters) new_filters.remove(filter_to_remove) return new_filters if self.active_type: type_query = self._create_term_expression('type', self.active_type) type_filters = [ dict( href=self.parameters.create_href(skip_type=True, force_filters=[]), label=unicode(self.active_type).capitalize(), query=type_query, ) ] else: type_filters = [] active_filter_queries = [ dict( href=self.parameters.create_href( force_filters=remove_filter_from_list(filter_query)), label=filter_query, query=filter_query, ) for filter_query in self.parameters.filter_queries ] active_query = dict(href=self.parameters.create_href(skip_query=True), label=u'"%s"' % self.parameters.query, query=self.parameters.query) self.data[self.DATA_ACTIVE_FILTER_QUERIES] = \ type_filters + active_filter_queries self.data[self.DATA_ACTIVE_QUERY] = active_query def _prepare_query_suggestion(self, suggestion): if self.query_suggestions and suggestion is not None: self.data[self.DATA_QUERY_SUGGESTION] = dict( query=suggestion, href=self.parameters.create_href(query=suggestion)) else: self.data[self.DATA_QUERY_SUGGESTION] = None
def test_can_create_href_with_single_sort(self): href = RequestParameters( self.req).create_href(sort=SortInstruction("field1", ASC)) href = unquote(href) self.assertIn("sort=field1+asc", href)