def test_single_project(self): skip_test() u_to = User("admin") u_from = User("testuser") to_proj = self._prj("test", "admin") self._add(to_proj, u_to, "README.md", "hi") from_proj = self._prj("testuser/test", "testuser", to_proj.id) self._add(from_proj, u_from, "README.md", "hello") pullreq = PullRequest.open(from_proj, "master", to_proj, "master") ticket = Ticket(None, None, to_proj.id, "title", "desc", "testuser", None, None) pullreq = add_pull(ticket, pullreq, u_from) iss = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=to_proj.id) IssuePRSearch.index_a_project(to_proj) res = IssueSearch.search_a_phrase('title1', to_proj.id) res = SearchEngine.decode(res, ('issue_id',)) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss.id res = PullRequestSearch.search_a_phrase('title', to_proj.id) res = SearchEngine.decode(res, ('issue_id',)) res = [id for id, in res] assert len(res) == 1
def test_single_project(self): skip_test() u_to = User("admin") u_from = User("testuser") to_proj = self._prj("test", "admin") self._add(to_proj, u_to, "README.md", "hi") from_proj = self._prj("testuser/test", "testuser", to_proj.id) self._add(from_proj, u_from, "README.md", "hello") pullreq = PullRequest.open(from_proj, "master", to_proj, "master") ticket = Ticket(None, None, to_proj.id, "title", "desc", "testuser", None, None) pullreq = add_pull(ticket, pullreq, u_from) iss = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=to_proj.id) IssuePRSearch.index_a_project(to_proj) res = IssueSearch.search_a_phrase('title1', to_proj.id) res = SearchEngine.decode(res, ('issue_id', )) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss.id res = PullRequestSearch.search_a_phrase('title', to_proj.id) res = SearchEngine.decode(res, ('issue_id', )) res = [id for id, in res] assert len(res) == 1
def count(self, request): request.response.set_content_type('application/json; charset=utf8') q = request.get_form_var('q', '') project = CodeDoubanProject.get_by_name(self.project) \ if self.project else None state = request.get_form_var('state', '') language = request.get_form_var('language', '') doctype = request.get_form_var('doctype', '') counts = {} for kind, cls in KIND_CLASS_MAP.iteritems(): kwargs = dict(phrase=q, from_=0, size=0) if project and kind in (K_DOC, K_CODE, K_PULL, K_ISSUE): kwargs.update(project_id=project.id) if language and kind == K_CODE: kwargs.update(language=language) if state and kind in (K_PULL, K_ISSUE): kwargs.update(state=state) if doctype and kind == K_DOC: kwargs.update(doctype=doctype) result = cls.search_a_phrase(**kwargs) counts[kind] = SearchEngine.get_count(result) tdt = { 'q': q, 'repos': counts[K_REPO], 'codes': counts[K_CODE], 'users': counts[K_USER], 'docs': counts[K_DOC], 'pulls': counts[K_PULL], 'issues': counts[K_ISSUE], } return json.dumps(tdt)
def test_multiple_project(self): skip_test() p1 = self._prj("test_1") p2 = self._prj("test_2") iss1 = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=p1.id) iss2 = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=p2.id) IssueSearch.index_a_project_issue(p1) IssueSearch.index_a_project_issue(p2) res = IssueSearch.search_a_phrase('title1', p1.id) res = SearchEngine.decode(res, ('issue_id',)) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss1.id res = IssueSearch.search_a_phrase('title1', p2.id) res = SearchEngine.decode(res, ('issue_id',)) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss2.id
def test_single_project(self): skip_test() p = self._prj("test") iss1 = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=p.id) IssueSearch.index_a_project_issue(p) res = IssueSearch.search_a_phrase('owner', p.id) res = SearchEngine.decode(res, ('issue_id',)) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss1.id iss2 = ProjectIssue.add(title='title2', description='desc2', creator='owner', project=p.id) IssueSearch.index_a_project_issue(p) res = IssueSearch.search_a_phrase('owner', p.id) res = SearchEngine.decode(res, ('issue_id',)) res = [id for id, in res] assert len(res) == 2 assert iss1.id in res assert iss2.id in res
def test_multiple_project(self): skip_test() p1 = self._prj("test_1") p2 = self._prj("test_2") iss1 = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=p1.id) iss2 = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=p2.id) IssueSearch.index_a_project_issue(p1) IssueSearch.index_a_project_issue(p2) res = IssueSearch.search_a_phrase('title1', p1.id) res = SearchEngine.decode(res, ('issue_id', )) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss1.id res = IssueSearch.search_a_phrase('title1', p2.id) res = SearchEngine.decode(res, ('issue_id', )) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss2.id
def test_single_project(self): skip_test() p = self._prj("test") iss1 = ProjectIssue.add(title='title1', description='desc1', creator='owner', project=p.id) IssueSearch.index_a_project_issue(p) res = IssueSearch.search_a_phrase('owner', p.id) res = SearchEngine.decode(res, ('issue_id', )) res = [id for id, in res] assert len(res) == 1 assert res[0] == iss1.id iss2 = ProjectIssue.add(title='title2', description='desc2', creator='owner', project=p.id) IssueSearch.index_a_project_issue(p) res = IssueSearch.search_a_phrase('owner', p.id) res = SearchEngine.decode(res, ('issue_id', )) res = [id for id, in res] assert len(res) == 2 assert iss1.id in res assert iss2.id in res
def test_single_project(self): skip_test() u_to = User("admin") u_from = User("testuser") to_proj = self._prj("test", "admin") self._add(to_proj, u_to, "README.md", "hi") from_proj = self._prj("testuser/test", "testuser", to_proj.id) self._add(from_proj, u_from, "README.md", "hello") pullreq = PullRequest.open(from_proj, "master", to_proj, "master") ticket = Ticket(None, None, to_proj.id, "title", "desc", "testuser", None, None) pullreq = add_pull(ticket, pullreq, u_from) ticket = pullreq.ticket PullRequestSearch.index_a_project_pr(to_proj) res = PullRequestSearch.search_a_phrase('title', to_proj.id) res = SearchEngine.decode(res, ('to_proj_id',)) res = [id for id, in res] assert len(res) == 1
def test_single_project(self): skip_test() u_to = User("admin") u_from = User("testuser") to_proj = self._prj("test", "admin") self._add(to_proj, u_to, "README.md", "hi") from_proj = self._prj("testuser/test", "testuser", to_proj.id) self._add(from_proj, u_from, "README.md", "hello") pullreq = PullRequest.open(from_proj, "master", to_proj, "master") ticket = Ticket(None, None, to_proj.id, "title", "desc", "testuser", None, None) pullreq = add_pull(ticket, pullreq, u_from) ticket = pullreq.ticket PullRequestSearch.index_a_project_pr(to_proj) res = PullRequestSearch.search_a_phrase('title', to_proj.id) res = SearchEngine.decode(res, ('to_proj_id', )) res = [id for id, in res] assert len(res) == 1
def search_a_phrase(cls, phrase, project_id=None, from_=0, size=20, state=None, sort_data=None): filter_list = [] if project_id: if cls.type_name == "issue": key = "project_id" else: key = "to_proj_id" filter_list.append({ "term": { key: project_id } }) if state: filter_list.append({"term": {"state": state}}) if filter_list: filter_data = {"and": filter_list} else: filter_data = None highlight_data = { "fields": { "description": {"number_of_fragments": 0} } } facets_data = { "state": { "terms": { "field": "state" } } } result = SearchEngine.search_a_phrase(cls.type_name, phrase, from_, size, sort_data=sort_data, filter_data=filter_data, highlight_data=highlight_data, facets_data=facets_data) return result
def format_search_result(cls, result): if not SearchEngine.check_result(result): return [] formatted = [] result = result['hits']['hits'] for r in result: _source = r['_source'] try: hl_description = r['highlight']['description'][0] except: logging.debug('No highlight for %s', _source) hl_description = '' description = _source.get('description') sr = dict( issue_id=_source.get('issue_id'), description=description if description else '', hl_description=hl_description, ) if not sr['issue_id']: logging.warn('Invaild issue search result, skip: %s', _source) continue sr = IssueResult(**sr) formatted.append(sr) return formatted
def format_search_result(cls, result): if not SearchEngine.check_result(result): return [] formatted = [] result = result['hits']['hits'] for r in result: _source = r['_source'] try: hl_description = r['highlight']['description'][0] except: logging.debug('No highlight for %s', _source) hl_description = '' sr = dict( ticket_number=_source.get('ticket_id'), project_id=_source.get('to_proj_id'), hl_description=hl_description, ) if not sr['project_id'] or not sr['ticket_number']: logging.warn( 'Invaild pullrequest search result, skip: %s', _source) continue sr = PullResult(**sr) formatted.append(sr) return formatted
def format_facets(cls, result): if not SearchEngine.check_result(result): return {} formatted = dict(state=result['facets']['state']['terms']) return formatted
def _q_index(self, request): q = request.get_form_var('q', '') kind = request.get_form_var('kind', str(K_CODE)) order = request.get_form_var('s', '') order = int(order) if order else '' page = request.get_form_var('page') page = int(page) if page and page.isdigit() else 1 project = CodeDoubanProject.get_by_name(self.project) \ if self.project else None state = request.get_form_var('state', '') language = request.get_form_var('language', '') doctype = request.get_form_var('doctype', '') if not kind.isdigit() and int(kind) not in SEARCH_KINDS: raise TraversalError() kind = int(kind) orders = KIND_ORDERS_MAP.get(kind, CODE_ORDERS) sort_data = orders.get(order) if sort_data: sort_data = sort_data[1] limit = PERPAGE_LIMIT total = 0 offset = (page - 1) * limit result = {} formated_result = [] facets = {} tdt = {} cls = KIND_CLASS_MAP.get(kind) by_project = by_language = by_state = by_doctype = False if project and kind in (K_DOC, K_CODE, K_PULL, K_ISSUE): by_project = True if language and kind == K_CODE: by_language = True if state and kind in (K_PULL, K_ISSUE): by_state = True if doctype and kind == K_DOC: by_doctype = True # for facets if kind in (K_CODE, K_PULL, K_ISSUE, K_DOC): project_id = project.id if by_project else None result = cls.search_a_phrase( phrase=q, from_=0, size=0, project_id=project_id) facets = cls.format_facets(result) for title, data in facets.iteritems(): for item in data: params = {'q': q, 'kind': kind, 's': order, title: item['term']} if by_project: params.update(project_id=project_id) item['url'] = '?' + urlencode(params) # highlight current term current = None if by_language: current = language elif by_state: current = state elif by_doctype: current = doctype if current and current == item['term']: item['selected'] = True del params[title] item['url'] = '?' + urlencode(params) # for search kwargs = dict(phrase=q, sort_data=sort_data, from_=offset, size=limit) if by_project: kwargs.update(project_id=project.id) if by_language: kwargs.update(language=language) if by_state: kwargs.update(state=state) if by_doctype: kwargs.update(doctype=doctype) result = cls.search_a_phrase(**kwargs) formated_result = cls.format_search_result(result) total = SearchEngine.get_count(result) pages = total / limit + 1 if total % limit > 0 else total / limit tdt.update(request=request, q=q, kind=kind, total=total, facets=facets, language=language, state=state, doctype=doctype, result=formated_result, orders=orders, s=order, page=page, pages=pages) # for menu and pagenation SEARCH_URLS = dict() for k in SEARCH_KINDS: if project and k in (K_CODE, K_DOC, K_PULL, K_ISSUE): url_root = '/%s/search' % project.name else: url_root = SEARCH_URL_ROOT params = {'q': q, 'kind': k, 's': order} if state and k in (K_PULL, K_ISSUE): params.update({'state': state}) if language and k == K_CODE: params.update({'language': language}) if doctype and k == K_DOC: params.update({'doctype': doctype}) url = '%s?' % url_root + urlencode(params) SEARCH_URLS[k] = url tdt.update(SEARCH_URLS=SEARCH_URLS) return st('search/base.html', **tdt)