def get_result(self, limit, cursor=None): assert limit > 0 offset = cursor.offset if cursor is not None else 0 # Request 1 more than limit so we can tell if there is another page data = self.data_fn(offset=offset, limit=limit + 1) if isinstance(data, list): has_more = len(data) == limit + 1 if has_more: data.pop() elif isinstance(data.get("data"), list): has_more = len(data["data"]) == limit + 1 if has_more: data["data"].pop() else: raise NotImplementedError # Since we are not issuing ranged queries, our cursors always have # `value=0` (ie. all rows have the same value), and so offset naturally # becomes the absolute row offset from the beginning of the entire # dataset, which is the same meaning as SQLs `OFFSET`. return CursorResult( data, prev=Cursor(0, max(0, offset - limit), True, offset > 0), next=Cursor(0, max(0, offset + limit), False, has_more), )
def get_result(self, limit=100, cursor=None): # offset is page # # value is page limit if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) queryset = self.queryset if self.key: queryset = queryset.order_by(*self.key) page = cursor.offset offset = cursor.offset * cursor.value stop = offset + (cursor.value or limit) + 1 if self.max_offset is not None and offset >= self.max_offset: raise BadPaginationError("Pagination offset too large") if offset < 0: raise BadPaginationError("Pagination offset cannot be negative") results = list(queryset[offset:stop]) if cursor.value != limit: results = results[-(limit + 1):] next_cursor = Cursor(limit, page + 1, False, len(results) > limit) prev_cursor = Cursor(limit, page - 1, True, page > 0) results = list(results[:limit]) if self.on_results: results = self.on_results(results) return CursorResult(results=results, next=next_cursor, prev=prev_cursor)
def get_result(self, limit=100, cursor=None): # offset is page # # value is page limit if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) queryset = self.queryset if self.key: if self.desc: queryset = queryset.order_by(u'-{}'.format(self.key)) else: queryset = queryset.order_by(self.key) page = cursor.offset offset = cursor.offset * cursor.value stop = offset + (cursor.value or limit) + 1 results = list(queryset[offset:stop]) if cursor.value != limit: results = results[-(limit + 1):] next_cursor = Cursor(limit, page + 1, False, len(results) > limit) prev_cursor = Cursor(limit, page - 1, True, page > 0) results = list(results[:limit]) if self.on_results: results = self.on_results(results) return CursorResult( results=results, next=next_cursor, prev=prev_cursor, )
def prep_search( cls: Any, request: Request, project: "Project", extra_query_kwargs: Optional[Mapping[str, Any]] = None, ) -> Tuple[CursorResult, Mapping[str, Any]]: try: environment = cls._get_environment_from_request( request, project.organization_id) except Environment.DoesNotExist: result = CursorResult([], None, None, hits=0, max_hits=SEARCH_MAX_HITS) query_kwargs: MutableMapping[str, Any] = {} else: environments = [environment ] if environment is not None else environment query_kwargs = build_query_params_from_request(request, project.organization, [project], environments) if extra_query_kwargs is not None: assert "environment" not in extra_query_kwargs query_kwargs.update(extra_query_kwargs) query_kwargs["environments"] = environments result = search.query(**query_kwargs) return result, query_kwargs
def _search(self, request, project, extra_query_kwargs=None): try: environment = self._get_environment_from_request( request, project.organization_id, ) except Environment.DoesNotExist: # XXX: The 1000 magic number for `max_hits` is an abstraction leak # from `sentry.api.paginator.BasePaginator.get_result`. result = CursorResult([], None, None, hits=0, max_hits=1000) query_kwargs = {} else: environments = [environment ] if environment is not None else environment query_kwargs = build_query_params_from_request( request, project.organization, [project], environments, ) if extra_query_kwargs is not None: assert 'environment' not in extra_query_kwargs query_kwargs.update(extra_query_kwargs) query_kwargs['environments'] = environments result = search.query(**query_kwargs) return result, query_kwargs
def get_result(self, limit, cursor=None, count_hits=False): limit = min(limit, self.max_limit) if cursor is None: cursor = Cursor(0, 0, False) assert cursor.offset > -1 if cursor.value == 0: position = len(self.scores) if cursor.is_prev else 0 else: position = self.search(cursor.value) position = position + cursor.offset if cursor.is_prev: # TODO: It might make sense to ensure that this hi value is at # least the length of the page + 1 if we want to ensure we return a # full page of results when paginating backwards while data is # being mutated. hi = min(position, len(self.scores)) lo = max(hi - limit, 0) else: lo = max(position, 0) hi = min(lo + limit, len(self.scores)) if self.scores: prev_score = self.scores[min(lo, len(self.scores) - 1)] prev_cursor = Cursor( prev_score, lo - self.search(prev_score, hi=lo), True, True if lo > 0 else False, ) next_score = self.scores[min(hi, len(self.scores) - 1)] next_cursor = Cursor( next_score, hi - self.search(next_score, hi=hi), False, True if hi < len(self.scores) else False, ) else: prev_cursor = Cursor(cursor.value, cursor.offset, True, False) next_cursor = Cursor(cursor.value, cursor.offset, False, False) results = self.values[lo:hi] if self.on_results: results = self.on_results(results) return CursorResult( results, prev=prev_cursor, next=next_cursor, hits=min(len(self.scores), MAX_HITS_LIMIT) if count_hits else None, max_hits=MAX_HITS_LIMIT if count_hits else None, )
def get_result(self, limit=100, cursor=None): if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) page = cursor.offset offset = cursor.offset * cursor.value limit = cursor.value or limit if self.max_offset is not None and offset >= self.max_offset: raise BadPaginationError("Pagination offset too large") if offset < 0: raise BadPaginationError("Pagination offset cannot be negative") primary_results = self.data_load_func(offset=offset, limit=self.max_limit + 1) queryset = self.apply_to_queryset(self.queryset, primary_results) mapping = {} for model in queryset: mapping[self.key_from_model(model)] = model results = [] for row in primary_results: model = mapping.get(self.key_from_data(row)) if model is not None: results.append(model) if self.queryset_load_func and self.data_count_func and len( results) < limit: # If we hit the end of the results from the data load func, check whether there are # any additional results in the queryset_load_func, if one is provided. extra_limit = limit - len(results) + 1 total_data_count = self.data_count_func() total_offset = offset + len(results) qs_offset = max(0, total_offset - total_data_count) qs_results = self.queryset_load_func(self.queryset, total_offset, qs_offset, extra_limit) results.extend(qs_results) has_more = len(qs_results) == extra_limit else: has_more = len(primary_results) > limit results = results[:limit] next_cursor = Cursor(limit, page + 1, False, has_more) prev_cursor = Cursor(limit, page - 1, True, page > 0) if self.on_results: results = self.on_results(results) return CursorResult(results=results, next=next_cursor, prev=prev_cursor)
def get_result(self, limit: int, cursor: Optional[Cursor] = None) -> CursorResult: assert limit > 0 offset = cursor.offset if cursor is not None else 0 # Request 1 more than limit so we can tell if there is another page data = self.data_fn(offset, limit + 1) has_more = any(len(result["examples"]) == limit + 1 for result in data) for result in data: result["examples"] = result["examples"][:limit] return CursorResult( data, prev=Cursor(0, max(0, offset - limit), True, offset > 0), next=Cursor(0, max(0, offset + limit), False, has_more), )
def get_result(self, limit, cursor=None): assert limit > 0 offset = cursor.offset if cursor is not None else 0 data = self.data_fn(offset=offset, limit=limit + 1) if isinstance(data.get("groups"), list): has_more = len(data["groups"]) == limit + 1 if has_more: data["groups"].pop() else: raise NotImplementedError return CursorResult( data, prev=Cursor(0, max(0, offset - limit), True, offset > 0), next=Cursor(0, max(0, offset + limit), False, has_more), )
def get_result(self, limit=100, cursor=None): # offset is page # # value is page limit if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) page = cursor.offset offset = cursor.offset * cursor.value if self.max_offset is not None and offset >= self.max_offset: raise BadPaginationError("Pagination offset too large") if limit <= 0: raise BadPaginationError("Limit must be positive") if offset < 0: raise BadPaginationError("Pagination offset cannot be negative") results = [] # note: we shouldn't use itertools.islice(itertools.chain.from_iterable(self.sources)) # because source may be a QuerySet which is much more efficient to slice directly for source in self.sources: # Get an additional item so we can check for a next page. remaining = limit - len(results) + 1 results.extend(source[offset:offset + remaining]) # don't do offset = max(0, offset - len(source)) because len(source) may be expensive if len(results) == 0: offset -= len(source) else: offset = 0 if len(results) > limit: assert len(results) == limit + 1 break next_cursor = Cursor(limit, page + 1, False, len(results) > limit) prev_cursor = Cursor(limit, page - 1, True, page > 0) if next_cursor.has_results: results.pop() if self.on_results: results = self.on_results(results) return CursorResult(results=results, next=next_cursor, prev=prev_cursor)
def get_result(self, limit=100, cursor=None): # offset is page # # value is page limit if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) page = cursor.offset offset = cursor.offset * cursor.value if self.max_offset is not None and offset >= self.max_offset: raise BadPaginationError("Pagination offset too large") if offset < 0: raise BadPaginationError("Pagination offset cannot be negative") results = [] # Get an addition item so we can check for a next page. remaining = limit + 1 for source in self.sources: source_results = list(source[offset:remaining]) results.extend(source_results) result_count = len(results) if result_count == 0 and result_count < remaining: # Advance the offset based on the rows we skipped. offset = offset - len(source) elif result_count > 0 and result_count < remaining: # Start at the beginning of the next source offset = 0 remaining = remaining - result_count elif result_count >= limit: break next_cursor = Cursor(limit, page + 1, False, len(results) > limit) prev_cursor = Cursor(limit, page - 1, True, page > 0) if next_cursor.has_results: results.pop() if self.on_results: results = self.on_results(results) return CursorResult(results=results, next=next_cursor, prev=prev_cursor)
def get_result(self, limit, cursor=None): assert limit > 0 offset = cursor.offset if cursor is not None else 0 # Request 1 more than limit so we can tell if there is another page # Use raw_limit for the histogram itself so bucket calculations are correct data = self.data_fn(offset=offset, limit=limit + 1, raw_limit=limit) if isinstance(data["tags"], list): has_more = len(data["tags"]) == limit + 1 if has_more: data["tags"].pop() else: raise NotImplementedError return CursorResult( data, prev=Cursor(0, max(0, offset - limit), True, offset > 0), next=Cursor(0, max(0, offset + limit), False, has_more), )
def _search(self, request, project, extra_query_kwargs=None): query_kwargs = self._build_query_params_from_request(request, project) if extra_query_kwargs is not None: assert 'environment' not in extra_query_kwargs query_kwargs.update(extra_query_kwargs) try: if features.has('organizations:environments', project.organization, actor=request.user): query_kwargs['environment'] = self._get_environment_from_request( request, project.organization_id, ) except Environment.DoesNotExist: # XXX: The 1000 magic number for `max_hits` is an abstraction leak # from `sentry.api.paginator.BasePaginator.get_result`. result = CursorResult([], None, None, hits=0, max_hits=1000) else: result = search.query(**query_kwargs) return result, query_kwargs
def get_result(self, limit=100, cursor=None): if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) page = cursor.offset offset = cursor.offset * cursor.value limit = (cursor.value or limit) + 1 if self.max_offset is not None and offset >= self.max_offset: raise BadPaginationError("Pagination offset too large") if offset < 0: raise BadPaginationError("Pagination offset cannot be negative") primary_results = self.data_load_func(offset=offset, limit=self.max_limit) queryset = self.apply_to_queryset(self.queryset, primary_results) mapping = {} for model in queryset: mapping[self.key_from_model(model)] = model results = [] for row in primary_results: model = mapping.get(self.key_from_data(row)) if model is not None: results.append(model) next_cursor = Cursor(limit, page + 1, False, len(primary_results) > limit) prev_cursor = Cursor(limit, page - 1, True, page > 0) results = list(results[:limit]) if self.on_results: results = self.on_results(results) return CursorResult(results=results, next=next_cursor, prev=prev_cursor)
def query( self, project, query=None, status=None, tags=None, bookmarked_by=None, assigned_to=None, sort_by="date", date_filter="last_seen", date_from=None, date_to=None, cursor=None, limit=100, ): query_body = {"filter": {"and": [{"term": {"project_id": project.id}}]}} if query: query_body["query"] = {"match": {"message": query}} if status is not None: query_body["filter"]["and"].append({"term": {"status": status}}) if tags: # TODO(dcramer): filter might be too expensive here, need to confirm query_body["filter"]["and"].append( { "has_child": { "type": "event", "filter": {"and": [{"term": {"tag:{0}".format(k): v}} for k, v in tags.iteritems()]}, } } ) # TODO(dcramer): filter might be too expensive here, need to confirm if date_to and date_from: query_body["filter"]["and"].append({"range": {date_filter: {"gte": date_from, "lte": date_to}}}) elif date_from: query_body["filter"]["and"].append({"range": {date_filter: {"gte": date_from}}}) elif date_to: query_body["filter"]["and"].append({"range": {date_filter: {"lte": date_to}}}) if bookmarked_by: # TODO(dcramer): we could store an array on each event similar to how # we are doing tags? should we just make bookmarked events a special # thing that isn't searchable? raise NotImplementedError if assigned_to: # TODO(dcramer): raise NotImplementedError if sort_by == "date": sort_clause = [{"last_seen": {"order": "desc"}}] elif sort_by == "new": sort_clause = [{"first_seen": {"order": "desc"}}] elif sort_by == "priority": sort_clause = [{"score": {"order": "desc"}}] elif sort_by == "freq": sort_clause = [{"times_seen": {"order": "desc"}}] elif sort_by == "tottime": raise NotImplementedError elif sort_by == "avgtime": raise NotImplementedError else: raise ValueError("Invalid sort_by: %s" % (sort_by,)) results = self.backend.search( index=self.index_prefix + "sentry-1", doc_type="group", body={ "query": {"filtered": query_body}, "sort": sort_clause, "size": limit, # 'from': offset, }, ) if not results.get("hits"): return CursorResult(results=[], cursor=cursor, limit=limit) instance_ids = [int(n["_id"]) for n in results["hits"]["hits"]] return CursorResult.from_ids( id_list=instance_ids, cursor=cursor, limit=limit, # TODO(dcramer): implement cursors key="id", )
def query(self, project, query=None, status=None, tags=None, bookmarked_by=None, assigned_to=None, sort_by='date', date_filter='last_seen', date_from=None, date_to=None, cursor=None, limit=100): query_body = { 'filter': { 'and': [ {'term': {'project_id': project.id}}, ], }, } if query: query_body['query'] = {'match': {'message': query}} if status is not None: query_body['filter']['and'].append({'term': {'status': status}}) if tags: # TODO(dcramer): filter might be too expensive here, need to confirm query_body['filter']['and'].append({'has_child': { 'type': 'event', 'filter': { 'and': [ {'term': {'tag:{0}'.format(k): v}} for k, v in tags.iteritems() ] }, }}) # TODO(dcramer): filter might be too expensive here, need to confirm if date_to and date_from: query_body['filter']['and'].append({ 'range': {date_filter: { 'gte': date_from, 'lte': date_to, }} }) elif date_from: query_body['filter']['and'].append({ 'range': {date_filter: { 'gte': date_from, }} }) elif date_to: query_body['filter']['and'].append({ 'range': {date_filter: { 'lte': date_to, }} }) if bookmarked_by: # TODO(dcramer): we could store an array on each event similar to how # we are doing tags? should we just make bookmarked events a special # thing that isn't searchable? raise NotImplementedError if assigned_to: # TODO(dcramer): raise NotImplementedError if sort_by == 'date': sort_clause = [{'last_seen': {'order': 'desc'}}] elif sort_by == 'new': sort_clause = [{'first_seen': {'order': 'desc'}}] elif sort_by == 'priority': sort_clause = [{'score': {'order': 'desc'}}] elif sort_by == 'freq': sort_clause = [{'times_seen': {'order': 'desc'}}] elif sort_by == 'tottime': raise NotImplementedError elif sort_by == 'avgtime': raise NotImplementedError else: raise ValueError('Invalid sort_by: %s' % (sort_by,)) results = self.backend.search( index=self.index_prefix + 'sentry-1', doc_type='group', body={ 'query': {'filtered': query_body}, 'sort': sort_clause, 'size': limit, # 'from': offset, }, ) if not results.get('hits'): return CursorResult( results=[], cursor=cursor, limit=limit, ) instance_ids = [int(n['_id']) for n in results['hits']['hits']] return CursorResult.from_ids( id_list=instance_ids, cursor=cursor, limit=limit, # TODO(dcramer): implement cursors key=lambda x: x.id, )
def get(self, request, project): """ List a Project's Issues ``````````````````````` Return a list of issues (groups) bound to a project. All parameters are supplied as query string parameters. A default query of ``is:unresolved`` is applied. To return results with other statuses send an new query value (i.e. ``?query=`` for all results). The ``statsPeriod`` parameter can be used to select the timeline stats which should be present. Possible values are: '' (disable), '24h', '14d' :qparam string statsPeriod: an optional stat period (can be one of ``"24h"``, ``"14d"``, and ``""``). :qparam bool shortIdLookup: if this is set to true then short IDs are looked up by this function as well. This can cause the return value of the function to return an event issue of a different project which is why this is an opt-in. Set to `1` to enable. :qparam querystring query: an optional Sentry structured search query. If not provided an implied ``"is:unresolved"`` is assumed.) :pparam string organization_slug: the slug of the organization the issues belong to. :pparam string project_slug: the slug of the project the issues belong to. :auth: required """ stats_period = request.GET.get('statsPeriod') if stats_period not in (None, '', '24h', '14d'): return Response({"detail": ERR_INVALID_STATS_PERIOD}, status=400) elif stats_period is None: # default stats_period = '24h' elif stats_period == '': # disable stats stats_period = None serializer = functools.partial( StreamGroupSerializer, environment_func=self._get_environment_func(request, project.organization_id), stats_period=stats_period, ) query = request.GET.get('query', '').strip() if query: matching_group = None matching_event = None if len(query) == 32: # check to see if we've got an event ID try: matching_group = Group.objects.from_event_id(project, query) except Group.DoesNotExist: pass else: try: matching_event = Event.objects.get( event_id=query, project_id=project.id) except Event.DoesNotExist: pass # If the query looks like a short id, we want to provide some # information about where that is. Note that this can return # results for another project. The UI deals with this. elif request.GET.get('shortIdLookup') == '1' and \ looks_like_short_id(query): try: matching_group = Group.objects.by_qualified_short_id( project.organization_id, query ) except Group.DoesNotExist: matching_group = None if matching_group is not None: response = Response( serialize( [matching_group], request.user, serializer( matching_event_id=getattr(matching_event, 'id', None), ) ) ) response['X-Sentry-Direct-Hit'] = '1' return response try: query_kwargs = self._build_query_params_from_request( request, project) except ValidationError as exc: return Response({'detail': six.text_type(exc)}, status=400) try: environment_id = self._get_environment_id_from_request( request, project.organization_id) except Environment.DoesNotExist: # XXX: The 1000 magic number for `max_hits` is an abstraction leak # from `sentry.api.paginator.BasePaginator.get_result`. cursor_result = CursorResult([], None, None, hits=0, max_hits=1000) else: cursor_result = search.query( count_hits=True, environment_id=environment_id, **query_kwargs) results = list(cursor_result) context = serialize(results, request.user, serializer()) # HACK: remove auto resolved entries if query_kwargs.get('status') == GroupStatus.UNRESOLVED: context = [r for r in context if r['status'] == 'unresolved'] response = Response(context) self.add_cursor_headers(request, response, cursor_result) if results and query not in SAVED_SEARCH_QUERIES: advanced_search.send(project=project, sender=request.user) return response