def test_build_cursor(): event1 = build_mock(id=1.1, message='one') event2 = build_mock(id=1.1, message='two') event3 = build_mock(id=2.1, message='three') results = [event1, event2, event3] cursor_kwargs = { 'key': lambda x: math.floor(x.id), 'limit': 1, } cursor = build_cursor(results, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert cursor.next assert isinstance(cursor.prev, Cursor) assert not cursor.prev assert list(cursor) == [event1] cursor = build_cursor(results[1:], cursor=cursor.next, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert cursor.next assert isinstance(cursor.prev, Cursor) assert cursor.prev assert list(cursor) == [event2] cursor = build_cursor(results[2:], cursor=cursor.next, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert not cursor.next assert isinstance(cursor.prev, Cursor) assert cursor.prev assert list(cursor) == [event3]
def test_build_cursor(): event1 = build_mock(id=1.1, message="one") event2 = build_mock(id=1.1, message="two") event3 = build_mock(id=2.1, message="three") results = [event1, event2, event3] def item_key(key, for_prev=False): return int(math.floor(key.id)) cursor_kwargs = {"key": item_key, "limit": 1} cursor = build_cursor(results, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert cursor.next assert isinstance(cursor.prev, Cursor) assert not cursor.prev assert list(cursor) == [event1] cursor = build_cursor(results[1:], cursor=cursor.next, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert cursor.next assert isinstance(cursor.prev, Cursor) assert cursor.prev assert list(cursor) == [event2] cursor = build_cursor(results[2:], cursor=cursor.next, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert not cursor.next assert isinstance(cursor.prev, Cursor) assert cursor.prev assert list(cursor) == [event3]
def test_build_cursor(): event1 = build_mock(id=1.1, message='one') event2 = build_mock(id=1.1, message='two') event3 = build_mock(id=2.1, message='three') results = [event1, event2, event3] def item_key(key, for_prev=False): return math.floor(key.id) cursor_kwargs = { 'key': item_key, 'limit': 1, } cursor = build_cursor(results, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert cursor.next assert isinstance(cursor.prev, Cursor) assert not cursor.prev assert list(cursor) == [event1] cursor = build_cursor(results[1:], cursor=cursor.next, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert cursor.next assert isinstance(cursor.prev, Cursor) assert cursor.prev assert list(cursor) == [event2] cursor = build_cursor(results[2:], cursor=cursor.next, **cursor_kwargs) assert isinstance(cursor.next, Cursor) assert not cursor.next assert isinstance(cursor.prev, Cursor) assert cursor.prev assert list(cursor) == [event3]
def get_result(self, limit=100, cursor=None): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) if cursor.value: cursor_value = self._value_from_cursor(cursor) else: cursor_value = 0 queryset = self._get_results_from_qs(cursor_value, cursor.is_prev) # this effectively gets us the before post, and the current (after) post # every time if cursor.is_prev: stop = cursor.offset + limit + 2 else: stop = cursor.offset + limit + 1 results = list(queryset[cursor.offset:stop]) if cursor.is_prev: results = results[1:][::-1] return build_cursor( results=results, limit=limit, cursor=cursor, key=self._get_item_key, )
def get_result(self, limit=100, cursor=None): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) if cursor.value: cursor_value = self.value_from_cursor(cursor) else: cursor_value = 0 queryset = self._build_queryset(cursor_value, cursor.is_prev) # TODO(dcramer): this does not yet work correctly for ``is_prev`` when # the key is not unique offset = cursor.offset if cursor.is_prev: offset += 1 stop = offset + limit + 1 results = list(queryset[offset:stop]) if cursor.is_prev: results.reverse() return build_cursor( results=results, limit=limit, cursor=cursor, key=self.get_item_key, )
def get_result(self, limit=100, cursor=None): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) if cursor.value: cursor_value = self.value_from_cursor(cursor) else: cursor_value = 0 queryset = self._build_queryset(cursor_value, cursor.is_prev) # TODO(dcramer): this does not yet work correctly for ``is_prev`` when # the key is not unique offset = cursor.offset if cursor.is_prev: offset += 1 stop = offset + limit + 1 results = list(queryset[offset:stop]) if cursor.is_prev: results.reverse() return build_cursor( results=results, limit=limit, cursor=cursor, key=self.get_item_key, )
def get_result(self, limit=100, cursor=None): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) queryset = self._get_results_from_qs(cursor.value, cursor.is_prev) # this effectively gets us the before post, and the current (after) post # every time if cursor.is_prev: stop = cursor.offset + limit + 2 else: stop = cursor.offset + limit + 1 results = list(queryset[cursor.offset:stop]) if cursor.is_prev: results = results[1:][::-1] return build_cursor( results=results, limit=limit, cursor=cursor, key=self._get_item_key, )
def get(self, request, project): """ Fetches alert rules and legacy rules for an organization """ if not features.has("organizations:incidents", project.organization, actor=request.user): raise ResourceDoesNotExist cursor_string = request.GET.get( "cursor", six.binary_type(int(time.time() * 1000000)) + ":0:0") try: limit = min(100, int(request.GET.get("limit", 25))) except ValueError as e: return Response( { "detail": "Invalid input for `limit`. Error: %s" % six.text_type(e) }, status=400) cursor = Cursor.from_string(cursor_string) cursor_date = datetime.fromtimestamp( float(cursor.value) / 1000000).replace(tzinfo=timezone.utc) alert_rule_queryset = ( AlertRule.objects.fetch_for_project(project).filter( date_added__lte=cursor_date).order_by("-date_added")[:limit + 1]) legacy_rule_queryset = (Rule.objects.filter( project=project, status__in=[ RuleStatus.ACTIVE, RuleStatus.INACTIVE ]).select_related("project").filter( date_added__lte=cursor_date).order_by("-date_added")[:(limit + 1)]) combined_rules = list(alert_rule_queryset) + list(legacy_rule_queryset) combined_rules.sort(key=lambda instance: (instance.date_added, type(instance)), reverse=True) combined_rules = combined_rules[cursor.offset:cursor.offset + limit + 1] def get_item_key(item, for_prev=False): return 1000000 * float(item.date_added.strftime("%s.%f")) cursor_result = build_cursor(results=combined_rules, cursor=cursor, key=get_item_key, limit=limit, is_desc=True) results = list(cursor_result) context = serialize(results, request.user, CombinedRuleSerializer()) response = Response(context) self.add_cursor_headers(request, response, cursor_result) return response
def get_result(self, limit=100, cursor=None, count_hits=False): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) if cursor.value: cursor_value = self.value_from_cursor(cursor) else: cursor_value = 0 queryset = self._build_queryset(cursor_value, cursor.is_prev) # TODO(dcramer): this does not yet work correctly for ``is_prev`` when # the key is not unique if count_hits: max_hits = 1000 hits = self.count_hits(max_hits) else: hits = None max_hits = None offset = cursor.offset # this effectively gets us the before row, and the current (after) row # every time. Do not offset if the provided cursor value was empty since # there is nothing to traverse past. if cursor.is_prev and cursor.value: offset += 1 # The + 1 is needed so we can decide in the ResultCursor if there is # more on the next page. stop = offset + limit + 1 results = list(queryset[offset:stop]) if cursor.is_prev: results.reverse() return build_cursor( results=results, limit=limit, hits=hits, max_hits=max_hits, cursor=cursor, is_desc=self.desc, key=self.get_item_key, )
def get_result(self, cursor=None, limit=100): if cursor is None: cursor = Cursor(0, 0, 0) if cursor.value: cursor_value = self.value_from_cursor(cursor) else: cursor_value = None limit = min(limit, MAX_LIMIT) offset = cursor.offset extra = 1 if cursor.is_prev and cursor.value: extra += 1 combined_querysets = self._build_combined_querysets( cursor_value, cursor.is_prev, limit, extra) stop = offset + limit + extra results = (list(combined_querysets[offset:stop]) if self.using_dates else list(combined_querysets[:(limit + extra)])) if cursor.is_prev and cursor.value: # If the first result is equal to the cursor_value then it's safe to filter # it out, since the value hasn't been updated if results and self.get_item_key(results[0], for_prev=True) == cursor.value: results = results[1:] # Otherwise we may have fetched an extra row, just drop it off the end if so. elif len(results) == offset + limit + extra: results = results[:-1] # We reversed the results when generating the querysets, so we need to reverse back now. if cursor.is_prev: results.reverse() return build_cursor( results=results, cursor=cursor, key=self.get_item_key, limit=limit, is_desc=self.desc, on_results=self.on_results, )
def get_result(self, limit=100, cursor=None, count_hits=False): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) if cursor.value: cursor_value = self.value_from_cursor(cursor) else: cursor_value = 0 queryset = self._build_queryset(cursor_value, cursor.is_prev) # TODO(dcramer): this does not yet work correctly for ``is_prev`` when # the key is not unique if count_hits: max_hits = 1000 hits = self.count_hits(max_hits) else: hits = None max_hits = None offset = cursor.offset # this effectively gets us the before row, and the current (after) row # every time if cursor.is_prev: offset += 1 stop = offset + limit + 1 results = list(queryset[offset:stop]) if cursor.is_prev: results.reverse() return build_cursor( results=results, limit=limit, hits=hits, max_hits=max_hits, cursor=cursor, key=self.get_item_key, )
def get_result(self, limit=100, cursor=None, count_hits=False, known_hits=None, max_hits=None): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) if cursor.value: cursor_value = self.value_from_cursor(cursor) else: cursor_value = 0 queryset = self.build_queryset(cursor_value, cursor.is_prev) # TODO(dcramer): this does not yet work correctly for ``is_prev`` when # the key is not unique # max_hits can be limited to speed up the query if max_hits is None: max_hits = MAX_HITS_LIMIT if count_hits: hits = self.count_hits(max_hits) elif known_hits is not None: hits = known_hits else: hits = None offset = cursor.offset # The extra amount is needed so we can decide in the ResultCursor if there is # more on the next page. extra = 1 # this effectively gets us the before row, and the current (after) row # every time. Do not offset if the provided cursor value was empty since # there is nothing to traverse past. # We need to actually fetch the before row so that we can compare it to the # cursor value. This allows us to handle an edge case where the first row # for a given cursor is the same row that generated the cursor on the # previous page, but we want to display since it has had its its sort value # updated. if cursor.is_prev and cursor.value: extra += 1 stop = offset + limit + extra results = list(queryset[offset:stop]) if cursor.is_prev and cursor.value: # If the first result is equal to the cursor_value then it's safe to filter # it out, since the value hasn't been updated if results and self.get_item_key(results[0], for_prev=True) == cursor.value: results = results[1:] # Otherwise we may have fetched an extra row, just drop it off the end if so. elif len(results) == offset + limit + extra: results = results[:-1] if cursor.is_prev: results.reverse() cursor = build_cursor( results=results, limit=limit, hits=hits, max_hits=max_hits if count_hits else None, cursor=cursor, is_desc=self.desc, key=self.get_item_key, on_results=self.on_results, ) # Note that this filter is just to remove unwanted rows from the result set. # This will reduce the number of rows returned rather than fill a full page, # and could result in an empty page being returned if self.post_query_filter: cursor.results = self.post_query_filter(cursor.results) return cursor
def get_result(self, limit=100, cursor=None, count_hits=False): # cursors are: # (identifier(integer), row offset, is_prev) if cursor is None: cursor = Cursor(0, 0, 0) limit = min(limit, self.max_limit) if cursor.value: cursor_value = self.value_from_cursor(cursor) else: cursor_value = 0 queryset = self._build_queryset(cursor_value, cursor.is_prev) # TODO(dcramer): this does not yet work correctly for ``is_prev`` when # the key is not unique if count_hits: hits = self.count_hits(MAX_HITS_LIMIT) else: hits = None offset = cursor.offset # The extra amount is needed so we can decide in the ResultCursor if there is # more on the next page. extra = 1 # this effectively gets us the before row, and the current (after) row # every time. Do not offset if the provided cursor value was empty since # there is nothing to traverse past. # We need to actually fetch the before row so that we can compare it to the # cursor value. This allows us to handle an edge case where the first row # for a given cursor is the same row that generated the cursor on the # previous page, but we want to display since it has had its its sort value # updated. if cursor.is_prev and cursor.value: extra += 1 stop = offset + limit + extra results = list(queryset[offset:stop]) if cursor.is_prev and cursor.value: # If the first result is equal to the cursor_value then it's safe to filter # it out, since the value hasn't been updated if results and self.get_item_key(results[0], for_prev=True) == cursor.value: results = results[1:] # Otherwise we may have fetched an extra row, just drop it off the end if so. elif len(results) == offset + limit + extra: results = results[:-1] if cursor.is_prev: results.reverse() return build_cursor( results=results, limit=limit, hits=hits, max_hits=MAX_HITS_LIMIT if count_hits else None, cursor=cursor, is_desc=self.desc, key=self.get_item_key, on_results=self.on_results, )