def _check_key_info(self, page_info: dict) -> None:
try:
info_key = page_info['key']
except KeyError as e:
raise InvalidPage('Missing "key".') from e
else:
if info_key != self.key:
raise InvalidPage(
'Invalid "key" (different from paginator key).')
def _offset_from_info(page_info: dict) -> int:
try:
offset = int(page_info.get('offset', 0))
except ValueError as e:
raise InvalidPage('Invalid "offset" (not integer).') from e
if offset < 0:
raise InvalidPage('Invalid "offset" (negative) .')
return offset
def validate_number(self, number):
"""Validates the given 1-based page number."""
try:
number = int(number)
except (TypeError, ValueError):
raise InvalidPage("That page number is not an integer")
if number < 1:
raise InvalidPage("That page number is less than 1")
return number
def validate_number(self, number):
if isinstance(number, text):
try:
number = json.loads(number)
except ValueError:
raise InvalidPage('Invalid key')
if not number or number == 1:
return None
if not isinstance(number, list):
raise InvalidPage('Invalid key')
if len(number) != 1 + len(self.keys):
raise InvalidPage('Key length mismatch')
return number
def validate_number(self, number):
"""
Validates the given 1-based page number.
We also check that the number isn't too large.
"""
try:
number = int(number)
except (TypeError, ValueError):
raise PageNotAnInteger("That page number is not an integer")
if number < 1:
raise EmptyPage("That page number is less than 1")
if number >= 1000:
# Anything >=1,000 will result in a hard error in
# Elasticsearch which would happen before we even get a chance
# to validate that the range is too big. The error you would
# get from Elasticsearch 6.x is something like this:
#
# Result window is too large, from + size must be less
# than or equal to: [10000] but was [11000].
#
# See https://github.com/mdn/kuma/issues/6092
raise InvalidPage("Page number too large")
return number
def page(self, number):
"""
Returns a page object.
This class overrides the default behavior and ignores "orphans" and
assigns the count from the ES result to the Paginator.
"""
number = self.validate_number(number)
bottom = (number - 1) * self.per_page
top = bottom + self.per_page
if bottom > self.max_result_window:
raise InvalidPage(
'That page number is too high for the current page size')
# Force the search to evaluate and then attach the count. We want to
# avoid an extra useless query even if there are no results, so we
# directly fetch the count from hits.
# Overwrite `object_list` with the list of ES results.
result = self.object_list[bottom:top].execute()
page = Page(result.hits, number, self)
# Update the `_count`.
self._count = page.object_list.total
# Now that we have the count validate that the page number isn't higher
# than the possible number of pages and adjust accordingly.
if number > self.num_pages:
if number == 1 and self.allow_empty_first_page:
pass
else:
raise EmptyPage('That page contains no results')
return page
def page_key(raw_page):
"""
Parse a raw page value of ``timestamp-pk`` format.
Return a tuple of (timestamp, pk)
:raises: ``InvalidPage``
"""
if not raw_page:
return None, None
m = re.match(PAGE_RE, raw_page)
if m is None:
raise InvalidPage('Bad key format')
try:
timestamp = _fromtimestamp(float(m.group('value')))
except (OverflowError, ValueError):
raise InvalidPage('Key out of range')
return _make_aware_maybe(timestamp), m.group('pk')
def page(self, number, softlimit=False):
try:
return super(ExPaginator, self).page(number)
except PageNotAnInteger as e:
raise InvalidPage(_('That page number is not an integer'))
except InvalidPage as e:
if number > self.num_pages and softlimit:
return self.page(self.num_pages, softlimit=False)
else:
raise e
def get_page(queryset, order_by, per_page, start=0):
if start < 0:
raise InvalidPage()
object_list = list(_slice_queryset(queryset, order_by, per_page, start))
if start and not object_list:
raise EmptyPage()
next_cursor = None
if len(object_list) > per_page:
next_slice_first_item = object_list.pop(-1)
attr_name = order_by.lstrip("-")
next_cursor = getattr(next_slice_first_item, attr_name)
return CursorPage(start, object_list, next_cursor)
def set_count(self, count):
"""
method used for seting and validating the count param
the count param changes PAGINATION_COUNT which control the
quantity of instances comming in one page
:param count: set count used for pagination
"""
try:
self.PAGINATION_COUNT = int(count)
except Exception:
raise PageNotAnInteger(self.PAGE_ERR_MSG_COUNT_ARG)
if self.PAGINATION_COUNT <= 0:
raise InvalidPage(self.PAGE_ERR_MSG_LOW_0)
def page(self, number):
"""
Returns a Page object for the given 1-based page number.
"""
number = self.validate_number(number)
offset = (number - 1) * self.per_page
limit = offset + self.per_page
object_list = self.object_list[offset:limit]
page = YTPage(object_list, number, self)
if not page.num_pages:
if number != 1 or not self.allow_empty_first_page:
raise InvalidPage("That page contains no results")
return page
def page(self, number):
"""
Returns a Page object for the given 1-based page number.
Retrieves objects for the given page number plus 1 additional to check
if there are more objects after this page.
"""
number = self.validate_number(number)
offset = (number - 1) * self.per_page
limit = offset + self.per_page
object_list = list(self.object_list[offset:limit])
if not object_list and number != 1:
raise InvalidPage("That page contains no results")
return YTPage(object_list, number, self)
def paginate_queryset(self, queryset, request_data):
"""
Paginate the search results. This is a simplified version of
Django's MultipleObjectMixin.paginate_queryset
"""
paginator = self.get_paginator(queryset)
page_kwarg = self.page_kwarg
page = request_data.get(page_kwarg, 1)
try:
page_number = int(page)
except ValueError:
if page == 'last':
page_number = paginator.num_pages
else:
raise InvalidPage(
_("Page is not 'last', nor can it be converted to an int.")
)
# This can also raise an InvalidPage exception.
return paginator, paginator.page(page_number)
def _get_qs(self, page_info: dict, reverse: bool) -> QuerySet:
value = page_info['value']
attr_name = self._attr_name
if value is None:
q = Q(**{attr_name + '__isnull': True}) if reverse else Q()
else:
op = '__lte' if reverse else '__gte'
q = Q(**{attr_name + op: value})
if reverse and any(f.null for f in self._key_field_info):
q |= Q(**{attr_name + '__isnull': True})
try:
qs = self.queryset.filter(q)
except (ValueError, ValidationError) as e:
raise InvalidPage(f'Invalid "value" [{e}].') from e
return qs
def paginate_query_object(self, query_object, page_size):
"""Paginate the query object."""
paginator = self.get_paginator(
query_object, page_size,
allow_empty_first_page=self.get_allow_empty())
page = self.kwargs.get('page') or self.request.GET.get('page') or 1
try:
page_number = int(page)
except ValueError:
if page == 'last':
page_number = paginator.num_pages
else:
raise InvalidPage("Page is not 'last', "
"nor can it be converted to an int.")
# DB2 fix for invalid 0 literal.
# Generates FETCH 0 rows if not done this way
if not paginator.count == 0:
page = paginator.page(page_number)
return (paginator, page, page.object_list, page.has_other_pages())
else:
return (paginator, Page([], 1, paginator), [], False)
def get_context(self, page, range_gap=5):
try:
page = int(page)
except (ValueError, TypeError) as exc:
raise InvalidPage(exc)
try:
paginator = self.page(page)
except EmptyPage:
return {
'EMPTY_PAGE': True,
}
if page > 5:
start = page-range_gap
else:
start = 1
if page < self.num_pages-range_gap:
end = page+range_gap+1
else:
end = self.num_pages+1
context = {
'page_range': list(range(start, end)),
'objects': paginator.object_list,
'num_pages': self.num_pages,
'page': page,
'has_pages': self.num_pages > 1,
'has_previous': paginator.has_previous(),
'has_next': paginator.has_next(),
'previous_page': paginator.previous_page_number() if paginator.has_previous() else None,
'next_page': paginator.next_page_number() if paginator.has_next() else None,
'is_first': page == 1,
'is_last': page == self.num_pages,
}
return context
def page(self, number):
"""
Returns a Page object for the given 1-based page number.
"""
bottom = (number - 1) * self.per_page
if bottom >= self.MAX_ES_OFFSET:
# Only validate if bigger than offset
number = self.validate_number(number)
bottom = (number - 1) * self.per_page
top = bottom + self.per_page
try:
self.object_list = self.object_list[bottom:top]
except ValueError:
raise InvalidPage()
# ignore top boundary
# if top + self.orphans >= self.count:
# top = self.count
# Validate number after limit/offset has been set
number = self.validate_number(number)
return self._get_page(self.object_list, number, self)
def validate_number(self, number):
try:
int(b64decode(number, validate=True))
return number
except (TypeError, binascii.Error):
raise InvalidPage("Page number is invalid")
def page(self, page_info: Optional[dict] = None) -> 'FlowPage':
"""Get the wanted page.
@param page_info: A dictionary returned by the methods
info()/next_page_info()/previous_page_info() of a page,
or None (which means 'first page').
@return An instance of FlowPage.
@raise FirstPage: the first page has been reached when going backward
(the page could be not complete).
@raise LastPage: it seems that the last page has been exceeded
(this page is empty).
@raise InvalidPage: page_info is invalid.
@see FlowPage.info()
"""
if page_info is None:
page_info = {'type': 'first'}
per_page = self._per_page
offset = 0
forward = True
first_page = False
move_type = page_info.get('type')
# PyCharm does not understand that it's not a problem to use list
# methods in local contexts...
# entities: Iterable[Model]
if move_type == 'first' or self.count <= per_page:
entities = [*self.queryset[:per_page + 1]]
next_item = None if len(entities) <= per_page else entities.pop()
first_page = True
elif move_type == 'last':
self._check_key_info(page_info)
entities = reversed(self.queryset.reverse()[:per_page])
next_item = None
forward = False
else:
self._check_key_info(page_info)
offset = self._offset_from_info(page_info)
if move_type == _FORWARD:
qs = self._get_qs(page_info, reverse=self._reverse_order)
entities = [*qs[offset:offset + per_page + 1]]
next_item = None if len(
entities) <= per_page else entities.pop()
if not entities:
raise LastPage()
elif move_type == _BACKWARD:
qs = self._get_qs(page_info, reverse=not self._reverse_order)
# NB: we get 2 additional items:
# - 1 will be the next_item of the page.
# - if the second one exists, it indicates that there is at
# least one item before the page (so it is not the first one).
size = per_page + 2
entities = [*qs.reverse()[offset:offset + size]]
if len(entities) != size:
raise FirstPage()
entities.pop()
entities.reverse()
next_item = entities.pop()
if self._key_field_info.value_from(
entities[-1]) != page_info['value']:
offset = 0
forward = False
else:
raise InvalidPage('Invalid or missing "type".')
return FlowPage(
object_list=entities,
paginator=self,
forward=forward,
key=self._key,
key_field_info=self._key_field_info,
attr_name=self._attr_name,
offset=offset,
max_size=per_page,
next_item=next_item,
first_page=first_page,
)
def serialize_stream(
graph, paginators, context, page=1, embed=False,
restrict_inclusion=True, restrict_embed=False
):
# restrict_inclusion: only public components of contents are included
# restrict_embed: only contents with no restrictions are embedded
from .models import UserComponent
if not isinstance(paginators, (tuple, list)):
paginators = [paginators]
assert isinstance(page, int)
if page <= 1:
num_pages = max(map(lambda p: p.num_pages, paginators))
per_page = sum(map(lambda p: p.per_page, paginators))
graph.add((
context["sourceref"],
spkcgraph["pages.num_pages"],
Literal(num_pages, datatype=XSD.positiveInteger)
))
graph.add((
context["sourceref"],
spkcgraph["pages.size_page"],
Literal(per_page, datatype=XSD.positiveInteger)
))
graph.add((
context["sourceref"],
spkcgraph["pages.current_page"],
Literal(page, datatype=XSD.positiveInteger)
))
invalid_pages = 0
for paginator in paginators:
try:
page_view = paginator.get_page(page)
# for mysql
object_list = list(page_view.object_list)
# error if page is out of bound
if page > paginator.num_pages:
raise InvalidPage()
except InvalidPage:
invalid_pages += 1
continue
if paginator.object_list.model == UserComponent:
if embed:
prefetch_related_objects(
object_list,
"contents",
"contents__ctype",
"contents__datacontent"
)
for component in object_list:
ref_component = serialize_component(
graph, component, context
)
list_features(graph, component, ref_component, context)
else:
# either start with invalid usercomponent which will be replaced
# or use bottom-1 usercomponent to detect split
if page <= 1:
usercomponent = None
ref_component = None
else:
_pos = page_view.start_index() - 1
usercomponent = paginator.object_list[_pos]
ref_component = URIRef("{}{}".format(
context["hostpart"],
usercomponent.get_absolute_url()
))
prefetch_related_objects(
object_list, "ctype", "datacontent"
)
for content in object_list:
if usercomponent != content.usercomponent:
usercomponent = content.usercomponent
ref_component = serialize_component(
graph, usercomponent, context,
visible=not restrict_inclusion
)
list_features(graph, usercomponent, ref_component, context)
_embed = embed
if restrict_embed and content.usercomponent.strength != 0:
_embed = False
ref_content = serialize_content(
graph, content, context, embed=_embed
)
if ref_component:
graph.add((
ref_component,
spkcgraph["contents"],
ref_content
))
if invalid_pages == len(paginators):
raise Http404('Invalid page (%(page_number)s)' % {
'page_number': page
})