def test_prev_desc_asc_sticky_pinned_non_unique(self):
articles = Article.objects.all().order_by('-is_pinned', 'is_sticky',
"-date", '-pk')
for a in articles[:5]:
a.is_pinned = True
a.save()
for a in articles[10:15]:
a.is_pinned = True
a.is_sticky = True
a.save()
for a in articles[20:]:
a.is_pinned = True
a.is_sticky = True
a.save()
paginator = SeekPaginator(Article.objects.all(),
per_page=10,
lookup_field=('-is_pinned', 'is_sticky',
'-date'))
page_2 = paginator.page(value=(list(articles)[20].is_pinned,
list(articles)[20].is_sticky,
list(articles)[20].date),
pk=list(articles)[20].pk,
move_to=inf_paginator.PREV_PAGE)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_1 = paginator.page(value=(page_2[0].is_pinned,
page_2[0].is_sticky, page_2[0].date),
pk=page_2[0].pk,
move_to=inf_paginator.PREV_PAGE)
self.assertListEqual(list(page_1), list(articles[:10]))
def test_reverse_date_for_pk(self):
"""
When the date increment does not match the pk increment,
we should still get the right results.
"""
Article.objects.all().delete()
self.assertFalse(list(Article.objects.all()))
# asc order date and desc order pk
date = timezone.now()
dates = reversed(
[date + datetime.timedelta(seconds=seconds)
for seconds in range(25)])
for i, d in enumerate(dates):
Article.objects.create(title="%s" % i, date=date, date_unique=d)
articles = Article.objects.all().order_by("-date_unique")
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="-date_unique")
page_1 = paginator.page(value=None)
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = paginator.page(value=page_1[-1].date_unique, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = paginator.page(value=page_2[-1].date_unique, pk=page_2[-1].pk)
self.assertListEqual(list(page_3), list(articles[20:]))
def test_next_desc_asc_sticky_pinned_non_unique(self):
articles = Article.objects.all().order_by('-is_pinned', 'is_sticky',
"-date", '-pk')
for a in articles[:5]:
a.is_pinned = True
a.save()
for a in articles[10:15]:
a.is_pinned = True
a.is_sticky = True
a.save()
for a in articles[20:]:
a.is_pinned = True
a.is_sticky = True
a.save()
paginator = SeekPaginator(Article.objects.all(),
per_page=10,
lookup_field=('-is_pinned', 'is_sticky',
'-date'))
page_1 = paginator.page(value=None, pk=None)
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = paginator.page(value=(page_1[-1].is_pinned,
page_1[-1].is_sticky, page_1[-1].date),
pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = paginator.page(value=(page_2[-1].is_pinned,
page_2[-1].is_sticky, page_2[-1].date),
pk=page_2[-1].pk)
self.assertListEqual(list(page_3), list(articles[20:]))
def test_prev_pages_left(self):
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="-date_unique")
page = paginator.page(value=None)
self.assertEqual(page.prev_pages_left(), 0)
page = paginator.page(**page.next_page())
self.assertEqual(page.prev_pages_left(), 1)
page = paginator.page(**page.next_page())
self.assertEqual(page.prev_pages_left(), 2)
def test_has_next_page(self):
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="date_unique")
page = paginator.page(value=None)
self.assertTrue(page.has_next())
page = paginator.page(**page.next_page())
self.assertTrue(page.has_next())
page = paginator.page(**page.next_page())
self.assertFalse(page.has_next())
def test_next_page(self):
articles = Article.objects.all().order_by("date_unique")
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="date_unique")
page = paginator.page(value=None)
self.assertListEqual(list(page), list(articles[:10]))
page = paginator.page(**page.next_page())
self.assertListEqual(list(page), list(articles[10:20]))
page = paginator.page(**page.next_page())
self.assertListEqual(list(page), list(articles[20:]))
def test_paginator_next_desc_non_unique(self):
articles = Article.objects.all().order_by("-date", "-pk")
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="-date")
page_1 = paginator.page(value=None, pk=None)
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = paginator.page(value=page_1[-1].date, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = paginator.page(value=page_2[-1].date, pk=page_2[-1].pk)
self.assertListEqual(list(page_3), list(articles[20:]))
def test_next_objects_left(self):
articles = Article.objects.all().order_by("-date_unique")
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="-date_unique")
page = paginator.page(value=None)
self.assertEqual(
page.next_objects_left(),
len(articles[paginator.per_page:]))
# last page
page_last = paginator.page(
value=list(articles)[-paginator.per_page].date_unique)
self.assertEqual(page_last.next_objects_left(), 0)
def test_has_prev_page(self):
articles = Article.objects.all().order_by("-date_unique")
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="-date_unique")
page = paginator.page(
value=list(articles)[20].date_unique,
move_to=inf_paginator.PREV_PAGE)
self.assertTrue(page.has_previous())
page = paginator.page(
move_to=inf_paginator.PREV_PAGE,
**page.prev_page())
self.assertFalse(page.has_previous())
def test_paginator_prev_asc(self):
articles = Article.objects.all().order_by("date_unique")
paginator = SeekPaginator(
Article.objects.all(), per_page=10, lookup_field="date_unique")
page_2 = paginator.page(
value=list(articles)[20].date_unique,
move_to=inf_paginator.PREV_PAGE)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_1 = paginator.page(
value=page_2[0].date_unique,
move_to=inf_paginator.PREV_PAGE)
self.assertListEqual(list(page_1), list(articles[:10]))
def test_prev_desc(self):
articles = Article.objects.all().order_by('-is_pinned', "-date_unique")
paginator = SeekPaginator(Article.objects.all(),
per_page=10,
lookup_field=('-is_pinned', '-date_unique'))
page_2 = paginator.page(value=(list(articles)[20].is_pinned,
list(articles)[20].date_unique),
move_to=inf_paginator.PREV_PAGE)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_1 = paginator.page(value=(page_2[0].is_pinned,
page_2[0].date_unique),
move_to=inf_paginator.PREV_PAGE)
self.assertListEqual(list(page_1), list(articles[:10]))
def test_next_desc(self):
articles = Article.objects.all().order_by('-is_pinned', "-date_unique")
paginator = SeekPaginator(Article.objects.all(),
per_page=10,
lookup_field=('-is_pinned', '-date_unique'))
page_1 = paginator.page(value=None)
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = paginator.page(value=(page_1[-1].is_pinned,
page_1[-1].date_unique))
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = paginator.page(value=(page_2[-1].is_pinned,
page_2[-1].date_unique))
self.assertListEqual(list(page_3), list(articles[20:]))
def object_pagination(object_list, start_object=None, per_page=50):
"""
Paginate a list of objects (e.g transactions or blocks). The `start_object` object should have attribute `pk` and
`time`.
:param object_list: A list of objects need to paginate
:param start_object: Starting object
:param per_page: The page size
:return: (page, objects). page is the pagination info. objects is the queried objects.
"""
pk = None
time = None
if start_object:
pk = start_object.pk
time = start_object.time
paginator = SeekPaginator(object_list,
per_page=per_page,
lookup_field='time')
try:
objects = paginator.page(value=time, pk=pk)
page = {
'starting_after': objects[0].hash if objects else None,
'ending_before': objects[-1].hash if objects else None,
'next_uri': None
}
except EmptyPage:
objects = []
page = {
'starting_after': None,
'ending_before': None,
'next_uri': None
}
return page, objects
def paginate(request, query_set, lookup_field, per_page=15, page_var='value'):
page_pk = request.GET.get(page_var, None)
paginator = SeekPaginator(query_set, per_page=per_page, lookup_field=lookup_field)
# First page
if page_pk is None:
return paginator.page()
try:
obj = query_set.model.objects.get(pk=page_pk)
except query_set.model.DoesNotExist:
raise Http404()
value = getattr(obj, lookup_field)
try:
page = paginator.page(value=value, pk=page_pk)
except EmptyPage:
raise Http404()
return page
def paginate(request, query_set, lookup_field, per_page=15, page_var='value'):
page_pk = request.GET.get(page_var, None)
paginator = SeekPaginator(query_set, per_page=per_page, lookup_field=lookup_field)
# First page
if page_pk is None:
return paginator.page()
try:
obj = query_set.model.objects.get(pk=page_pk)
except query_set.model.DoesNotExist:
raise Http404()
value = getattr(obj, lookup_field)
try:
page = paginator.page(value=value, pk=page_pk)
except EmptyPage:
raise Http404()
return page
def test_empty_first_page(self):
paginator = SeekPaginator(
Article.objects.none(), per_page=10, lookup_field="-date_unique")
page = paginator.page(value=None)
self.assertFalse(list(page))
self.assertFalse(page.has_next())
self.assertFalse(page.has_previous())
self.assertEqual(page.next_objects_left(), 0)
self.assertEqual(page.prev_objects_left(), 0)
self.assertEqual(page.next_pages_left(), 0)
self.assertEqual(page.next_page(), {})
self.assertEqual(page.prev_page(), {})
class PageTest(TestCase):
def setUp(self):
date = timezone.now()
for i in range(25):
seconds = datetime.timedelta(seconds=i)
Article.objects.create(title="%s" % i,
date=date,
date_unique=date + seconds)
self.paginator = SeekPaginator(Article.objects.all(),
per_page=10,
lookup_field="date_unique")
def test_unimplemented(self):
page = self.paginator.page()
self.assertRaises(NotImplementedError, page.has_previous)
self.assertRaises(NotImplementedError, page.next_page_number)
self.assertRaises(NotImplementedError, page.previous_page_number)
self.assertRaises(NotImplementedError, page.start_index)
self.assertRaises(NotImplementedError, page.end_index)
def test_objects_left(self):
articles = Article.objects.all().order_by("-date_unique")
page = self.paginator.page()
self.assertEqual(page.objects_left,
len(articles[self.paginator.per_page:]))
# last page
art = list(articles)[-self.paginator.per_page]
page_last = self.paginator.page(value=art.date_unique, pk=art.pk)
self.assertEqual(page_last.objects_left, 0)
def test_pages_left(self):
page = self.paginator.page()
self.assertEqual(page.pages_left, 2)
def test_next_page_pk(self):
page = self.paginator.page()
self.assertEqual(page.next_page_pk(), page[-1].pk)
class PageTest(TestCase):
def setUp(self):
date = timezone.now()
for i in range(25):
seconds = datetime.timedelta(seconds=i)
Article.objects.create(title="%s" % i, date=date, date_unique=date + seconds)
self.paginator = SeekPaginator(Article.objects.all(), per_page=10, lookup_field="date_unique")
def test_unimplemented(self):
page = self.paginator.page()
self.assertRaises(NotImplementedError, page.has_previous)
self.assertRaises(NotImplementedError, page.next_page_number)
self.assertRaises(NotImplementedError, page.previous_page_number)
self.assertRaises(NotImplementedError, page.start_index)
self.assertRaises(NotImplementedError, page.end_index)
def test_objects_left(self):
articles = Article.objects.all().order_by("-date_unique")
page = self.paginator.page()
self.assertEqual(page.objects_left, len(articles[self.paginator.per_page:]))
# last page
art = list(articles)[-self.paginator.per_page]
page_last = self.paginator.page(value=art.date_unique, pk=art.pk)
self.assertEqual(page_last.objects_left, 0)
def test_pages_left(self):
page = self.paginator.page()
self.assertEqual(page.pages_left, 2)
def test_next_page_pk(self):
page = self.paginator.page()
self.assertEqual(page.next_page_pk(), page[-1].pk)
def paginate(request, query_set, lookup_field, per_page=15, page_var='value'):
# remove
value = None
page_pk = request.GET.get(page_var, None)
# It's not the first page
if page_pk is not None:
obj = get_object_or_404(query_set.model, pk=page_pk)
value = getattr(obj, lookup_field)
paginator = SeekPaginator(query_set, per_page=per_page, lookup_field=lookup_field)
try:
page = paginator.page(value=value, pk=page_pk)
except EmptyPage:
raise Http404()
return page
def paginate(request, query_set, lookup_field, per_page=15, page_var='value'):
# TODO: remove
value = None
page_pk = request.GET.get(page_var, None)
# It's not the first page
if page_pk is not None:
obj = get_object_or_404(query_set.model, pk=page_pk)
value = getattr(obj, lookup_field)
paginator = SeekPaginator(query_set, per_page=per_page, lookup_field=lookup_field)
try:
page = paginator.page(value=value, pk=page_pk)
except EmptyPage:
raise Http404()
return page
def pagination_ajax(request, pk=None):
if not request.is_ajax():
return Http404()
if pk is not None:
date = get_object_or_404(Article, pk=pk).date
else:
date = None
articles = Article.objects.all()
paginator = SeekPaginator(articles, per_page=20, lookup_field="date")
try:
page = paginator.page(value=date, pk=pk)
except EmptyPage:
return Http404()
articles_list = [{"title": a.title, } for a in page]
data = {'articles': articles_list,
'has_next': page.has_next(),
'pk': page[-1].pk}
return HttpResponse(json.dumps(data), content_type="application/json")
class PaginatorTest(TestCase):
def setUp(self):
date = timezone.now()
for i in range(25):
seconds = datetime.timedelta(seconds=i)
Article.objects.create(title="%s" % i,
date=date,
date_unique=date + seconds)
self.paginator = SeekPaginator(Article.objects.all(),
per_page=10,
lookup_field="date_unique")
def test_prepare_order(self):
self.assertListEqual(self.paginator.prepare_order(),
["-date_unique", "-pk"])
self.paginator.lookup_field = "pk"
self.assertListEqual(self.paginator.prepare_order(), [
"-pk",
])
self.paginator.lookup_field = "id"
self.assertListEqual(self.paginator.prepare_order(), [
"-id",
])
def test_prepare_lookup(self):
lookup_f, lookup_e = self.paginator.prepare_lookup(value=1, pk=2)
self.assertDictEqual(lookup_f, {
"date_unique__lte": 1,
})
self.assertDictEqual(lookup_e, {"date_unique": 1, "pk__gte": 2})
self.paginator.lookup_field = "pk"
lookup_f, lookup_e = self.paginator.prepare_lookup(value=2, pk=2)
self.assertDictEqual(lookup_f, {
"pk__lt": 2,
})
self.assertIsNone(lookup_e)
self.paginator.lookup_field = "id"
lookup_f, lookup_e = self.paginator.prepare_lookup(value=2, pk=2)
self.assertDictEqual(lookup_f, {
"id__lt": 2,
})
self.assertIsNone(lookup_e)
def test_paginator(self):
articles = Article.objects.all().order_by("-date_unique")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].date_unique,
pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = self.paginator.page(value=page_2[-1].date_unique,
pk=page_2[-1].pk)
self.assertListEqual(list(page_3), list(articles[20:]))
def test_first_page(self):
page = self.paginator.page()
self.assertTrue(page.has_next())
def test_last_page(self):
articles = Article.objects.all().order_by("-date_unique")
next_to_last = list(articles)[-2]
page = self.paginator.page(value=next_to_last.date_unique,
pk=next_to_last.pk)
self.assertFalse(page.has_next())
def test_lookup_not_unique(self):
self.paginator.lookup_field = "date"
articles = Article.objects.all().order_by("-date", "-pk")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].date, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
def test_lookup_pk(self):
self.paginator.lookup_field = "pk"
articles = Article.objects.all().order_by("-pk")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].pk, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
def test_lookup_id(self):
self.paginator.lookup_field = "id"
articles = Article.objects.all().order_by("-id")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].id, pk=page_1[-1].id)
self.assertListEqual(list(page_2), list(articles[10:20]))
def test_reverse_date_for_pk(self):
"""
When the date increment does not match the pk increment,
we should still get the right results.
"""
Article.objects.all().delete()
# asc order date and desc order pk
date = timezone.now()
dates = reversed([
date + datetime.timedelta(seconds=seconds) for seconds in range(25)
])
for i, d in enumerate(dates):
Article.objects.create(title="%s" % i, date=date, date_unique=d)
articles = Article.objects.all().order_by("-date_unique")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].date_unique,
pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = self.paginator.page(value=page_2[-1].date_unique,
pk=page_2[-1].pk)
self.assertListEqual(list(page_3), list(articles[20:]))
article1 = list(articles[800_000:800_010])[0]
#article1 = list(articles[4_000_000:4_000_010])[0]
end = timer()
print("Offset/Limit", end - start)
start = timer()
paginator = SeekPaginator(Article.objects.all(),
per_page=10,
lookup_field=(
'-is_pinned',
'-is_sticky',
'-date',
))
page = paginator.page(value=(
article1.is_pinned,
article1.is_sticky,
article1.date,
),
pk=article1.pk)
assert list(page)
end = timer()
print("Seek Method", end - start)
def start():
django_setup()
print('Populating DB')
populate_db()
print('Running bench')
bench()
class PaginatorTest(TestCase):
def setUp(self):
date = timezone.now()
for i in range(25):
seconds = datetime.timedelta(seconds=i)
Article.objects.create(title="%s" % i, date=date, date_unique=date + seconds)
self.paginator = SeekPaginator(Article.objects.all(), per_page=10, lookup_field="date_unique")
def test_prepare_order(self):
self.assertListEqual(self.paginator.prepare_order(), ["-date_unique", "-pk"])
self.paginator.lookup_field = "pk"
self.assertListEqual(self.paginator.prepare_order(), ["-pk", ])
self.paginator.lookup_field = "id"
self.assertListEqual(self.paginator.prepare_order(), ["-id", ])
def test_prepare_lookup(self):
lookup_f, lookup_e = self.paginator.prepare_lookup(value=1, pk=2)
self.assertDictEqual(lookup_f, {"date_unique__lte": 1, })
self.assertDictEqual(lookup_e, {"date_unique": 1, "pk__gte": 2})
self.paginator.lookup_field = "pk"
lookup_f, lookup_e = self.paginator.prepare_lookup(value=2, pk=2)
self.assertDictEqual(lookup_f, {"pk__lt": 2, })
self.assertIsNone(lookup_e)
self.paginator.lookup_field = "id"
lookup_f, lookup_e = self.paginator.prepare_lookup(value=2, pk=2)
self.assertDictEqual(lookup_f, {"id__lt": 2, })
self.assertIsNone(lookup_e)
def test_paginator(self):
articles = Article.objects.all().order_by("-date_unique")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].date_unique, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = self.paginator.page(value=page_2[-1].date_unique, pk=page_2[-1].pk)
self.assertListEqual(list(page_3), list(articles[20:]))
def test_first_page(self):
page = self.paginator.page()
self.assertTrue(page.has_next())
def test_last_page(self):
articles = Article.objects.all().order_by("-date_unique")
next_to_last = list(articles)[-2]
page = self.paginator.page(value=next_to_last.date_unique, pk=next_to_last.pk)
self.assertFalse(page.has_next())
def test_lookup_not_unique(self):
self.paginator.lookup_field = "date"
articles = Article.objects.all().order_by("-date", "-pk")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].date, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
def test_lookup_pk(self):
self.paginator.lookup_field = "pk"
articles = Article.objects.all().order_by("-pk")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].pk, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
def test_lookup_id(self):
self.paginator.lookup_field = "id"
articles = Article.objects.all().order_by("-id")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].id, pk=page_1[-1].id)
self.assertListEqual(list(page_2), list(articles[10:20]))
def test_reverse_date_for_pk(self):
"""
When the date increment does not match the pk increment,
we should still get the right results.
"""
Article.objects.all().delete()
# asc order date and desc order pk
date = timezone.now()
dates = reversed([date + datetime.timedelta(seconds=seconds)
for seconds in range(25)])
for i, d in enumerate(dates):
Article.objects.create(title="%s" % i, date=date, date_unique=d)
articles = Article.objects.all().order_by("-date_unique")
page_1 = self.paginator.page()
self.assertListEqual(list(page_1), list(articles[:10]))
page_2 = self.paginator.page(value=page_1[-1].date_unique, pk=page_1[-1].pk)
self.assertListEqual(list(page_2), list(articles[10:20]))
page_3 = self.paginator.page(value=page_2[-1].date_unique, pk=page_2[-1].pk)
self.assertListEqual(list(page_3), list(articles[20:]))
