def test_start_with(self):
last_pk = 333
for item in iterator(self.qs, 9, start_with=last_pk, limit=100):
self.assertEqual(item.pk - last_pk, 1)
last_pk = item.pk
last_pk = 333
for item in iterator(self.qs, 9, '-pk', last_pk, limit=100):
self.assertEqual(item.pk - last_pk, -1)
last_pk = item.pk
def test_values(self):
qs = self.qs.values()
last_pk = 0
for item in iterator(qs, 9, 'id', limit=100):
self.assertEqual(item['id'] - last_pk, 1)
last_pk = item['id']
qs = self.qs.values('pk', 'created')
last_pk = 0
for item in iterator(qs, 9, limit=100):
self.assertEqual(item['pk'] - last_pk, 1)
last_pk = item['pk']
qs = self.qs.values()
msg = "`pk` field should be in returned objects"
with self.assertRaisesRegex(ValueError, msg):
for item in iterator(qs, 9, limit=100):
pass
def test_named_values_list(self):
qs = self.qs.values_list('pk', 'created', named=True)
last_pk = 0
for item in iterator(qs, 9, limit=100):
self.assertEqual(item.pk - last_pk, 1)
last_pk = item.pk
qs = self.qs.values_list('pk', 'created', named=True)
last_created = datetime.max
for item in iterator(qs, 9, '-created', limit=100):
self.assertLess(item.created, last_created)
last_created = item.created
qs = self.qs.values_list('id', 'created', named=True)
msg = "`pk` field should be in returned objects"
with self.assertRaisesRegex(ValueError, msg):
for item in iterator(qs, 9, limit=100):
pass
def test_order_by(self):
# default is pk
last_pk = 0
for item in iterator(self.qs, 9, limit=100):
self.assertEqual(item.pk - last_pk, 1)
last_pk = item.pk
last_pk = 1e100
for item in iterator(self.qs, 9, order_by='-pk', limit=100):
self.assertLess(item.pk, last_pk)
last_pk = item.pk
last_created = datetime.min
for item in iterator(self.qs, 9, 'created', limit=100):
self.assertGreater(item.created, last_created)
last_created = item.created
last_created = datetime.max
for item in iterator(self.qs, 9, '-created', limit=100):
self.assertLess(item.created, last_created)
last_created = item.created
def test_first_start_time(self):
start = time.time()
for item in self.qs.all():
break
default_time = time.time() - start
print('>>> Default', default_time)
start = time.time()
for item in self.qs.iterator():
break
iterator_time = time.time() - start
print('>>> Iterator', iterator_time)
start = time.time()
for item in iterator(self.qs, 100):
break
chunked_iterator_time = time.time() - start
print('>>> Chunked iterator', chunked_iterator_time)
self.assertLess(chunked_iterator_time * 10, iterator_time)
self.assertLess(chunked_iterator_time * 20, default_time)
def test_total_time(self):
start = time.time()
for item in self.qs.all():
pass
default_time = time.time() - start
print('>>> Default', default_time)
start = time.time()
for item in self.qs.iterator():
pass
iterator_time = time.time() - start
print('>>> Iterator', iterator_time)
start = time.time()
for item in iterator(self.qs, 200):
pass
chunked_iterator_time = time.time() - start
print('>>> Chunked iterator', chunked_iterator_time)
self.assertLess(chunked_iterator_time, iterator_time * 1.6)
self.assertLess(chunked_iterator_time, default_time * 1.6)
def get_courses(user, org=None, filter_=None, batch_size=100):
"""
Return a LazySequence of courses available, optionally filtered by org code (case-insensitive).
"""
courses_qset = branding.get_visible_courses(
org=org,
filter_=filter_,
).select_related('image_set').prefetch_related(
Prefetch(
'modes',
queryset=CourseMode.objects.exclude(
mode_slug__in=CourseMode.CREDIT_MODES),
to_attr='selectable_modes',
), )
permission_name = configuration_helpers.get_value(
'COURSE_CATALOG_VISIBILITY_PERMISSION',
settings.COURSE_CATALOG_VISIBILITY_PERMISSION)
if user.is_authenticated:
prefetch_related_objects([user], 'roles', 'experimentdata_set')
def with_prefetched_users():
for courses in batch_iterator(courses_qset, batch_size=batch_size):
prefetch_related_objects(
[user],
Prefetch(
'courseenrollment_set',
queryset=CourseEnrollment.objects.filter(
user=user,
course__in=courses).select_related('schedule')))
for course in courses:
if has_access(user, permission_name, course):
yield course
course_iterator = with_prefetched_users()
else:
course_iterator = (c for c in iterator(courses_qset)
if has_access(user, permission_name, c))
return LazySequence(course_iterator, est_len=courses_qset.count())
def test_manager(self):
for item in iterator(Item.objects, limit=10):
self.assertIsInstance(item, Item)
def test_slice(self):
with self.assertRaises(AssertionError):
for item in iterator(self.qs[:10000]):
break