def test_start_with(self): last_pk = 333 for item in iterator(self.qs, 9, start_with=last_pk, limit=100): self.assertEqual(item.pk - last_pk, 1) last_pk = item.pk last_pk = 333 for item in iterator(self.qs, 9, '-pk', last_pk, limit=100): self.assertEqual(item.pk - last_pk, -1) last_pk = item.pk
def test_values(self): qs = self.qs.values() last_pk = 0 for item in iterator(qs, 9, 'id', limit=100): self.assertEqual(item['id'] - last_pk, 1) last_pk = item['id'] qs = self.qs.values('pk', 'created') last_pk = 0 for item in iterator(qs, 9, limit=100): self.assertEqual(item['pk'] - last_pk, 1) last_pk = item['pk'] qs = self.qs.values() msg = "`pk` field should be in returned objects" with self.assertRaisesRegex(ValueError, msg): for item in iterator(qs, 9, limit=100): pass
def test_named_values_list(self): qs = self.qs.values_list('pk', 'created', named=True) last_pk = 0 for item in iterator(qs, 9, limit=100): self.assertEqual(item.pk - last_pk, 1) last_pk = item.pk qs = self.qs.values_list('pk', 'created', named=True) last_created = datetime.max for item in iterator(qs, 9, '-created', limit=100): self.assertLess(item.created, last_created) last_created = item.created qs = self.qs.values_list('id', 'created', named=True) msg = "`pk` field should be in returned objects" with self.assertRaisesRegex(ValueError, msg): for item in iterator(qs, 9, limit=100): pass
def test_order_by(self): # default is pk last_pk = 0 for item in iterator(self.qs, 9, limit=100): self.assertEqual(item.pk - last_pk, 1) last_pk = item.pk last_pk = 1e100 for item in iterator(self.qs, 9, order_by='-pk', limit=100): self.assertLess(item.pk, last_pk) last_pk = item.pk last_created = datetime.min for item in iterator(self.qs, 9, 'created', limit=100): self.assertGreater(item.created, last_created) last_created = item.created last_created = datetime.max for item in iterator(self.qs, 9, '-created', limit=100): self.assertLess(item.created, last_created) last_created = item.created
def test_first_start_time(self): start = time.time() for item in self.qs.all(): break default_time = time.time() - start print('>>> Default', default_time) start = time.time() for item in self.qs.iterator(): break iterator_time = time.time() - start print('>>> Iterator', iterator_time) start = time.time() for item in iterator(self.qs, 100): break chunked_iterator_time = time.time() - start print('>>> Chunked iterator', chunked_iterator_time) self.assertLess(chunked_iterator_time * 10, iterator_time) self.assertLess(chunked_iterator_time * 20, default_time)
def test_total_time(self): start = time.time() for item in self.qs.all(): pass default_time = time.time() - start print('>>> Default', default_time) start = time.time() for item in self.qs.iterator(): pass iterator_time = time.time() - start print('>>> Iterator', iterator_time) start = time.time() for item in iterator(self.qs, 200): pass chunked_iterator_time = time.time() - start print('>>> Chunked iterator', chunked_iterator_time) self.assertLess(chunked_iterator_time, iterator_time * 1.6) self.assertLess(chunked_iterator_time, default_time * 1.6)
def get_courses(user, org=None, filter_=None, batch_size=100): """ Return a LazySequence of courses available, optionally filtered by org code (case-insensitive). """ courses_qset = branding.get_visible_courses( org=org, filter_=filter_, ).select_related('image_set').prefetch_related( Prefetch( 'modes', queryset=CourseMode.objects.exclude( mode_slug__in=CourseMode.CREDIT_MODES), to_attr='selectable_modes', ), ) permission_name = configuration_helpers.get_value( 'COURSE_CATALOG_VISIBILITY_PERMISSION', settings.COURSE_CATALOG_VISIBILITY_PERMISSION) if user.is_authenticated: prefetch_related_objects([user], 'roles', 'experimentdata_set') def with_prefetched_users(): for courses in batch_iterator(courses_qset, batch_size=batch_size): prefetch_related_objects( [user], Prefetch( 'courseenrollment_set', queryset=CourseEnrollment.objects.filter( user=user, course__in=courses).select_related('schedule'))) for course in courses: if has_access(user, permission_name, course): yield course course_iterator = with_prefetched_users() else: course_iterator = (c for c in iterator(courses_qset) if has_access(user, permission_name, c)) return LazySequence(course_iterator, est_len=courses_qset.count())
def test_manager(self): for item in iterator(Item.objects, limit=10): self.assertIsInstance(item, Item)
def test_slice(self): with self.assertRaises(AssertionError): for item in iterator(self.qs[:10000]): break