def __iter__(self): max_value = self.max_value if self.min_value is not None: cur_value = self.min_value elif not self.sorted: cur_value = 0 else: cur_value = None num = 0 limit = self.limit queryset = self.queryset if max_value: queryset = queryset.filter(**{'%s__lte' % self.order_by: max_value}) # Adjust the sort order if we're stepping through reverse if self.sorted: if self.desc: queryset = queryset.order_by('-%s' % self.order_by) else: queryset = queryset.order_by(self.order_by) # we implement basic cursor pagination for columns that are not unique last_value = None offset = 0 has_results = True while ((max_value and cur_value <= max_value) or has_results) and (not self.limit or num < self.limit): start = num if cur_value is None: results = queryset elif self.desc: results = queryset.filter(**{'%s__lte' % self.order_by: cur_value}) elif not self.desc: results = queryset.filter(**{'%s__gte' % self.order_by: cur_value}) results = results[offset:offset + self.step].iterator() # hash maps to pull in select_related columns if self.select_related: # we have to pull them all into memory to do the select_related results = list(results) for fkey in self.select_related: if '__' in fkey: fkey, related = fkey.split('__') else: related = [] attach_foreignkey(results, getattr(self.queryset.model, fkey, related)) if self.callbacks: results = list(results) for callback in self.callbacks: callback(results) for result in results: yield result num += 1 cur_value = getattr(result, self.order_by) if cur_value == last_value: offset += 1 else: # offset needs to be based at 1 so we dont return a row # that was already selected last_value = cur_value offset = 1 if (max_value and cur_value >= max_value) or (limit and num >= limit): break if cur_value is None: break has_results = num > start
def __iter__(self): max_id = self.max_id if self.min_id is not None: at = self.min_id elif not self.sorted: at = 0 else: at = None num = 0 limit = self.limit or max_id queryset = self.queryset # Adjust the sort order if we're stepping through reverse if self.sorted: if self.desc: queryset = queryset.order_by('-id') else: queryset = queryset.order_by('id') if self.max_id: queryset = queryset.filter(id__lte=max_id) has_results = True while ((max_id and at <= max_id) or has_results) and (not self.limit or num < self.limit): start = num if at is None: results = queryset elif self.desc: results = queryset.filter(id__lte=at) elif not self.desc: results = queryset.filter(id__gte=at) results = results[:self.step].iterator() # We treat select_related as a special clause if it's passed, and do a hash-join # at the application level for each chunk of results if self.select_related: results = list(results) for fkey in self.select_related: # TODO: We only handle one level of nesting if '__' in fkey: fkey, related = fkey.split('__') else: related = [] attach_foreignkey(results, getattr(self.queryset.model, fkey, related)) # Callbacks operate on a buffered chunk if self.callbacks: results = list(results) for callback in self.callbacks: callback(results) for result in results: yield result num += 1 at = result.id if (max_id and result.id >= max_id) or (limit and num >= limit): break if at is None: break has_results = num > start if self.desc: at -= 1 else: at += 1