def _seek_internal(self, target_index): if (self._raw_data_iter is not None and self._raw_data_iter.get_index() == target_index): return target_index, self._raw_data_iter.get_item() assert len(self._raw_data_reps) == len(self._raw_data_start_index) idx = bisect.bisect_left(self._raw_data_start_index, target_index) if (len(self._raw_data_reps) > 0 and (len(self._raw_data_reps) == idx or self._raw_data_reps[idx].index.start_index > target_index)): idx -= 1 if idx < 0: raise IndexError( "target_index {} is too samll".format(target_index)) if idx < len(self._raw_data_reps): if self._raw_data_iter is None: self._raw_data_iter = create_raw_data_iter() self._raw_data_iter.reset_iter(self._raw_data_reps[idx]) self._raw_data_iter.seek_to_target(target_index) assert self._raw_data_iter.get_index() <= target_index if self._raw_data_iter.get_index() == target_index: item = self._raw_data_iter.get_item() return target_index, item idx += 1 if idx != len(self._raw_data_reps): raise IndexError('raw data index is not consecutive') next_rep = self._raw_data_manager.get_raw_data_rep_by_index(idx) if next_rep is None: self._finished = True raise StopIteration("all raw data finished") if next_rep.HasField('unindexed'): start_index = 0 if self._raw_data_iter is not None: start_index = self._raw_data_iter.get_index() + 1 self._index_raw_data_rep(idx, start_index) return self._seek_internal(target_index)
def _next_internal(self): if not self._finished: try: item = None if self._fiter is None: self._fiter = create_raw_data_iter(self._reader_options) meta = visitor.IndexMeta(0, 0, self._fpath) self._fiter.reset_iter(meta, True) item = self._fiter.get_item() else: _, item = next(self._fiter) assert item is not None return SortRunReader.MergeItem(item, self._reader_index, self._comparator) except StopIteration: self._finished = True raise StopIteration("%s has been iter finished" % self._fpath)
def _new_iter(self): return create_raw_data_iter(self._raw_data_options)