def idx_to_cdx(self, idx_iter, query): blocks = None ranges = [] for idx in idx_iter: idx = IDXObject(idx) if (blocks and blocks.part == idx['part'] and blocks.offset + blocks.length == idx['offset'] and blocks.count < self.max_blocks): blocks.length += idx['length'] blocks.count += 1 ranges.append(idx['length']) else: if blocks: yield self.block_to_cdx_iter(blocks, ranges, query) blocks = ZipBlocks(idx['part'], idx['offset'], idx['length'], 1) ranges = [blocks.length] if blocks: yield self.block_to_cdx_iter(blocks, ranges, query)
def gen_idx(): for idx in idx_iter: yield IDXObject(idx)
def compute_page_range(self, reader, query): pagesize = query.page_size if not pagesize: pagesize = self.max_blocks else: pagesize = int(pagesize) last_line = None # Get End end_iter = search(reader, query.end_key, prev_size=1) try: end_line = six.next(end_iter) except StopIteration: last_line = read_last_line(reader) end_line = last_line # Get Start first_iter = iter_range(reader, query.key, query.end_key, prev_size=1) try: first_line = six.next(first_iter) except StopIteration: if end_line == last_line and query.key >= last_line: first_line = last_line else: reader.close() if query.page_count: yield self._page_info(0, pagesize, 0) return first = IDXObject(first_line) end = IDXObject(end_line) try: blocks = end['lineno'] - first['lineno'] total_pages = int(blocks / pagesize) + 1 except: blocks = -1 total_pages = 1 if query.page_count: # same line, so actually need to look at cdx # to determine if it exists if blocks == 0: try: block_cdx_iter = self.idx_to_cdx([first_line], query) block = six.next(block_cdx_iter) cdx = six.next(block) except StopIteration: total_pages = 0 blocks = -1 yield self._page_info(total_pages, pagesize, blocks + 1) reader.close() return curr_page = query.page if curr_page >= total_pages or curr_page < 0: msg = 'Page {0} invalid: First Page is 0, Last Page is {1}' reader.close() raise CDXException(msg.format(curr_page, total_pages - 1)) startline = curr_page * pagesize endline = startline + pagesize - 1 if blocks >= 0: endline = min(endline, blocks) if curr_page == 0: yield first_line else: startline -= 1 idxiter = itertools.islice(first_iter, startline, endline) for idx in idxiter: yield idx reader.close()
def test_invalid_idx_format(): with raises(CDXException): x = IDXObject(b'a b c')