def search_by_line_num(self, reader, line): # pragma: no cover def line_cmp(line1, line2): line1_no = int(line1.rsplit('\t', 1)[-1]) line2_no = int(line2.rsplit('\t', 1)[-1]) return cmp(line1_no, line2_no) line_iter = search(reader, line, compare_func=line_cmp) yield line_iter.next()
def search_by_line_num(self, reader, line): # pragma: no cover def line_cmp(line1, line2): line1_no = int(line1.rsplit(b'\t', 1)[-1]) line2_no = int(line2.rsplit(b'\t', 1)[-1]) return cmp(line1_no, line2_no) line_iter = search(reader, line, compare_func=line_cmp) yield six.next(line_iter)
def compute_page_range(self, reader, query): # Get End end_iter = search(reader, query.end_key, prev_size=1) try: end_line = end_iter.next() except StopIteration: end_line = read_last_line(reader) # Get Start first_iter = iter_range(reader, query.key, query.end_key, prev_size=1) try: first_line = first_iter.next() except StopIteration: reader.close() raise first = IDXObject(first_line) end = IDXObject(end_line) diff = end['lineno'] - first['lineno'] pagesize = query.page_size if not pagesize: pagesize = self.max_blocks total_pages = diff / pagesize + 1 if query.page_count: info = dict(pages=total_pages, pageSize=pagesize, blocks=diff + 1) yield json.dumps(info) reader.close() return curr_page = query.page if curr_page >= total_pages or curr_page < 0: msg = 'Page {0} invalid: First Page is 0, Last Page is {1}' reader.close() raise CDXException(msg.format(curr_page, total_pages - 1)) startline = curr_page * pagesize endline = min(startline + pagesize - 1, diff) if curr_page == 0: yield first_line else: startline -= 1 idxiter = itertools.islice(first_iter, startline, endline) for idx in idxiter: yield idx reader.close()
def _do_iter(self, fh, params): """Iterates over the supplied file handle to an access control list yielding the results of the search for the params key :param TextIO fh: The file handle to an access control list :param dict params: The params of the :return: A generator yielding the results of the param search """ exact_suffix = params.get('exact_match_suffix') key = params['key'] if exact_suffix: key += exact_suffix for line in search(fh, key, prev_size=1, compare_func=self.rev_cmp): yield line
def compute_page_range(self, reader, query): pagesize = query.page_size if not pagesize: pagesize = self.max_blocks else: pagesize = int(pagesize) last_line = None # Get End end_iter = search(reader, query.end_key, prev_size=1) try: end_line = six.next(end_iter) except StopIteration: last_line = read_last_line(reader) end_line = last_line # Get Start first_iter = iter_range(reader, query.key, query.end_key, prev_size=1) try: first_line = six.next(first_iter) except StopIteration: if end_line == last_line and query.key >= last_line: first_line = last_line else: reader.close() if query.page_count: yield self._page_info(0, pagesize, 0) return first = IDXObject(first_line) end = IDXObject(end_line) try: blocks = end['lineno'] - first['lineno'] total_pages = int(blocks / pagesize) + 1 except: blocks = -1 total_pages = 1 if query.page_count: # same line, so actually need to look at cdx # to determine if it exists if blocks == 0: try: block_cdx_iter = self.idx_to_cdx([first_line], query) block = six.next(block_cdx_iter) cdx = six.next(block) except StopIteration: total_pages = 0 blocks = -1 yield self._page_info(total_pages, pagesize, blocks + 1) reader.close() return curr_page = query.page if curr_page >= total_pages or curr_page < 0: msg = 'Page {0} invalid: First Page is 0, Last Page is {1}' reader.close() raise CDXException(msg.format(curr_page, total_pages - 1)) startline = curr_page * pagesize endline = startline + pagesize - 1 if blocks >= 0: endline = min(endline, blocks) if curr_page == 0: yield first_line else: startline -= 1 idxiter = itertools.islice(first_iter, startline, endline) for idx in idxiter: yield idx reader.close()
def compute_page_range(self, reader, query): pagesize = query.page_size if not pagesize: pagesize = self.max_blocks else: pagesize = int(pagesize) last_line = None # Get End end_iter = search(reader, query.end_key, prev_size=1) try: end_line = end_iter.next() except StopIteration: last_line = read_last_line(reader) end_line = last_line # Get Start first_iter = iter_range(reader, query.key, query.end_key, prev_size=1) try: first_line = first_iter.next() except StopIteration: if end_line == last_line and query.key >= last_line: first_line = last_line else: reader.close() if query.page_count: yield self._page_info(0, pagesize, 0) return else: raise first = IDXObject(first_line) end = IDXObject(end_line) try: blocks = end['lineno'] - first['lineno'] total_pages = blocks / pagesize + 1 except: blocks = -1 total_pages = 1 if query.page_count: # same line, so actually need to look at cdx # to determine if it exists if blocks == 0: try: block_cdx_iter = self.idx_to_cdx([first_line], query) block = block_cdx_iter.next() cdx = block.next() except StopIteration: total_pages = 0 blocks = -1 yield self._page_info(total_pages, pagesize, blocks + 1) reader.close() return curr_page = query.page if curr_page >= total_pages or curr_page < 0: msg = 'Page {0} invalid: First Page is 0, Last Page is {1}' reader.close() raise CDXException(msg.format(curr_page, total_pages - 1)) startline = curr_page * pagesize endline = startline + pagesize - 1 if blocks >= 0: endline = min(endline, blocks) if curr_page == 0: yield first_line else: startline -= 1 idxiter = itertools.islice(first_iter, startline, endline) for idx in idxiter: yield idx reader.close()