def mergeResults(results, has_sort_keys, reverse): """Sort/merge sub-results, generating a flat sequence. results is a list of result set sequences, all with or without sort keys """ if not has_sort_keys: return LazyCat(results) else: # Concatenate the catalog results into one list and sort it # Each result record consists of a list of tuples with three values: # (sortkey, docid, catalog__getitem__) combined = [] if len(results) > 1: for r in results: combined.extend(r) elif len(results) == 1: combined = results[0] else: return [] if reverse: combined.sort(reverse=True) else: combined.sort() return LazyMap(lambda rec: rec[2](rec[1]), combined, len(combined))
def sortResults(self, rs, sort_index, reverse=False, limit=None, merge=True, actual_result_count=None, b_start=0, b_size=None): # Sort a result set using one or more sort indexes. Both sort_index # and reverse can be lists of indexes and reverse specifications. # Return a lazy result set in sorted order if merge is true otherwise # returns a list of (sortkey, uid, getter_function) tuples, where # sortkey can be a tuple on its own. second_indexes = None second_indexes_key_map = None sort_index_length = 1 if isinstance(sort_index, list): sort_index_length = len(sort_index) if sort_index_length > 1: second_indexes = sort_index[1:] second_indexes_key_map = [] for si in second_indexes: second_indexes_key_map.append(si.documentToKeyMap()) sort_index = sort_index[0] result = [] if hasattr(rs, 'keys'): rs = rs.keys() if actual_result_count is None: rlen = len(rs) actual_result_count = rlen else: rlen = actual_result_count # don't limit to more than what we have if limit is not None and limit >= rlen: limit = rlen # if we want a batch from the end of the result set, reverse sorting # order and limit it, then reverse the result set again switched_reverse = False if b_size and b_start and b_start > rlen / 2: if isinstance(reverse, list): reverse = [not r for r in reverse] else: reverse = not reverse switched_reverse = True b_end = b_start + b_size if b_end >= rlen: overrun = rlen - b_end if b_start >= rlen: # bail out, we are outside the possible range return LazyCat([], 0, actual_result_count) else: b_size += overrun b_start = 0 else: b_start = rlen - b_end limit = b_start + b_size # determine sort_spec if isinstance(reverse, list): sort_spec = [r and -1 or 1 for r in reverse] # limit to current maximum of sort indexes sort_spec = sort_spec[:sort_index_length] # use first sort order for choosing the algorithm first_reverse = reverse[0] else: sort_spec = [] for i in xrange(sort_index_length): sort_spec.append(reverse and -1 or 1) first_reverse = reverse # Special first condition, as it changes post-processing. iterate_sort_index = (merge and limit is None and (rlen > (len(sort_index) * (rlen / 100 + 1)))) # Choose one of the sort algorithms. if iterate_sort_index: sort_func = self._sort_iterate_index elif limit is None or (limit * 4 > rlen): sort_func = self._sort_iterate_resultset elif first_reverse: sort_func = self._sort_nbest else: sort_func = self._sort_nbest_reverse actual_result_count, length, result = sort_func( actual_result_count, result, rs, limit, merge, reverse, sort_index, sort_index_length, sort_spec, second_indexes_key_map) sequence, slen = self._limit_sequence(result, length, b_start, b_size, switched_reverse) if iterate_sort_index: result = LazyCat(LazyValues(sequence), slen, actual_result_count) else: if not merge: return sequence result = LazyValues(sequence) result.actual_result_count = actual_result_count return LazyMap(self.__getitem__, result, len(result), actual_result_count=actual_result_count)
def search(self, query, sort_index=None, reverse=False, limit=None, merge=True): """Iterate through the indexes, applying the query to each one. If merge is true then return a lazy result set (sorted if appropriate) otherwise return the raw (possibly scored) results for later merging. Limit is used in conjunction with sorting or scored results to inform the catalog how many results you are really interested in. The catalog can then use optimizations to save time and memory. The number of results is not guaranteed to fall within the limit however, you should still slice or batch the results as usual.""" # Indexes fulfill a fairly large contract here. We hand each # index the query mapping we are given (which may be composed # of some combination of web request, kw mappings or plain old dicts) # and the index decides what to do with it. If the index finds work # for itself in the query, it returns the results and a tuple of # the attributes that were used. If the index finds nothing for it # to do then it returns None. # Canonicalize the request into a sensible query before passing it on query = self.make_query(query) cr = self.getCatalogPlan(query) cr.start() plan = cr.plan() if not plan: plan = self._sorted_search_indexes(query) rs = None # result set for index_id in plan: # The actual core loop over all indices. if index_id not in self.indexes: # We can have bogus keys or the plan can contain index names # that have been removed in the meantime. continue rs = self._search_index(cr, index_id, query, rs) if not rs: break if not rs: # None of the indexes found anything to do with the query. result = LazyCat([]) cr.stop() return result # Try to deduce the sort limit from batching arguments. b_start, b_size, limit, sort_report_name = self._sort_limit_arguments( query, sort_index, reverse, limit) # We got some results from the indexes, sort and convert to sequences. rlen = len(rs) if sort_index is None and hasattr(rs, 'items'): # Having a 'items' means we have a data structure with # scores. Build a new result set, sort it by score, reverse # it, compute the normalized score, and Lazify it. if not merge: # Don't bother to sort here, return a list of # three tuples to be passed later to mergeResults. # Note that data_record_normalized_score_ cannot be # calculated and will always be 1 in this case. result = [(score, (1, score, rid), self.__getitem__) for rid, score in rs.items()] else: cr.start_split('sort_on#score') # Sort it by score. rs = rs.byValue(0) max = float(rs[0][0]) # Here we define our getter function inline so that # we can conveniently store the max value as a default arg # and make the normalized score computation lazy def getScoredResult(item, max=max, self=self): """ Returns instances of self._v_brains, or whatever is passed into self.useBrains. """ score, key = item norm_score = int(100.0 * score / max) return self.instantiate((key, self.data[key]), score_data=(score, norm_score)) sequence, slen = self._limit_sequence(rs, rlen, b_start, b_size) result = LazyMap(getScoredResult, sequence, slen, actual_result_count=rlen) cr.stop_split('sort_on#score', None) elif sort_index is None and not hasattr(rs, 'values'): # no scores if hasattr(rs, 'keys'): rs = rs.keys() sequence, slen = self._limit_sequence(rs, rlen, b_start, b_size) result = LazyMap(self.__getitem__, sequence, slen, actual_result_count=rlen) else: # Sort. If there are scores, then this block is not # reached, therefore 'sort-on' does not happen in the # context of a text index query. This should probably # sort by relevance first, then the 'sort-on' attribute. cr.start_split(sort_report_name) result = self.sortResults(rs, sort_index, reverse, limit, merge, actual_result_count=rlen, b_start=b_start, b_size=b_size) cr.stop_split(sort_report_name, None) cr.stop() return result
def sortResults(self, rs, sort_index, reverse=False, limit=None, merge=True, actual_result_count=None, b_start=0, b_size=None): # Sort a result set using one or more sort indexes. Both sort_index # and reverse can be lists of indexes and reverse specifications. # Return a lazy result set in sorted order if merge is true otherwise # returns a list of (sortkey, uid, getter_function) tuples, where # sortkey can be a tuple on its own. second_indexes = None second_indexes_key_map = None sort_index_length = 1 if isinstance(sort_index, list): sort_index_length = len(sort_index) if sort_index_length > 1: second_indexes = sort_index[1:] second_indexes_key_map = [] for si in second_indexes: second_indexes_key_map.append(si.documentToKeyMap()) sort_index = sort_index[0] result = [] if hasattr(rs, 'keys'): rs = rs.keys() if actual_result_count is None: rlen = len(rs) actual_result_count = rlen else: rlen = actual_result_count # don't limit to more than what we have if limit is not None and limit >= rlen: limit = rlen # if we want a batch from the end of the result set, reverse sorting # order and limit it, then reverse the result set again switched_reverse = False if b_size and b_start and b_start > rlen / 2: if isinstance(reverse, list): reverse = [not r for r in reverse] else: reverse = not reverse switched_reverse = True b_end = b_start + b_size if b_end >= rlen: overrun = rlen - b_end if b_start >= rlen: # bail out, we are outside the possible range return LazyCat([], 0, actual_result_count) else: b_size += overrun b_start = 0 else: b_start = rlen - b_end limit = b_start + b_size # determine sort_spec if isinstance(reverse, list): sort_spec = [r and -1 or 1 for r in reverse] # limit to current maximum of sort indexes sort_spec = sort_spec[:sort_index_length] # use first sort order for choosing the algorithm first_reverse = reverse[0] else: sort_spec = [] for i in xrange(sort_index_length): sort_spec.append(reverse and -1 or 1) first_reverse = reverse # Special first condition, as it changes post-processing. iterate_sort_index = ( merge and limit is None and ( rlen > (len(sort_index) * (rlen / 100 + 1)))) # Choose one of the sort algorithms. if iterate_sort_index: sort_func = self._sort_iterate_index elif limit is None or (limit * 4 > rlen): sort_func = self._sort_iterate_resultset elif first_reverse: sort_func = self._sort_nbest else: sort_func = self._sort_nbest_reverse actual_result_count, length, result = sort_func( actual_result_count, result, rs, limit, merge, reverse, sort_index, sort_index_length, sort_spec, second_indexes_key_map) sequence, slen = self._limit_sequence( result, length, b_start, b_size, switched_reverse) if iterate_sort_index: result = LazyCat(LazyValues(sequence), slen, actual_result_count) else: if not merge: return sequence result = LazyValues(sequence) result.actual_result_count = actual_result_count return LazyMap(self.__getitem__, result, len(result), actual_result_count=actual_result_count)
def _createLSeq(self, *sequences): from ZTUtils.Lazy import LazyCat return LazyCat(sequences)