def _execute_multi_search(self, page, num_results): indices = self.samples_by_family_index.keys() if not self.previous_search_results.get('loaded_variant_counts'): self.previous_search_results['loaded_variant_counts'] = {} ms = MultiSearch() for index_name in indices: start_index = 0 if self.previous_search_results['loaded_variant_counts'].get(index_name): index_total = self.previous_search_results['loaded_variant_counts'][index_name]['total'] start_index = self.previous_search_results['loaded_variant_counts'][index_name]['loaded'] if start_index >= index_total: continue else: self.previous_search_results['loaded_variant_counts'][index_name] = {'loaded': 0, 'total': 0} searches = self._get_paginated_searches(index_name, page, num_results, start_index=start_index) ms = ms.index(index_name) for search in searches: ms = ms.add(search) responses = self._execute_search(ms) new_results = [] compound_het_results = self.previous_search_results.get('compound_het_results', []) for response in responses: response_hits, response_total, is_compound_het = self._parse_response(response) if not response_total: continue index_name = response.hits[0].meta.index if is_compound_het: compound_het_results += response_hits self.previous_search_results['loaded_variant_counts']['{}_compound_het'.format(index_name)] = {'total': response_total} else: new_results += response_hits self.previous_search_results['loaded_variant_counts'][index_name]['total'] = response_total self.previous_search_results['loaded_variant_counts'][index_name]['loaded'] += len(response_hits) self.previous_search_results['total_results'] = sum(counts['total'] for counts in self.previous_search_results['loaded_variant_counts'].values()) # combine new results with unsorted previously loaded results to correctly sort/paginate all_loaded_results = self.previous_search_results.get('all_results', []) previous_page_record_count = (page - 1) * num_results if len(all_loaded_results) >= previous_page_record_count: loaded_results = all_loaded_results[:previous_page_record_count] new_results += all_loaded_results[previous_page_record_count:] else: loaded_results = [] new_results += self.previous_search_results.get('variant_results', []) new_results = sorted(new_results, key=lambda variant: variant['_sort']) variant_results = self._deduplicate_results(new_results) if compound_het_results or self.previous_search_results.get('grouped_results'): if compound_het_results: compound_het_results = self._deduplicate_compound_het_results(compound_het_results) return self._process_compound_hets(compound_het_results, variant_results, num_results) else: self.previous_search_results['all_results'] = loaded_results + variant_results return variant_results[:num_results]