def execute_queries(self, queries):
results = []
executed_queries = set()
psearch = IndexerSearch(indexer=self.indexer)
psearch.save()
total_results = 0
total_known = False
has_more = False
while len(queries) > 0:
query = queries.pop()
if query in executed_queries:
# To make sure that in case an offset is reported wrong or we have a bug we don't get stuck in an endless loop
continue
try:
request, papiaccess = self.get_url_with_papi_access(query, "search")
papiaccess.indexer_search = psearch
executed_queries.add(query)
papiaccess.save()
if request is not None:
self.check_auth(request.text)
self.logger.debug("Successfully loaded URL %s" % request.url)
try:
parsed_results = self.process_query_result(request.content, query)
results.extend(parsed_results.entries) # Retrieve the processed results
queries.extend(parsed_results.queries) # Add queries that were added as a result of the parsing, e.g. when the next result page should also be loaded
total_results += parsed_results.total
total_known = parsed_results.total_known
has_more = parsed_results.has_more
papiaccess.response_successful = True
self.handle_indexer_success()
except Exception as e:
self.logger.exception("Error while processing search results from indexer %s" % self)
raise IndexerResultParsingException("Error while parsing the results from indexer", self)
except IndexerAuthException as e:
self.logger.error("Unable to authorize with %s: %s" % (e.search_module, e.message))
papiaccess.error = "Authorization error :%s" % e.message
self.handle_indexer_failure(reason="Authentication failed", disable_permanently=True)
papiaccess.response_successful = False
except IndexerAccessException as e:
self.logger.exception("Unable to access %s: %s" % (e.search_module, e.message))
papiaccess.error = "Access error: %s" % e.message
self.handle_indexer_failure(reason="Access failed")
papiaccess.response_successful = False
except IndexerResultParsingException as e:
papiaccess.exception = "Access error: %s" % e.message
self.handle_indexer_failure(reason="Parsing results failed")
papiaccess.response_successful = False
except Exception as e:
self.logger.exception("An error error occurred while searching: %s", e)
papiaccess.error = "Unknown error :%s" % e
papiaccess.response_successful = False
finally:
papiaccess.save()
psearch.results = total_results
psearch.successful = papiaccess.response_successful
psearch.save()
return QueriesExecutionResult(results=results, dbentry=psearch, total=total_results, loaded_results=len(results), total_known=total_known, has_more=has_more)
