def _request(self, query):
"""
Issues a single request to the API_ENDPOINT and returns the result as
an ifind Response.
Args:
query (ifind Query): object encapsulating details of a search query.
Returns:
ifind Response: object encapsulating a search request's results.
Raises:
EngineException
Usage:
Private method.
"""
query_string = self._create_query_string(query)
try:
response = requests.get(query_string)
except requests.exceptions.ConnectionError:
raise EngineConnectionException(self.name, "Unable to send request, check connectivity")
if response.status_code != 200:
raise EngineConnectionException(self.name, "", code=response.status_code)
return Twitter._parse_json_response(query, response)
def _get_person_info(self, user_id):
"""
Could be in its own class, or result type for later handling.
Gets person specific information by making an additional API call.
"""
query_string = "{}people/{}?key={}".format(API_ENDPOINT, user_id,
self.api_key)
try:
personResponse = requests.get(query_string)
except requests.exceptions.ConnectionError:
raise EngineConnectionException(
self.name, "Unable to send request, check connectivity.")
if personResponse.status_code != 200:
raise EngineConnectionException(self.name,
"",
code=personResponse.status_code)
return personResponse
def __init__(self, whoosh_index_dir='', stopwords_file='', cache_host='localhost', cache_port=6379, **kwargs):
Engine.__init__(self, **kwargs)
self.whoosh_index_dir = whoosh_index_dir
if not self.whoosh_index_dir:
raise EngineConnectionException(self.name, "'whoosh_index_dir=' keyword argument not specified")
self.stopwords_file = stopwords_file
if self.stopwords_file:
self.stopwords = ListReader(self.stopwords_file) # Open the stopwords file, read into a ListReader
else:
raise EngineConnectionException(self.name, "'stopwords_file=' keyword argument not specified")
self.scoring_model_identifier = 1
self.scoring_model = scoring.PL2(c=10.0)
self.__verbose = False
try:
self.doc_index = open_dir(self.whoosh_index_dir)
self.reader = self.doc_index.reader()
self.parser = QueryParser('content', self.doc_index.schema) # By default, we use AND grouping.
# Use the grouping parameter and specify whoosh.qparser.OrGroup, etc...
# Objects required for document snippet generation
self.analyzer = self.doc_index.schema[self.parser.fieldname].analyzer
self.fragmenter = ContextFragmenter(maxchars=200, surround=40)
self.formatter = HtmlFormatter()
except EmptyIndexError:
message = "Could not open Whoosh index at '{0}'".format(self.whoosh_index_dir)
raise EngineConnectionException(self.name, message)
except OSError:
message = "Could not open Whoosh index at '{0}' - directory does not exist".format(self.whoosh_index_dir)
raise EngineConnectionException(self.name, message)
# Attempt to connect to the specified Redis cache.
self.cache = RedisConn(host=cache_host, port=cache_port)
self.cache.connect()
def _request(self, query):
"""
Issues a single request to the API_ENDPOINT and returns the result as
an ifind Response.
Args:
query (ifind Query): object encapsulating details of a search query.
Returns:
ifind Response: object encapsulating a search request's results.
Raises:
EngineException
Usage:
Private method.
"""
search_params = {
'format': 'xml',
'search': query.terms,
'action': 'opensearch',
'limit': query.top
}
try:
response = requests.get(API_ENDPOINT, params=search_params)
except requests.exceptions.ConnectionError:
raise EngineConnectionException(
self.name, "Unable to send request, check connectivity")
if response.status_code != 200:
raise EngineConnectionException(self.name,
"",
code=response.status_code)
return Wikipedia._parse_xml_response(query, response)
def __init__(self,
whoosh_index_dir='',
use_cache=True,
cache_host='localhost',
cache_port=6379,
**kwargs):
"""
Constructor for the engine.
"""
Engine.__init__(self, **kwargs)
self.whoosh_index_dir = whoosh_index_dir
if not self.whoosh_index_dir:
raise EngineConnectionException(
self.name,
"'whoosh_index_dir=' keyword argument not specified")
# Only put PL2 in for now (for more, add the model parameter to the constructor to specify!)
self.scoring_model_identifier = 1
self.scoring_model = scoring.PL2(c=10.0)
try:
self.doc_index = open_dir(self.whoosh_index_dir)
self.reader = self.doc_index.reader()
self.parser = QueryParser(
'content',
self.doc_index.schema) # By default, we use AND grouping.
# Use the grouping parameter and specify whoosh.qparser.OrGroup, etc...
# Objects required for document snippet generation
self.analyzer = self.doc_index.schema[
self.parser.fieldname].analyzer
self.fragmenter = ContextFragmenter(maxchars=200, surround=40)
self.formatter = HtmlFormatter()
except EmptyIndexError:
message = "Could not open Whoosh index at '{0}'".format(
self.whoosh_index_dir)
raise EngineConnectionException(self.name, message)
except OSError:
message = "Could not open Whoosh index at '{0}' - directory does not exist".format(
self.whoosh_index_dir)
raise EngineConnectionException(self.name, message)
self.use_cache = use_cache
if self.use_cache:
self.cache = RedisConn(host=cache_host, port=cache_port)
self.cache.connect()
self.page_cache_forward_look = 40 # How many additional pages to cache when required.
self.page_cache_when = 4 # When the user is x pages away from the end of the page cache, cache more pages.
self.page_cache_controller = PageCacheController(
cache_host=self.cache.host,
cache_port=self.cache.port,
whoosh_index=self.doc_index,
scoring_model_identifier=self.scoring_model_identifier,
parser=self.parser,
analyzer=self.analyzer,
fragmenter=self.fragmenter,
formatter=self.formatter,
cache_forward_look=self.page_cache_forward_look)