def __init__(self, url=POS_URL, usr=None, pwd=None):
'''
:param url: URL of the jeremia web service
:param usr: optional user name
:param pwd: optional password
'''
RESTClient.__init__(self, url, usr, pwd)
def __init__(self, url=SENTIMENT_ANALYSIS_URL, usr=None, pwd=None):
'''
:param url: URL of the jeremia web service
:param usr: optional user name
:param pwd: optional password
'''
RESTClient.__init__(self, url, usr, pwd)
def test_get_url(self):
assert RESTClient.get_request_url(self.TEST_URL, 'execute', '12') \
.endswith("/execute/12")
assert RESTClient.get_request_url(
self.TEST_URL,
'execute',
'12',
{'debug': True}).endswith("/execute/12?debug=True")
def test_retrieve(self):
r = RESTClient(self.TEST_URL, self.TEST_USER, self.TEST_PASS)
try:
r._json_request(self.TEST_URL)
except HTTPError as e:
# authentification succeeded, but no object could
# be found
assert '404: Not Found' in str(e)
def test_retrieve(self):
r = RESTClient(self.TEST_URL, self.TEST_USER, self.TEST_PASS)
try:
r._json_request(self.TEST_URL)
except HTTPError as e:
# authentification succeeded, but no object could
# be found
assert '404: Not Found' in str(e)
def test_get_url(self):
assert RESTClient.get_request_url(self.TEST_URL, 'execute', '12') \
.endswith("/execute/12")
assert RESTClient.get_request_url(
self.TEST_URL,
'execute',
'12',
{'debug': True}).endswith("/execute/12?debug=True")
def __init__(self, api_key, engine_id, api_url=ROOT_URL):
"""fixes the credentials and initiates the RESTClient"""
assert(api_key)
self.api_key = api_key
assert(engine_id)
self.engine_id = engine_id
self.api_url = api_url
self.client = RESTClient(self.api_url, authentification_method='basic')
def __init__(self, api_key, username, api_url=ROOT_URL):
"""fixes the credentials and initiates the RESTClient"""
assert (api_key)
self.api_key = api_key
self.api_url = api_url
self.username = username
self.client = RESTClient(self.api_url,
password=self.api_key,
user=self.username,
authentification_method='basic')
def __init__(self, api_key, username, api_url=ROOT_URL):
"""fixes the credentials and initiates the RESTClient"""
assert(api_key)
self.api_key = api_key
self.api_url = api_url
self.username = username
self.client = RESTClient(
self.api_url, password=self.api_key, user=self.username,
authentification_method='basic')
class BingSearch(AbstractIterableWebSource):
"""wrapper for the Bing Search API"""
NAME = "Bing Search"
ROOT_URL = 'https://api.datamarket.azure.com/Bing/Search'
DEFAULT_MAX_RESULTS = 50 # requires only 1 api access
SUPPORTED_PARAMS = ['command', 'output_format']
DEFAULT_COMMAND = 'Web' # Image, News
DEFAULT_FORMAT = 'json'
DEFAULT_START_INDEX = 0
RESULT_PATH = lambda x: x['d']['results'] # path to the results in json
MAPPING = {'date': ('valid_from', 'convert_date'),
'text': ('content', None),
'title': 'Title',
}
def __init__(self, api_key, username, api_url=ROOT_URL):
"""fixes the credentials and initiates the RESTClient"""
assert(api_key)
self.api_key = api_key
self.api_url = api_url
self.username = username
self.client = RESTClient(
self.api_url, password=self.api_key, user=self.username,
authentification_method='basic')
def search_documents(self, search_terms, max_results=DEFAULT_MAX_RESULTS,
from_date=None, to_date=None, command=DEFAULT_COMMAND,
output_format=DEFAULT_FORMAT):
"""calls iterator and results' post-processor"""
# Web search is by default
fetched = self.invoke_iterator(search_terms, max_results, from_date,
to_date, command, output_format)
result_path = lambda x: x['d']['results']
return self.process_output(fetched, result_path)
def request(self, search_term, current_index,
max_results=DEFAULT_MAX_RESULTS, from_date=None,
to_date=None, command=DEFAULT_COMMAND,
output_format=DEFAULT_FORMAT):
"""calls Bing Search API"""
parameters = {'Query': search_term,
'$format': output_format,
'$top': max_results,
'$skip': current_index}
# for testing purposes
# print(current_index, max_results, search_term)
response = self.client.execute(command, query_parameters=parameters)
return response
@classmethod
def convert_item(cls, item):
"""output convertor: applies a mapping to convert
the result to the required format
"""
result = {'url': item['Url'],
'title': item['Title'],
}
return result
class BingSearch(AbstractIterableWebSource):
"""wrapper for the Bing Search API"""
NAME = "Bing Search"
ROOT_URL = 'https://api.datamarket.azure.com/Bing/Search'
DEFAULT_MAX_RESULTS = 50 # requires only 1 api access
SUPPORTED_PARAMS = ['command', 'output_format']
DEFAULT_COMMAND = 'Web' # Image, News
DEFAULT_FORMAT = 'json'
DEFAULT_START_INDEX = 0
RESULT_PATH = lambda x: x['d']['results'] # path to the results in json
MAPPING = {
'date': ('valid_from', 'convert_date'),
'text': ('content', None),
'title': 'Title',
}
def __init__(self, api_key, username, api_url=ROOT_URL):
"""fixes the credentials and initiates the RESTClient"""
assert (api_key)
self.api_key = api_key
self.api_url = api_url
self.username = username
self.client = RESTClient(self.api_url,
password=self.api_key,
user=self.username,
authentification_method='basic')
def search_documents(self,
search_terms,
max_results=DEFAULT_MAX_RESULTS,
from_date=None,
to_date=None,
command=DEFAULT_COMMAND,
output_format=DEFAULT_FORMAT):
"""calls iterator and results' post-processor"""
# Web search is by default
fetched = self.invoke_iterator(search_terms, max_results, from_date,
to_date, command, output_format)
result_path = lambda x: x['d']['results']
return self.process_output(fetched, result_path)
def request(self,
search_term,
current_index,
max_results=DEFAULT_MAX_RESULTS,
from_date=None,
to_date=None,
command=DEFAULT_COMMAND,
output_format=DEFAULT_FORMAT):
"""calls Bing Search API"""
parameters = {
'Query': search_term,
'$format': output_format,
'$top': max_results,
'$skip': current_index
}
# for testing purposes
# print(current_index, max_results, search_term)
response = self.client.execute(command, query_parameters=parameters)
return response
@classmethod
def convert_item(cls, item):
"""output convertor: applies a mapping to convert
the result to the required format
"""
result = {
'url': item['Url'],
'title': item['Title'],
}
return result
class CustomSearch(AbstractIterableWebSource):
"""wrapper for the Google Custom Search API"""
NAME = "Google Custom Search"
ROOT_URL = 'https://www.googleapis.com/customsearch'
DEFAULT_MAX_RESULTS = 10 # requires only 1 api access
SUPPORTED_PARAMS = ['command', 'output_format', 'language']
DEFAULT_COMMAND = 'v1'
DEFAULT_FORMAT = 'json'
DEFAULT_START_INDEX = 1
DEFAULT_RESULT_LANGUAGE = 'lang_de' # lang_en
MAPPING = {'date': ('valid_from', 'convert_date'),
'text': ('content', None),
'title': 'Title',
}
def __init__(self, api_key, engine_id, api_url=ROOT_URL):
"""fixes the credentials and initiates the RESTClient"""
assert(api_key)
self.api_key = api_key
assert(engine_id)
self.engine_id = engine_id
self.api_url = api_url
self.client = RESTClient(self.api_url, authentification_method='basic')
def search_documents(self, search_terms, max_results=DEFAULT_MAX_RESULTS,
from_date=None, to_date=None, command=DEFAULT_COMMAND,
output_format=DEFAULT_FORMAT,
language=DEFAULT_RESULT_LANGUAGE):
"""calls iterator and results' post-processor"""
fetched = self.invoke_iterator(search_terms, max_results, from_date,
to_date, command, output_format, language)
return self.process_output(fetched, RESULT_PATH)
def request(self, search_term, current_index,
max_results=DEFAULT_MAX_RESULTS, from_date=None, to_date=None,
command=DEFAULT_COMMAND, output_format=DEFAULT_FORMAT,
language=DEFAULT_RESULT_LANGUAGE):
"""calls Google Custom Search API"""
parameters = {'q': '"%s"' % search_term,
'alt': output_format,
'cx': self.engine_id,
'key': self.api_key,
'num': max_results,
'start': current_index,
# 'tbm': 'nws',
'sort': 'date'}
# set language
if language is not None and not language.startswith('lang_') and len(language) == 2:
language = 'lang_{}'.format(language)
if language is not None and len(language) == 7:
parameters['lr'] = language
response = self.client.execute(command, query_parameters=parameters)
return response
@classmethod
def convert_item(cls, item):
"""output convertor: applies a mapping to convert
the result to the required format
"""
result = {'url': item['link'],
'title': item['title'],
}
return result