def request(self,
endpoint,
data=None,
params=None,
method='get',
stream=False):
assert method in _VALID_METHODS, \
'Method {used} not accepted. Please choose one of {valid_methods}'.format(
used=method, valid_methods=", ".join(_VALID_METHODS)
)
requests_call = getattr(requests, method)
logger.debug("Request {method} to {endpoint}".format(
method=method, endpoint=endpoint))
response = requests_call(
url=self._BASE_URL + endpoint,
headers=self.headers,
data=json.dumps(data) if data else None,
params=params,
stream=stream,
)
if self.log_curl_commands:
logger.info("API query to %s" % to_curl(response.request))
if response.status_code != 200:
logger.error("Error calling the API, we tried: %s" %
to_curl(response.request))
raise APIException(
'Got an error {status}: body was \'{error_message}\''.format(
status=response.status_code, error_message=response.text),
response=response)
return response
def save_to_file(self, filename):
api = self.api
job = self # just to be clear
with open(filename, 'wb') as output:
job.wait_for_completion()
logger.info(u"Writing to %s" % filename)
# this is a different request than the normal API
# streaming the file in chunks
response = requests.get(
job.url,
headers=api.headers,
stream=True,
)
if response.status_code != 200:
logger.error("Error calling the API, we tried: %s" %
to_curl(response.request))
raise APIException(
'Got an error {status}: body was \'{error_message}\''.
format(status=response.status_code,
error_message=response.text),
response=response)
for chunk in response.iter_content(chunk_size=self._CHUNK_SIZE):
if chunk:
output.write(chunk)
def request_realtime(self):
api = self.api
endpoint = "{base}/{dataset_id}?keep_alive".format(
base=api._FEED_BASE_URL, dataset_id=self.id)
logger.debug("Connecting with RT feed: %s" % endpoint)
try:
response = api.session.get(endpoint,
headers=api.headers,
stream=True,
**api.common_request_params)
if response.status_code != 200:
logger.error("Error calling the API, we tried: %s" %
to_curl(response.request))
raise APIException(
'Got an error {status}: body was \'{error_message}\''.
format(status=response.status_code,
error_message=response.text),
response=response)
response.encoding = 'utf-8'
for line in response.iter_lines(decode_unicode=True, chunk_size=1):
if line: # skip empty lines to support keep-alive
yield Result(line)
except (requests.exceptions.ConnectionError,
requests.exceptions.ChunkedEncodingError,
urllib3.exceptions.ProtocolError, requests.exceptions.Timeout):
raise ApiConnectionError()
def test_normal_curl(self):
request = AttrDict(method='GET',
headers={'API-KEY': 'test'},
body=None,
url='http://test')
assert to_curl(
request) == 'curl -X GET -H \'API-KEY:test\' \'http://test\''
def request(self,
endpoint,
data=None,
json=None,
params=None,
method='get',
stream=False,
request_params=None,
headers=None,
except_on_fail=True):
assert method in _VALID_METHODS, \
'Method {used} not accepted. Please choose one of {valid_methods}'.format(
used=method, valid_methods=", ".join(_VALID_METHODS)
)
requests_call = getattr(requests, method)
logger.debug("Request {method} to {endpoint}".format(
method=method, endpoint=endpoint))
if endpoint.startswith("http://") or endpoint.startswith("https://"):
# calling another host
url = endpoint
else:
url = self._BASE_URL + endpoint
extra_params = self.common_request_params.copy()
if request_params:
extra_params.update(request_params)
response = requests_call(
url=url,
headers=self.headers if headers is None else headers,
data=data,
json=json,
params=params,
stream=stream,
**extra_params)
if self.log_curl_commands:
logger.info("API query to %s" % to_curl(response.request))
if except_on_fail and response.status_code != 200:
logger.error("Error calling the API, we tried: %s" %
to_curl(response.request))
raise APIException(
'Got an error {status}: body was \'{error_message}\''.format(
status=response.status_code, error_message=response.text),
response=response)
return response
def request_realtime(self):
api = self.api
endpoint = "{base}/{dataset_id}".format(base=api._FEED_BASE_URL,
dataset_id=self.id)
logger.debug("Connecting with RT feed: %s" % endpoint)
response = requests.get(
endpoint,
headers=api.headers,
stream=True,
)
if response.status_code != 200:
logger.error("Error calling the API, we tried: %s" %
to_curl(response.request))
raise APIException(
'Got an error {status}: body was \'{error_message}\''.format(
status=response.status_code, error_message=response.text),
response=response)
response.encoding = 'utf-8'
for line in response.iter_lines(decode_unicode=True):
yield Result(line)
def test_missing_request(self):
request = None
assert to_curl(request) == 'No request'
