def test_request_with_timeout_failure(self):
def immediately_timeout(*args, **kwargs):
raise requests.exceptions.Timeout("I give up")
with pytest.raises(RequestTimedOut) as excinfo:
HTTP._request_with_timeout("http://url/", immediately_timeout, "a", "b")
assert "Timeout accessing http://url/: I give up" in str(excinfo.value)
def test_request_with_network_failure(self):
def immediately_fail(*args, **kwargs):
raise requests.exceptions.ConnectionError("a disaster")
with pytest.raises(RequestNetworkException) as excinfo:
HTTP._request_with_timeout("http://url/", immediately_fail, "a", "b")
assert "Network error contacting http://url/: a disaster" in str(excinfo.value)
def test_request_with_response_indicative_of_failure(self):
def fake_500_response(*args, **kwargs):
return MockRequestsResponse(500, content="Failure!")
with pytest.raises(BadResponseException) as excinfo:
HTTP._request_with_timeout("http://url/", fake_500_response, "a", "b")
assert (
"Bad response from http://url/: Got status code 500 from external server"
in str(excinfo.value)
)
def _make_request(self, url, method, headers, data=None, params=None,
**kwargs):
"""Actually make an HTTP request."""
try:
return HTTP.request_with_timeout(
method, url, headers=headers, data=data,
params=params, timeout=90, **kwargs
)
except RequestTimedOut, e:
self.log.info("Request to %s timed out once. Trying a second time.", url)
return HTTP.request_with_timeout(
method, url, headers=headers, data=data,
params=params, timeout=90, **kwargs
)
def patron_request(self, patron, pin, url, extra_headers={}, data=None,
exception_on_401=False, method=None):
"""Make an HTTP request on behalf of a patron.
The results are never cached.
"""
patron_credential = self.get_patron_credential(patron, pin)
headers = dict(Authorization="Bearer %s" % patron_credential.credential)
headers.update(extra_headers)
if method and method.lower() in ('get', 'post', 'put', 'delete'):
method = method.lower()
else:
if data:
method = 'post'
else:
method = 'get'
response = HTTP.request_with_timeout(
method, url, headers=headers, data=data
)
if response.status_code == 401:
if exception_on_401:
# This is our second try. Give up.
raise Exception("Something's wrong with the patron OAuth Bearer Token!")
else:
# Refresh the token and try again.
self.refresh_patron_access_token(
patron_credential, patron, pin)
return self.patron_request(
patron, pin, url, extra_headers, data, True)
else:
# This is commented out because it may expose patron
# information.
#
# self.log.debug("%s: %s", url, response.status_code)
return response
def _request_with_timeout(self, method, url, *args, **kwargs):
self.requests.append([method, url, args, kwargs])
response = self.responses.pop()
return HTTP._process_response(
url, response, kwargs.get('allowed_response_codes'),
kwargs.get('disallowed_response_codes')
)
def _make_request(self, url, *args, **kwargs):
response = self.responses.pop()
self.requests.append((url, args, kwargs))
return HTTP._process_response(
url, response, kwargs.get('allowed_response_codes'),
kwargs.get('disallowed_response_codes')
)
def patron_request(self, patron, pin, url, extra_headers={}, data=None, exception_on_401=False, method=None):
"""Make an HTTP request on behalf of a patron.
The results are never cached.
"""
headers = dict(Authorization="Bearer %s" % self.token)
headers['Content-Type'] = 'application/json'
headers.update(extra_headers)
if method and method.lower() in ('get', 'post', 'put', 'delete'):
method = method.lower()
else:
if data:
method = 'post'
else:
method = 'get'
url = self._make_absolute_url(url)
response = HTTP.request_with_timeout(
method, url, headers=headers, data=data,
timeout=60
)
if response.status_code == 401:
if exception_on_401:
# This is our second try. Give up.
raise Exception("Something's wrong with the patron OAuth Bearer Token!")
else:
# Refresh the token and try again.
self.check_creds(True)
return self.patron_request(patron, pin, url, extra_headers, data, True)
else:
return response
def patron_request(self, patron, pin, url, extra_headers={}, data=None,
exception_on_401=False, method=None):
"""Make an HTTP request on behalf of a patron.
The results are never cached.
"""
patron_credential = self.get_patron_credential(patron, pin)
headers = dict(Authorization="Bearer %s" % patron_credential.credential)
headers.update(extra_headers)
if method and method.lower() in ('get', 'post', 'put', 'delete'):
method = method.lower()
else:
if data:
method = 'post'
else:
method = 'get'
response = HTTP.request_with_timeout(
method, url, headers=headers, data=data
)
if response.status_code == 401:
if exception_on_401:
# This is our second try. Give up.
raise Exception("Something's wrong with the patron OAuth Bearer Token!")
else:
# Refresh the token and try again.
self.refresh_patron_access_token(
patron_credential, patron, pin)
return self.patron_request(
patron, pin, url, extra_headers, data, True)
else:
# This is commented out because it may expose patron
# information.
#
# self.log.debug("%s: %s", url, response.status_code)
return response
def _request_with_timeout(self, method, url, *args, **kwargs):
self.requests.append([method, url, args, kwargs])
response = self.responses.pop()
return HTTP._process_response(
url, response, kwargs.get('allowed_response_codes'),
kwargs.get('disallowed_response_codes')
)
def test_unicode_converted_to_utf8(self):
"""Any Unicode that sneaks into the URL, headers or body is
converted to UTF-8.
"""
class ResponseGenerator(object):
def __init__(self):
self.requests = []
def response(self, *args, **kwargs):
self.requests.append((args, kwargs))
return MockRequestsResponse(200, content="Success!")
generator = ResponseGenerator()
url = "http://foo"
response = HTTP._request_with_timeout(
url,
generator.response,
"POST",
headers={"unicode header": "unicode value"},
data="unicode data",
)
[(args, kwargs)] = generator.requests
url, method = args
headers = kwargs["headers"]
data = kwargs["data"]
# All the Unicode data was converted to bytes before being sent
# "over the wire".
for k, v in list(headers.items()):
assert isinstance(k, bytes)
assert isinstance(v, bytes)
assert isinstance(data, bytes)
def token_post(self, url, payload, headers={}, **kwargs):
"""Mock the request for an OAuth token.
"""
self.access_token_requests.append((url, payload, headers, kwargs))
response = self.access_token_response
return HTTP._process_response(url, response, **kwargs)
def put(self, url, headers, **kwargs):
data = kwargs.get('data')
if 'data' in kwargs:
del kwargs['data']
response = HTTP.put_with_timeout(url, data, headers=headers, **kwargs)
return response
def put(self, url, headers, **kwargs):
data = kwargs.get('data')
if 'data' in kwargs:
del kwargs['data']
# This might take a very long time -- disable the normal
# timeout.
kwargs['timeout'] = None
response = HTTP.put_with_timeout(url, data, headers=headers, **kwargs)
return response
def _request(self, method, url, headers, data, params, **kwargs):
"""Actually make an HTTP request.
MockEnkiAPI overrides this method.
"""
return HTTP.request_with_timeout(
method, url, headers=headers, data=data,
params=params, timeout=90, disallowed_response_codes=None,
**kwargs
)
def _request(self, method, url, headers, data, params, **kwargs):
"""Override EnkiAPI._request to pull responses from a
queue instead of making real HTTP requests
"""
self.requests.append([method, url, headers, data, params, kwargs])
response = self.responses.pop()
return HTTP._process_response(
url, response, kwargs.get('allowed_response_codes'),
kwargs.get('disallowed_response_codes'),
)
def _request(self, method, url, headers, data, params, **kwargs):
"""Actually make an HTTP request.
MockEnkiAPI overrides this method.
"""
return HTTP.request_with_timeout(
method, url, headers=headers, data=data,
params=params, timeout=90, disallowed_response_codes=None,
**kwargs
)
def _request(self, method, url, headers, data, params, **kwargs):
"""Override EnkiAPI._request to pull responses from a
queue instead of making real HTTP requests
"""
self.requests.append([method, url, headers, data, params, kwargs])
response = self.responses.pop()
return HTTP._process_response(
url, response, kwargs.get('allowed_response_codes'),
kwargs.get('disallowed_response_codes'),
)
def _do_get(url, headers, **kwargs):
# More time please
if 'timeout' not in kwargs:
kwargs['timeout'] = 60
if 'allow_redirects' not in kwargs:
kwargs['allow_redirects'] = True
response = HTTP.get_with_timeout(url, headers=headers, **kwargs)
return response.status_code, response.headers, response.content
def post_request(self, data):
"""Make an HTTP request.
Defined solely so it can be overridden in the mock.
"""
return HTTP.post_with_timeout(
self.base_url,
data,
headers={"Content-Type": "application/xml"},
allowed_response_codes=['2xx'],
)
def __init__(self, _db, collection, new_css=None, *args, **kwargs):
kwargs['content_modifier'] = self.replace_css
if new_css:
self.new_css = new_css
else:
self.new_css = HTTP.get_with_timeout("http://www.daisy.org/z3986/2005/dtbook.2005.basic.css").content
super(FeedbooksOPDSImporter, self).__init__(
_db, collection, **kwargs
)
def put(self, url, headers, **kwargs):
data = kwargs.get('data')
if 'data' in kwargs:
del kwargs['data']
# This might take a very long time -- disable the normal
# timeout.
kwargs['timeout'] = None
response = HTTP.put_with_timeout(
url, data, headers=headers, **kwargs
)
return response
def get_catalogue(self, name):
response = HTTP.get_with_timeout(
self.CATALOGUES_ENDPOINT, headers=self.default_headers
)
if response.status_code == 200:
catalogues = response.json().get('results')
catalogue = filter(lambda c: c.get('name') == name, catalogues)
if catalogue:
return catalogue[0]
else:
return None
def refresh_credential(self, credential):
url = self.API_ENDPOINT + 'token'
headers = {'Content-Type': self.TOKEN_CONTENT_TYPE}
client_details = dict(client_id=self.client_id, client_secret=self.client_secret)
response = HTTP.post_with_timeout(url, client_details, headers=headers)
data = response.json()
credential.credential = data.get('access_token')
expires_in = data.get('expires_in')
credential.expires = datetime.utcnow() + timedelta(0, expires_in * 0.9)
self._credential = credential
def _get(self, url, headers=None):
"""Make a normal HTTP request, but include an authentication
header with the credentials for the collection.
"""
username = self.username
password = self.password
headers = dict(headers or {})
auth_header = "Basic %s" % base64.b64encode("%s:%s" %
(username, password))
headers['Authorization'] = auth_header
return HTTP.get_with_timeout(url, headers=headers)
def _make_request(self,
url,
method,
headers,
data=None,
params=None,
**kwargs):
"""Actually make an HTTP request."""
return HTTP.request_with_timeout(method,
url,
headers=headers,
data=data,
params=params,
**kwargs)
def patron_request(
self,
patron,
pin,
url,
extra_headers={},
data=None,
exception_on_401=False,
method=None,
):
"""Make an HTTP request on behalf of a patron.
The results are never cached.
"""
headers = dict(Authorization="Bearer %s" % self.token)
headers["Content-Type"] = "application/json"
headers.update(extra_headers)
if method and method.lower() in ("get", "post", "put", "delete"):
method = method.lower()
else:
if data:
method = "post"
else:
method = "get"
url = self._make_absolute_url(url)
response = HTTP.request_with_timeout(method,
url,
headers=headers,
data=data,
timeout=60)
# TODO: If Odilo doesn't recognize the patron it will send
# 404 in this case.
if response.status_code == 401:
if exception_on_401:
# This is our second try. Give up.
raise Exception(
"Something's wrong with the patron OAuth Bearer Token!")
else:
# Refresh the token and try again.
self.check_creds(True)
return self.patron_request(patron, pin, url, extra_headers,
data, True)
else:
return response
def create_content_item(self, content_item):
content_item = self.set_timestamp(content_item, create=True)
content_item = json.dumps(content_item)
response = HTTP.post_with_timeout(
self.CONTENT_ITEMS_ENDPOINT, content_item,
headers=self.default_headers,
allowed_response_codes=[201]
)
content_item = response.json()
name = content_item.get('name')
content_item_id = content_item.get('contentItemId')
self.log.info(
"New content item created for '%s': '%s'", name, content_item_id)
return content_item
def delete_content_item(self, identifier):
content_item_id = identifier
if isinstance(identifier, Identifier):
if not identifier.type == Identifier.BIBBLIO_CONTENT_ITEM_ID:
raise TypeError('Identifier is not a Bibblio Content Item')
content_item_id = identifier.identifier
delete_url = self.CONTENT_ITEMS_ENDPOINT + content_item_id
response = HTTP.request_with_timeout(
'DELETE', delete_url, headers=self.default_headers,
allowed_response_codes=[200]
)
if not isinstance(identifier, basestring) and response.status_code == 200:
self._db.delete(identifier)
self.log.info("DELETED: Bibblio Content Item '%s'" % content_item_id)
def create_catalogue(self, name, description=None):
catalogue = dict(name=name)
if description:
catalogue['description'] = description
catalogue = json.dumps(catalogue)
response = HTTP.post_with_timeout(
self.CATALOGUES_ENDPOINT, catalogue,
headers=self.default_headers,
allowed_response_codes=[201]
)
catalogue = response.json()
name = catalogue.get('name')
catalogue_id = catalogue.get('catalogueId')
self.log.info(
"New catalogue '%s' created with ID: %s", name, catalogue_id)
return catalogue
def test_request_with_timeout_success(self):
called_with = None
def fake_200_response(*args, **kwargs):
# The HTTP method and URL are passed in the order
# requests.request would expect.
assert ("GET", "http://url/") == args
# Keyword arguments to _request_with_timeout are passed in
# as-is.
assert "value" == kwargs["kwarg"]
# A default timeout is added.
assert 20 == kwargs["timeout"]
return MockRequestsResponse(200, content="Success!")
response = HTTP._request_with_timeout(
"http://url/", fake_200_response, "GET", kwarg="value"
)
assert 200 == response.status_code
assert b"Success!" == response.content
def _request_with_timeout(self, method, url, *args, **kwargs):
"""Wrapper around HTTP.request_with_timeout to be overridden for tests."""
return HTTP.request_with_timeout(method, url, *args, **kwargs)
def _get(self, url, headers=None):
self.requests.append([url, headers])
response = self.responses.pop()
return HTTP._process_response(url, response)
def post(self, url, params):
response = HTTP.post_with_timeout(url, params)
def request(self, url, *args, **kwargs):
"""Actually make an HTTP request. This method exists only so the mock
can override it.
"""
self._update_request_kwargs(kwargs)
return HTTP.request_with_timeout("GET", url, *args, **kwargs)
def _get(self, url, headers):
return HTTP.get_with_timeout(url, headers=headers).json()
def _get_token(self, payload, headers):
response = HTTP.post_with_timeout(self.CLEVER_TOKEN_URL,
json.dumps(payload),
headers=headers)
return response.json()
def _get(self, url, headers):
return HTTP.get_with_timeout(url, headers=headers).json()
def _do_post(url, payload, headers, **kwargs):
# More time please
if 'timeout' not in kwargs:
kwargs['timeout'] = 60
return HTTP.post_with_timeout(url, payload, headers=headers, **kwargs)
def _get_token(self, payload, headers):
response = HTTP.post_with_timeout(
self.CLEVER_TOKEN_URL, json.dumps(payload), headers=headers
)
return response.json()
def post(self, url, params):
response = HTTP.post_with_timeout(url, params)
def _request_with_timeout(self, method, url, *args, **kwargs):
"""Wrapper around HTTP.request_with_timeout to be overridden for tests."""
return HTTP.request_with_timeout(method, url, *args, **kwargs)