def build_url(url, params=None):
"""
Builds the full url when provided the base ``url`` and some
url parameters:
>>> build_url("/foobar", {"first": "foo", "second": "bar"})
'/foobar?first=foo&second=bar'
>>> build_url("/foobar bar/")
''/foobar%20bar/'
:param str url:
The url to build off of.
:param dict params:
A dictionary of parameters that should be added on to ``url``. If
this value is not provided ``url`` will be returned by itself.
Arguments to a url are unordered by default however they will be
sorted alphabetically so the results are repeatable from call to call.
"""
assert isinstance(url, STRING_TYPES)
# append url arguments
if isinstance(params, (dict, ImmutableDict, UserDict)) and params:
url += "?" + "&".join([
"%s=%s" % (key, value)for key, value in sorted(params.items())])
return quote_url(url)
def request(method, url, **kwargs):
"""
Wrapper around :func:`treq.request` with some added arguments
and validation.
:param str method:
The HTTP method to use when making the request.
:param str url:
The url this request will be made to.
:type data: str, list, tuple, set, dict
:keyword data:
The data to send along with some types of requests
such as ``POST`` or ``PUT``
:keyword dict headers:
The headers to send along with the request to
``url``. Currently only single values per header
are supported.
:keyword function callback:
The function to deliver an instance of :class:`Response`
once we receive and unpack a response.
:keyword function errback:
The function to deliver an error message to. By default
this will use :func:`.log.err`.
:keyword class response_class:
The class to use to unpack the internal response. This is mainly
used by the unittests but could be used elsewhere to add some
custom behavior to the unpack process for the incoming response.
:raises NotImplementedError:
Raised whenever a request is made of this function that we can't
implement such as an invalid http scheme, request method or a problem
constructing data to an api.
"""
assert isinstance(url, STRING_TYPES)
direct = kwargs.pop("direct", False)
# We only support http[s]
parsed_url = urlparse(url)
if not parsed_url.hostname:
raise NotImplementedError("No hostname present in url")
if not parsed_url.path:
raise NotImplementedError("No path provided in url")
if not direct:
original_request = Request(
method=method, url=url, kwargs=ImmutableDict(kwargs.copy()))
# Headers
headers = kwargs.pop("headers", {})
headers.setdefault("Content-Type", ["application/json"])
headers.setdefault("User-Agent", [USERAGENT])
# Twisted requires lists for header values
for header, value in headers.items():
if isinstance(value, STRING_TYPES):
headers[header] = [value]
elif isinstance(value, (list, tuple, set)):
continue
else:
raise NotImplementedError(
"Cannot handle header values with type %s" % type(value))
# Handle request data
data = kwargs.pop("data", NOTSET)
if isinstance(data, dict):
data = json_safe(data)
if (data is not NOTSET and
headers["Content-Type"] == ["application/json"]):
data = json.dumps(data)
elif data is not NOTSET:
raise NotImplementedError(
"Don't know how to dump data for %s" % headers["Content-Type"])
# prepare keyword arguments
kwargs.update(
headers=headers,
persistent=config["agent_http_persistent_connections"])
if data is not NOTSET:
kwargs.update(data=data)
if direct:
# We don't support these with direct request
# types.
assert "callback" not in kwargs
assert "errback" not in kwargs
assert "response_class" not in kwargs
# Construct the request and attach some loggers
# to callback/errback.
uid = uuid4()
treq_request = treq.request(method, url, **kwargs)
treq_request.addCallback(HTTPLog.response, uid=uid)
treq_request.addErrback(HTTPLog.error, uid=uid, method=method, url=url)
return treq_request
else:
callback = kwargs.pop("callback", None)
errback = kwargs.pop("errback", log.err)
response_class = kwargs.pop("response_class", Response)
# check assumptions for keywords
assert callback is not None, "callback not provided"
assert callable(callback) and callable(errback)
assert data is NOTSET or \
isinstance(data, tuple(list(STRING_TYPES) + [dict, list]))
def unpack_response(response):
deferred = Deferred()
deferred.addCallback(callback)
# Deliver the body onto an instance of the response
# object along with the original request. Finally
# the request and response via an instance of `Response`
# to the outer scope's callback function.
response.deliverBody(
response_class(deferred, response, original_request))
return deferred
debug_kwargs = kwargs.copy()
debug_url = build_url(url, debug_kwargs.pop("params", None))
logger.debug(
"Queued %s %s, kwargs: %r", method, debug_url, debug_kwargs)
try:
deferred = treq.request(method, quote_url(url), **kwargs)
except NotImplementedError: # pragma: no cover
logger.error(
"Attempting to access a url over SSL but you don't have the "
"proper libraries installed. Please install the PyOpenSSL and "
"service_identity Python packages.")
raise
deferred.addCallback(unpack_response)
deferred.addErrback(errback)
return deferred
def test_fragment(self):
self.assertEqual(
quote_url("/foobar?first=1&second=2#abcd"),
"/foobar?first=1&second=2#abcd")
def test_parameters(self):
self.assertEqual(
quote_url("/foobar?first=1&second=2"),
"/foobar?first=1&second=2")
def test_simple_without_scheme(self):
self.assertEqual(quote_url("/foobar"), "/foobar")
def test_simple_with_scheme(self):
self.assertEqual(quote_url("http://foobar"), "http://foobar")
