def build_url(url, params=None): """ Builds the full url when provided the base ``url`` and some url parameters: >>> build_url("/foobar", {"first": "foo", "second": "bar"}) '/foobar?first=foo&second=bar' >>> build_url("/foobar bar/") ''/foobar%20bar/' :param str url: The url to build off of. :param dict params: A dictionary of parameters that should be added on to ``url``. If this value is not provided ``url`` will be returned by itself. Arguments to a url are unordered by default however they will be sorted alphabetically so the results are repeatable from call to call. """ assert isinstance(url, STRING_TYPES) # append url arguments if isinstance(params, (dict, ImmutableDict, UserDict)) and params: url += "?" + "&".join([ "%s=%s" % (key, value)for key, value in sorted(params.items())]) return quote_url(url)
def request(method, url, **kwargs): """ Wrapper around :func:`treq.request` with some added arguments and validation. :param str method: The HTTP method to use when making the request. :param str url: The url this request will be made to. :type data: str, list, tuple, set, dict :keyword data: The data to send along with some types of requests such as ``POST`` or ``PUT`` :keyword dict headers: The headers to send along with the request to ``url``. Currently only single values per header are supported. :keyword function callback: The function to deliver an instance of :class:`Response` once we receive and unpack a response. :keyword function errback: The function to deliver an error message to. By default this will use :func:`.log.err`. :keyword class response_class: The class to use to unpack the internal response. This is mainly used by the unittests but could be used elsewhere to add some custom behavior to the unpack process for the incoming response. :raises NotImplementedError: Raised whenever a request is made of this function that we can't implement such as an invalid http scheme, request method or a problem constructing data to an api. """ assert isinstance(url, STRING_TYPES) direct = kwargs.pop("direct", False) # We only support http[s] parsed_url = urlparse(url) if not parsed_url.hostname: raise NotImplementedError("No hostname present in url") if not parsed_url.path: raise NotImplementedError("No path provided in url") if not direct: original_request = Request( method=method, url=url, kwargs=ImmutableDict(kwargs.copy())) # Headers headers = kwargs.pop("headers", {}) headers.setdefault("Content-Type", ["application/json"]) headers.setdefault("User-Agent", [USERAGENT]) # Twisted requires lists for header values for header, value in headers.items(): if isinstance(value, STRING_TYPES): headers[header] = [value] elif isinstance(value, (list, tuple, set)): continue else: raise NotImplementedError( "Cannot handle header values with type %s" % type(value)) # Handle request data data = kwargs.pop("data", NOTSET) if isinstance(data, dict): data = json_safe(data) if (data is not NOTSET and headers["Content-Type"] == ["application/json"]): data = json.dumps(data) elif data is not NOTSET: raise NotImplementedError( "Don't know how to dump data for %s" % headers["Content-Type"]) # prepare keyword arguments kwargs.update( headers=headers, persistent=config["agent_http_persistent_connections"]) if data is not NOTSET: kwargs.update(data=data) if direct: # We don't support these with direct request # types. assert "callback" not in kwargs assert "errback" not in kwargs assert "response_class" not in kwargs # Construct the request and attach some loggers # to callback/errback. uid = uuid4() treq_request = treq.request(method, url, **kwargs) treq_request.addCallback(HTTPLog.response, uid=uid) treq_request.addErrback(HTTPLog.error, uid=uid, method=method, url=url) return treq_request else: callback = kwargs.pop("callback", None) errback = kwargs.pop("errback", log.err) response_class = kwargs.pop("response_class", Response) # check assumptions for keywords assert callback is not None, "callback not provided" assert callable(callback) and callable(errback) assert data is NOTSET or \ isinstance(data, tuple(list(STRING_TYPES) + [dict, list])) def unpack_response(response): deferred = Deferred() deferred.addCallback(callback) # Deliver the body onto an instance of the response # object along with the original request. Finally # the request and response via an instance of `Response` # to the outer scope's callback function. response.deliverBody( response_class(deferred, response, original_request)) return deferred debug_kwargs = kwargs.copy() debug_url = build_url(url, debug_kwargs.pop("params", None)) logger.debug( "Queued %s %s, kwargs: %r", method, debug_url, debug_kwargs) try: deferred = treq.request(method, quote_url(url), **kwargs) except NotImplementedError: # pragma: no cover logger.error( "Attempting to access a url over SSL but you don't have the " "proper libraries installed. Please install the PyOpenSSL and " "service_identity Python packages.") raise deferred.addCallback(unpack_response) deferred.addErrback(errback) return deferred
def test_fragment(self): self.assertEqual( quote_url("/foobar?first=1&second=2#abcd"), "/foobar?first=1&second=2#abcd")
def test_parameters(self): self.assertEqual( quote_url("/foobar?first=1&second=2"), "/foobar?first=1&second=2")
def test_simple_without_scheme(self): self.assertEqual(quote_url("/foobar"), "/foobar")
def test_simple_with_scheme(self): self.assertEqual(quote_url("http://foobar"), "http://foobar")