class MarconiClient(object): USER_AGENT = 'txmarconi/{version}'.format(version=__version__) RETRYABLE_ERRORS = [RequestTransmissionFailed] def __init__(self, base_url='http://localhost:8888', quiet_requests=True, **kwargs): self.client_id = str(uuid4()) self.base_url = base_url pool = HTTPConnectionPool(reactor, persistent=True) agent = ContentDecoderAgent(RedirectAgent(Agent(reactor, pool=pool)), [('gzip', GzipDecoder)]) if quiet_requests: pool._factory = QuieterHTTP11ClientFactory auth_url = kwargs.get('auth_url') if auth_url: username = kwargs.get('username') password = kwargs.get('password') api_key = kwargs.get('api_key') if not username: raise RuntimeError('Marconi "auth_url" specified with no username') if api_key: cred = api_key auth_type = 'api_key' elif password: cred = password auth_type = 'password' else: raise RuntimeError('Marconi "auth_url" specified with no "password" or "api_key"') agent = KeystoneAgent(agent, auth_url, (username, cred), auth_type=auth_type) self.http_client = HTTPClient(agent) def _wrap_error(self, failure): if not failure.check(MarconiError): log.err(failure) raise MarconiError(failure.value) log.err(failure.value) return failure def _handle_error_response(self, response): def _raise_error(content_str): content_str = content_str.strip() if len(content_str) > 0: raise MarconiError(json.loads(content_str)) else: msg = 'Received {code} response with empty body'.format(code=response.code) raise MarconiError(msg) d = content(response) d.addCallback(_raise_error) return d def _request(self, method, path, params=None, data=None): url = '{base_url}{path}'.format( base_url=self.base_url, path=path, ) headers = { 'Content-Type': 'application/json', 'Accept': 'application/json', 'User-Agent': self.USER_AGENT, 'Client-ID': self.client_id, } def _possibly_retry(failure): # Either I'm doing something wrong (likely) or Marconi is doing # something unpleasant to connections after it returns a 201 to us, # because the next request always seems to get one of these. if failure.check(*self.RETRYABLE_ERRORS): return self._request(method, path, params=params, data=data) else: return failure if data: body = QuieterFileBodyProducer(StringIO(json.dumps(data))) else: body = None d = self.http_client.request(method, url, headers=headers, data=body, params=params) d.addErrback(_possibly_retry) return d def _expect_204(self, response): if response.code == 204: return None else: return self._handle_error_response(response) def ensure_queue(self, queue_name): path = '/v1/queues/{queue_name}'.format(queue_name=queue_name) def _on_response(response): if response.code in (201, 204): return path else: return self._handle_error_response(response) d = self._request('PUT', path) d.addCallback(_on_response) d.addErrback(self._wrap_error) return d def push_message(self, queue_name, body, ttl): path = '/v1/queues/{queue_name}/messages'.format(queue_name=queue_name) data = [ { 'ttl': ttl, 'body': body, } ] def _construct_message(obj): return MarconiMessage(body=body, ttl=ttl, age=0, href=obj['resources'][0]) def _on_response(response): if response.code == 201: return json_content(response).addCallback(_construct_message) else: return self._handle_error_response(response) d = self._request('POST', path, data=data) d.addCallback(_on_response) d.addErrback(self._wrap_error) return d def claim_message(self, queue_name, ttl, grace, polling_interval=1): path = '/v1/queues/{queue_name}/claims'.format(queue_name=queue_name) data = { 'ttl': ttl, 'grace': grace, } params = { 'limit': 1, } d = defer.Deferred() def _construct_message(obj, response): claim_href = response.headers.getRawHeaders('location')[0] d.callback(ClaimedMarconiMessage(claim_href=claim_href, **obj[0])) def _on_response(response): if response.code == 201: json_content(response).addCallback(_construct_message, response) elif response.code == 204: reactor.callLater(polling_interval, _perform_call) else: return self._handle_error_response(response) def _perform_call(): d1 = self._request('POST', path, data=data, params=params) d1.addCallback(_on_response) d1.addErrback(self._wrap_error) d1.addErrback(d.errback) _perform_call() return d def update_claim(self, claimed_message, ttl): data = { 'ttl': ttl, } d = self._request('PATCH', claimed_message.claim_href, data=data) d.addCallback(self._expect_204) d.addErrback(self._wrap_error) return d def release_claim(self, claimed_message): d = self._request('DELETE', claimed_message.claim_href) d.addCallback(self._expect_204) d.addErrback(self._wrap_error) return d def delete_message(self, message): d = self._request('DELETE', message.href) d.addCallback(self._expect_204) d.addErrback(self._wrap_error) return d
def reverse_proxy(self, request, protected=True): if protected: sess = request.getSession() valid_sessions = self.valid_sessions sess_uid = sess.uid username = valid_sessions[sess_uid]['username'] # Normal reverse proxying. kwds = {} cookiejar = {} kwds['allow_redirects'] = False kwds['cookies'] = cookiejar req_headers = self.mod_headers(dict(request.requestHeaders.getAllRawHeaders())) kwds['headers'] = req_headers if protected: kwds['headers'][self.remoteUserHeader] = [username] if request.method in ('PUT', 'POST'): kwds['data'] = request.content.read() url = self.proxied_url + request.uri # Determine if a plugin wants to intercept this URL. interceptors = self.interceptors for interceptor in interceptors: if interceptor.should_resource_be_intercepted(url, request.method, req_headers, request): return interceptor.handle_resource(url, request.method, req_headers, request) # Check if this is a request for a websocket. d = self.checkForWebsocketUpgrade(request) if d is not None: return d # Typical reverse proxying. self.log("Proxying URL => {0}".format(url)) http_client = HTTPClient(self.proxy_agent) d = http_client.request(request.method, url, **kwds) def process_response(response, request): req_resp_headers = request.responseHeaders resp_code = response.code resp_headers = response.headers resp_header_map = dict(resp_headers.getAllRawHeaders()) # Rewrite Location headers for redirects as required. if resp_code in (301, 302, 303, 307, 308) and "Location" in resp_header_map: values = resp_header_map["Location"] if len(values) == 1: location = values[0] if request.isSecure(): proxy_scheme = 'https' else: proxy_scheme = 'http' new_location = self.proxied_url_to_proxy_url(proxy_scheme, location) if new_location is not None: resp_header_map['Location'] = [new_location] request.setResponseCode(response.code, message=response.phrase) for k,v in resp_header_map.iteritems(): if k == 'Set-Cookie': v = self.mod_cookies(v) req_resp_headers.setRawHeaders(k, v) return response def mod_content(body, request): """ Modify response content before returning it to the user agent. """ d = None for content_modifier in self.content_modifiers: if d is None: d = content_modifier.transform_content(body, request) else: d.addCallback(content_modifier.transform_content, request) if d is None: return body else: return d d.addCallback(process_response, request) d.addCallback(treq.content) d.addCallback(mod_content, request) return d
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch('treq.client.FileBodyProducer') self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer') self.MultiPartProducer = self.mbp_patcher.start() self.addCleanup(self.mbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_post(self): self.client.post('http://example.com/') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_idn(self): self.client.request('GET', u'http://č.net') self.agent.request.assert_called_once_with( b'GET', b'http://xn--bea.net', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_case_insensitive_methods(self): self.client.request('gEt', 'http://example.com/') self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': ['bar']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_tuple_query_values(self): self.client.request('GET', 'http://example.com/', params={'foo': ('bar',)}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_merge_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_merge_tuple_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_dict_single_value_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_data_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar&foo=baz') def test_request_data_single_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_tuple(self): self.client.request('POST', 'http://example.com/', data=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "wb") as temp_file: temp_file.write(b'hello') self.client.request('POST', 'http://example.com/', data=open(temp_fn, 'rb')) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'hello') def test_request_json_dict(self): self.client.request('POST', 'http://example.com/', json={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'{"foo":"bar"}') def test_request_json_tuple(self): self.client.request('POST', 'http://example.com/', json=('foo', 1)) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'["foo",1]') def test_request_json_number(self): self.client.request('POST', 'http://example.com/', json=1.) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'1.0') def test_request_json_string(self): self.client.request('POST', 'http://example.com/', json='hello') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'"hello"') def test_request_json_bool(self): self.client.request('POST', 'http://example.com/', json=True) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'true') def test_request_json_none(self): self.client.request('POST', 'http://example.com/', json=None) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'null') @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_no_name_attachment(self): self.client.request( 'POST', 'http://example.com/', files={"name": BytesIO(b"hello")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'image/jpeg', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment_and_ctype(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', 'text/plain', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'text/plain', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params(self): class NamedFile(BytesIO): def __init__(self, val): BytesIO.__init__(self, val) self.name = "image.png" self.client.request( 'POST', 'http://example.com/', data=[("a", "b"), ("key", "val")], files=[ ("file1", ('image.jpg', BytesIO(b"hello"))), ("file2", NamedFile(b"yo"))]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('a', 'b'), ('key', 'val'), ('file1', ('image.jpg', 'image/jpeg', FP)), ('file2', ('image.png', 'image/png', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params_dict(self): self.client.request( 'POST', 'http://example.com/', data={"key": "a", "key2": "b"}, files={"file1": BytesIO(b"hey")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('key', 'a'), ('key2', 'b'), ('file1', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) def test_request_unsupported_params_combination(self): self.assertRaises(ValueError, self.client.request, 'POST', 'http://example.com/', data=BytesIO(b"yo"), files={"file1": BytesIO(b"hey")}) def test_request_dict_headers(self): self.client.request('GET', 'http://example.com/', headers={ 'User-Agent': 'treq/0.1dev', 'Accept': ['application/json', 'text/plain'] }) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({b'User-Agent': [b'treq/0.1dev'], b'accept-encoding': [b'gzip'], b'Accept': [b'application/json', b'text/plain']}), None) @with_clock def test_request_timeout_fired(self, clock): """ Verify the request is cancelled if a response is not received within specified timeout period. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate we haven't gotten a response within timeout seconds clock.advance(3) # a deferred should have been cancelled self.failureResultOf(d, CancelledError) @with_clock def test_request_timeout_cancelled(self, clock): """ Verify timeout is cancelled if a response is received before timeout period elapses. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate a response d.callback(mock.Mock(code=200, headers=Headers({}))) # now advance the clock but since we already got a result, # a cancellation timer should have been cancelled clock.advance(3) self.successResultOf(d) def test_response_is_buffered(self): response = mock.Mock(deliverBody=mock.Mock(), headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com') result = self.successResultOf(d) protocol = mock.Mock(Protocol) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) def test_response_buffering_is_disabled_with_unbufferred_arg(self): response = mock.Mock(headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com', unbuffered=True) # YOLO public attribute. self.assertEqual(self.successResultOf(d).original, response) def test_request_post_redirect_denied(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) d = self.client.post('http://www.example.com') self.failureResultOf(d, ResponseFailed) def test_request_browser_like_redirects(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) raw = mock.Mock(return_value=[]) final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw)) with mock.patch('twisted.web.client.RedirectAgent._handleRedirect', return_value=final_resp): d = self.client.post('http://www.google.com', browser_like_redirects=True, unbuffered=True) self.assertEqual(self.successResultOf(d).original, final_resp)
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch('treq.client.FileBodyProducer') self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_request_case_insensitive_methods(self): self.client.request('gEt', 'http://example.com/') self.agent.request.assert_called_once_with( 'GET', 'http://example.com/', headers=Headers({}), bodyProducer=None) def test_request_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': ['bar']}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_tuple_query_values(self): self.client.request('GET', 'http://example.com/', params={'foo': ('bar',)}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_merge_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?baz=bax&foo=bar&foo=baz', headers=Headers({}), bodyProducer=None) def test_request_merge_tuple_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params=[('foo', 'bar')]) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?baz=bax&foo=bar', headers=Headers({}), bodyProducer=None) def test_request_dict_single_value_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_data_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar&foo=baz') def test_request_data_single_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': 'bar'}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar') def test_request_data_tuple(self): self.client.request('POST', 'http://example.com/', data=[('foo', 'bar')]) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar') def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "w") as temp_file: temp_file.write('hello') self.client.request('POST', 'http://example.com/', data=file(temp_fn)) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('hello') def test_request_dict_headers(self): self.client.request('GET', 'http://example.com/', headers={ 'User-Agent': 'treq/0.1dev', 'Accept': ['application/json', 'text/plain'] }) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/', headers=Headers({'User-Agent': ['treq/0.1dev'], 'Accept': ['application/json', 'text/plain']}), bodyProducer=None)
class JWSClient(object): """ HTTP client using JWS-signed messages for ACME. """ timeout = _DEFAULT_TIMEOUT def __init__(self, agent, key, alg, new_nonce_url, kid, user_agent=u'txacme/{}'.format(__version__).encode('ascii')): self._treq = HTTPClient(agent=agent) self._agent = agent self._current_request = None self._key = key self._alg = alg self._user_agent = user_agent self._nonces = set() self._new_nonce = new_nonce_url self._kid = kid @classmethod def from_directory(cls, agent, key, alg, directory): """ Prepare for ACME operations based on 'directory' url. :param str directory: The URL to the ACME v2 directory. :return: When operation is done. :rtype: Deferred[None] """ # Provide invalid new_nonce_url & kid, but don't expose it to the # caller. self = cls(agent, key, alg, None, None) def cb_extract_new_nonce(directory): try: self._new_nonce = directory.newNonce except AttributeError: raise errors.ClientError('Directory has no newNonce URL', directory) return directory return (self.get(directory).addCallback(json_content).addCallback( messages.Directory.from_json).addCallback(cb_extract_new_nonce)) @classmethod def _check_response(cls, response, content_type=JSON_CONTENT_TYPE): """ Check response content and its type. .. note:: Unlike :mod:content_type`acme.client`, checking is strict. :param bytes content_type: Expected Content-Type response header. If the response Content-Type does not match, :exc:`ClientError` is raised. :raises .ServerError: If server response body carries HTTP Problem (draft-ietf-appsawg-http-problem-00). :raises ~acme.errors.ClientError: In case of other networking errors. """ def _got_failure(f): f.trap(ValueError) return None def _got_json(jobj): if 400 <= response.code < 600: if (response_ct.lower().startswith(JSON_ERROR_CONTENT_TYPE) and jobj is not None): raise ServerError(messages.Error.from_json(jobj), response) else: # response is not JSON object return _fail_and_consume( response, errors.ClientError('Response is not JSON.')) elif content_type not in response_ct.lower(): return _fail_and_consume( response, errors.ClientError( 'Unexpected response Content-Type: {0!r}. ' 'Expecting {1!r}.'.format(response_ct, content_type))) elif JSON_CONTENT_TYPE in content_type.lower() and jobj is None: return _fail_and_consume( response, errors.ClientError('Missing JSON body.')) return response response_ct = response.headers.getRawHeaders(b'Content-Type', [None])[0] action = LOG_JWS_CHECK_RESPONSE(expected_content_type=content_type, response_content_type=response_ct) with action.context(): # TODO: response.json() is called twice, once here, and # once in _get and _post clients return (DeferredContext(response.json()).addErrback( _got_failure).addCallback(_got_json).addActionFinish()) def _send_request(self, method, url, *args, **kwargs): """ Send HTTP request. :param str method: The HTTP method to use. :param str url: The URL to make the request to. :return: Deferred firing with the HTTP response. """ if self._current_request is not None: return defer.fail(RuntimeError('Overlapped HTTP request')) def cb_request_done(result): """ Called when we got a response from the request. """ self._current_request = None return result action = LOG_JWS_REQUEST(url=url) with action.context(): headers = kwargs.setdefault('headers', Headers()) headers.setRawHeaders(b'user-agent', [self._user_agent]) kwargs.setdefault('timeout', self.timeout) self._current_request = self._treq.request(method, url, *args, **kwargs) return (DeferredContext(self._current_request).addCallback( cb_request_done).addCallback( tap(lambda r: action.add_success_fields( code=r.code, content_type=r.headers.getRawHeaders( b'content-type', [None])[0]))).addActionFinish()) def stop(self): """ Stops the operation. This cancels pending operations and does cleanup. :return: A deferred which fires when the client is stopped. """ if self._current_request is not None: self._current_request.addErrback(lambda _: None) self._current_request.cancel() self._current_request = None agent_pool = getattr(self._agent, '_pool', None) if agent_pool: return agent_pool.closeCachedConnections() return defer.succeed(None) def head(self, url, *args, **kwargs): """ Send HEAD request without checking the response. Note that ``_check_response`` is not called, as there will be no response body to check. :param str url: The URL to make the request to. """ with LOG_JWS_HEAD().context(): return DeferredContext( self._send_request(u'HEAD', url, *args, **kwargs)).addActionFinish() def get(self, url, content_type=JSON_CONTENT_TYPE, **kwargs): """ Send GET request and check response. :param str method: The HTTP method to use. :param str url: The URL to make the request to. :raises txacme.client.ServerError: If server response body carries HTTP Problem (draft-ietf-appsawg-http-problem-00). :raises acme.errors.ClientError: In case of other protocol errors. :return: Deferred firing with the checked HTTP response. """ with LOG_JWS_GET().context(): return (DeferredContext(self._send_request( u'GET', url, **kwargs)).addCallback( self._check_response, content_type=content_type).addActionFinish()) def _add_nonce(self, response): """ Store a nonce from a response we received. :param twisted.web.iweb.IResponse response: The HTTP response. :return: The response, unmodified. """ nonce = response.headers.getRawHeaders(REPLAY_NONCE_HEADER, [None])[0] with LOG_JWS_ADD_NONCE(raw_nonce=nonce) as action: if nonce is None: return _fail_and_consume( response, errors.ClientError(str(errors.MissingNonce(response))), ) else: try: decoded_nonce = Header._fields['nonce'].decode( nonce.decode('ascii')) action.add_success_fields(nonce=decoded_nonce) except DeserializationError as error: return _fail_and_consume(response, errors.BadNonce(nonce, error)) self._nonces.add(decoded_nonce) return response def _get_nonce(self, url): """ Get a nonce to use in a request, removing it from the nonces on hand. """ action = LOG_JWS_GET_NONCE() if len(self._nonces) > 0: with action: nonce = self._nonces.pop() action.add_success_fields(nonce=nonce) return defer.succeed(nonce) else: with action.context(): return (DeferredContext(self.head( self._new_nonce)).addCallback(self._add_nonce).addCallback( lambda _: self._nonces.pop()).addCallback( tap(lambda nonce: action.add_success_fields( nonce=nonce))).addActionFinish()) def _post(self, url, obj, content_type, response_type=JSON_CONTENT_TYPE, kid=None, **kwargs): """ POST an object and check the response. :param str url: The URL to request. :param ~josepy.interfaces.JSONDeSerializable obj: The serializable payload of the request. :param bytes content_type: The expected content type of the response. :raises txacme.client.ServerError: If server response body carries HTTP Problem (draft-ietf-appsawg-http-problem-00). :raises acme.errors.ClientError: In case of other protocol errors. """ if kid is None: kid = self._kid def cb_wrap_in_jws(nonce): with LOG_JWS_SIGN(key_type=self._key.typ, alg=self._alg.name, nonce=nonce): if obj is None: jobj = b'' else: jobj = obj.json_dumps().encode() result = (JWS.sign( payload=jobj, key=self._key, alg=self._alg, nonce=nonce, url=url, kid=kid, ).json_dumps().encode()) return result with LOG_JWS_POST().context(): headers = kwargs.setdefault('headers', Headers()) headers.setRawHeaders(b'content-type', [JOSE_CONTENT_TYPE]) return (DeferredContext(self._get_nonce(url)).addCallback( cb_wrap_in_jws).addCallback(lambda data: self._send_request( u'POST', url, data=data, **kwargs)).addCallback( self._add_nonce).addCallback( self._check_response, content_type=response_type).addActionFinish()) def post(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs): """ POST an object and check the response. Retry once if a badNonce error is received. :param str url: The URL to request. :param ~josepy.interfaces.JSONDeSerializable obj: The serializable payload of the request. :param bytes content_type: The expected content type of the response. By default, JSON. :raises txacme.client.ServerError: If server response body carries HTTP Problem (draft-ietf-appsawg-http-problem-00). :raises acme.errors.ClientError: In case of other protocol errors. """ def retry_bad_nonce(f): f.trap(ServerError) # The current RFC draft defines the namespace as # urn:ietf:params:acme:error:<code>, but earlier drafts (and some # current implementations) use urn:acme:error:<code> instead. We # don't really care about the namespace here, just the error code. if f.value.message.typ.split(':')[-1] == 'badNonce': # If one nonce is bad, others likely are too. Let's clear them # and re-add the one we just got. self._nonces.clear() self._add_nonce(f.value.response) return self._post(url, obj, content_type, **kwargs) return f return (self._post(url, obj, content_type, **kwargs).addErrback(retry_bad_nonce))
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch('treq.client.FileBodyProducer') self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer') self.MultiPartProducer = self.mbp_patcher.start() self.addCleanup(self.mbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_post(self): self.client.post('http://example.com/') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_idn(self): self.client.request('GET', u'http://‽.net') self.agent.request.assert_called_once_with( b'GET', b'http://xn--fwg.net', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_case_insensitive_methods(self): self.client.request('gEt', 'http://example.com/') self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': ['bar']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_tuple_query_values(self): self.client.request('GET', 'http://example.com/', params={'foo': ('bar', )}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_merge_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_merge_tuple_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_dict_single_value_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_data_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar&foo=baz') def test_request_data_single_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_tuple(self): self.client.request('POST', 'http://example.com/', data=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "wb") as temp_file: temp_file.write(b'hello') self.client.request('POST', 'http://example.com/', data=open(temp_fn, 'rb')) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'hello') def test_request_json_dict(self): self.client.request('POST', 'http://example.com/', json={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'{"foo":"bar"}') def test_request_json_tuple(self): self.client.request('POST', 'http://example.com/', json=('foo', 1)) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'["foo",1]') def test_request_json_number(self): self.client.request('POST', 'http://example.com/', json=1.) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'1.0') def test_request_json_string(self): self.client.request('POST', 'http://example.com/', json='hello') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'"hello"') def test_request_json_bool(self): self.client.request('POST', 'http://example.com/', json=True) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'true') def test_request_json_none(self): self.client.request('POST', 'http://example.com/', json=None) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'null') @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_no_name_attachment(self): self.client.request('POST', 'http://example.com/', files={"name": BytesIO(b"hello")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('name', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment(self): self.client.request('POST', 'http://example.com/', files={"name": ('image.jpg', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('name', ('image.jpg', 'image/jpeg', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment_and_ctype(self): self.client.request( 'POST', 'http://example.com/', files={"name": ('image.jpg', 'text/plain', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('name', ('image.jpg', 'text/plain', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params(self): class NamedFile(BytesIO): def __init__(self, val): BytesIO.__init__(self, val) self.name = "image.png" self.client.request('POST', 'http://example.com/', data=[("a", "b"), ("key", "val")], files=[("file1", ('image.jpg', BytesIO(b"hello"))), ("file2", NamedFile(b"yo"))]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('a', 'b'), ('key', 'val'), ('file1', ('image.jpg', 'image/jpeg', FP)), ('file2', ('image.png', 'image/png', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params_dict(self): self.client.request('POST', 'http://example.com/', data={ "key": "a", "key2": "b" }, files={"file1": BytesIO(b"hey")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('key', 'a'), ('key2', 'b'), ('file1', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) def test_request_unsupported_params_combination(self): self.assertRaises(ValueError, self.client.request, 'POST', 'http://example.com/', data=BytesIO(b"yo"), files={"file1": BytesIO(b"hey")}) def test_request_dict_headers(self): self.client.request('GET', 'http://example.com/', headers={ 'User-Agent': 'treq/0.1dev', 'Accept': ['application/json', 'text/plain'] }) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({ b'User-Agent': [b'treq/0.1dev'], b'accept-encoding': [b'gzip'], b'Accept': [b'application/json', b'text/plain'] }), None) @with_clock def test_request_timeout_fired(self, clock): """ Verify the request is cancelled if a response is not received within specified timeout period. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate we haven't gotten a response within timeout seconds clock.advance(3) # a deferred should have been cancelled self.failureResultOf(d, CancelledError) @with_clock def test_request_timeout_cancelled(self, clock): """ Verify timeout is cancelled if a response is received before timeout period elapses. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate a response d.callback(mock.Mock(code=200, headers=Headers({}))) # now advance the clock but since we already got a result, # a cancellation timer should have been cancelled clock.advance(3) self.successResultOf(d) def test_response_is_buffered(self): response = mock.Mock(deliverBody=mock.Mock(), headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com') result = self.successResultOf(d) protocol = mock.Mock(Protocol) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) def test_response_buffering_is_disabled_with_unbufferred_arg(self): response = mock.Mock(headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com', unbuffered=True) # YOLO public attribute. self.assertEqual(self.successResultOf(d).original, response) def test_request_post_redirect_denied(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) d = self.client.post('http://www.example.com') self.failureResultOf(d, ResponseFailed) def test_request_browser_like_redirects(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) raw = mock.Mock(return_value=[]) final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw)) with mock.patch('twisted.web.client.RedirectAgent._handleRedirect', return_value=final_resp): d = self.client.post('http://www.google.com', browser_like_redirects=True, unbuffered=True) self.assertEqual(self.successResultOf(d).original, final_resp)
def reverse_proxy(self, request, protected=True): if protected: sess = request.getSession() valid_sessions = self.valid_sessions sess_uid = sess.uid username = valid_sessions[sess_uid]['username'] # Normal reverse proxying. kwds = {} cookiejar = {} kwds['allow_redirects'] = False kwds['cookies'] = cookiejar req_headers = self.mod_headers( dict(request.requestHeaders.getAllRawHeaders())) kwds['headers'] = req_headers if protected: kwds['headers'][self.remoteUserHeader] = [username] if request.method in ('PUT', 'POST'): kwds['data'] = request.content.read() url = self.proxied_url + request.uri # Determine if a plugin wants to intercept this URL. interceptors = self.interceptors for interceptor in interceptors: if interceptor.should_resource_be_intercepted( url, request.method, req_headers, request): return interceptor.handle_resource(url, request.method, req_headers, request) # Check if this is a request for a websocket. d = self.checkForWebsocketUpgrade(request) if d is not None: return d # Typical reverse proxying. self.log("Proxying URL => {0}".format(url)) http_client = HTTPClient(self.proxy_agent) d = http_client.request(request.method, url, **kwds) def process_response(response, request): req_resp_headers = request.responseHeaders resp_code = response.code resp_headers = response.headers resp_header_map = dict(resp_headers.getAllRawHeaders()) # Rewrite Location headers for redirects as required. if resp_code in (301, 302, 303, 307, 308) and "Location" in resp_header_map: values = resp_header_map["Location"] if len(values) == 1: location = values[0] if request.isSecure(): proxy_scheme = 'https' else: proxy_scheme = 'http' new_location = self.proxied_url_to_proxy_url( proxy_scheme, location) if new_location is not None: resp_header_map['Location'] = [new_location] request.setResponseCode(response.code, message=response.phrase) for k, v in resp_header_map.iteritems(): if k == 'Set-Cookie': v = self.mod_cookies(v) req_resp_headers.setRawHeaders(k, v) return response def mod_content(body, request): """ Modify response content before returning it to the user agent. """ d = None for content_modifier in self.content_modifiers: if d is None: d = content_modifier.transform_content(body, request) else: d.addCallback(content_modifier.transform_content, request) if d is None: return body else: return d d.addCallback(process_response, request) d.addCallback(treq.content) d.addCallback(mod_content, request) return d
def http_dlr_callback(self, message): msgid = message.content.properties['message-id'] url = message.content.properties['headers']['url'] method = message.content.properties['headers']['method'] level = message.content.properties['headers']['level'] self.log.debug('Got one message (msgid:%s) to throw', msgid) # If any, clear requeuing timer self.clearRequeueTimer(msgid) # Build mandatory arguments args = { 'id': msgid, 'level': level, 'message_status': message.content.properties['headers']['message_status'], 'connector': message.content.properties['headers']['connector'] } # Level 2 extra args if level in [2, 3]: args['id_smsc'] = message.content.properties['headers']['id_smsc'] args['sub'] = message.content.properties['headers']['sub'] args['dlvrd'] = message.content.properties['headers']['dlvrd'] args['subdate'] = message.content.properties['headers']['subdate'] args['donedate'] = message.content.properties['headers']['donedate'] args['err'] = message.content.properties['headers']['err'] args['text'] = message.content.properties['headers']['text'] try: # Throw the message to http endpoint postdata = None params = None baseurl = url if method == 'GET': params = args else: postdata = args self.log.debug('Calling %s with args %s using %s method.', baseurl, args, method) agent = Agent(reactor) client = HTTPClient(agent) response = yield client.request( method, baseurl, params=params, data=postdata, timeout=self.config.timeout, agent='Jasmin gateway/1.0 %s' % self.name, headers={'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'text/plain'}) self.log.info('Throwed DLR [msgid:%s] to %s.', msgid, baseurl) content = yield text_content(response) if response.code >= 400: raise HttpApiError(response.code, content) self.log.debug('Destination end replied to message [msgid:%s]: %r', msgid, content) # Check for acknowledgement if content.strip() != 'ACK': raise MessageAcknowledgementError( 'Destination end did not acknowledge receipt of the DLR message.') # Everything is okay ? then: yield self.ackMessage(message) except Exception as e: self.log.error('Throwing HTTP/DLR [msgid:%s] to (%s): %r.', msgid, baseurl, e) # List of errors after which, no further retrying shall be made noRetryErrors = ['404'] # Requeue message for later retry if (str(e) not in noRetryErrors and self.getThrowingRetrials(message) <= self.config.max_retries): self.log.debug('Message try-count is %s [msgid:%s]: requeuing', self.getThrowingRetrials(message), msgid) yield self.rejectAndRequeueMessage(message) elif str(e) in noRetryErrors: self.log.warn('Message is no more processed after receiving "%s" error', str(e)) yield self.rejectMessage(message) else: self.log.warn('Message try-count is %s [msgid:%s]: purged from queue', self.getThrowingRetrials(message), msgid) yield self.rejectMessage(message)
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch('treq.client.FileBodyProducer') self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer') self.MultiPartProducer = self.mbp_patcher.start() self.addCleanup(self.mbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_request_case_insensitive_methods(self): self.client.request('gEt', 'http://example.com/') self.agent.request.assert_called_once_with( 'GET', 'http://example.com/', headers=Headers({}), bodyProducer=None) def test_request_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': ['bar']}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_tuple_query_values(self): self.client.request('GET', 'http://example.com/', params={'foo': ('bar',)}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_merge_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?baz=bax&foo=bar&foo=baz', headers=Headers({}), bodyProducer=None) def test_request_merge_tuple_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params=[('foo', 'bar')]) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?baz=bax&foo=bar', headers=Headers({}), bodyProducer=None) def test_request_dict_single_value_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_data_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar&foo=baz') def test_request_data_single_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': 'bar'}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar') def test_request_data_tuple(self): self.client.request('POST', 'http://example.com/', data=[('foo', 'bar')]) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar') def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "w") as temp_file: temp_file.write('hello') self.client.request('POST', 'http://example.com/', data=file(temp_fn)) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('hello') @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_no_name_attachment(self): self.client.request( 'POST', 'http://example.com/', files={"name": StringIO("hello")}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', (None, 'application/octet-stream', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', StringIO("hello"))}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'image/jpeg', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment_and_ctype(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', 'text/plain', StringIO("hello"))}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'text/plain', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params(self): class NamedFile(StringIO): def __init__(self, val): StringIO.__init__(self, val) self.name = "image.png" self.client.request( 'POST', 'http://example.com/', data=[("a", "b"), ("key", "val")], files=[ ("file1", ('image.jpg', StringIO("hello"))), ("file2", NamedFile("yo"))]) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('a', 'b'), ('key', 'val'), ('file1', ('image.jpg', 'image/jpeg', FP)), ('file2', ('image.png', 'image/png', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params_dict(self): self.client.request( 'POST', 'http://example.com/', data={"key": "a", "key2": "b"}, files={"file1": StringIO("hey")}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('key', 'a'), ('key2', 'b'), ('file1', (None, 'application/octet-stream', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) def test_request_unsupported_params_combination(self): self.assertRaises(ValueError, self.client.request, 'POST', 'http://example.com/', data=StringIO("yo"), files={"file1": StringIO("hey")}) def test_request_dict_headers(self): self.client.request('GET', 'http://example.com/', headers={ 'User-Agent': 'treq/0.1dev', 'Accept': ['application/json', 'text/plain'] }) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/', headers=Headers({'User-Agent': ['treq/0.1dev'], 'Accept': ['application/json', 'text/plain']}), bodyProducer=None)
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch("treq.client.FileBodyProducer") self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) self.mbp_patcher = mock.patch("treq.multipart.MultiPartProducer") self.MultiPartProducer = self.mbp_patcher.start() self.addCleanup(self.mbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_request_case_insensitive_methods(self): self.client.request("gEt", "http://example.com/") self.agent.request.assert_called_once_with("GET", "http://example.com/", headers=Headers({}), bodyProducer=None) def test_request_query_params(self): self.client.request("GET", "http://example.com/", params={"foo": ["bar"]}) self.agent.request.assert_called_once_with( "GET", "http://example.com/?foo=bar", headers=Headers({}), bodyProducer=None ) def test_request_tuple_query_values(self): self.client.request("GET", "http://example.com/", params={"foo": ("bar",)}) self.agent.request.assert_called_once_with( "GET", "http://example.com/?foo=bar", headers=Headers({}), bodyProducer=None ) def test_request_merge_query_params(self): self.client.request("GET", "http://example.com/?baz=bax", params={"foo": ["bar", "baz"]}) self.agent.request.assert_called_once_with( "GET", "http://example.com/?baz=bax&foo=bar&foo=baz", headers=Headers({}), bodyProducer=None ) def test_request_merge_tuple_query_params(self): self.client.request("GET", "http://example.com/?baz=bax", params=[("foo", "bar")]) self.agent.request.assert_called_once_with( "GET", "http://example.com/?baz=bax&foo=bar", headers=Headers({}), bodyProducer=None ) def test_request_dict_single_value_query_params(self): self.client.request("GET", "http://example.com/", params={"foo": "bar"}) self.agent.request.assert_called_once_with( "GET", "http://example.com/?foo=bar", headers=Headers({}), bodyProducer=None ) def test_request_data_dict(self): self.client.request("POST", "http://example.com/", data={"foo": ["bar", "baz"]}) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["application/x-www-form-urlencoded"]}), bodyProducer=self.FileBodyProducer.return_value, ) self.assertBody("foo=bar&foo=baz") def test_request_data_single_dict(self): self.client.request("POST", "http://example.com/", data={"foo": "bar"}) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["application/x-www-form-urlencoded"]}), bodyProducer=self.FileBodyProducer.return_value, ) self.assertBody("foo=bar") def test_request_data_tuple(self): self.client.request("POST", "http://example.com/", data=[("foo", "bar")]) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["application/x-www-form-urlencoded"]}), bodyProducer=self.FileBodyProducer.return_value, ) self.assertBody("foo=bar") def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "w") as temp_file: temp_file.write("hello") self.client.request("POST", "http://example.com/", data=file(temp_fn)) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({}), bodyProducer=self.FileBodyProducer.return_value ) self.assertBody("hello") @mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid")) def test_request_no_name_attachment(self): self.client.request("POST", "http://example.com/", files={"name": StringIO("hello")}) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}), bodyProducer=self.MultiPartProducer.return_value, ) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([("name", (None, "application/octet-stream", FP))], boundary="heyDavid"), self.MultiPartProducer.call_args, ) @mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid")) def test_request_named_attachment(self): self.client.request("POST", "http://example.com/", files={"name": ("image.jpg", StringIO("hello"))}) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}), bodyProducer=self.MultiPartProducer.return_value, ) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([("name", ("image.jpg", "image/jpeg", FP))], boundary="heyDavid"), self.MultiPartProducer.call_args, ) @mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid")) def test_request_named_attachment_and_ctype(self): self.client.request( "POST", "http://example.com/", files={"name": ("image.jpg", "text/plain", StringIO("hello"))} ) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}), bodyProducer=self.MultiPartProducer.return_value, ) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([("name", ("image.jpg", "text/plain", FP))], boundary="heyDavid"), self.MultiPartProducer.call_args, ) @mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid")) def test_request_mixed_params(self): class NamedFile(StringIO): def __init__(self, val): StringIO.__init__(self, val) self.name = "image.png" self.client.request( "POST", "http://example.com/", data=[("a", "b"), ("key", "val")], files=[("file1", ("image.jpg", StringIO("hello"))), ("file2", NamedFile("yo"))], ) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}), bodyProducer=self.MultiPartProducer.return_value, ) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [ ("a", "b"), ("key", "val"), ("file1", ("image.jpg", "image/jpeg", FP)), ("file2", ("image.png", "image/png", FP)), ], boundary="heyDavid", ), self.MultiPartProducer.call_args, ) @mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid")) def test_request_mixed_params_dict(self): self.client.request( "POST", "http://example.com/", data={"key": "a", "key2": "b"}, files={"file1": StringIO("hey")} ) self.agent.request.assert_called_once_with( "POST", "http://example.com/", headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}), bodyProducer=self.MultiPartProducer.return_value, ) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [("key", "a"), ("key2", "b"), ("file1", (None, "application/octet-stream", FP))], boundary="heyDavid" ), self.MultiPartProducer.call_args, ) def test_request_unsupported_params_combination(self): self.assertRaises( ValueError, self.client.request, "POST", "http://example.com/", data=StringIO("yo"), files={"file1": StringIO("hey")}, ) def test_request_dict_headers(self): self.client.request( "GET", "http://example.com/", headers={"User-Agent": "treq/0.1dev", "Accept": ["application/json", "text/plain"]}, ) self.agent.request.assert_called_once_with( "GET", "http://example.com/", headers=Headers({"User-Agent": ["treq/0.1dev"], "Accept": ["application/json", "text/plain"]}), bodyProducer=None, ) @with_clock def test_request_timeout_fired(self, clock): """ Verify the request is cancelled if a response is not received within specified timeout period. """ self.client.request("GET", "http://example.com", timeout=2) # simulate we haven't gotten a response within timeout seconds clock.advance(3) deferred = self.agent.request.return_value # a deferred should have been cancelled self.assertTrue(deferred.cancel.called) @with_clock def test_request_timeout_cancelled(self, clock): """ Verify timeout is cancelled if a response is received before timeout period elapses. """ self.client.request("GET", "http://example.com", timeout=2) # simulate a response deferred = self.agent.request.return_value gotResult = deferred.addBoth.call_args[0][0] gotResult("result") # now advance the clock but since we already got a result, # a cancellation timer should have been cancelled clock.advance(3) self.assertFalse(deferred.cancel.called) def test_response_is_buffered(self): response = mock.Mock(deliverBody=mock.Mock()) self.agent.request.return_value = succeed(response) d = self.client.get("http://www.example.com") result = self.successResultOf(d) protocol = mock.Mock(Protocol) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) def test_response_buffering_is_disabled_with_unbufferred_arg(self): response = mock.Mock() self.agent.request.return_value = succeed(response) d = self.client.get("http://www.example.com", unbuffered=True) self.assertEqual(self.successResultOf(d), response)
def reverse_proxy(self, request, protected=True): if protected: sess = request.getSession() valid_sessions = self.valid_sessions sess_uid = sess.uid username = valid_sessions[sess_uid]['username'] # Normal reverse proxying. kwds = {} #cookiejar = cookielib.CookieJar() cookiejar = {} kwds['allow_redirects'] = False kwds['cookies'] = cookiejar req_headers = self.mod_headers(dict(request.requestHeaders.getAllRawHeaders())) kwds['headers'] = req_headers if protected: kwds['headers']['REMOTE_USER'] = [username] #print "** HEADERS **" #pprint.pprint(self.mod_headers(dict(request.requestHeaders.getAllRawHeaders()))) #print if request.method in ('PUT', 'POST'): kwds['data'] = request.content.read() #print "request.method", request.method #print "url", self.proxied_url + request.uri #print "kwds:" #pprint.pprint(kwds) #print url = self.proxied_url + request.uri # Determine if a plugin wants to intercept this URL. interceptors = self.interceptors for interceptor in interceptors: if interceptor.should_resource_be_intercepted(url, request.method, req_headers, request): return interceptor.handle_resource(url, request.method, req_headers, request) log.msg("[INFO] Proxying URL: %s" % url) http_client = HTTPClient(self.agent) d = http_client.request(request.method, url, **kwds) #print "** Requesting %s %s" % (request.method, self.proxied_url + request.uri) def process_response(response, request): req_resp_headers = request.responseHeaders resp_code = response.code resp_headers = response.headers resp_header_map = dict(resp_headers.getAllRawHeaders()) # Rewrite Location headers for redirects as required. if resp_code in (301, 302, 303, 307, 308) and "Location" in resp_header_map: values = resp_header_map["Location"] if len(values) == 1: location = values[0] if request.isSecure(): proxy_scheme = 'https' else: proxy_scheme = 'http' new_location = self.proxied_url_to_proxy_url(proxy_scheme, location) if new_location is not None: resp_header_map['Location'] = [new_location] log.msg("[DEBUG] Re-wrote Location header: '%s' => '%s'" % (location, new_location)) request.setResponseCode(response.code, message=response.phrase) for k,v in resp_header_map.iteritems(): if k == 'Set-Cookie': v = self.mod_cookies(v) print("Browser Response >>> Setting response header: %s: %s" % (k, v)) req_resp_headers.setRawHeaders(k, v) return response def show_cookies(resp): jar = resp.cookies() print("Cookie Jar:") pprint.pprint(cookiejar) print("") return resp def mod_content(body, request): """ Modify response content before returning it to the user agent. """ d = None for content_modifier in self.content_modifiers: if d is None: d = content_modifier.transform_content(body, request) else: d.addCallback(content_modifier.transform_content, request) if d is None: return body else: return d d.addCallback(show_cookies) d.addCallback(process_response, request) d.addCallback(treq.content) d.addCallback(mod_content, request) return d
class HTTPClient(object): def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, contextFactory=None, **kwargs): self.host = host self.port = port self.scheme = scheme self.base_uri = '%s://%s:%s' % (self.scheme, self.host, self.port) agent_kwargs = dict( reactor=reactor, pool=HTTPConnectionPool(reactor), **kwargs) if contextFactory is not None: # use the provided context factory agent_kwargs['contextFactory'] = contextFactory elif not verify: # if no context is provided and verify is set to false, use the # insecure context factory implementation agent_kwargs['contextFactory'] = InsecureContextFactory() self.client = TreqHTTPClient(Agent(**agent_kwargs)) def uri(self, path, params=None): uri = self.base_uri + path if not params: return uri return '%s?%s' % (uri, urllib.parse.urlencode(params)) @staticmethod def response(code, headers, text): return base.Response(code, headers, text) @staticmethod def compat_string(value): """ Provide a python2/3 compatible string representation of the value :type value: :rtype : """ if isinstance(value, bytes): return value.decode(encoding='utf-8') return str(value) @inlineCallbacks def _get_resp(self, response): # Merge multiple header values as per RFC2616 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 headers = { self.compat_string(k): ','.join(map(self.compat_string, v)) for k, v in dict(response.headers.getAllRawHeaders()).items() } body = yield response.text(encoding='utf-8') returnValue((response.code, headers, body)) @inlineCallbacks def request(self, callback, method, url, **kwargs): if 'data' in kwargs and not isinstance(kwargs['data'], bytes): # python2/3 compatibility data = kwargs.pop('data') kwargs['data'] = data.encode(encoding='utf-8') \ if hasattr(data, 'encode') else b(data) try: response = yield self.client.request(method, url, **kwargs) parsed = yield self._get_resp(response) returnValue(callback(self.response(*parsed))) except ConnectError as e: raise ConsulException( '{}: {}'.format(e.__class__.__name__, e.message)) except ResponseNeverReceived: # this exception is raised if the connection to the server is lost # when yielding a response, this could be due to network issues or # server restarts raise ConsulException( 'Server connection lost: {} {}'.format(method.upper(), url)) except RequestTransmissionFailed: # this exception is expected if the reactor is stopped mid request raise ConsulException( 'Request incomplete: {} {}'.format(method.upper(), url)) @inlineCallbacks def get(self, callback, path, params=None): uri = self.uri(path, params) response = yield self.request(callback, 'get', uri, params=params) returnValue(response) @inlineCallbacks def put(self, callback, path, params=None, data=''): uri = self.uri(path, params) response = yield self.request(callback, 'put', uri, data=data) returnValue(response) @inlineCallbacks def post(self, callback, path, params=None, data=''): uri = self.uri(path, params) response = yield self.request(callback, 'post', uri, data=data) returnValue(response) @inlineCallbacks def delete(self, callback, path, params=None): uri = self.uri(path, params) response = yield self.request(callback, 'delete', uri, params=params) returnValue(response)
class HTTPClient(object): def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True): self.host = host self.port = port self.scheme = scheme self.base_uri = '%s://%s:%s' % (self.scheme, self.host, self.port) self.verify = SSLSpec.CERT_NONE \ if not verify else SSLSpec.CERT_REQUIRED agent = Agent(reactor=reactor, pool=HTTPConnectionPool(reactor), contextFactory=AsyncClientSSLContextFactory( verify=self.verify)) self.client = TreqHTTPClient(agent) def uri(self, path, params=None): uri = self.base_uri + path if not params: return uri return '%s?%s' % (uri, urllib.parse.urlencode(params)) def response(self, code, headers, text): return base.Response(code, headers, text) @inlineCallbacks def _get_resp(self, response): # Merge multiple header values as per RFC2616 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 headers = { k: ','.join(v) for k, v in dict(response.headers.getAllRawHeaders()).items() } body = yield response.text(encoding='utf-8') returnValue((response.code, headers, body)) @inlineCallbacks def request(self, callback, method, url, **kwargs): try: response = yield self.client.request(method, url, **kwargs) parsed = yield self._get_resp(response) returnValue(callback(self.response(*parsed))) except ConnectError as e: raise ConsulException( '{}: {}'.format(e.__class__.__name__, e.message)) @inlineCallbacks def get(self, callback, path, params=None): uri = self.uri(path, params) response = yield self.request(callback, 'get', uri, params=params) returnValue(response) @inlineCallbacks def put(self, callback, path, params=None, data=''): uri = self.uri(path, params) # handle unicode data failure in the zope interface # twisted.web.iweb.IBodyProducer data = str(data) response = yield self.request(callback, 'put', uri, data=data) returnValue(response) @inlineCallbacks def delete(self, callback, path, params=None): uri = self.uri(path, params) response = yield self.request(callback, 'delete', uri, params=params) returnValue(response)
class HTTPClient(base.HTTPClient): def __init__(self, contextFactory, *args, **kwargs): super(HTTPClient, self).__init__(*args, **kwargs) agent_kwargs = dict(reactor=reactor, pool=HTTPConnectionPool(reactor)) if contextFactory is not None: # use the provided context factory agent_kwargs['contextFactory'] = contextFactory elif not self.verify: # if no context is provided and verify is set to false, use the # insecure context factory implementation agent_kwargs['contextFactory'] = InsecureContextFactory() self.client = TreqHTTPClient(Agent(**agent_kwargs)) @staticmethod def response(code, headers, text): return base.Response(code, headers, text) @staticmethod def compat_string(value): """ Provide a python2/3 compatible string representation of the value :type value: :rtype : """ if isinstance(value, bytes): return value.decode(encoding='utf-8') return str(value) @inlineCallbacks def _get_resp(self, response): # Merge multiple header values as per RFC2616 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 headers = dict([ (self.compat_string(k), ','.join(map(self.compat_string, v))) for k, v in dict(response.headers.getAllRawHeaders()).items() ]) body = yield response.text(encoding='utf-8') returnValue((response.code, headers, body)) @inlineCallbacks def request(self, callback, method, url, **kwargs): if 'data' in kwargs and not isinstance(kwargs['data'], bytes): # python2/3 compatibility data = kwargs.pop('data') kwargs['data'] = data.encode(encoding='utf-8') \ if hasattr(data, 'encode') else bytes(data) try: response = yield self.client.request(method, url, **kwargs) parsed = yield self._get_resp(response) returnValue(callback(self.response(*parsed))) except ConnectError as e: raise ConsulException('{}: {}'.format(e.__class__.__name__, e.message)) except ResponseNeverReceived: # this exception is raised if the connection to the server is lost # when yielding a response, this could be due to network issues or # server restarts raise ConsulException('Server connection lost: {} {}'.format( method.upper(), url)) except RequestTransmissionFailed: # this exception is expected if the reactor is stopped mid request raise ConsulException('Request incomplete: {} {}'.format( method.upper(), url)) @inlineCallbacks def get(self, callback, path, params=None): uri = self.uri(path, params) response = yield self.request(callback, 'get', uri, params=params) returnValue(response) @inlineCallbacks def put(self, callback, path, params=None, data=''): uri = self.uri(path, params) response = yield self.request(callback, 'put', uri, data=data) returnValue(response) @inlineCallbacks def post(self, callback, path, params=None, data=''): uri = self.uri(path, params) response = yield self.request(callback, 'post', uri, data=data) returnValue(response) @inlineCallbacks def delete(self, callback, path, params=None): uri = self.uri(path, params) response = yield self.request(callback, 'delete', uri, params=params) returnValue(response) @inlineCallbacks def close(self): pass
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch('treq.client.FileBodyProducer') self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer') self.MultiPartProducer = self.mbp_patcher.start() self.addCleanup(self.mbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_post(self): self.client.post('http://example.com/') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_idn(self): self.client.request('GET', u'http://č.net') self.agent.request.assert_called_once_with( b'GET', b'http://xn--bea.net', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_decodedurl(self): """ A URL may be passed as a `hyperlink.DecodedURL` object. It is converted to bytes when passed to the underlying agent. """ url = DecodedURL.from_text(u"https://example.org/foo") self.client.request("GET", url) self.agent.request.assert_called_once_with( b"GET", b"https://example.org/foo", Headers({b"accept-encoding": [b"gzip"]}), None, ) def test_request_uri_encodedurl(self): """ A URL may be passed as a `hyperlink.EncodedURL` object. It is converted to bytes when passed to the underlying agent. """ url = EncodedURL.from_text(u"https://example.org/foo") self.client.request("GET", url) self.agent.request.assert_called_once_with( b"GET", b"https://example.org/foo", Headers({b"accept-encoding": [b"gzip"]}), None, ) def test_request_uri_idn_params(self): """ A URL that contains non-ASCII characters can be augmented with querystring parameters. This reproduces treq #264. """ self.client.request('GET', u'http://č.net', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'GET', b'http://xn--bea.net/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_hyperlink_params(self): """ The *params* argument augments an instance of `hyperlink.DecodedURL` passed as the *url* parameter, just as if it were a string. """ self.client.request( method="GET", url=DecodedURL.from_text(u"http://č.net"), params={"foo": "bar"}, ) self.agent.request.assert_called_once_with( b"GET", b"http://xn--bea.net/?foo=bar", Headers({b"accept-encoding": [b"gzip"]}), None, ) def test_request_case_insensitive_methods(self): self.client.request('gEt', 'http://example.com/') self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': ['bar']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_tuple_query_values(self): self.client.request('GET', 'http://example.com/', params={'foo': ('bar',)}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_tuple_query_value_coercion(self): """ treq coerces non-string values passed to *params* like `urllib.urlencode()` """ self.client.request('GET', 'http://example.com/', params=[ ('text', u'A\u03a9'), ('text-seq', [u'A\u03a9']), ('bytes', [b'ascii']), ('bytes-seq', [b'ascii']), ('native', ['native']), ('native-seq', ['aa', 'bb']), ('int', 1), ('int-seq', (1, 2, 3)), ('none', None), ('none-seq', [None, None]), ]) self.agent.request.assert_called_once_with( b'GET', ( b'http://example.com/?' b'text=A%CE%A9&text-seq=A%CE%A9' b'&bytes=ascii&bytes-seq=ascii' b'&native=native&native-seq=aa&native-seq=bb' b'&int=1&int-seq=1&int-seq=2&int-seq=3' b'&none=None&none-seq=None&none-seq=None' ), Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_tuple_query_param_coercion(self): """ treq coerces non-string param names passed to *params* like `urllib.urlencode()` """ self.client.request('GET', 'http://example.com/', params=[ (u'text', u'A\u03a9'), (b'bytes', ['ascii']), ('native', 'native'), (1, 'int'), (None, ['none']), ]) self.agent.request.assert_called_once_with( b'GET', ( b'http://example.com/' b'?text=A%CE%A9&bytes=ascii' b'&native=native&1=int&None=none' ), Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_query_param_seps(self): """ When the characters ``&`` and ``#`` are passed to *params* as param names or values they are percent-escaped in the URL. This reproduces https://github.com/twisted/treq/issues/282 """ self.client.request('GET', 'http://example.com/', params=( ('ampersand', '&'), ('&', 'ampersand'), ('octothorpe', '#'), ('#', 'octothorpe'), )) self.agent.request.assert_called_once_with( b'GET', ( b'http://example.com/' b'?ampersand=%26' b'&%26=ampersand' b'&octothorpe=%23' b'&%23=octothorpe' ), Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_merge_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_merge_tuple_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_dict_single_value_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_data_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar&foo=baz') def test_request_data_single_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_tuple(self): self.client.request('POST', 'http://example.com/', data=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "wb") as temp_file: temp_file.write(b'hello') self.client.request('POST', 'http://example.com/', data=open(temp_fn, 'rb')) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'hello') def test_request_json_dict(self): self.client.request('POST', 'http://example.com/', json={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'{"foo":"bar"}') def test_request_json_tuple(self): self.client.request('POST', 'http://example.com/', json=('foo', 1)) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'["foo",1]') def test_request_json_number(self): self.client.request('POST', 'http://example.com/', json=1.) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'1.0') def test_request_json_string(self): self.client.request('POST', 'http://example.com/', json='hello') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'"hello"') def test_request_json_bool(self): self.client.request('POST', 'http://example.com/', json=True) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'true') def test_request_json_none(self): self.client.request('POST', 'http://example.com/', json=None) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'null') @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_no_name_attachment(self): self.client.request( 'POST', 'http://example.com/', files={"name": BytesIO(b"hello")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'image/jpeg', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment_and_ctype(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', 'text/plain', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'text/plain', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params(self): class NamedFile(BytesIO): def __init__(self, val): BytesIO.__init__(self, val) self.name = "image.png" self.client.request( 'POST', 'http://example.com/', data=[("a", "b"), ("key", "val")], files=[ ("file1", ('image.jpg', BytesIO(b"hello"))), ("file2", NamedFile(b"yo"))]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('a', 'b'), ('key', 'val'), ('file1', ('image.jpg', 'image/jpeg', FP)), ('file2', ('image.png', 'image/png', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params_dict(self): self.client.request( 'POST', 'http://example.com/', data={"key": "a", "key2": "b"}, files={"file1": BytesIO(b"hey")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('key', 'a'), ('key2', 'b'), ('file1', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) def test_request_unsupported_params_combination(self): self.assertRaises(ValueError, self.client.request, 'POST', 'http://example.com/', data=BytesIO(b"yo"), files={"file1": BytesIO(b"hey")}) def test_request_dict_headers(self): self.client.request('GET', 'http://example.com/', headers={ 'User-Agent': 'treq/0.1dev', 'Accept': ['application/json', 'text/plain'] }) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({b'User-Agent': [b'treq/0.1dev'], b'accept-encoding': [b'gzip'], b'Accept': [b'application/json', b'text/plain']}), None) @with_clock def test_request_timeout_fired(self, clock): """ Verify the request is cancelled if a response is not received within specified timeout period. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate we haven't gotten a response within timeout seconds clock.advance(3) # a deferred should have been cancelled self.failureResultOf(d, CancelledError) @with_clock def test_request_timeout_cancelled(self, clock): """ Verify timeout is cancelled if a response is received before timeout period elapses. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate a response d.callback(mock.Mock(code=200, headers=Headers({}))) # now advance the clock but since we already got a result, # a cancellation timer should have been cancelled clock.advance(3) self.successResultOf(d) def test_response_is_buffered(self): response = mock.Mock(deliverBody=mock.Mock(), headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com') result = self.successResultOf(d) protocol = mock.Mock(Protocol) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) def test_response_buffering_is_disabled_with_unbufferred_arg(self): response = mock.Mock(headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com', unbuffered=True) # YOLO public attribute. self.assertEqual(self.successResultOf(d).original, response) def test_request_post_redirect_denied(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) d = self.client.post('http://www.example.com') self.failureResultOf(d, ResponseFailed) def test_request_browser_like_redirects(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) raw = mock.Mock(return_value=[]) final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw)) with mock.patch('twisted.web.client.RedirectAgent._handleRedirect', return_value=final_resp): d = self.client.post('http://www.google.com', browser_like_redirects=True, unbuffered=True) self.assertEqual(self.successResultOf(d).original, final_resp)
def http_deliver_sm_callback(self, message): msgid = message.content.properties['message-id'] route_type = message.content.properties['headers']['route-type'] dcs = pickle.loads(message.content.properties['headers']['dst-connectors']) RoutedDeliverSmContent = pickle.loads(message.content.body) self.log.debug('Got one message (msgid:%s) to throw: %s', msgid, RoutedDeliverSmContent) # If any, clear requeuing timer self.clearRequeueTimer(msgid) if dcs[0]._type != 'http': self.log.error( 'Rejecting message [msgid:%s] because destination connector is not http (type were %s)', msgid, dcs[0]._type) yield self.rejectMessage(message) defer.returnValue(None) # Build mandatory arguments args = { 'id': msgid, 'from': RoutedDeliverSmContent.params['source_addr'], 'to': RoutedDeliverSmContent.params['destination_addr'], 'origin-connector': message.content.properties['headers']['src-connector-id']} # Content can be short_message or message_payload: if 'short_message' in RoutedDeliverSmContent.params and len(RoutedDeliverSmContent.params['short_message']) > 0: args['content'] = RoutedDeliverSmContent.params['short_message'] elif 'message_payload' in RoutedDeliverSmContent.params: args['content'] = RoutedDeliverSmContent.params['message_payload'] elif 'short_message' in RoutedDeliverSmContent.params: args['content'] = RoutedDeliverSmContent.params['short_message'] else: self.log.error('Cannot find content in pdu (msgid:%s): %s', msgid, RoutedDeliverSmContent) yield self.rejectMessage(message) defer.returnValue(None) # Set the binary arg after deciding where to pick the content from if isinstance(args['content'], bytes): args['binary'] = binascii.hexlify(args['content']) else: args['binary'] = binascii.hexlify(args['content'].encode()) # Build optional arguments if ('priority_flag' in RoutedDeliverSmContent.params and RoutedDeliverSmContent.params['priority_flag'] is not None): args['priority'] = priority_flag_name_map[RoutedDeliverSmContent.params['priority_flag'].name] if ('data_coding' in RoutedDeliverSmContent.params and RoutedDeliverSmContent.params['data_coding'] is not None): args['coding'] = DataCodingEncoder().encode(RoutedDeliverSmContent.params['data_coding']) if ('validity_period' in RoutedDeliverSmContent.params and RoutedDeliverSmContent.params['validity_period'] is not None): args['validity'] = RoutedDeliverSmContent.params['validity_period'] counter = 0 for dc in dcs: counter += 1 self.log.debug('DCS Iteration %s/%s taking [cid:%s] (%s)', counter, len(dcs), dc.cid, dc) last_dc = True if route_type == 'failover' and counter < len(dcs): last_dc = False try: # Throw the message to http endpoint postdata = None params = None baseurl = dc.baseurl _method = dc.method if isinstance(_method, bytes): _method = _method.decode().upper() else: _method = _method.upper() if _method == 'GET': params = args else: postdata = args self.log.debug('Calling %s with args %s using %s method.', dc.baseurl, args, _method) agent = Agent(reactor) client = HTTPClient(agent) response = yield client.request( _method, baseurl, params=params, data=postdata, timeout=self.config.timeout, agent='Jasmin gateway/1.0 deliverSmHttpThrower', headers={'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'text/plain'}) self.log.info('Throwed message [msgid:%s] to connector (%s %s/%s)[cid:%s] using http to %s.', msgid, route_type, counter, len(dcs), dc.cid, dc.baseurl) content = yield text_content(response) if response.code >= 400: raise HttpApiError(response.code, content) self.log.debug('Destination end replied to message [msgid:%s]: %r', msgid, content) # Check for acknowledgement if content.strip() != 'ACK': raise MessageAcknowledgementError( 'Destination end did not acknowledge receipt of the message.') except Exception as e: self.log.error('Throwing message [msgid:%s] to (%s %s/%s)[cid:%s] (%s), %s: %s.', msgid, route_type, counter, len(dcs), dc.cid, dc.baseurl, type(e), e) # List of errors after which, no further retrying shall be made noRetryErrors = ['404'] if route_type == 'simple': # Requeue message for later retry if (str(e) not in noRetryErrors and self.getThrowingRetrials(message) <= self.config.max_retries): self.log.debug('Message try-count is %s [msgid:%s]: requeuing', self.getThrowingRetrials(message), msgid) yield self.rejectAndRequeueMessage(message) elif str(e) in noRetryErrors: self.log.warn('Message [msgid:%s] is no more processed after receiving "%s" error', msgid, str(e)) yield self.rejectMessage(message) else: self.log.warn('Message try-count is %s [msgid:%s]: purged from queue', self.getThrowingRetrials(message), msgid) yield self.rejectMessage(message) elif route_type == 'failover': # The route has multiple connectors, we will not retry throwing to same connector if last_dc: self.log.warn( 'Message [msgid:%s] is no more processed after receiving "%s" error on this fo/connector', msgid, str(e)) else: # Everything is okay ? then: yield self.ackMessage(message) if route_type == 'failover': self.log.debug('Stopping iteration for failover route.') break finally: if route_type == 'simple': # There's only one connector for simple routes break elif route_type == 'failover' and last_dc: self.log.debug('Break (last dc) iteration for failover route.') break elif route_type == 'failover' and not last_dc: self.log.debug('Continue iteration for failover route.')
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch('treq.client.FileBodyProducer') self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer') self.MultiPartProducer = self.mbp_patcher.start() self.addCleanup(self.mbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_request_case_insensitive_methods(self): self.client.request('gEt', 'http://example.com/') self.agent.request.assert_called_once_with( 'GET', 'http://example.com/', headers=Headers({}), bodyProducer=None) def test_request_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': ['bar']}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_tuple_query_values(self): self.client.request('GET', 'http://example.com/', params={'foo': ('bar',)}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_merge_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?baz=bax&foo=bar&foo=baz', headers=Headers({}), bodyProducer=None) def test_request_merge_tuple_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params=[('foo', 'bar')]) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?baz=bax&foo=bar', headers=Headers({}), bodyProducer=None) def test_request_dict_single_value_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/?foo=bar', headers=Headers({}), bodyProducer=None) def test_request_data_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar&foo=baz') def test_request_data_single_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': 'bar'}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar') def test_request_data_tuple(self): self.client.request('POST', 'http://example.com/', data=[('foo', 'bar')]) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers( {'Content-Type': ['application/x-www-form-urlencoded']}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('foo=bar') def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "w") as temp_file: temp_file.write('hello') self.client.request('POST', 'http://example.com/', data=file(temp_fn)) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({}), bodyProducer=self.FileBodyProducer.return_value) self.assertBody('hello') @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_no_name_attachment(self): self.client.request( 'POST', 'http://example.com/', files={"name": StringIO("hello")}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', (None, 'application/octet-stream', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', StringIO("hello"))}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'image/jpeg', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment_and_ctype(self): self.client.request( 'POST', 'http://example.com/', files={ "name": ('image.jpg', 'text/plain', StringIO("hello"))}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call( [('name', ('image.jpg', 'text/plain', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params(self): class NamedFile(StringIO): def __init__(self, val): StringIO.__init__(self, val) self.name = "image.png" self.client.request( 'POST', 'http://example.com/', data=[("a", "b"), ("key", "val")], files=[ ("file1", ('image.jpg', StringIO("hello"))), ("file2", NamedFile("yo"))]) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('a', 'b'), ('key', 'val'), ('file1', ('image.jpg', 'image/jpeg', FP)), ('file2', ('image.png', 'image/png', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params_dict(self): self.client.request( 'POST', 'http://example.com/', data={"key": "a", "key2": "b"}, files={"file1": StringIO("hey")}) self.agent.request.assert_called_once_with( 'POST', 'http://example.com/', headers=Headers({ 'Content-Type': [ 'multipart/form-data; boundary=heyDavid']}), bodyProducer=self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([ ('key', 'a'), ('key2', 'b'), ('file1', (None, 'application/octet-stream', FP))], boundary='heyDavid'), self.MultiPartProducer.call_args) def test_request_unsupported_params_combination(self): self.assertRaises(ValueError, self.client.request, 'POST', 'http://example.com/', data=StringIO("yo"), files={"file1": StringIO("hey")}) def test_request_dict_headers(self): self.client.request('GET', 'http://example.com/', headers={ 'User-Agent': 'treq/0.1dev', 'Accept': ['application/json', 'text/plain'] }) self.agent.request.assert_called_once_with( 'GET', 'http://example.com/', headers=Headers({'User-Agent': ['treq/0.1dev'], 'Accept': ['application/json', 'text/plain']}), bodyProducer=None) @with_clock def test_request_timeout_fired(self, clock): """ Verify the request is cancelled if a response is not received within specified timeout period. """ self.client.request('GET', 'http://example.com', timeout=2) # simulate we haven't gotten a response within timeout seconds clock.advance(3) deferred = self.agent.request.return_value # a deferred should have been cancelled self.assertTrue(deferred.cancel.called) @with_clock def test_request_timeout_cancelled(self, clock): """ Verify timeout is cancelled if a response is received before timeout period elapses. """ self.client.request('GET', 'http://example.com', timeout=2) # simulate a response deferred = self.agent.request.return_value gotResult = deferred.addBoth.call_args[0][0] gotResult('result') # now advance the clock but since we already got a result, # a cancellation timer should have been cancelled clock.advance(3) self.assertFalse(deferred.cancel.called)
class HTTPClientTests(TestCase): def setUp(self): self.agent = mock.Mock(Agent) self.client = HTTPClient(self.agent) self.fbp_patcher = mock.patch('treq.client.FileBodyProducer') self.FileBodyProducer = self.fbp_patcher.start() self.addCleanup(self.fbp_patcher.stop) self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer') self.MultiPartProducer = self.mbp_patcher.start() self.addCleanup(self.mbp_patcher.stop) def assertBody(self, expected): body = self.FileBodyProducer.mock_calls[0][1][0] self.assertEqual(body.read(), expected) def test_post(self): self.client.post('http://example.com/') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_idn(self): self.client.request('GET', u'http://č.net') self.agent.request.assert_called_once_with( b'GET', b'http://xn--bea.net', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_decodedurl(self): """ A URL may be passed as a `hyperlink.DecodedURL` object. It is converted to bytes when passed to the underlying agent. """ url = DecodedURL.from_text(u"https://example.org/foo") self.client.request("GET", url) self.agent.request.assert_called_once_with( b"GET", b"https://example.org/foo", Headers({b"accept-encoding": [b"gzip"]}), None, ) def test_request_uri_encodedurl(self): """ A URL may be passed as a `hyperlink.EncodedURL` object. It is converted to bytes when passed to the underlying agent. """ url = EncodedURL.from_text(u"https://example.org/foo") self.client.request("GET", url) self.agent.request.assert_called_once_with( b"GET", b"https://example.org/foo", Headers({b"accept-encoding": [b"gzip"]}), None, ) def test_request_uri_bytes_pass(self): """ The URL parameter may contain path segments or querystring parameters that are not valid UTF-8. These pass through. """ # This URL is http://example.com/hello?who=you, but "hello", "who", and # "you" are encoded as UTF-16. The particulars of the encoding aren't # important; what matters is that those segments can't be decoded by # Hyperlink's UTF-8 default. self.client.request( "GET", ("http://example.com/%FF%FEh%00e%00l%00l%00o%00" "?%FF%FEw%00h%00o%00=%FF%FEy%00o%00u%00"), ) self.agent.request.assert_called_once_with( b'GET', (b'http://example.com/%FF%FEh%00e%00l%00l%00o%00' b'?%FF%FEw%00h%00o%00=%FF%FEy%00o%00u%00'), Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_uri_plus_pass(self): """ URL parameters may contain spaces encoded as ``+``. These remain as such and are not mangled. This reproduces `Klein #339 <https://github.com/twisted/klein/issues/339>`_. """ self.client.request( "GET", "https://example.com/?foo+bar=baz+biff", ) self.agent.request.assert_called_once_with( b'GET', b"https://example.com/?foo+bar=baz+biff", Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_uri_idn_params(self): """ A URL that contains non-ASCII characters can be augmented with querystring parameters. This reproduces treq #264. """ self.client.request('GET', u'http://č.net', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'GET', b'http://xn--bea.net/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_uri_hyperlink_params(self): """ The *params* argument augments an instance of `hyperlink.DecodedURL` passed as the *url* parameter, just as if it were a string. """ self.client.request( method="GET", url=DecodedURL.from_text(u"http://č.net"), params={"foo": "bar"}, ) self.agent.request.assert_called_once_with( b"GET", b"http://xn--bea.net/?foo=bar", Headers({b"accept-encoding": [b"gzip"]}), None, ) def test_request_case_insensitive_methods(self): self.client.request('gEt', 'http://example.com/') self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': ['bar']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_tuple_query_values(self): self.client.request('GET', 'http://example.com/', params={'foo': ('bar', )}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_tuple_query_value_coercion(self): """ treq coerces non-string values passed to *params* like `urllib.urlencode()` """ self.client.request('GET', 'http://example.com/', params=[ ('text', u'A\u03a9'), ('text-seq', [u'A\u03a9']), ('bytes', [b'ascii']), ('bytes-seq', [b'ascii']), ('native', ['native']), ('native-seq', ['aa', 'bb']), ('int', 1), ('int-seq', (1, 2, 3)), ('none', None), ('none-seq', [None, None]), ]) self.agent.request.assert_called_once_with( b'GET', (b'http://example.com/?' b'text=A%CE%A9&text-seq=A%CE%A9' b'&bytes=ascii&bytes-seq=ascii' b'&native=native&native-seq=aa&native-seq=bb' b'&int=1&int-seq=1&int-seq=2&int-seq=3' b'&none=None&none-seq=None&none-seq=None'), Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_tuple_query_param_coercion(self): """ treq coerces non-string param names passed to *params* like `urllib.urlencode()` """ # A value used to test that it is never encoded or decoded. # It should be invalid UTF-8 or UTF-32 (at least). raw_bytes = b"\x00\xff\xfb" self.client.request('GET', 'http://example.com/', params=[ (u'text', u'A\u03a9'), (b'bytes', ['ascii', raw_bytes]), ('native', 'native'), (1, 'int'), (None, ['none']), ]) self.agent.request.assert_called_once_with( b'GET', (b'http://example.com/' b'?text=A%CE%A9&bytes=ascii&bytes=%00%FF%FB' b'&native=native&1=int&None=none'), Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_query_param_seps(self): """ When the characters ``&`` and ``#`` are passed to *params* as param names or values they are percent-escaped in the URL. This reproduces https://github.com/twisted/treq/issues/282 """ self.client.request('GET', 'http://example.com/', params=( ('ampersand', '&'), ('&', 'ampersand'), ('octothorpe', '#'), ('#', 'octothorpe'), )) self.agent.request.assert_called_once_with( b'GET', (b'http://example.com/' b'?ampersand=%26' b'&%26=ampersand' b'&octothorpe=%23' b'&%23=octothorpe'), Headers({b'accept-encoding': [b'gzip']}), None, ) def test_request_merge_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_merge_tuple_query_params(self): self.client.request('GET', 'http://example.com/?baz=bax', params=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?baz=bax&foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_dict_single_value_query_params(self): self.client.request('GET', 'http://example.com/', params={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/?foo=bar', Headers({b'accept-encoding': [b'gzip']}), None) def test_request_data_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': ['bar', 'baz']}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar&foo=baz') def test_request_data_single_dict(self): self.client.request('POST', 'http://example.com/', data={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_tuple(self): self.client.request('POST', 'http://example.com/', data=[('foo', 'bar')]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/x-www-form-urlencoded'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'foo=bar') def test_request_data_file(self): temp_fn = self.mktemp() with open(temp_fn, "wb") as temp_file: temp_file.write(b'hello') self.client.request('POST', 'http://example.com/', data=open(temp_fn, 'rb')) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({b'accept-encoding': [b'gzip']}), self.FileBodyProducer.return_value) self.assertBody(b'hello') def test_request_json_dict(self): self.client.request('POST', 'http://example.com/', json={'foo': 'bar'}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'{"foo":"bar"}') def test_request_json_tuple(self): self.client.request('POST', 'http://example.com/', json=('foo', 1)) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'["foo",1]') def test_request_json_number(self): self.client.request('POST', 'http://example.com/', json=1.) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'1.0') def test_request_json_string(self): self.client.request('POST', 'http://example.com/', json='hello') self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'"hello"') def test_request_json_bool(self): self.client.request('POST', 'http://example.com/', json=True) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'true') def test_request_json_none(self): self.client.request('POST', 'http://example.com/', json=None) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'Content-Type': [b'application/json; charset=UTF-8'], b'accept-encoding': [b'gzip'] }), self.FileBodyProducer.return_value) self.assertBody(b'null') @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_no_name_attachment(self): self.client.request('POST', 'http://example.com/', files={"name": BytesIO(b"hello")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('name', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment(self): self.client.request('POST', 'http://example.com/', files={"name": ('image.jpg', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('name', ('image.jpg', 'image/jpeg', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_named_attachment_and_ctype(self): self.client.request( 'POST', 'http://example.com/', files={"name": ('image.jpg', 'text/plain', BytesIO(b"hello"))}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('name', ('image.jpg', 'text/plain', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) def test_request_files_tuple_too_short(self): """ The `HTTPClient.request()` *files* argument requires tuples of length 2 or 3. It raises `TypeError` when the tuple is too short. """ with self.assertRaises(TypeError) as c: self.client.request( "POST", b"http://example.com/", files=[("t1", ("foo.txt", ))], ) self.assertIn("'t1' tuple has length 1", str(c.exception)) def test_request_files_tuple_too_long(self): """ The `HTTPClient.request()` *files* argument requires tuples of length 2 or 3. It raises `TypeError` when the tuple is too long. """ with self.assertRaises(TypeError) as c: self.client.request( "POST", b"http://example.com/", files=[ ("t4", ("foo.txt", "text/plain", BytesIO(b"...\n"), "extra!")), ], ) self.assertIn("'t4' tuple has length 4", str(c.exception)) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params(self): class NamedFile(BytesIO): def __init__(self, val): BytesIO.__init__(self, val) self.name = "image.png" self.client.request('POST', 'http://example.com/', data=[("a", "b"), ("key", "val")], files=[("file1", ('image.jpg', BytesIO(b"hello"))), ("file2", NamedFile(b"yo"))]) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('a', 'b'), ('key', 'val'), ('file1', ('image.jpg', 'image/jpeg', FP)), ('file2', ('image.png', 'image/png', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) @mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid")) def test_request_mixed_params_dict(self): self.client.request('POST', 'http://example.com/', data={ "key": "a", "key2": "b" }, files={"file1": BytesIO(b"hey")}) self.agent.request.assert_called_once_with( b'POST', b'http://example.com/', Headers({ b'accept-encoding': [b'gzip'], b'Content-Type': [b'multipart/form-data; boundary=heyDavid'] }), self.MultiPartProducer.return_value) FP = self.FileBodyProducer.return_value self.assertEqual( mock.call([('key', 'a'), ('key2', 'b'), ('file1', (None, 'application/octet-stream', FP))], boundary=b'heyDavid'), self.MultiPartProducer.call_args) def test_request_unsupported_params_combination(self): self.assertRaises(ValueError, self.client.request, 'POST', 'http://example.com/', data=BytesIO(b"yo"), files={"file1": BytesIO(b"hey")}) def test_request_json_with_data(self): """ Passing `HTTPClient.request()` both *data* and *json* parameters is invalid because *json* is ignored. This behavior is deprecated. """ self.client.request( "POST", "http://example.com/", data=BytesIO(b"..."), json=None, # NB: None is a valid value. It encodes to b'null'. ) [w] = self.flushWarnings([self.test_request_json_with_data]) self.assertEqual(DeprecationWarning, w["category"]) self.assertEqual( ("Argument 'json' will be ignored because 'data' was also passed." " This will raise TypeError in the next treq release."), w['message'], ) def test_request_json_with_files(self): """ Passing `HTTPClient.request()` both *files* and *json* parameters is invalid because *json* is ignored. This behavior is deprecated. """ self.client.request( "POST", "http://example.com/", files={"f1": ("foo.txt", "text/plain", BytesIO(b"...\n"))}, json=["this is ignored"], ) [w] = self.flushWarnings([self.test_request_json_with_files]) self.assertEqual(DeprecationWarning, w["category"]) self.assertEqual( ("Argument 'json' will be ignored because 'files' was also passed." " This will raise TypeError in the next treq release."), w['message'], ) def test_request_dict_headers(self): self.client.request('GET', 'http://example.com/', headers={ 'User-Agent': 'treq/0.1dev', 'Accept': ['application/json', 'text/plain'] }) self.agent.request.assert_called_once_with( b'GET', b'http://example.com/', Headers({ b'User-Agent': [b'treq/0.1dev'], b'accept-encoding': [b'gzip'], b'Accept': [b'application/json', b'text/plain'] }), None) def test_request_headers_object(self): """ The *headers* parameter accepts a `twisted.web.http_headers.Headers` instance. """ self.client.request( "GET", "https://example.com", headers=Headers({"X-Foo": ["bar"]}), ) self.agent.request.assert_called_once_with( b"GET", b"https://example.com", Headers({ "X-Foo": ["bar"], "Accept-Encoding": ["gzip"], }), None, ) def test_request_headers_invalid_type(self): """ `HTTPClient.request()` warns that headers of an unexpected type are invalid and that this behavior is deprecated. """ self.client.request('GET', 'http://example.com', headers=[]) [w] = self.flushWarnings([self.test_request_headers_invalid_type]) self.assertEqual(DeprecationWarning, w['category']) self.assertIn( "headers must be a dict, twisted.web.http_headers.Headers, or None,", w['message'], ) def test_request_dict_headers_invalid_values(self): """ `HTTPClient.request()` warns that non-string header values are dropped and that this behavior is deprecated. """ self.client.request('GET', 'http://example.com', headers=OrderedDict([ ('none', None), ('one', 1), ('ok', 'string'), ])) [w1, w2] = self.flushWarnings( [self.test_request_dict_headers_invalid_values]) self.assertEqual(DeprecationWarning, w1['category']) self.assertEqual(DeprecationWarning, w2['category']) self.assertIn( "The value of headers key 'none' has non-string type", w1['message'], ) self.assertIn( "The value of headers key 'one' has non-string type", w2['message'], ) def test_request_invalid_param(self): """ `HTTPClient.request()` rejects invalid keyword parameters with `TypeError`. """ self.assertRaises( TypeError, self.client.request, "GET", b"http://example.com", invalid=True, ) @with_clock def test_request_timeout_fired(self, clock): """ Verify the request is cancelled if a response is not received within specified timeout period. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate we haven't gotten a response within timeout seconds clock.advance(3) # a deferred should have been cancelled self.failureResultOf(d, CancelledError) @with_clock def test_request_timeout_cancelled(self, clock): """ Verify timeout is cancelled if a response is received before timeout period elapses. """ self.agent.request.return_value = d = Deferred() self.client.request('GET', 'http://example.com', timeout=2) # simulate a response d.callback(mock.Mock(code=200, headers=Headers({}))) # now advance the clock but since we already got a result, # a cancellation timer should have been cancelled clock.advance(3) self.successResultOf(d) def test_response_is_buffered(self): response = mock.Mock(deliverBody=mock.Mock(), headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com') result = self.successResultOf(d) protocol = mock.Mock(Protocol) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) result.deliverBody(protocol) self.assertEqual(response.deliverBody.call_count, 1) def test_response_buffering_is_disabled_with_unbufferred_arg(self): response = mock.Mock(headers=Headers({})) self.agent.request.return_value = succeed(response) d = self.client.get('http://www.example.com', unbuffered=True) # YOLO public attribute. self.assertEqual(self.successResultOf(d).original, response) def test_request_post_redirect_denied(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) d = self.client.post('http://www.example.com') self.failureResultOf(d, ResponseFailed) def test_request_browser_like_redirects(self): response = mock.Mock(code=302, headers=Headers({'Location': ['/']})) self.agent.request.return_value = succeed(response) raw = mock.Mock(return_value=[]) final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw)) with mock.patch('twisted.web.client.RedirectAgent._handleRedirect', return_value=final_resp): d = self.client.post('http://www.google.com', browser_like_redirects=True, unbuffered=True) self.assertEqual(self.successResultOf(d).original, final_resp)
class _SCPDCommand(object): def __init__(self, gateway_address, service_port, control_url, service_id, method, param_names, returns, reactor=None): if not reactor: from twisted.internet import reactor self._reactor = reactor self._pool = HTTPConnectionPool(reactor) self.agent = Agent(reactor, connectTimeout=1) self._http_client = HTTPClient(self.agent, data_to_body_producer=StringProducer) self.gateway_address = gateway_address self.service_port = service_port self.control_url = control_url self.service_id = service_id self.method = method self.param_names = param_names self.returns = returns def extract_body(self, xml_response, service_key=IP_SCHEMA): content_dict = etree_to_dict(ElementTree.fromstring(xml_response)) envelope = content_dict[ENVELOPE] return flatten_keys(envelope[BODY], "{%s}" % service_key) def extract_response(self, body): body = handle_fault(body) # raises UPnPError if there is a fault if '%sResponse' % self.method in body: response_key = '%sResponse' % self.method else: log.error(body.keys()) raise UPnPError("unknown response fields") response = body[response_key] extracted_response = tuple([response[n] for n in self.returns]) if len(extracted_response) == 1: return extracted_response[0] return extracted_response @defer.inlineCallbacks def send_upnp_soap(self, **kwargs): soap_body = get_soap_body(self.service_id, self.method, self.param_names, **kwargs).encode() headers = OrderedDict(( ('SOAPAction', '%s#%s' % (self.service_id, self.method)), ('Host', ('%s:%i' % (SSDP_IP_ADDRESS, self.service_port))), ('Content-Type', 'text/xml'), ('Content-Length', len(soap_body)) )) response = yield self._http_client.request( POST, url=self.control_url, data=soap_body, headers=headers ) xml_response = yield response.content() response = self.extract_response(self.extract_body(xml_response)) defer.returnValue(response) @staticmethod def _process_result(results): """ this method gets decorated automatically with a function that maps result types to the types defined in the @return_types decorator """ return results @defer.inlineCallbacks def __call__(self, **kwargs): if set(kwargs.keys()) != set(self.param_names): raise Exception("argument mismatch") response = yield self.send_upnp_soap(**kwargs) result = self._process_result(response) defer.returnValue(result)