class MarconiClient(object):
USER_AGENT = 'txmarconi/{version}'.format(version=__version__)
RETRYABLE_ERRORS = [RequestTransmissionFailed]
def __init__(self, base_url='http://localhost:8888', quiet_requests=True, **kwargs):
self.client_id = str(uuid4())
self.base_url = base_url
pool = HTTPConnectionPool(reactor, persistent=True)
agent = ContentDecoderAgent(RedirectAgent(Agent(reactor, pool=pool)), [('gzip', GzipDecoder)])
if quiet_requests:
pool._factory = QuieterHTTP11ClientFactory
auth_url = kwargs.get('auth_url')
if auth_url:
username = kwargs.get('username')
password = kwargs.get('password')
api_key = kwargs.get('api_key')
if not username:
raise RuntimeError('Marconi "auth_url" specified with no username')
if api_key:
cred = api_key
auth_type = 'api_key'
elif password:
cred = password
auth_type = 'password'
else:
raise RuntimeError('Marconi "auth_url" specified with no "password" or "api_key"')
agent = KeystoneAgent(agent, auth_url, (username, cred), auth_type=auth_type)
self.http_client = HTTPClient(agent)
def _wrap_error(self, failure):
if not failure.check(MarconiError):
log.err(failure)
raise MarconiError(failure.value)
log.err(failure.value)
return failure
def _handle_error_response(self, response):
def _raise_error(content_str):
content_str = content_str.strip()
if len(content_str) > 0:
raise MarconiError(json.loads(content_str))
else:
msg = 'Received {code} response with empty body'.format(code=response.code)
raise MarconiError(msg)
d = content(response)
d.addCallback(_raise_error)
return d
def _request(self, method, path, params=None, data=None):
url = '{base_url}{path}'.format(
base_url=self.base_url,
path=path,
)
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
'User-Agent': self.USER_AGENT,
'Client-ID': self.client_id,
}
def _possibly_retry(failure):
# Either I'm doing something wrong (likely) or Marconi is doing
# something unpleasant to connections after it returns a 201 to us,
# because the next request always seems to get one of these.
if failure.check(*self.RETRYABLE_ERRORS):
return self._request(method, path, params=params, data=data)
else:
return failure
if data:
body = QuieterFileBodyProducer(StringIO(json.dumps(data)))
else:
body = None
d = self.http_client.request(method, url, headers=headers, data=body, params=params)
d.addErrback(_possibly_retry)
return d
def _expect_204(self, response):
if response.code == 204:
return None
else:
return self._handle_error_response(response)
def ensure_queue(self, queue_name):
path = '/v1/queues/{queue_name}'.format(queue_name=queue_name)
def _on_response(response):
if response.code in (201, 204):
return path
else:
return self._handle_error_response(response)
d = self._request('PUT', path)
d.addCallback(_on_response)
d.addErrback(self._wrap_error)
return d
def push_message(self, queue_name, body, ttl):
path = '/v1/queues/{queue_name}/messages'.format(queue_name=queue_name)
data = [
{
'ttl': ttl,
'body': body,
}
]
def _construct_message(obj):
return MarconiMessage(body=body, ttl=ttl, age=0, href=obj['resources'][0])
def _on_response(response):
if response.code == 201:
return json_content(response).addCallback(_construct_message)
else:
return self._handle_error_response(response)
d = self._request('POST', path, data=data)
d.addCallback(_on_response)
d.addErrback(self._wrap_error)
return d
def claim_message(self, queue_name, ttl, grace, polling_interval=1):
path = '/v1/queues/{queue_name}/claims'.format(queue_name=queue_name)
data = {
'ttl': ttl,
'grace': grace,
}
params = {
'limit': 1,
}
d = defer.Deferred()
def _construct_message(obj, response):
claim_href = response.headers.getRawHeaders('location')[0]
d.callback(ClaimedMarconiMessage(claim_href=claim_href, **obj[0]))
def _on_response(response):
if response.code == 201:
json_content(response).addCallback(_construct_message, response)
elif response.code == 204:
reactor.callLater(polling_interval, _perform_call)
else:
return self._handle_error_response(response)
def _perform_call():
d1 = self._request('POST', path, data=data, params=params)
d1.addCallback(_on_response)
d1.addErrback(self._wrap_error)
d1.addErrback(d.errback)
_perform_call()
return d
def update_claim(self, claimed_message, ttl):
data = {
'ttl': ttl,
}
d = self._request('PATCH', claimed_message.claim_href, data=data)
d.addCallback(self._expect_204)
d.addErrback(self._wrap_error)
return d
def release_claim(self, claimed_message):
d = self._request('DELETE', claimed_message.claim_href)
d.addCallback(self._expect_204)
d.addErrback(self._wrap_error)
return d
def delete_message(self, message):
d = self._request('DELETE', message.href)
d.addCallback(self._expect_204)
d.addErrback(self._wrap_error)
return d
def reverse_proxy(self, request, protected=True):
if protected:
sess = request.getSession()
valid_sessions = self.valid_sessions
sess_uid = sess.uid
username = valid_sessions[sess_uid]['username']
# Normal reverse proxying.
kwds = {}
cookiejar = {}
kwds['allow_redirects'] = False
kwds['cookies'] = cookiejar
req_headers = self.mod_headers(dict(request.requestHeaders.getAllRawHeaders()))
kwds['headers'] = req_headers
if protected:
kwds['headers'][self.remoteUserHeader] = [username]
if request.method in ('PUT', 'POST'):
kwds['data'] = request.content.read()
url = self.proxied_url + request.uri
# Determine if a plugin wants to intercept this URL.
interceptors = self.interceptors
for interceptor in interceptors:
if interceptor.should_resource_be_intercepted(url, request.method, req_headers, request):
return interceptor.handle_resource(url, request.method, req_headers, request)
# Check if this is a request for a websocket.
d = self.checkForWebsocketUpgrade(request)
if d is not None:
return d
# Typical reverse proxying.
self.log("Proxying URL => {0}".format(url))
http_client = HTTPClient(self.proxy_agent)
d = http_client.request(request.method, url, **kwds)
def process_response(response, request):
req_resp_headers = request.responseHeaders
resp_code = response.code
resp_headers = response.headers
resp_header_map = dict(resp_headers.getAllRawHeaders())
# Rewrite Location headers for redirects as required.
if resp_code in (301, 302, 303, 307, 308) and "Location" in resp_header_map:
values = resp_header_map["Location"]
if len(values) == 1:
location = values[0]
if request.isSecure():
proxy_scheme = 'https'
else:
proxy_scheme = 'http'
new_location = self.proxied_url_to_proxy_url(proxy_scheme, location)
if new_location is not None:
resp_header_map['Location'] = [new_location]
request.setResponseCode(response.code, message=response.phrase)
for k,v in resp_header_map.iteritems():
if k == 'Set-Cookie':
v = self.mod_cookies(v)
req_resp_headers.setRawHeaders(k, v)
return response
def mod_content(body, request):
"""
Modify response content before returning it to the user agent.
"""
d = None
for content_modifier in self.content_modifiers:
if d is None:
d = content_modifier.transform_content(body, request)
else:
d.addCallback(content_modifier.transform_content, request)
if d is None:
return body
else:
return d
d.addCallback(process_response, request)
d.addCallback(treq.content)
d.addCallback(mod_content, request)
return d
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch('treq.client.FileBodyProducer')
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer')
self.MultiPartProducer = self.mbp_patcher.start()
self.addCleanup(self.mbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_post(self):
self.client.post('http://example.com/')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_idn(self):
self.client.request('GET', u'http://č.net')
self.agent.request.assert_called_once_with(
b'GET', b'http://xn--bea.net',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ['bar']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_tuple_query_values(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ('bar',)})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_merge_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_merge_tuple_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_dict_single_value_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_data_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar&foo=baz')
def test_request_data_single_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_tuple(self):
self.client.request('POST', 'http://example.com/',
data=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "wb") as temp_file:
temp_file.write(b'hello')
self.client.request('POST', 'http://example.com/',
data=open(temp_fn, 'rb'))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'hello')
def test_request_json_dict(self):
self.client.request('POST', 'http://example.com/', json={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'{"foo":"bar"}')
def test_request_json_tuple(self):
self.client.request('POST', 'http://example.com/', json=('foo', 1))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'["foo",1]')
def test_request_json_number(self):
self.client.request('POST', 'http://example.com/', json=1.)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'1.0')
def test_request_json_string(self):
self.client.request('POST', 'http://example.com/', json='hello')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'"hello"')
def test_request_json_bool(self):
self.client.request('POST', 'http://example.com/', json=True)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'true')
def test_request_json_none(self):
self.client.request('POST', 'http://example.com/', json=None)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'null')
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={"name": BytesIO(b"hello")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'image/jpeg', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment_and_ctype(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', 'text/plain', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'text/plain', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params(self):
class NamedFile(BytesIO):
def __init__(self, val):
BytesIO.__init__(self, val)
self.name = "image.png"
self.client.request(
'POST', 'http://example.com/',
data=[("a", "b"), ("key", "val")],
files=[
("file1", ('image.jpg', BytesIO(b"hello"))),
("file2", NamedFile(b"yo"))])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('a', 'b'),
('key', 'val'),
('file1', ('image.jpg', 'image/jpeg', FP)),
('file2', ('image.png', 'image/png', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params_dict(self):
self.client.request(
'POST', 'http://example.com/',
data={"key": "a", "key2": "b"},
files={"file1": BytesIO(b"hey")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('key', 'a'),
('key2', 'b'),
('file1', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
def test_request_unsupported_params_combination(self):
self.assertRaises(ValueError,
self.client.request,
'POST', 'http://example.com/',
data=BytesIO(b"yo"),
files={"file1": BytesIO(b"hey")})
def test_request_dict_headers(self):
self.client.request('GET', 'http://example.com/', headers={
'User-Agent': 'treq/0.1dev',
'Accept': ['application/json', 'text/plain']
})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({b'User-Agent': [b'treq/0.1dev'],
b'accept-encoding': [b'gzip'],
b'Accept': [b'application/json', b'text/plain']}),
None)
@with_clock
def test_request_timeout_fired(self, clock):
"""
Verify the request is cancelled if a response is not received
within specified timeout period.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate we haven't gotten a response within timeout seconds
clock.advance(3)
# a deferred should have been cancelled
self.failureResultOf(d, CancelledError)
@with_clock
def test_request_timeout_cancelled(self, clock):
"""
Verify timeout is cancelled if a response is received before
timeout period elapses.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate a response
d.callback(mock.Mock(code=200, headers=Headers({})))
# now advance the clock but since we already got a result,
# a cancellation timer should have been cancelled
clock.advance(3)
self.successResultOf(d)
def test_response_is_buffered(self):
response = mock.Mock(deliverBody=mock.Mock(),
headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com')
result = self.successResultOf(d)
protocol = mock.Mock(Protocol)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
def test_response_buffering_is_disabled_with_unbufferred_arg(self):
response = mock.Mock(headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com', unbuffered=True)
# YOLO public attribute.
self.assertEqual(self.successResultOf(d).original, response)
def test_request_post_redirect_denied(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
d = self.client.post('http://www.example.com')
self.failureResultOf(d, ResponseFailed)
def test_request_browser_like_redirects(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
raw = mock.Mock(return_value=[])
final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw))
with mock.patch('twisted.web.client.RedirectAgent._handleRedirect',
return_value=final_resp):
d = self.client.post('http://www.google.com',
browser_like_redirects=True,
unbuffered=True)
self.assertEqual(self.successResultOf(d).original, final_resp)
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch('treq.client.FileBodyProducer')
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/',
headers=Headers({}), bodyProducer=None)
def test_request_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ['bar']})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_tuple_query_values(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ('bar',)})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_merge_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?baz=bax&foo=bar&foo=baz',
headers=Headers({}), bodyProducer=None)
def test_request_merge_tuple_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?baz=bax&foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_dict_single_value_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_data_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar&foo=baz')
def test_request_data_single_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': 'bar'})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar')
def test_request_data_tuple(self):
self.client.request('POST', 'http://example.com/',
data=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar')
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "w") as temp_file:
temp_file.write('hello')
self.client.request('POST', 'http://example.com/', data=file(temp_fn))
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('hello')
def test_request_dict_headers(self):
self.client.request('GET', 'http://example.com/', headers={
'User-Agent': 'treq/0.1dev',
'Accept': ['application/json', 'text/plain']
})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/',
headers=Headers({'User-Agent': ['treq/0.1dev'],
'Accept': ['application/json', 'text/plain']}),
bodyProducer=None)
class JWSClient(object):
"""
HTTP client using JWS-signed messages for ACME.
"""
timeout = _DEFAULT_TIMEOUT
def __init__(self,
agent,
key,
alg,
new_nonce_url,
kid,
user_agent=u'txacme/{}'.format(__version__).encode('ascii')):
self._treq = HTTPClient(agent=agent)
self._agent = agent
self._current_request = None
self._key = key
self._alg = alg
self._user_agent = user_agent
self._nonces = set()
self._new_nonce = new_nonce_url
self._kid = kid
@classmethod
def from_directory(cls, agent, key, alg, directory):
"""
Prepare for ACME operations based on 'directory' url.
:param str directory: The URL to the ACME v2 directory.
:return: When operation is done.
:rtype: Deferred[None]
"""
# Provide invalid new_nonce_url & kid, but don't expose it to the
# caller.
self = cls(agent, key, alg, None, None)
def cb_extract_new_nonce(directory):
try:
self._new_nonce = directory.newNonce
except AttributeError:
raise errors.ClientError('Directory has no newNonce URL',
directory)
return directory
return (self.get(directory).addCallback(json_content).addCallback(
messages.Directory.from_json).addCallback(cb_extract_new_nonce))
@classmethod
def _check_response(cls, response, content_type=JSON_CONTENT_TYPE):
"""
Check response content and its type.
.. note::
Unlike :mod:content_type`acme.client`, checking is strict.
:param bytes content_type: Expected Content-Type response header. If
the response Content-Type does not match, :exc:`ClientError` is
raised.
:raises .ServerError: If server response body carries HTTP Problem
(draft-ietf-appsawg-http-problem-00).
:raises ~acme.errors.ClientError: In case of other networking errors.
"""
def _got_failure(f):
f.trap(ValueError)
return None
def _got_json(jobj):
if 400 <= response.code < 600:
if (response_ct.lower().startswith(JSON_ERROR_CONTENT_TYPE)
and jobj is not None):
raise ServerError(messages.Error.from_json(jobj), response)
else:
# response is not JSON object
return _fail_and_consume(
response, errors.ClientError('Response is not JSON.'))
elif content_type not in response_ct.lower():
return _fail_and_consume(
response,
errors.ClientError(
'Unexpected response Content-Type: {0!r}. '
'Expecting {1!r}.'.format(response_ct, content_type)))
elif JSON_CONTENT_TYPE in content_type.lower() and jobj is None:
return _fail_and_consume(
response, errors.ClientError('Missing JSON body.'))
return response
response_ct = response.headers.getRawHeaders(b'Content-Type',
[None])[0]
action = LOG_JWS_CHECK_RESPONSE(expected_content_type=content_type,
response_content_type=response_ct)
with action.context():
# TODO: response.json() is called twice, once here, and
# once in _get and _post clients
return (DeferredContext(response.json()).addErrback(
_got_failure).addCallback(_got_json).addActionFinish())
def _send_request(self, method, url, *args, **kwargs):
"""
Send HTTP request.
:param str method: The HTTP method to use.
:param str url: The URL to make the request to.
:return: Deferred firing with the HTTP response.
"""
if self._current_request is not None:
return defer.fail(RuntimeError('Overlapped HTTP request'))
def cb_request_done(result):
"""
Called when we got a response from the request.
"""
self._current_request = None
return result
action = LOG_JWS_REQUEST(url=url)
with action.context():
headers = kwargs.setdefault('headers', Headers())
headers.setRawHeaders(b'user-agent', [self._user_agent])
kwargs.setdefault('timeout', self.timeout)
self._current_request = self._treq.request(method, url, *args,
**kwargs)
return (DeferredContext(self._current_request).addCallback(
cb_request_done).addCallback(
tap(lambda r: action.add_success_fields(
code=r.code,
content_type=r.headers.getRawHeaders(
b'content-type', [None])[0]))).addActionFinish())
def stop(self):
"""
Stops the operation.
This cancels pending operations and does cleanup.
:return: A deferred which fires when the client is stopped.
"""
if self._current_request is not None:
self._current_request.addErrback(lambda _: None)
self._current_request.cancel()
self._current_request = None
agent_pool = getattr(self._agent, '_pool', None)
if agent_pool:
return agent_pool.closeCachedConnections()
return defer.succeed(None)
def head(self, url, *args, **kwargs):
"""
Send HEAD request without checking the response.
Note that ``_check_response`` is not called, as there will be no
response body to check.
:param str url: The URL to make the request to.
"""
with LOG_JWS_HEAD().context():
return DeferredContext(
self._send_request(u'HEAD', url, *args,
**kwargs)).addActionFinish()
def get(self, url, content_type=JSON_CONTENT_TYPE, **kwargs):
"""
Send GET request and check response.
:param str method: The HTTP method to use.
:param str url: The URL to make the request to.
:raises txacme.client.ServerError: If server response body carries HTTP
Problem (draft-ietf-appsawg-http-problem-00).
:raises acme.errors.ClientError: In case of other protocol errors.
:return: Deferred firing with the checked HTTP response.
"""
with LOG_JWS_GET().context():
return (DeferredContext(self._send_request(
u'GET', url, **kwargs)).addCallback(
self._check_response,
content_type=content_type).addActionFinish())
def _add_nonce(self, response):
"""
Store a nonce from a response we received.
:param twisted.web.iweb.IResponse response: The HTTP response.
:return: The response, unmodified.
"""
nonce = response.headers.getRawHeaders(REPLAY_NONCE_HEADER, [None])[0]
with LOG_JWS_ADD_NONCE(raw_nonce=nonce) as action:
if nonce is None:
return _fail_and_consume(
response,
errors.ClientError(str(errors.MissingNonce(response))),
)
else:
try:
decoded_nonce = Header._fields['nonce'].decode(
nonce.decode('ascii'))
action.add_success_fields(nonce=decoded_nonce)
except DeserializationError as error:
return _fail_and_consume(response,
errors.BadNonce(nonce, error))
self._nonces.add(decoded_nonce)
return response
def _get_nonce(self, url):
"""
Get a nonce to use in a request, removing it from the nonces on hand.
"""
action = LOG_JWS_GET_NONCE()
if len(self._nonces) > 0:
with action:
nonce = self._nonces.pop()
action.add_success_fields(nonce=nonce)
return defer.succeed(nonce)
else:
with action.context():
return (DeferredContext(self.head(
self._new_nonce)).addCallback(self._add_nonce).addCallback(
lambda _: self._nonces.pop()).addCallback(
tap(lambda nonce: action.add_success_fields(
nonce=nonce))).addActionFinish())
def _post(self,
url,
obj,
content_type,
response_type=JSON_CONTENT_TYPE,
kid=None,
**kwargs):
"""
POST an object and check the response.
:param str url: The URL to request.
:param ~josepy.interfaces.JSONDeSerializable obj: The serializable
payload of the request.
:param bytes content_type: The expected content type of the response.
:raises txacme.client.ServerError: If server response body carries HTTP
Problem (draft-ietf-appsawg-http-problem-00).
:raises acme.errors.ClientError: In case of other protocol errors.
"""
if kid is None:
kid = self._kid
def cb_wrap_in_jws(nonce):
with LOG_JWS_SIGN(key_type=self._key.typ,
alg=self._alg.name,
nonce=nonce):
if obj is None:
jobj = b''
else:
jobj = obj.json_dumps().encode()
result = (JWS.sign(
payload=jobj,
key=self._key,
alg=self._alg,
nonce=nonce,
url=url,
kid=kid,
).json_dumps().encode())
return result
with LOG_JWS_POST().context():
headers = kwargs.setdefault('headers', Headers())
headers.setRawHeaders(b'content-type', [JOSE_CONTENT_TYPE])
return (DeferredContext(self._get_nonce(url)).addCallback(
cb_wrap_in_jws).addCallback(lambda data: self._send_request(
u'POST', url, data=data, **kwargs)).addCallback(
self._add_nonce).addCallback(
self._check_response,
content_type=response_type).addActionFinish())
def post(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs):
"""
POST an object and check the response. Retry once if a badNonce error
is received.
:param str url: The URL to request.
:param ~josepy.interfaces.JSONDeSerializable obj: The serializable
payload of the request.
:param bytes content_type: The expected content type of the response.
By default, JSON.
:raises txacme.client.ServerError: If server response body carries HTTP
Problem (draft-ietf-appsawg-http-problem-00).
:raises acme.errors.ClientError: In case of other protocol errors.
"""
def retry_bad_nonce(f):
f.trap(ServerError)
# The current RFC draft defines the namespace as
# urn:ietf:params:acme:error:<code>, but earlier drafts (and some
# current implementations) use urn:acme:error:<code> instead. We
# don't really care about the namespace here, just the error code.
if f.value.message.typ.split(':')[-1] == 'badNonce':
# If one nonce is bad, others likely are too. Let's clear them
# and re-add the one we just got.
self._nonces.clear()
self._add_nonce(f.value.response)
return self._post(url, obj, content_type, **kwargs)
return f
return (self._post(url, obj, content_type,
**kwargs).addErrback(retry_bad_nonce))
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch('treq.client.FileBodyProducer')
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer')
self.MultiPartProducer = self.mbp_patcher.start()
self.addCleanup(self.mbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_post(self):
self.client.post('http://example.com/')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_idn(self):
self.client.request('GET', u'http://‽.net')
self.agent.request.assert_called_once_with(
b'GET', b'http://xn--fwg.net',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_query_params(self):
self.client.request('GET',
'http://example.com/',
params={'foo': ['bar']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_tuple_query_values(self):
self.client.request('GET',
'http://example.com/',
params={'foo': ('bar', )})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_merge_query_params(self):
self.client.request('GET',
'http://example.com/?baz=bax',
params={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_merge_tuple_query_params(self):
self.client.request('GET',
'http://example.com/?baz=bax',
params=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_dict_single_value_query_params(self):
self.client.request('GET',
'http://example.com/',
params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_data_dict(self):
self.client.request('POST',
'http://example.com/',
data={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar&foo=baz')
def test_request_data_single_dict(self):
self.client.request('POST', 'http://example.com/', data={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_tuple(self):
self.client.request('POST',
'http://example.com/',
data=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "wb") as temp_file:
temp_file.write(b'hello')
self.client.request('POST',
'http://example.com/',
data=open(temp_fn, 'rb'))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'hello')
def test_request_json_dict(self):
self.client.request('POST', 'http://example.com/', json={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'{"foo":"bar"}')
def test_request_json_tuple(self):
self.client.request('POST', 'http://example.com/', json=('foo', 1))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'["foo",1]')
def test_request_json_number(self):
self.client.request('POST', 'http://example.com/', json=1.)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'1.0')
def test_request_json_string(self):
self.client.request('POST', 'http://example.com/', json='hello')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'"hello"')
def test_request_json_bool(self):
self.client.request('POST', 'http://example.com/', json=True)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'true')
def test_request_json_none(self):
self.client.request('POST', 'http://example.com/', json=None)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'null')
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
self.client.request('POST',
'http://example.com/',
files={"name": BytesIO(b"hello")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('name', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment(self):
self.client.request('POST',
'http://example.com/',
files={"name": ('image.jpg', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('name', ('image.jpg', 'image/jpeg', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment_and_ctype(self):
self.client.request(
'POST',
'http://example.com/',
files={"name": ('image.jpg', 'text/plain', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('name', ('image.jpg', 'text/plain', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params(self):
class NamedFile(BytesIO):
def __init__(self, val):
BytesIO.__init__(self, val)
self.name = "image.png"
self.client.request('POST',
'http://example.com/',
data=[("a", "b"), ("key", "val")],
files=[("file1", ('image.jpg', BytesIO(b"hello"))),
("file2", NamedFile(b"yo"))])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('a', 'b'), ('key', 'val'),
('file1', ('image.jpg', 'image/jpeg', FP)),
('file2', ('image.png', 'image/png', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params_dict(self):
self.client.request('POST',
'http://example.com/',
data={
"key": "a",
"key2": "b"
},
files={"file1": BytesIO(b"hey")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('key', 'a'), ('key2', 'b'),
('file1', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
def test_request_unsupported_params_combination(self):
self.assertRaises(ValueError,
self.client.request,
'POST',
'http://example.com/',
data=BytesIO(b"yo"),
files={"file1": BytesIO(b"hey")})
def test_request_dict_headers(self):
self.client.request('GET',
'http://example.com/',
headers={
'User-Agent': 'treq/0.1dev',
'Accept': ['application/json', 'text/plain']
})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({
b'User-Agent': [b'treq/0.1dev'],
b'accept-encoding': [b'gzip'],
b'Accept': [b'application/json', b'text/plain']
}), None)
@with_clock
def test_request_timeout_fired(self, clock):
"""
Verify the request is cancelled if a response is not received
within specified timeout period.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate we haven't gotten a response within timeout seconds
clock.advance(3)
# a deferred should have been cancelled
self.failureResultOf(d, CancelledError)
@with_clock
def test_request_timeout_cancelled(self, clock):
"""
Verify timeout is cancelled if a response is received before
timeout period elapses.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate a response
d.callback(mock.Mock(code=200, headers=Headers({})))
# now advance the clock but since we already got a result,
# a cancellation timer should have been cancelled
clock.advance(3)
self.successResultOf(d)
def test_response_is_buffered(self):
response = mock.Mock(deliverBody=mock.Mock(), headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com')
result = self.successResultOf(d)
protocol = mock.Mock(Protocol)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
def test_response_buffering_is_disabled_with_unbufferred_arg(self):
response = mock.Mock(headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com', unbuffered=True)
# YOLO public attribute.
self.assertEqual(self.successResultOf(d).original, response)
def test_request_post_redirect_denied(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
d = self.client.post('http://www.example.com')
self.failureResultOf(d, ResponseFailed)
def test_request_browser_like_redirects(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
raw = mock.Mock(return_value=[])
final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw))
with mock.patch('twisted.web.client.RedirectAgent._handleRedirect',
return_value=final_resp):
d = self.client.post('http://www.google.com',
browser_like_redirects=True,
unbuffered=True)
self.assertEqual(self.successResultOf(d).original, final_resp)
def reverse_proxy(self, request, protected=True):
if protected:
sess = request.getSession()
valid_sessions = self.valid_sessions
sess_uid = sess.uid
username = valid_sessions[sess_uid]['username']
# Normal reverse proxying.
kwds = {}
cookiejar = {}
kwds['allow_redirects'] = False
kwds['cookies'] = cookiejar
req_headers = self.mod_headers(
dict(request.requestHeaders.getAllRawHeaders()))
kwds['headers'] = req_headers
if protected:
kwds['headers'][self.remoteUserHeader] = [username]
if request.method in ('PUT', 'POST'):
kwds['data'] = request.content.read()
url = self.proxied_url + request.uri
# Determine if a plugin wants to intercept this URL.
interceptors = self.interceptors
for interceptor in interceptors:
if interceptor.should_resource_be_intercepted(
url, request.method, req_headers, request):
return interceptor.handle_resource(url, request.method,
req_headers, request)
# Check if this is a request for a websocket.
d = self.checkForWebsocketUpgrade(request)
if d is not None:
return d
# Typical reverse proxying.
self.log("Proxying URL => {0}".format(url))
http_client = HTTPClient(self.proxy_agent)
d = http_client.request(request.method, url, **kwds)
def process_response(response, request):
req_resp_headers = request.responseHeaders
resp_code = response.code
resp_headers = response.headers
resp_header_map = dict(resp_headers.getAllRawHeaders())
# Rewrite Location headers for redirects as required.
if resp_code in (301, 302, 303, 307,
308) and "Location" in resp_header_map:
values = resp_header_map["Location"]
if len(values) == 1:
location = values[0]
if request.isSecure():
proxy_scheme = 'https'
else:
proxy_scheme = 'http'
new_location = self.proxied_url_to_proxy_url(
proxy_scheme, location)
if new_location is not None:
resp_header_map['Location'] = [new_location]
request.setResponseCode(response.code, message=response.phrase)
for k, v in resp_header_map.iteritems():
if k == 'Set-Cookie':
v = self.mod_cookies(v)
req_resp_headers.setRawHeaders(k, v)
return response
def mod_content(body, request):
"""
Modify response content before returning it to the user agent.
"""
d = None
for content_modifier in self.content_modifiers:
if d is None:
d = content_modifier.transform_content(body, request)
else:
d.addCallback(content_modifier.transform_content, request)
if d is None:
return body
else:
return d
d.addCallback(process_response, request)
d.addCallback(treq.content)
d.addCallback(mod_content, request)
return d
def http_dlr_callback(self, message):
msgid = message.content.properties['message-id']
url = message.content.properties['headers']['url']
method = message.content.properties['headers']['method']
level = message.content.properties['headers']['level']
self.log.debug('Got one message (msgid:%s) to throw', msgid)
# If any, clear requeuing timer
self.clearRequeueTimer(msgid)
# Build mandatory arguments
args = {
'id': msgid,
'level': level,
'message_status': message.content.properties['headers']['message_status'],
'connector': message.content.properties['headers']['connector']
}
# Level 2 extra args
if level in [2, 3]:
args['id_smsc'] = message.content.properties['headers']['id_smsc']
args['sub'] = message.content.properties['headers']['sub']
args['dlvrd'] = message.content.properties['headers']['dlvrd']
args['subdate'] = message.content.properties['headers']['subdate']
args['donedate'] = message.content.properties['headers']['donedate']
args['err'] = message.content.properties['headers']['err']
args['text'] = message.content.properties['headers']['text']
try:
# Throw the message to http endpoint
postdata = None
params = None
baseurl = url
if method == 'GET':
params = args
else:
postdata = args
self.log.debug('Calling %s with args %s using %s method.', baseurl, args, method)
agent = Agent(reactor)
client = HTTPClient(agent)
response = yield client.request(
method,
baseurl,
params=params,
data=postdata,
timeout=self.config.timeout,
agent='Jasmin gateway/1.0 %s' % self.name,
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain'})
self.log.info('Throwed DLR [msgid:%s] to %s.', msgid, baseurl)
content = yield text_content(response)
if response.code >= 400:
raise HttpApiError(response.code, content)
self.log.debug('Destination end replied to message [msgid:%s]: %r', msgid, content)
# Check for acknowledgement
if content.strip() != 'ACK':
raise MessageAcknowledgementError(
'Destination end did not acknowledge receipt of the DLR message.')
# Everything is okay ? then:
yield self.ackMessage(message)
except Exception as e:
self.log.error('Throwing HTTP/DLR [msgid:%s] to (%s): %r.', msgid, baseurl, e)
# List of errors after which, no further retrying shall be made
noRetryErrors = ['404']
# Requeue message for later retry
if (str(e) not in noRetryErrors
and self.getThrowingRetrials(message) <= self.config.max_retries):
self.log.debug('Message try-count is %s [msgid:%s]: requeuing',
self.getThrowingRetrials(message), msgid)
yield self.rejectAndRequeueMessage(message)
elif str(e) in noRetryErrors:
self.log.warn('Message is no more processed after receiving "%s" error', str(e))
yield self.rejectMessage(message)
else:
self.log.warn('Message try-count is %s [msgid:%s]: purged from queue',
self.getThrowingRetrials(message), msgid)
yield self.rejectMessage(message)
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch('treq.client.FileBodyProducer')
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer')
self.MultiPartProducer = self.mbp_patcher.start()
self.addCleanup(self.mbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/',
headers=Headers({}), bodyProducer=None)
def test_request_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ['bar']})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_tuple_query_values(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ('bar',)})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_merge_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?baz=bax&foo=bar&foo=baz',
headers=Headers({}), bodyProducer=None)
def test_request_merge_tuple_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?baz=bax&foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_dict_single_value_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_data_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar&foo=baz')
def test_request_data_single_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': 'bar'})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar')
def test_request_data_tuple(self):
self.client.request('POST', 'http://example.com/',
data=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar')
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "w") as temp_file:
temp_file.write('hello')
self.client.request('POST', 'http://example.com/', data=file(temp_fn))
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('hello')
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={"name": StringIO("hello")})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', (None, 'application/octet-stream', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', StringIO("hello"))})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'image/jpeg', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment_and_ctype(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', 'text/plain', StringIO("hello"))})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'text/plain', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params(self):
class NamedFile(StringIO):
def __init__(self, val):
StringIO.__init__(self, val)
self.name = "image.png"
self.client.request(
'POST', 'http://example.com/',
data=[("a", "b"), ("key", "val")],
files=[
("file1", ('image.jpg', StringIO("hello"))),
("file2", NamedFile("yo"))])
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('a', 'b'),
('key', 'val'),
('file1', ('image.jpg', 'image/jpeg', FP)),
('file2', ('image.png', 'image/png', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params_dict(self):
self.client.request(
'POST', 'http://example.com/',
data={"key": "a", "key2": "b"},
files={"file1": StringIO("hey")})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('key', 'a'),
('key2', 'b'),
('file1', (None, 'application/octet-stream', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
def test_request_unsupported_params_combination(self):
self.assertRaises(ValueError,
self.client.request,
'POST', 'http://example.com/',
data=StringIO("yo"),
files={"file1": StringIO("hey")})
def test_request_dict_headers(self):
self.client.request('GET', 'http://example.com/', headers={
'User-Agent': 'treq/0.1dev',
'Accept': ['application/json', 'text/plain']
})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/',
headers=Headers({'User-Agent': ['treq/0.1dev'],
'Accept': ['application/json', 'text/plain']}),
bodyProducer=None)
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch("treq.client.FileBodyProducer")
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
self.mbp_patcher = mock.patch("treq.multipart.MultiPartProducer")
self.MultiPartProducer = self.mbp_patcher.start()
self.addCleanup(self.mbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_request_case_insensitive_methods(self):
self.client.request("gEt", "http://example.com/")
self.agent.request.assert_called_once_with("GET", "http://example.com/", headers=Headers({}), bodyProducer=None)
def test_request_query_params(self):
self.client.request("GET", "http://example.com/", params={"foo": ["bar"]})
self.agent.request.assert_called_once_with(
"GET", "http://example.com/?foo=bar", headers=Headers({}), bodyProducer=None
)
def test_request_tuple_query_values(self):
self.client.request("GET", "http://example.com/", params={"foo": ("bar",)})
self.agent.request.assert_called_once_with(
"GET", "http://example.com/?foo=bar", headers=Headers({}), bodyProducer=None
)
def test_request_merge_query_params(self):
self.client.request("GET", "http://example.com/?baz=bax", params={"foo": ["bar", "baz"]})
self.agent.request.assert_called_once_with(
"GET", "http://example.com/?baz=bax&foo=bar&foo=baz", headers=Headers({}), bodyProducer=None
)
def test_request_merge_tuple_query_params(self):
self.client.request("GET", "http://example.com/?baz=bax", params=[("foo", "bar")])
self.agent.request.assert_called_once_with(
"GET", "http://example.com/?baz=bax&foo=bar", headers=Headers({}), bodyProducer=None
)
def test_request_dict_single_value_query_params(self):
self.client.request("GET", "http://example.com/", params={"foo": "bar"})
self.agent.request.assert_called_once_with(
"GET", "http://example.com/?foo=bar", headers=Headers({}), bodyProducer=None
)
def test_request_data_dict(self):
self.client.request("POST", "http://example.com/", data={"foo": ["bar", "baz"]})
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["application/x-www-form-urlencoded"]}),
bodyProducer=self.FileBodyProducer.return_value,
)
self.assertBody("foo=bar&foo=baz")
def test_request_data_single_dict(self):
self.client.request("POST", "http://example.com/", data={"foo": "bar"})
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["application/x-www-form-urlencoded"]}),
bodyProducer=self.FileBodyProducer.return_value,
)
self.assertBody("foo=bar")
def test_request_data_tuple(self):
self.client.request("POST", "http://example.com/", data=[("foo", "bar")])
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["application/x-www-form-urlencoded"]}),
bodyProducer=self.FileBodyProducer.return_value,
)
self.assertBody("foo=bar")
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "w") as temp_file:
temp_file.write("hello")
self.client.request("POST", "http://example.com/", data=file(temp_fn))
self.agent.request.assert_called_once_with(
"POST", "http://example.com/", headers=Headers({}), bodyProducer=self.FileBodyProducer.return_value
)
self.assertBody("hello")
@mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
self.client.request("POST", "http://example.com/", files={"name": StringIO("hello")})
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}),
bodyProducer=self.MultiPartProducer.return_value,
)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([("name", (None, "application/octet-stream", FP))], boundary="heyDavid"),
self.MultiPartProducer.call_args,
)
@mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid"))
def test_request_named_attachment(self):
self.client.request("POST", "http://example.com/", files={"name": ("image.jpg", StringIO("hello"))})
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}),
bodyProducer=self.MultiPartProducer.return_value,
)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([("name", ("image.jpg", "image/jpeg", FP))], boundary="heyDavid"),
self.MultiPartProducer.call_args,
)
@mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid"))
def test_request_named_attachment_and_ctype(self):
self.client.request(
"POST", "http://example.com/", files={"name": ("image.jpg", "text/plain", StringIO("hello"))}
)
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}),
bodyProducer=self.MultiPartProducer.return_value,
)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([("name", ("image.jpg", "text/plain", FP))], boundary="heyDavid"),
self.MultiPartProducer.call_args,
)
@mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid"))
def test_request_mixed_params(self):
class NamedFile(StringIO):
def __init__(self, val):
StringIO.__init__(self, val)
self.name = "image.png"
self.client.request(
"POST",
"http://example.com/",
data=[("a", "b"), ("key", "val")],
files=[("file1", ("image.jpg", StringIO("hello"))), ("file2", NamedFile("yo"))],
)
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}),
bodyProducer=self.MultiPartProducer.return_value,
)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[
("a", "b"),
("key", "val"),
("file1", ("image.jpg", "image/jpeg", FP)),
("file2", ("image.png", "image/png", FP)),
],
boundary="heyDavid",
),
self.MultiPartProducer.call_args,
)
@mock.patch("treq.client.uuid.uuid4", mock.Mock(return_value="heyDavid"))
def test_request_mixed_params_dict(self):
self.client.request(
"POST", "http://example.com/", data={"key": "a", "key2": "b"}, files={"file1": StringIO("hey")}
)
self.agent.request.assert_called_once_with(
"POST",
"http://example.com/",
headers=Headers({"Content-Type": ["multipart/form-data; boundary=heyDavid"]}),
bodyProducer=self.MultiPartProducer.return_value,
)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[("key", "a"), ("key2", "b"), ("file1", (None, "application/octet-stream", FP))], boundary="heyDavid"
),
self.MultiPartProducer.call_args,
)
def test_request_unsupported_params_combination(self):
self.assertRaises(
ValueError,
self.client.request,
"POST",
"http://example.com/",
data=StringIO("yo"),
files={"file1": StringIO("hey")},
)
def test_request_dict_headers(self):
self.client.request(
"GET",
"http://example.com/",
headers={"User-Agent": "treq/0.1dev", "Accept": ["application/json", "text/plain"]},
)
self.agent.request.assert_called_once_with(
"GET",
"http://example.com/",
headers=Headers({"User-Agent": ["treq/0.1dev"], "Accept": ["application/json", "text/plain"]}),
bodyProducer=None,
)
@with_clock
def test_request_timeout_fired(self, clock):
"""
Verify the request is cancelled if a response is not received
within specified timeout period.
"""
self.client.request("GET", "http://example.com", timeout=2)
# simulate we haven't gotten a response within timeout seconds
clock.advance(3)
deferred = self.agent.request.return_value
# a deferred should have been cancelled
self.assertTrue(deferred.cancel.called)
@with_clock
def test_request_timeout_cancelled(self, clock):
"""
Verify timeout is cancelled if a response is received before
timeout period elapses.
"""
self.client.request("GET", "http://example.com", timeout=2)
# simulate a response
deferred = self.agent.request.return_value
gotResult = deferred.addBoth.call_args[0][0]
gotResult("result")
# now advance the clock but since we already got a result,
# a cancellation timer should have been cancelled
clock.advance(3)
self.assertFalse(deferred.cancel.called)
def test_response_is_buffered(self):
response = mock.Mock(deliverBody=mock.Mock())
self.agent.request.return_value = succeed(response)
d = self.client.get("http://www.example.com")
result = self.successResultOf(d)
protocol = mock.Mock(Protocol)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
def test_response_buffering_is_disabled_with_unbufferred_arg(self):
response = mock.Mock()
self.agent.request.return_value = succeed(response)
d = self.client.get("http://www.example.com", unbuffered=True)
self.assertEqual(self.successResultOf(d), response)
def reverse_proxy(self, request, protected=True):
if protected:
sess = request.getSession()
valid_sessions = self.valid_sessions
sess_uid = sess.uid
username = valid_sessions[sess_uid]['username']
# Normal reverse proxying.
kwds = {}
#cookiejar = cookielib.CookieJar()
cookiejar = {}
kwds['allow_redirects'] = False
kwds['cookies'] = cookiejar
req_headers = self.mod_headers(dict(request.requestHeaders.getAllRawHeaders()))
kwds['headers'] = req_headers
if protected:
kwds['headers']['REMOTE_USER'] = [username]
#print "** HEADERS **"
#pprint.pprint(self.mod_headers(dict(request.requestHeaders.getAllRawHeaders())))
#print
if request.method in ('PUT', 'POST'):
kwds['data'] = request.content.read()
#print "request.method", request.method
#print "url", self.proxied_url + request.uri
#print "kwds:"
#pprint.pprint(kwds)
#print
url = self.proxied_url + request.uri
# Determine if a plugin wants to intercept this URL.
interceptors = self.interceptors
for interceptor in interceptors:
if interceptor.should_resource_be_intercepted(url, request.method, req_headers, request):
return interceptor.handle_resource(url, request.method, req_headers, request)
log.msg("[INFO] Proxying URL: %s" % url)
http_client = HTTPClient(self.agent)
d = http_client.request(request.method, url, **kwds)
#print "** Requesting %s %s" % (request.method, self.proxied_url + request.uri)
def process_response(response, request):
req_resp_headers = request.responseHeaders
resp_code = response.code
resp_headers = response.headers
resp_header_map = dict(resp_headers.getAllRawHeaders())
# Rewrite Location headers for redirects as required.
if resp_code in (301, 302, 303, 307, 308) and "Location" in resp_header_map:
values = resp_header_map["Location"]
if len(values) == 1:
location = values[0]
if request.isSecure():
proxy_scheme = 'https'
else:
proxy_scheme = 'http'
new_location = self.proxied_url_to_proxy_url(proxy_scheme, location)
if new_location is not None:
resp_header_map['Location'] = [new_location]
log.msg("[DEBUG] Re-wrote Location header: '%s' => '%s'" % (location, new_location))
request.setResponseCode(response.code, message=response.phrase)
for k,v in resp_header_map.iteritems():
if k == 'Set-Cookie':
v = self.mod_cookies(v)
print("Browser Response >>> Setting response header: %s: %s" % (k, v))
req_resp_headers.setRawHeaders(k, v)
return response
def show_cookies(resp):
jar = resp.cookies()
print("Cookie Jar:")
pprint.pprint(cookiejar)
print("")
return resp
def mod_content(body, request):
"""
Modify response content before returning it to the user agent.
"""
d = None
for content_modifier in self.content_modifiers:
if d is None:
d = content_modifier.transform_content(body, request)
else:
d.addCallback(content_modifier.transform_content, request)
if d is None:
return body
else:
return d
d.addCallback(show_cookies)
d.addCallback(process_response, request)
d.addCallback(treq.content)
d.addCallback(mod_content, request)
return d
class HTTPClient(object):
def __init__(self,
host='127.0.0.1',
port=8500,
scheme='http',
verify=True,
contextFactory=None,
**kwargs):
self.host = host
self.port = port
self.scheme = scheme
self.base_uri = '%s://%s:%s' % (self.scheme, self.host, self.port)
agent_kwargs = dict(
reactor=reactor, pool=HTTPConnectionPool(reactor), **kwargs)
if contextFactory is not None:
# use the provided context factory
agent_kwargs['contextFactory'] = contextFactory
elif not verify:
# if no context is provided and verify is set to false, use the
# insecure context factory implementation
agent_kwargs['contextFactory'] = InsecureContextFactory()
self.client = TreqHTTPClient(Agent(**agent_kwargs))
def uri(self, path, params=None):
uri = self.base_uri + path
if not params:
return uri
return '%s?%s' % (uri, urllib.parse.urlencode(params))
@staticmethod
def response(code, headers, text):
return base.Response(code, headers, text)
@staticmethod
def compat_string(value):
"""
Provide a python2/3 compatible string representation of the value
:type value:
:rtype :
"""
if isinstance(value, bytes):
return value.decode(encoding='utf-8')
return str(value)
@inlineCallbacks
def _get_resp(self, response):
# Merge multiple header values as per RFC2616
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
headers = {
self.compat_string(k): ','.join(map(self.compat_string, v))
for k, v in dict(response.headers.getAllRawHeaders()).items()
}
body = yield response.text(encoding='utf-8')
returnValue((response.code, headers, body))
@inlineCallbacks
def request(self, callback, method, url, **kwargs):
if 'data' in kwargs and not isinstance(kwargs['data'], bytes):
# python2/3 compatibility
data = kwargs.pop('data')
kwargs['data'] = data.encode(encoding='utf-8') \
if hasattr(data, 'encode') else b(data)
try:
response = yield self.client.request(method, url, **kwargs)
parsed = yield self._get_resp(response)
returnValue(callback(self.response(*parsed)))
except ConnectError as e:
raise ConsulException(
'{}: {}'.format(e.__class__.__name__, e.message))
except ResponseNeverReceived:
# this exception is raised if the connection to the server is lost
# when yielding a response, this could be due to network issues or
# server restarts
raise ConsulException(
'Server connection lost: {} {}'.format(method.upper(), url))
except RequestTransmissionFailed:
# this exception is expected if the reactor is stopped mid request
raise ConsulException(
'Request incomplete: {} {}'.format(method.upper(), url))
@inlineCallbacks
def get(self, callback, path, params=None):
uri = self.uri(path, params)
response = yield self.request(callback, 'get', uri, params=params)
returnValue(response)
@inlineCallbacks
def put(self, callback, path, params=None, data=''):
uri = self.uri(path, params)
response = yield self.request(callback, 'put', uri, data=data)
returnValue(response)
@inlineCallbacks
def post(self, callback, path, params=None, data=''):
uri = self.uri(path, params)
response = yield self.request(callback, 'post', uri, data=data)
returnValue(response)
@inlineCallbacks
def delete(self, callback, path, params=None):
uri = self.uri(path, params)
response = yield self.request(callback, 'delete', uri, params=params)
returnValue(response)
class HTTPClient(object):
def __init__(self, host='127.0.0.1', port=8500, scheme='http',
verify=True):
self.host = host
self.port = port
self.scheme = scheme
self.base_uri = '%s://%s:%s' % (self.scheme, self.host, self.port)
self.verify = SSLSpec.CERT_NONE \
if not verify else SSLSpec.CERT_REQUIRED
agent = Agent(reactor=reactor, pool=HTTPConnectionPool(reactor),
contextFactory=AsyncClientSSLContextFactory(
verify=self.verify))
self.client = TreqHTTPClient(agent)
def uri(self, path, params=None):
uri = self.base_uri + path
if not params:
return uri
return '%s?%s' % (uri, urllib.parse.urlencode(params))
def response(self, code, headers, text):
return base.Response(code, headers, text)
@inlineCallbacks
def _get_resp(self, response):
# Merge multiple header values as per RFC2616
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
headers = {
k: ','.join(v)
for k, v in dict(response.headers.getAllRawHeaders()).items()
}
body = yield response.text(encoding='utf-8')
returnValue((response.code, headers, body))
@inlineCallbacks
def request(self, callback, method, url, **kwargs):
try:
response = yield self.client.request(method, url, **kwargs)
parsed = yield self._get_resp(response)
returnValue(callback(self.response(*parsed)))
except ConnectError as e:
raise ConsulException(
'{}: {}'.format(e.__class__.__name__, e.message))
@inlineCallbacks
def get(self, callback, path, params=None):
uri = self.uri(path, params)
response = yield self.request(callback, 'get', uri, params=params)
returnValue(response)
@inlineCallbacks
def put(self, callback, path, params=None, data=''):
uri = self.uri(path, params)
# handle unicode data failure in the zope interface
# twisted.web.iweb.IBodyProducer
data = str(data)
response = yield self.request(callback, 'put', uri, data=data)
returnValue(response)
@inlineCallbacks
def delete(self, callback, path, params=None):
uri = self.uri(path, params)
response = yield self.request(callback, 'delete', uri, params=params)
returnValue(response)
class HTTPClient(base.HTTPClient):
def __init__(self, contextFactory, *args, **kwargs):
super(HTTPClient, self).__init__(*args, **kwargs)
agent_kwargs = dict(reactor=reactor, pool=HTTPConnectionPool(reactor))
if contextFactory is not None:
# use the provided context factory
agent_kwargs['contextFactory'] = contextFactory
elif not self.verify:
# if no context is provided and verify is set to false, use the
# insecure context factory implementation
agent_kwargs['contextFactory'] = InsecureContextFactory()
self.client = TreqHTTPClient(Agent(**agent_kwargs))
@staticmethod
def response(code, headers, text):
return base.Response(code, headers, text)
@staticmethod
def compat_string(value):
"""
Provide a python2/3 compatible string representation of the value
:type value:
:rtype :
"""
if isinstance(value, bytes):
return value.decode(encoding='utf-8')
return str(value)
@inlineCallbacks
def _get_resp(self, response):
# Merge multiple header values as per RFC2616
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
headers = dict([
(self.compat_string(k), ','.join(map(self.compat_string, v)))
for k, v in dict(response.headers.getAllRawHeaders()).items()
])
body = yield response.text(encoding='utf-8')
returnValue((response.code, headers, body))
@inlineCallbacks
def request(self, callback, method, url, **kwargs):
if 'data' in kwargs and not isinstance(kwargs['data'], bytes):
# python2/3 compatibility
data = kwargs.pop('data')
kwargs['data'] = data.encode(encoding='utf-8') \
if hasattr(data, 'encode') else bytes(data)
try:
response = yield self.client.request(method, url, **kwargs)
parsed = yield self._get_resp(response)
returnValue(callback(self.response(*parsed)))
except ConnectError as e:
raise ConsulException('{}: {}'.format(e.__class__.__name__,
e.message))
except ResponseNeverReceived:
# this exception is raised if the connection to the server is lost
# when yielding a response, this could be due to network issues or
# server restarts
raise ConsulException('Server connection lost: {} {}'.format(
method.upper(), url))
except RequestTransmissionFailed:
# this exception is expected if the reactor is stopped mid request
raise ConsulException('Request incomplete: {} {}'.format(
method.upper(), url))
@inlineCallbacks
def get(self, callback, path, params=None):
uri = self.uri(path, params)
response = yield self.request(callback, 'get', uri, params=params)
returnValue(response)
@inlineCallbacks
def put(self, callback, path, params=None, data=''):
uri = self.uri(path, params)
response = yield self.request(callback, 'put', uri, data=data)
returnValue(response)
@inlineCallbacks
def post(self, callback, path, params=None, data=''):
uri = self.uri(path, params)
response = yield self.request(callback, 'post', uri, data=data)
returnValue(response)
@inlineCallbacks
def delete(self, callback, path, params=None):
uri = self.uri(path, params)
response = yield self.request(callback, 'delete', uri, params=params)
returnValue(response)
@inlineCallbacks
def close(self):
pass
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch('treq.client.FileBodyProducer')
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer')
self.MultiPartProducer = self.mbp_patcher.start()
self.addCleanup(self.mbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_post(self):
self.client.post('http://example.com/')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_idn(self):
self.client.request('GET', u'http://č.net')
self.agent.request.assert_called_once_with(
b'GET', b'http://xn--bea.net',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_decodedurl(self):
"""
A URL may be passed as a `hyperlink.DecodedURL` object. It is converted
to bytes when passed to the underlying agent.
"""
url = DecodedURL.from_text(u"https://example.org/foo")
self.client.request("GET", url)
self.agent.request.assert_called_once_with(
b"GET", b"https://example.org/foo",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)
def test_request_uri_encodedurl(self):
"""
A URL may be passed as a `hyperlink.EncodedURL` object. It is converted
to bytes when passed to the underlying agent.
"""
url = EncodedURL.from_text(u"https://example.org/foo")
self.client.request("GET", url)
self.agent.request.assert_called_once_with(
b"GET", b"https://example.org/foo",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)
def test_request_uri_idn_params(self):
"""
A URL that contains non-ASCII characters can be augmented with
querystring parameters.
This reproduces treq #264.
"""
self.client.request('GET', u'http://č.net', params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'GET', b'http://xn--bea.net/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_hyperlink_params(self):
"""
The *params* argument augments an instance of `hyperlink.DecodedURL`
passed as the *url* parameter, just as if it were a string.
"""
self.client.request(
method="GET",
url=DecodedURL.from_text(u"http://č.net"),
params={"foo": "bar"},
)
self.agent.request.assert_called_once_with(
b"GET", b"http://xn--bea.net/?foo=bar",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ['bar']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_tuple_query_values(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ('bar',)})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_tuple_query_value_coercion(self):
"""
treq coerces non-string values passed to *params* like
`urllib.urlencode()`
"""
self.client.request('GET', 'http://example.com/', params=[
('text', u'A\u03a9'),
('text-seq', [u'A\u03a9']),
('bytes', [b'ascii']),
('bytes-seq', [b'ascii']),
('native', ['native']),
('native-seq', ['aa', 'bb']),
('int', 1),
('int-seq', (1, 2, 3)),
('none', None),
('none-seq', [None, None]),
])
self.agent.request.assert_called_once_with(
b'GET',
(
b'http://example.com/?'
b'text=A%CE%A9&text-seq=A%CE%A9'
b'&bytes=ascii&bytes-seq=ascii'
b'&native=native&native-seq=aa&native-seq=bb'
b'&int=1&int-seq=1&int-seq=2&int-seq=3'
b'&none=None&none-seq=None&none-seq=None'
),
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_tuple_query_param_coercion(self):
"""
treq coerces non-string param names passed to *params* like
`urllib.urlencode()`
"""
self.client.request('GET', 'http://example.com/', params=[
(u'text', u'A\u03a9'),
(b'bytes', ['ascii']),
('native', 'native'),
(1, 'int'),
(None, ['none']),
])
self.agent.request.assert_called_once_with(
b'GET',
(
b'http://example.com/'
b'?text=A%CE%A9&bytes=ascii'
b'&native=native&1=int&None=none'
),
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_query_param_seps(self):
"""
When the characters ``&`` and ``#`` are passed to *params* as param
names or values they are percent-escaped in the URL.
This reproduces https://github.com/twisted/treq/issues/282
"""
self.client.request('GET', 'http://example.com/', params=(
('ampersand', '&'),
('&', 'ampersand'),
('octothorpe', '#'),
('#', 'octothorpe'),
))
self.agent.request.assert_called_once_with(
b'GET',
(
b'http://example.com/'
b'?ampersand=%26'
b'&%26=ampersand'
b'&octothorpe=%23'
b'&%23=octothorpe'
),
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_merge_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_merge_tuple_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_dict_single_value_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_data_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar&foo=baz')
def test_request_data_single_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_tuple(self):
self.client.request('POST', 'http://example.com/',
data=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "wb") as temp_file:
temp_file.write(b'hello')
self.client.request('POST', 'http://example.com/',
data=open(temp_fn, 'rb'))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'hello')
def test_request_json_dict(self):
self.client.request('POST', 'http://example.com/', json={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'{"foo":"bar"}')
def test_request_json_tuple(self):
self.client.request('POST', 'http://example.com/', json=('foo', 1))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'["foo",1]')
def test_request_json_number(self):
self.client.request('POST', 'http://example.com/', json=1.)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'1.0')
def test_request_json_string(self):
self.client.request('POST', 'http://example.com/', json='hello')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'"hello"')
def test_request_json_bool(self):
self.client.request('POST', 'http://example.com/', json=True)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'true')
def test_request_json_none(self):
self.client.request('POST', 'http://example.com/', json=None)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'null')
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={"name": BytesIO(b"hello")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'image/jpeg', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment_and_ctype(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', 'text/plain', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'text/plain', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params(self):
class NamedFile(BytesIO):
def __init__(self, val):
BytesIO.__init__(self, val)
self.name = "image.png"
self.client.request(
'POST', 'http://example.com/',
data=[("a", "b"), ("key", "val")],
files=[
("file1", ('image.jpg', BytesIO(b"hello"))),
("file2", NamedFile(b"yo"))])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('a', 'b'),
('key', 'val'),
('file1', ('image.jpg', 'image/jpeg', FP)),
('file2', ('image.png', 'image/png', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params_dict(self):
self.client.request(
'POST', 'http://example.com/',
data={"key": "a", "key2": "b"},
files={"file1": BytesIO(b"hey")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']}),
self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('key', 'a'),
('key2', 'b'),
('file1', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'),
self.MultiPartProducer.call_args)
def test_request_unsupported_params_combination(self):
self.assertRaises(ValueError,
self.client.request,
'POST', 'http://example.com/',
data=BytesIO(b"yo"),
files={"file1": BytesIO(b"hey")})
def test_request_dict_headers(self):
self.client.request('GET', 'http://example.com/', headers={
'User-Agent': 'treq/0.1dev',
'Accept': ['application/json', 'text/plain']
})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({b'User-Agent': [b'treq/0.1dev'],
b'accept-encoding': [b'gzip'],
b'Accept': [b'application/json', b'text/plain']}),
None)
@with_clock
def test_request_timeout_fired(self, clock):
"""
Verify the request is cancelled if a response is not received
within specified timeout period.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate we haven't gotten a response within timeout seconds
clock.advance(3)
# a deferred should have been cancelled
self.failureResultOf(d, CancelledError)
@with_clock
def test_request_timeout_cancelled(self, clock):
"""
Verify timeout is cancelled if a response is received before
timeout period elapses.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate a response
d.callback(mock.Mock(code=200, headers=Headers({})))
# now advance the clock but since we already got a result,
# a cancellation timer should have been cancelled
clock.advance(3)
self.successResultOf(d)
def test_response_is_buffered(self):
response = mock.Mock(deliverBody=mock.Mock(),
headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com')
result = self.successResultOf(d)
protocol = mock.Mock(Protocol)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
def test_response_buffering_is_disabled_with_unbufferred_arg(self):
response = mock.Mock(headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com', unbuffered=True)
# YOLO public attribute.
self.assertEqual(self.successResultOf(d).original, response)
def test_request_post_redirect_denied(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
d = self.client.post('http://www.example.com')
self.failureResultOf(d, ResponseFailed)
def test_request_browser_like_redirects(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
raw = mock.Mock(return_value=[])
final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw))
with mock.patch('twisted.web.client.RedirectAgent._handleRedirect',
return_value=final_resp):
d = self.client.post('http://www.google.com',
browser_like_redirects=True,
unbuffered=True)
self.assertEqual(self.successResultOf(d).original, final_resp)
def http_deliver_sm_callback(self, message):
msgid = message.content.properties['message-id']
route_type = message.content.properties['headers']['route-type']
dcs = pickle.loads(message.content.properties['headers']['dst-connectors'])
RoutedDeliverSmContent = pickle.loads(message.content.body)
self.log.debug('Got one message (msgid:%s) to throw: %s', msgid, RoutedDeliverSmContent)
# If any, clear requeuing timer
self.clearRequeueTimer(msgid)
if dcs[0]._type != 'http':
self.log.error(
'Rejecting message [msgid:%s] because destination connector is not http (type were %s)',
msgid,
dcs[0]._type)
yield self.rejectMessage(message)
defer.returnValue(None)
# Build mandatory arguments
args = {
'id': msgid,
'from': RoutedDeliverSmContent.params['source_addr'],
'to': RoutedDeliverSmContent.params['destination_addr'],
'origin-connector': message.content.properties['headers']['src-connector-id']}
# Content can be short_message or message_payload:
if 'short_message' in RoutedDeliverSmContent.params and len(RoutedDeliverSmContent.params['short_message']) > 0:
args['content'] = RoutedDeliverSmContent.params['short_message']
elif 'message_payload' in RoutedDeliverSmContent.params:
args['content'] = RoutedDeliverSmContent.params['message_payload']
elif 'short_message' in RoutedDeliverSmContent.params:
args['content'] = RoutedDeliverSmContent.params['short_message']
else:
self.log.error('Cannot find content in pdu (msgid:%s): %s', msgid, RoutedDeliverSmContent)
yield self.rejectMessage(message)
defer.returnValue(None)
# Set the binary arg after deciding where to pick the content from
if isinstance(args['content'], bytes):
args['binary'] = binascii.hexlify(args['content'])
else:
args['binary'] = binascii.hexlify(args['content'].encode())
# Build optional arguments
if ('priority_flag' in RoutedDeliverSmContent.params and
RoutedDeliverSmContent.params['priority_flag'] is not None):
args['priority'] = priority_flag_name_map[RoutedDeliverSmContent.params['priority_flag'].name]
if ('data_coding' in RoutedDeliverSmContent.params and
RoutedDeliverSmContent.params['data_coding'] is not None):
args['coding'] = DataCodingEncoder().encode(RoutedDeliverSmContent.params['data_coding'])
if ('validity_period' in RoutedDeliverSmContent.params and
RoutedDeliverSmContent.params['validity_period'] is not None):
args['validity'] = RoutedDeliverSmContent.params['validity_period']
counter = 0
for dc in dcs:
counter += 1
self.log.debug('DCS Iteration %s/%s taking [cid:%s] (%s)', counter, len(dcs), dc.cid, dc)
last_dc = True
if route_type == 'failover' and counter < len(dcs):
last_dc = False
try:
# Throw the message to http endpoint
postdata = None
params = None
baseurl = dc.baseurl
_method = dc.method
if isinstance(_method, bytes):
_method = _method.decode().upper()
else:
_method = _method.upper()
if _method == 'GET':
params = args
else:
postdata = args
self.log.debug('Calling %s with args %s using %s method.', dc.baseurl, args, _method)
agent = Agent(reactor)
client = HTTPClient(agent)
response = yield client.request(
_method,
baseurl,
params=params,
data=postdata,
timeout=self.config.timeout,
agent='Jasmin gateway/1.0 deliverSmHttpThrower',
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain'})
self.log.info('Throwed message [msgid:%s] to connector (%s %s/%s)[cid:%s] using http to %s.',
msgid, route_type, counter, len(dcs), dc.cid, dc.baseurl)
content = yield text_content(response)
if response.code >= 400:
raise HttpApiError(response.code, content)
self.log.debug('Destination end replied to message [msgid:%s]: %r',
msgid, content)
# Check for acknowledgement
if content.strip() != 'ACK':
raise MessageAcknowledgementError(
'Destination end did not acknowledge receipt of the message.')
except Exception as e:
self.log.error('Throwing message [msgid:%s] to (%s %s/%s)[cid:%s] (%s), %s: %s.',
msgid, route_type, counter, len(dcs), dc.cid, dc.baseurl, type(e), e)
# List of errors after which, no further retrying shall be made
noRetryErrors = ['404']
if route_type == 'simple':
# Requeue message for later retry
if (str(e) not in noRetryErrors
and self.getThrowingRetrials(message) <= self.config.max_retries):
self.log.debug('Message try-count is %s [msgid:%s]: requeuing',
self.getThrowingRetrials(message), msgid)
yield self.rejectAndRequeueMessage(message)
elif str(e) in noRetryErrors:
self.log.warn('Message [msgid:%s] is no more processed after receiving "%s" error',
msgid, str(e))
yield self.rejectMessage(message)
else:
self.log.warn('Message try-count is %s [msgid:%s]: purged from queue',
self.getThrowingRetrials(message), msgid)
yield self.rejectMessage(message)
elif route_type == 'failover':
# The route has multiple connectors, we will not retry throwing to same connector
if last_dc:
self.log.warn(
'Message [msgid:%s] is no more processed after receiving "%s" error on this fo/connector',
msgid, str(e))
else:
# Everything is okay ? then:
yield self.ackMessage(message)
if route_type == 'failover':
self.log.debug('Stopping iteration for failover route.')
break
finally:
if route_type == 'simple':
# There's only one connector for simple routes
break
elif route_type == 'failover' and last_dc:
self.log.debug('Break (last dc) iteration for failover route.')
break
elif route_type == 'failover' and not last_dc:
self.log.debug('Continue iteration for failover route.')
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch('treq.client.FileBodyProducer')
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer')
self.MultiPartProducer = self.mbp_patcher.start()
self.addCleanup(self.mbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/',
headers=Headers({}), bodyProducer=None)
def test_request_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ['bar']})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_tuple_query_values(self):
self.client.request('GET', 'http://example.com/',
params={'foo': ('bar',)})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_merge_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?baz=bax&foo=bar&foo=baz',
headers=Headers({}), bodyProducer=None)
def test_request_merge_tuple_query_params(self):
self.client.request('GET', 'http://example.com/?baz=bax',
params=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?baz=bax&foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_dict_single_value_query_params(self):
self.client.request('GET', 'http://example.com/',
params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/?foo=bar',
headers=Headers({}), bodyProducer=None)
def test_request_data_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar&foo=baz')
def test_request_data_single_dict(self):
self.client.request('POST', 'http://example.com/',
data={'foo': 'bar'})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar')
def test_request_data_tuple(self):
self.client.request('POST', 'http://example.com/',
data=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers(
{'Content-Type': ['application/x-www-form-urlencoded']}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('foo=bar')
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "w") as temp_file:
temp_file.write('hello')
self.client.request('POST', 'http://example.com/', data=file(temp_fn))
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({}),
bodyProducer=self.FileBodyProducer.return_value)
self.assertBody('hello')
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={"name": StringIO("hello")})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', (None, 'application/octet-stream', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', StringIO("hello"))})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'image/jpeg', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment_and_ctype(self):
self.client.request(
'POST', 'http://example.com/', files={
"name": ('image.jpg', 'text/plain', StringIO("hello"))})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call(
[('name', ('image.jpg', 'text/plain', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params(self):
class NamedFile(StringIO):
def __init__(self, val):
StringIO.__init__(self, val)
self.name = "image.png"
self.client.request(
'POST', 'http://example.com/',
data=[("a", "b"), ("key", "val")],
files=[
("file1", ('image.jpg', StringIO("hello"))),
("file2", NamedFile("yo"))])
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('a', 'b'),
('key', 'val'),
('file1', ('image.jpg', 'image/jpeg', FP)),
('file2', ('image.png', 'image/png', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params_dict(self):
self.client.request(
'POST', 'http://example.com/',
data={"key": "a", "key2": "b"},
files={"file1": StringIO("hey")})
self.agent.request.assert_called_once_with(
'POST', 'http://example.com/',
headers=Headers({
'Content-Type': [
'multipart/form-data; boundary=heyDavid']}),
bodyProducer=self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([
('key', 'a'),
('key2', 'b'),
('file1', (None, 'application/octet-stream', FP))],
boundary='heyDavid'),
self.MultiPartProducer.call_args)
def test_request_unsupported_params_combination(self):
self.assertRaises(ValueError,
self.client.request,
'POST', 'http://example.com/',
data=StringIO("yo"),
files={"file1": StringIO("hey")})
def test_request_dict_headers(self):
self.client.request('GET', 'http://example.com/', headers={
'User-Agent': 'treq/0.1dev',
'Accept': ['application/json', 'text/plain']
})
self.agent.request.assert_called_once_with(
'GET', 'http://example.com/',
headers=Headers({'User-Agent': ['treq/0.1dev'],
'Accept': ['application/json', 'text/plain']}),
bodyProducer=None)
@with_clock
def test_request_timeout_fired(self, clock):
"""
Verify the request is cancelled if a response is not received
within specified timeout period.
"""
self.client.request('GET', 'http://example.com', timeout=2)
# simulate we haven't gotten a response within timeout seconds
clock.advance(3)
deferred = self.agent.request.return_value
# a deferred should have been cancelled
self.assertTrue(deferred.cancel.called)
@with_clock
def test_request_timeout_cancelled(self, clock):
"""
Verify timeout is cancelled if a response is received before
timeout period elapses.
"""
self.client.request('GET', 'http://example.com', timeout=2)
# simulate a response
deferred = self.agent.request.return_value
gotResult = deferred.addBoth.call_args[0][0]
gotResult('result')
# now advance the clock but since we already got a result,
# a cancellation timer should have been cancelled
clock.advance(3)
self.assertFalse(deferred.cancel.called)
class HTTPClientTests(TestCase):
def setUp(self):
self.agent = mock.Mock(Agent)
self.client = HTTPClient(self.agent)
self.fbp_patcher = mock.patch('treq.client.FileBodyProducer')
self.FileBodyProducer = self.fbp_patcher.start()
self.addCleanup(self.fbp_patcher.stop)
self.mbp_patcher = mock.patch('treq.multipart.MultiPartProducer')
self.MultiPartProducer = self.mbp_patcher.start()
self.addCleanup(self.mbp_patcher.stop)
def assertBody(self, expected):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
def test_post(self):
self.client.post('http://example.com/')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_idn(self):
self.client.request('GET', u'http://č.net')
self.agent.request.assert_called_once_with(
b'GET', b'http://xn--bea.net',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_decodedurl(self):
"""
A URL may be passed as a `hyperlink.DecodedURL` object. It is converted
to bytes when passed to the underlying agent.
"""
url = DecodedURL.from_text(u"https://example.org/foo")
self.client.request("GET", url)
self.agent.request.assert_called_once_with(
b"GET",
b"https://example.org/foo",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)
def test_request_uri_encodedurl(self):
"""
A URL may be passed as a `hyperlink.EncodedURL` object. It is converted
to bytes when passed to the underlying agent.
"""
url = EncodedURL.from_text(u"https://example.org/foo")
self.client.request("GET", url)
self.agent.request.assert_called_once_with(
b"GET",
b"https://example.org/foo",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)
def test_request_uri_bytes_pass(self):
"""
The URL parameter may contain path segments or querystring parameters
that are not valid UTF-8. These pass through.
"""
# This URL is http://example.com/hello?who=you, but "hello", "who", and
# "you" are encoded as UTF-16. The particulars of the encoding aren't
# important; what matters is that those segments can't be decoded by
# Hyperlink's UTF-8 default.
self.client.request(
"GET",
("http://example.com/%FF%FEh%00e%00l%00l%00o%00"
"?%FF%FEw%00h%00o%00=%FF%FEy%00o%00u%00"),
)
self.agent.request.assert_called_once_with(
b'GET',
(b'http://example.com/%FF%FEh%00e%00l%00l%00o%00'
b'?%FF%FEw%00h%00o%00=%FF%FEy%00o%00u%00'),
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_uri_plus_pass(self):
"""
URL parameters may contain spaces encoded as ``+``. These remain as
such and are not mangled.
This reproduces `Klein #339 <https://github.com/twisted/klein/issues/339>`_.
"""
self.client.request(
"GET",
"https://example.com/?foo+bar=baz+biff",
)
self.agent.request.assert_called_once_with(
b'GET',
b"https://example.com/?foo+bar=baz+biff",
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_uri_idn_params(self):
"""
A URL that contains non-ASCII characters can be augmented with
querystring parameters.
This reproduces treq #264.
"""
self.client.request('GET', u'http://č.net', params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'GET', b'http://xn--bea.net/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_uri_hyperlink_params(self):
"""
The *params* argument augments an instance of `hyperlink.DecodedURL`
passed as the *url* parameter, just as if it were a string.
"""
self.client.request(
method="GET",
url=DecodedURL.from_text(u"http://č.net"),
params={"foo": "bar"},
)
self.agent.request.assert_called_once_with(
b"GET",
b"http://xn--bea.net/?foo=bar",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_query_params(self):
self.client.request('GET',
'http://example.com/',
params={'foo': ['bar']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_tuple_query_values(self):
self.client.request('GET',
'http://example.com/',
params={'foo': ('bar', )})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_tuple_query_value_coercion(self):
"""
treq coerces non-string values passed to *params* like
`urllib.urlencode()`
"""
self.client.request('GET',
'http://example.com/',
params=[
('text', u'A\u03a9'),
('text-seq', [u'A\u03a9']),
('bytes', [b'ascii']),
('bytes-seq', [b'ascii']),
('native', ['native']),
('native-seq', ['aa', 'bb']),
('int', 1),
('int-seq', (1, 2, 3)),
('none', None),
('none-seq', [None, None]),
])
self.agent.request.assert_called_once_with(
b'GET',
(b'http://example.com/?'
b'text=A%CE%A9&text-seq=A%CE%A9'
b'&bytes=ascii&bytes-seq=ascii'
b'&native=native&native-seq=aa&native-seq=bb'
b'&int=1&int-seq=1&int-seq=2&int-seq=3'
b'&none=None&none-seq=None&none-seq=None'),
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_tuple_query_param_coercion(self):
"""
treq coerces non-string param names passed to *params* like
`urllib.urlencode()`
"""
# A value used to test that it is never encoded or decoded.
# It should be invalid UTF-8 or UTF-32 (at least).
raw_bytes = b"\x00\xff\xfb"
self.client.request('GET',
'http://example.com/',
params=[
(u'text', u'A\u03a9'),
(b'bytes', ['ascii', raw_bytes]),
('native', 'native'),
(1, 'int'),
(None, ['none']),
])
self.agent.request.assert_called_once_with(
b'GET',
(b'http://example.com/'
b'?text=A%CE%A9&bytes=ascii&bytes=%00%FF%FB'
b'&native=native&1=int&None=none'),
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_query_param_seps(self):
"""
When the characters ``&`` and ``#`` are passed to *params* as param
names or values they are percent-escaped in the URL.
This reproduces https://github.com/twisted/treq/issues/282
"""
self.client.request('GET',
'http://example.com/',
params=(
('ampersand', '&'),
('&', 'ampersand'),
('octothorpe', '#'),
('#', 'octothorpe'),
))
self.agent.request.assert_called_once_with(
b'GET',
(b'http://example.com/'
b'?ampersand=%26'
b'&%26=ampersand'
b'&octothorpe=%23'
b'&%23=octothorpe'),
Headers({b'accept-encoding': [b'gzip']}),
None,
)
def test_request_merge_query_params(self):
self.client.request('GET',
'http://example.com/?baz=bax',
params={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar&foo=baz',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_merge_tuple_query_params(self):
self.client.request('GET',
'http://example.com/?baz=bax',
params=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?baz=bax&foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_dict_single_value_query_params(self):
self.client.request('GET',
'http://example.com/',
params={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/?foo=bar',
Headers({b'accept-encoding': [b'gzip']}), None)
def test_request_data_dict(self):
self.client.request('POST',
'http://example.com/',
data={'foo': ['bar', 'baz']})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar&foo=baz')
def test_request_data_single_dict(self):
self.client.request('POST', 'http://example.com/', data={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_tuple(self):
self.client.request('POST',
'http://example.com/',
data=[('foo', 'bar')])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/x-www-form-urlencoded'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'foo=bar')
def test_request_data_file(self):
temp_fn = self.mktemp()
with open(temp_fn, "wb") as temp_file:
temp_file.write(b'hello')
self.client.request('POST',
'http://example.com/',
data=open(temp_fn, 'rb'))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({b'accept-encoding': [b'gzip']}),
self.FileBodyProducer.return_value)
self.assertBody(b'hello')
def test_request_json_dict(self):
self.client.request('POST', 'http://example.com/', json={'foo': 'bar'})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'{"foo":"bar"}')
def test_request_json_tuple(self):
self.client.request('POST', 'http://example.com/', json=('foo', 1))
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'["foo",1]')
def test_request_json_number(self):
self.client.request('POST', 'http://example.com/', json=1.)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'1.0')
def test_request_json_string(self):
self.client.request('POST', 'http://example.com/', json='hello')
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'"hello"')
def test_request_json_bool(self):
self.client.request('POST', 'http://example.com/', json=True)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'true')
def test_request_json_none(self):
self.client.request('POST', 'http://example.com/', json=None)
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'Content-Type': [b'application/json; charset=UTF-8'],
b'accept-encoding': [b'gzip']
}), self.FileBodyProducer.return_value)
self.assertBody(b'null')
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
self.client.request('POST',
'http://example.com/',
files={"name": BytesIO(b"hello")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('name', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment(self):
self.client.request('POST',
'http://example.com/',
files={"name": ('image.jpg', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('name', ('image.jpg', 'image/jpeg', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_named_attachment_and_ctype(self):
self.client.request(
'POST',
'http://example.com/',
files={"name": ('image.jpg', 'text/plain', BytesIO(b"hello"))})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('name', ('image.jpg', 'text/plain', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
def test_request_files_tuple_too_short(self):
"""
The `HTTPClient.request()` *files* argument requires tuples of length
2 or 3. It raises `TypeError` when the tuple is too short.
"""
with self.assertRaises(TypeError) as c:
self.client.request(
"POST",
b"http://example.com/",
files=[("t1", ("foo.txt", ))],
)
self.assertIn("'t1' tuple has length 1", str(c.exception))
def test_request_files_tuple_too_long(self):
"""
The `HTTPClient.request()` *files* argument requires tuples of length
2 or 3. It raises `TypeError` when the tuple is too long.
"""
with self.assertRaises(TypeError) as c:
self.client.request(
"POST",
b"http://example.com/",
files=[
("t4", ("foo.txt", "text/plain", BytesIO(b"...\n"),
"extra!")),
],
)
self.assertIn("'t4' tuple has length 4", str(c.exception))
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params(self):
class NamedFile(BytesIO):
def __init__(self, val):
BytesIO.__init__(self, val)
self.name = "image.png"
self.client.request('POST',
'http://example.com/',
data=[("a", "b"), ("key", "val")],
files=[("file1", ('image.jpg', BytesIO(b"hello"))),
("file2", NamedFile(b"yo"))])
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('a', 'b'), ('key', 'val'),
('file1', ('image.jpg', 'image/jpeg', FP)),
('file2', ('image.png', 'image/png', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_mixed_params_dict(self):
self.client.request('POST',
'http://example.com/',
data={
"key": "a",
"key2": "b"
},
files={"file1": BytesIO(b"hey")})
self.agent.request.assert_called_once_with(
b'POST', b'http://example.com/',
Headers({
b'accept-encoding': [b'gzip'],
b'Content-Type': [b'multipart/form-data; boundary=heyDavid']
}), self.MultiPartProducer.return_value)
FP = self.FileBodyProducer.return_value
self.assertEqual(
mock.call([('key', 'a'), ('key2', 'b'),
('file1', (None, 'application/octet-stream', FP))],
boundary=b'heyDavid'), self.MultiPartProducer.call_args)
def test_request_unsupported_params_combination(self):
self.assertRaises(ValueError,
self.client.request,
'POST',
'http://example.com/',
data=BytesIO(b"yo"),
files={"file1": BytesIO(b"hey")})
def test_request_json_with_data(self):
"""
Passing `HTTPClient.request()` both *data* and *json* parameters is
invalid because *json* is ignored. This behavior is deprecated.
"""
self.client.request(
"POST",
"http://example.com/",
data=BytesIO(b"..."),
json=None, # NB: None is a valid value. It encodes to b'null'.
)
[w] = self.flushWarnings([self.test_request_json_with_data])
self.assertEqual(DeprecationWarning, w["category"])
self.assertEqual(
("Argument 'json' will be ignored because 'data' was also passed."
" This will raise TypeError in the next treq release."),
w['message'],
)
def test_request_json_with_files(self):
"""
Passing `HTTPClient.request()` both *files* and *json* parameters is
invalid because *json* is ignored. This behavior is deprecated.
"""
self.client.request(
"POST",
"http://example.com/",
files={"f1": ("foo.txt", "text/plain", BytesIO(b"...\n"))},
json=["this is ignored"],
)
[w] = self.flushWarnings([self.test_request_json_with_files])
self.assertEqual(DeprecationWarning, w["category"])
self.assertEqual(
("Argument 'json' will be ignored because 'files' was also passed."
" This will raise TypeError in the next treq release."),
w['message'],
)
def test_request_dict_headers(self):
self.client.request('GET',
'http://example.com/',
headers={
'User-Agent': 'treq/0.1dev',
'Accept': ['application/json', 'text/plain']
})
self.agent.request.assert_called_once_with(
b'GET', b'http://example.com/',
Headers({
b'User-Agent': [b'treq/0.1dev'],
b'accept-encoding': [b'gzip'],
b'Accept': [b'application/json', b'text/plain']
}), None)
def test_request_headers_object(self):
"""
The *headers* parameter accepts a `twisted.web.http_headers.Headers`
instance.
"""
self.client.request(
"GET",
"https://example.com",
headers=Headers({"X-Foo": ["bar"]}),
)
self.agent.request.assert_called_once_with(
b"GET",
b"https://example.com",
Headers({
"X-Foo": ["bar"],
"Accept-Encoding": ["gzip"],
}),
None,
)
def test_request_headers_invalid_type(self):
"""
`HTTPClient.request()` warns that headers of an unexpected type are
invalid and that this behavior is deprecated.
"""
self.client.request('GET', 'http://example.com', headers=[])
[w] = self.flushWarnings([self.test_request_headers_invalid_type])
self.assertEqual(DeprecationWarning, w['category'])
self.assertIn(
"headers must be a dict, twisted.web.http_headers.Headers, or None,",
w['message'],
)
def test_request_dict_headers_invalid_values(self):
"""
`HTTPClient.request()` warns that non-string header values are dropped
and that this behavior is deprecated.
"""
self.client.request('GET',
'http://example.com',
headers=OrderedDict([
('none', None),
('one', 1),
('ok', 'string'),
]))
[w1, w2] = self.flushWarnings(
[self.test_request_dict_headers_invalid_values])
self.assertEqual(DeprecationWarning, w1['category'])
self.assertEqual(DeprecationWarning, w2['category'])
self.assertIn(
"The value of headers key 'none' has non-string type",
w1['message'],
)
self.assertIn(
"The value of headers key 'one' has non-string type",
w2['message'],
)
def test_request_invalid_param(self):
"""
`HTTPClient.request()` rejects invalid keyword parameters with
`TypeError`.
"""
self.assertRaises(
TypeError,
self.client.request,
"GET",
b"http://example.com",
invalid=True,
)
@with_clock
def test_request_timeout_fired(self, clock):
"""
Verify the request is cancelled if a response is not received
within specified timeout period.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate we haven't gotten a response within timeout seconds
clock.advance(3)
# a deferred should have been cancelled
self.failureResultOf(d, CancelledError)
@with_clock
def test_request_timeout_cancelled(self, clock):
"""
Verify timeout is cancelled if a response is received before
timeout period elapses.
"""
self.agent.request.return_value = d = Deferred()
self.client.request('GET', 'http://example.com', timeout=2)
# simulate a response
d.callback(mock.Mock(code=200, headers=Headers({})))
# now advance the clock but since we already got a result,
# a cancellation timer should have been cancelled
clock.advance(3)
self.successResultOf(d)
def test_response_is_buffered(self):
response = mock.Mock(deliverBody=mock.Mock(), headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com')
result = self.successResultOf(d)
protocol = mock.Mock(Protocol)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
result.deliverBody(protocol)
self.assertEqual(response.deliverBody.call_count, 1)
def test_response_buffering_is_disabled_with_unbufferred_arg(self):
response = mock.Mock(headers=Headers({}))
self.agent.request.return_value = succeed(response)
d = self.client.get('http://www.example.com', unbuffered=True)
# YOLO public attribute.
self.assertEqual(self.successResultOf(d).original, response)
def test_request_post_redirect_denied(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
d = self.client.post('http://www.example.com')
self.failureResultOf(d, ResponseFailed)
def test_request_browser_like_redirects(self):
response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
self.agent.request.return_value = succeed(response)
raw = mock.Mock(return_value=[])
final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw))
with mock.patch('twisted.web.client.RedirectAgent._handleRedirect',
return_value=final_resp):
d = self.client.post('http://www.google.com',
browser_like_redirects=True,
unbuffered=True)
self.assertEqual(self.successResultOf(d).original, final_resp)
class _SCPDCommand(object):
def __init__(self, gateway_address, service_port, control_url, service_id, method, param_names, returns,
reactor=None):
if not reactor:
from twisted.internet import reactor
self._reactor = reactor
self._pool = HTTPConnectionPool(reactor)
self.agent = Agent(reactor, connectTimeout=1)
self._http_client = HTTPClient(self.agent, data_to_body_producer=StringProducer)
self.gateway_address = gateway_address
self.service_port = service_port
self.control_url = control_url
self.service_id = service_id
self.method = method
self.param_names = param_names
self.returns = returns
def extract_body(self, xml_response, service_key=IP_SCHEMA):
content_dict = etree_to_dict(ElementTree.fromstring(xml_response))
envelope = content_dict[ENVELOPE]
return flatten_keys(envelope[BODY], "{%s}" % service_key)
def extract_response(self, body):
body = handle_fault(body) # raises UPnPError if there is a fault
if '%sResponse' % self.method in body:
response_key = '%sResponse' % self.method
else:
log.error(body.keys())
raise UPnPError("unknown response fields")
response = body[response_key]
extracted_response = tuple([response[n] for n in self.returns])
if len(extracted_response) == 1:
return extracted_response[0]
return extracted_response
@defer.inlineCallbacks
def send_upnp_soap(self, **kwargs):
soap_body = get_soap_body(self.service_id, self.method, self.param_names, **kwargs).encode()
headers = OrderedDict((
('SOAPAction', '%s#%s' % (self.service_id, self.method)),
('Host', ('%s:%i' % (SSDP_IP_ADDRESS, self.service_port))),
('Content-Type', 'text/xml'),
('Content-Length', len(soap_body))
))
response = yield self._http_client.request(
POST, url=self.control_url, data=soap_body, headers=headers
)
xml_response = yield response.content()
response = self.extract_response(self.extract_body(xml_response))
defer.returnValue(response)
@staticmethod
def _process_result(results):
"""
this method gets decorated automatically with a function that maps result types to the types
defined in the @return_types decorator
"""
return results
@defer.inlineCallbacks
def __call__(self, **kwargs):
if set(kwargs.keys()) != set(self.param_names):
raise Exception("argument mismatch")
response = yield self.send_upnp_soap(**kwargs)
result = self._process_result(response)
defer.returnValue(result)