def request(self, url, method='GET', data=None):
d = defer.Deferred()
_headers = {}
headers = None
if method == 'GET':
data = None
elif method == 'POST':
_headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
d.errback(Exception('Not Support Method:%s.' % method))
return
if _headers:
headers = Headers()
for k, v in _headers.iteritems():
headers.setRawHeaders(urllib.quote(k), [urllib.quote(v)])
if data:
data = StrBodyProducer(data)
#print 'Req url:', url, ' Head:', headers
self.agent.request(
method,
url,
headers,
data).addCallback(self.received, d).addErrback(
self.errback, d)
return d
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders("test\u00E1", ["foo\u2603"])
i = h.copy()
# The copy contains the same value as the original
self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"])
self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"])
# Add a header to the original
h.addRawHeader("test\u00E1", "bar")
# Verify that the copy has not changed
self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"])
self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"])
# Add a header to the copy
i.addRawHeader("test\u00E1", b"baz")
# Verify that the orignal does not have it
self.assertEqual(h.getRawHeaders("test\u00E1"), ["foo\u2603", "bar"])
self.assertEqual(h.getRawHeaders(b"test\xe1"),
[b"foo\xe2\x98\x83", b"bar"])
def fetch(self, uri, headers=None):
headers = Headers() if headers is None else headers.copy()
headers.setRawHeaders(b'referer', ['https://www.pixiv.net/'])
headers.setRawHeaders(b'host', [self.host])
if (self.port == 80 and self.scheme == 'http://') \
or (self.port == 443 and self.scheme == 'https://') \
or self.port is None:
url = self.scheme + self.host
else:
url = self.scheme + self.host + ':' + str(self.port)
url = url + uri
dfd = self.agent.request('GET', url, headers)
dfd.addErrback(self.on_failure)
def _wait_body(response):
body_dfd = readBody(response)
body_dfd.addCallback(lambda b: setattr(response, 'body', b))
body_dfd.addCallback(lambda _: response)
return body_dfd
dfd.addCallback(_wait_body)
return dfd
def assertSanitized(testCase, components, expected):
"""
Assert that the components are sanitized to the expected value as
both a header name and value, across all of L{Header}'s setters
and getters.
@param testCase: A test case.
@param components: A sequence of values that contain linear
whitespace to use as header names and values; see
C{textLinearWhitespaceComponents} and
C{bytesLinearWhitespaceComponents}
@param expected: The expected sanitized form of the component for
both headers names and their values.
"""
for component in components:
headers = []
headers.append(Headers({component: [component]}))
added = Headers()
added.addRawHeader(component, component)
headers.append(added)
setHeader = Headers()
setHeader.setRawHeaders(component, [component])
headers.append(setHeader)
for header in headers:
testCase.assertEqual(list(header.getAllRawHeaders()),
[(expected, [expected])])
testCase.assertEqual(header.getRawHeaders(expected), [expected])
def request(testCase,
rootResource,
method,
uri,
body=b"",
baseURI=b'http://localhost:8900/',
headers=None):
"""
Issue a request and return a synchronous response.
"""
# allow for relative or absolute URIs, since we're going to the same
# resource no matter what
if headers is not None:
headers_object = Headers()
for key, value in headers.items():
headers_object.setRawHeaders(key, value)
else:
headers_object = None
base = makeURLPath(baseURI)
if isinstance(uri, text_type):
uri = uri.encode("ascii")
relative = base.click(uri)
bytesURI = text_type(relative).encode("utf-8")
return (RequestTraversalAgent(testCase, rootResource).request(
method,
bytesURI,
bodyProducer=SynchronousProducer(body),
headers=headers_object))
def request(rq):
headers = Headers(dict((k, [v]) for k, v in rq.headers.iteritems()))
scheme, host, port, path = _parse(rq.url)
headers.setRawHeaders('Host', [host])
return (rq.method, rq.url,
headers,
StringProducer(rq.data) if rq.data else None)
def headers(self):
h = Headers({b'content-type': [self.content_type]})
if self.nonce is not None:
h.setRawHeaders(b'replay-nonce', [self.nonce])
if self.links is not None:
h.setRawHeaders(b'link', self.links)
return h
def request(self, url, method='GET', data=None):
d = defer.Deferred()
_headers = {}
headers = None
if method == 'GET':
data = None
elif method == 'POST':
_headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
d.errback(Exception('Not Support Method:%s.' % method))
return
if _headers:
headers = Headers()
for k, v in _headers.iteritems():
headers.setRawHeaders(urllib.quote(k), [urllib.quote(v)])
if data:
data = StrBodyProducer(data)
#print 'Req url:', url, ' Head:', headers
self.agent.request(method, url, headers,
data).addCallback(self.received,
d).addErrback(self.errback, d)
return d
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(u'test\u00E1', [u'foo\u2603'])
i = h.copy()
# The copy contains the same value as the original
self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603'])
self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83'])
# Add a header to the original
h.addRawHeader(u'test\u00E1', u'bar')
# Verify that the copy has not changed
self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603'])
self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83'])
# Add a header to the copy
i.addRawHeader(u'test\u00E1', b'baz')
# Verify that the orignal does not have it
self.assertEqual(
h.getRawHeaders(u'test\u00E1'), [u'foo\u2603', u'bar'])
self.assertEqual(
h.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83', b'bar'])
def _create_headers(self, headers_dict):
headers = Headers()
for key, value in headers_dict.items():
if isinstance(value, list):
headers.setRawHeaders(key, value)
else:
headers.addRawHeader(key, value)
return headers
def request(self, method, path, headers=None, body=None, expected_code=200,
retry=0, delay=0):
"""
Send an http request to the gerrit service for the given path.
If 'retry' is specified, transient errors (http response code 500-599) will
be retried after an exponentially-increasing delay.
Returns a Deferred which will call back with the parsed json body of the
gerrit server's response.
Args:
method: 'GET', 'POST', etc.
path: Path element of the url.
headers: dict of http request headers.
body: json-encodable body of http request.
expected_code: http response code expected in reply.
retry: How many times to retry transient errors.
delay: Wait this many seconds before sending the request.
"""
retry_delay = delay * 2 if delay else 0.5
retry_args = (
method, path, headers, body, expected_code, retry - 1, retry_delay)
if not path.startswith('/'):
path = '/' + path
if not headers:
headers = Headers()
else:
# Make a copy so mutations don't affect retry attempts.
headers = Headers(dict(headers.getAllRawHeaders()))
if self.auth_token:
if not path.startswith('/a/'):
path = '/a' + path
headers.setRawHeaders('Authorization', [self.auth_token])
url = '%s://%s%s' % (self.gerrit_protocol, self.gerrit_host, path)
if body:
body = JsonBodyProducer(json.dumps(body))
headers.setRawHeaders('Content-Type', ['application/json'])
if DEBUG:
log.msg(url)
if delay:
d = defer.succeed(None)
d.addCallback(
reactor.callLater, delay, Agent.request, self, method, str(url),
headers, body)
else:
d = Agent.request(self, method, str(url), headers, body)
def _check_code(response):
if response.code == expected_code:
return response
if retry > 0 and response.code >= 500 and response.code < 600:
return self.request(*retry_args)
msg = 'Failed gerrit request (code %s, expected %s): %s' % (
response.code, expected_code, url)
raise GerritError(msg, response.code)
d.addCallback(_check_code)
d.addCallback(JsonResponse.Get, url=url)
return d
def test_getRawHeaders(self):
"""
L{Headers.getRawHeaders} returns the values which have been set for a
given header.
"""
h = Headers()
h.setRawHeaders(b"test", [b"lemur"])
self.assertEqual(h.getRawHeaders(b"test"), [b"lemur"])
self.assertEqual(h.getRawHeaders(b"Test"), [b"lemur"])
def test_hasHeaderTrue(self):
"""
Check that L{Headers.hasHeader} returns C{True} when the given header
is found.
"""
h = Headers()
h.setRawHeaders(b"test", [b"lemur"])
self.assertTrue(h.hasHeader(b"test"))
self.assertTrue(h.hasHeader(b"Test"))
def test_getRawHeaders(self):
"""
L{Headers.getRawHeaders} returns the values which have been set for a
given header.
"""
h = Headers()
h.setRawHeaders("test", ["lemur"])
self.assertEqual(h.getRawHeaders("test"), ["lemur"])
self.assertEqual(h.getRawHeaders("Test"), ["lemur"])
def test_rawHeadersValueEncoding(self):
"""
Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
ISO-8859-1 and values as UTF-8.
"""
h = Headers()
h.setRawHeaders(u"\u00E1", [u"\u2603", b"foo"])
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b'\xe2\x98\x83', b'foo'])
def test_rawHeadersValueEncoding(self):
"""
Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
ISO-8859-1 and values as UTF-8.
"""
h = Headers()
h.setRawHeaders("\u00E1", ["\u2603", b"foo"])
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b"\xe2\x98\x83", b"foo"])
def test_hasHeaderTrue(self):
"""
Check that L{Headers.hasHeader} returns C{True} when the given header
is found.
"""
h = Headers()
h.setRawHeaders("test", ["lemur"])
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
def headers(self, **kw):
"""
Create a L{Headers} instance populated with the header name/values
specified by C{kw} and a L{_DictHeaders} wrapped around it and return
them both.
"""
h = Headers()
for k, v in kw.iteritems():
h.setRawHeaders(k, v)
return h, _DictHeaders(h)
def headers(self, **kw):
"""
Create a L{Headers} instance populated with the header name/values
specified by C{kw} and a L{_DictHeaders} wrapped around it and return
them both.
"""
h = Headers()
for k, v in kw.items():
h.setRawHeaders(k.encode('ascii'), v)
return h, _DictHeaders(h)
def test_getRawHeadersWithDefaultMatchingValue(self):
"""
If the object passed as the value list to L{Headers.setRawHeaders}
is later passed as a default to L{Headers.getRawHeaders}, the
result nevertheless contains decoded values.
"""
h = Headers()
default = [b"value"]
h.setRawHeaders(b"key", default)
self.assertIsInstance(h.getRawHeaders("key", default)[0], str)
self.assertEqual(h.getRawHeaders("key", default), ["value"])
def test_getRawHeadersWithDefaultMatchingValue(self):
"""
If the object passed as the value list to L{Headers.setRawHeaders}
is later passed as a default to L{Headers.getRawHeaders}, the
result nevertheless contains encoded values.
"""
h = Headers()
default = [u"value"]
h.setRawHeaders(b"key", default)
self.assertIsInstance(h.getRawHeaders(b"key", default)[0], bytes)
self.assertEqual(h.getRawHeaders(b"key", default), [b"value"])
def test_hasHeaderTrue(self):
"""
Check that L{Headers.hasHeader} returns C{True} when the given header
is found.
"""
h = Headers()
h.setRawHeaders(u"test\u00E1", [u"lemur"])
self.assertTrue(h.hasHeader(u"test\u00E1"))
self.assertTrue(h.hasHeader(u"Test\u00E1"))
self.assertTrue(h.hasHeader(b"test\xe1"))
self.assertTrue(h.hasHeader(b"Test\xe1"))
def test_getRawHeaders(self):
"""
L{Headers.getRawHeaders} returns the values which have been set for a
given header.
"""
h = Headers()
h.setRawHeaders(u"test\u00E1", [u"lemur"])
self.assertEqual(h.getRawHeaders(u"test\u00E1"), [u"lemur"])
self.assertEqual(h.getRawHeaders(u"Test\u00E1"), [u"lemur"])
self.assertEqual(h.getRawHeaders(b"test\xe1"), [b"lemur"])
self.assertEqual(h.getRawHeaders(b"Test\xe1"), [b"lemur"])
def test_setRawHeaders(self):
"""
L{Headers.setRawHeaders} sets the header values for the given
header name to the sequence of string values.
"""
rawValue = ["value1", "value2"]
h = Headers()
h.setRawHeaders("test", rawValue)
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
self.assertEqual(h.getRawHeaders("test"), rawValue)
def test_setRawHeaders(self):
"""
L{Headers.setRawHeaders} sets the header values for the given
header name to the sequence of byte string values.
"""
rawValue = [b"value1", b"value2"]
h = Headers()
h.setRawHeaders(b"test", rawValue)
self.assertTrue(h.hasHeader(b"test"))
self.assertTrue(h.hasHeader(b"Test"))
self.assertEqual(h.getRawHeaders(b"test"), rawValue)
def check_range(requested_range, expected_response):
headers = Headers()
headers.setRawHeaders("range", [requested_range])
response = result_of(
self.http.client.request(
"GET",
self.http.client.relative_url("/v1/{}/{}/1".format(
self.KIND, _encode_si(storage_index))),
headers=headers,
))
self.assertEqual(response.headers.getRawHeaders("content-range"),
[expected_response])
def request(self, method, url, headers=None, bodyProducer=None):
"""
Issue a request with some extra headers.
:see: ``twisted.web.iweb.IAgent.request``
"""
if headers is None:
headers = Headers()
else:
headers = headers.copy()
for k, vs in self._to_inject.getAllRawHeaders():
headers.setRawHeaders(k, vs)
return self._agent.request(method, url, headers, bodyProducer)
def test_getAllRawHeaders(self):
"""
L{Headers.getAllRawHeaders} returns an iterable of (k, v) pairs, where
C{k} is the canonicalized representation of the header name, and C{v}
is a sequence of values.
"""
h = Headers()
h.setRawHeaders(b"test", [b"lemurs"])
h.setRawHeaders(b"www-authenticate", [b"basic aksljdlk="])
allHeaders = set([(k, tuple(v)) for k, v in h.getAllRawHeaders()])
self.assertEqual(allHeaders, set([(b"WWW-Authenticate", (b"basic aksljdlk=",)), (b"Test", (b"lemurs",))]))
def submitFileRequest(recipient, data, producer=StringProducer):
"""
This is a generic agent meant to make post requests.
:param recipient: a web or IP address and port combination.
:param data: all of the data, packaged, and ready to be sent as request.
:returns: a deferred that will fire when a response body is available.
"""
log.msg("submitFileRequest:: data:", recipient, system="httpFileTransfer")
headers = Headers()
headers.setRawHeaders('Content-type', ['application/json'])
return Agent(reactor).request("POST", recipient.encode('utf-8'), headers,
producer(data))
def test_nameNotEncodable(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1, and if it cannot be encoded, it will
raise a L{UnicodeDecodeError}.
"""
h = Headers()
# Only these two functions take names
with self.assertRaises(UnicodeEncodeError):
h.setRawHeaders(u"\u2603", [u"val"])
with self.assertRaises(UnicodeEncodeError):
h.hasHeader(u"\u2603")
def test_copy(self):
"""
L{Headers.copy} creates a new independant copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b'test', [b'foo'])
i = h.copy()
self.assertEqual(i.getRawHeaders(b'test'), [b'foo'])
h.addRawHeader(b'test', b'bar')
self.assertEqual(i.getRawHeaders(b'test'), [b'foo'])
i.addRawHeader(b'test', b'baz')
self.assertEqual(h.getRawHeaders(b'test'), [b'foo', b'bar'])
def test_copy(self):
"""
L{Headers.copy} creates a new independant copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders('test', ['foo'])
i = h.copy()
self.assertEquals(i.getRawHeaders('test'), ['foo'])
h.addRawHeader('test', 'bar')
self.assertEquals(i.getRawHeaders('test'), ['foo'])
i.addRawHeader('test', 'baz')
self.assertEquals(h.getRawHeaders('test'), ['foo', 'bar'])
def test_nameNotEncodable(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1, and if it cannot be encoded, it will
raise a L{UnicodeDecodeError}.
"""
h = Headers()
# Only these two functions take names
with self.assertRaises(UnicodeEncodeError):
h.setRawHeaders("\u2603", ["val"])
with self.assertRaises(UnicodeEncodeError):
h.hasHeader("\u2603")
def test_copy(self):
"""
L{Headers.copy} creates a new independant copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b"test", [b"foo"])
i = h.copy()
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
h.addRawHeader(b"test", b"bar")
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
i.addRawHeader(b"test", b"baz")
self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b"test", [b"foo"])
i = h.copy()
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
h.addRawHeader(b"test", b"bar")
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
i.addRawHeader(b"test", b"baz")
self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def test_removeHeader(self):
"""
Check that L{Headers.removeHeader} removes the given header.
"""
h = Headers()
h.setRawHeaders("foo", ["lemur"])
self.assertTrue(h.hasHeader("foo"))
h.removeHeader("foo")
self.assertFalse(h.hasHeader("foo"))
h.setRawHeaders("bar", ["panda"])
self.assertTrue(h.hasHeader("bar"))
h.removeHeader("Bar")
self.assertFalse(h.hasHeader("bar"))
def test_getAllRawHeaders(self):
"""
L{Headers.getAllRawHeaders} returns an iterable of (k, v) pairs, where
C{k} is the canonicalized representation of the header name, and C{v}
is a sequence of values.
"""
h = Headers()
h.setRawHeaders("test", ["lemurs"])
h.setRawHeaders("www-authenticate", ["basic aksljdlk="])
allHeaders = set([(k, tuple(v)) for k, v in h.getAllRawHeaders()])
self.assertEqual(allHeaders,
set([("WWW-Authenticate", ("basic aksljdlk=",)),
("Test", ("lemurs",))]))
def test_removeHeader(self):
"""
Check that L{Headers.removeHeader} removes the given header.
"""
h = Headers()
h.setRawHeaders(b"foo", [b"lemur"])
self.assertTrue(h.hasHeader(b"foo"))
h.removeHeader(b"foo")
self.assertFalse(h.hasHeader(b"foo"))
h.setRawHeaders(b"bar", [b"panda"])
self.assertTrue(h.hasHeader(b"bar"))
h.removeHeader(b"Bar")
self.assertFalse(h.hasHeader(b"bar"))
class FakeRequest(object):
"""
A fake request suitable for use in place of C{twisted.web.http.Request}.
@param method: The HTTP method being invoked such as C{GET} or C{POST}.
"""
def __init__(self, method):
self.method = method
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self._fluidDB_reqid = 'xxx'
self.finished = False
def finish(self):
"""
Indicate that all response data has been written to this request.
"""
self.finished = True
def getResponseHeader(self, name):
"""
Get the value of an HTTP response header.
@param name: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.responseHeaders.getRawHeaders(name)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set an HTTP header to include in the response returned to the client.
@param name: The name of the header to set.
@param value: The value to set.
"""
self.responseHeaders.setRawHeaders(name, [value])
def setResponseCode(self, code):
"""
Set the HTTP response code to return to the client.
@param code: An HTTP response code as defined in C{twisted.web.http}.
"""
self.status = code
def test_nameEncoding(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1.
"""
h = Headers()
# We set it using a Unicode string.
h.setRawHeaders(u"\u00E1", [b"foo"])
# It's encoded to the ISO-8859-1 value, which we can use to access it
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b'foo'])
# We can still access it using the Unicode string..
self.assertTrue(h.hasHeader(u"\u00E1"))
def submitFileRequest(recipient, data, producer=StringProducer):
"""
This is a generic agent meant to make post requests.
:param recipient: a web or IP address and port combination.
:param data: all of the data, packaged, and ready to be sent as request.
:returns: a deferred that will fire when a response body is available.
"""
log.msg("submitFileRequest:: data:", recipient, system="httpFileTransfer")
headers = Headers()
headers.setRawHeaders('Content-type', ['application/json'])
return Agent(reactor).request("POST",
recipient.encode('utf-8'),
headers,
producer(data))
def test_nameEncoding(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1.
"""
h = Headers()
# We set it using a Unicode string.
h.setRawHeaders("\u00E1", [b"foo"])
# It's encoded to the ISO-8859-1 value, which we can use to access it
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b"foo"])
# We can still access it using the Unicode string..
self.assertTrue(h.hasHeader("\u00E1"))
class FakeRequest(object):
"""
A fake request suitable for use in place of C{twisted.web.http.Request}.
@param method: The HTTP method being invoked such as C{GET} or C{POST}.
"""
def __init__(self, method):
self.method = method
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self._fluidDB_reqid = 'xxx'
self.finished = False
def finish(self):
"""
Indicate that all response data has been written to this request.
"""
self.finished = True
def getResponseHeader(self, name):
"""
Get the value of an HTTP response header.
@param name: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.responseHeaders.getRawHeaders(name)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set an HTTP header to include in the response returned to the client.
@param name: The name of the header to set.
@param value: The value to set.
"""
self.responseHeaders.setRawHeaders(name, [value])
def setResponseCode(self, code):
"""
Set the HTTP response code to return to the client.
@param code: An HTTP response code as defined in C{twisted.web.http}.
"""
self.status = code
async def get() -> IResponse:
headers = Headers({})
if self._auth.token:
headers.setRawHeaders("Authorization",
[f"Bearer {self._auth.token}"])
self.log.info("GET: {url}\nHeaders: {headers}",
url=url,
headers=headers)
response = await self._httpGET(url, headers=headers)
self.log.info(
"Response: {response}\nHeaders: {response.headers}",
response=response,
)
return response
def authorized_request(api_auth_token,
agent,
method,
uri,
headers=None,
data=None):
"""
Issue a request with the required token-based authorization header value.
:param bytes api_auth_token: The API authorization token to include.
:param IAgent agent: The agent to use to issue the request.
:param bytes method: The HTTP method for the request.
:param bytes uri: The URI for the request.
:param ({bytes: [bytes]})|None headers: If not ``None``, extra request
headers to include. The **Authorization** header will be overwritten
if it is present.
:param BytesIO|None data: If not ``None``, the request body.
:return: A ``Deferred`` like the one returned by ``IAgent.request``.
"""
if data is None:
bodyProducer = None
else:
bodyProducer = FileBodyProducer(data, cooperator=uncooperator())
if headers is None:
headers = Headers()
else:
headers = Headers(headers)
headers.setRawHeaders(
u"authorization",
[b"tahoe-lafs {}".format(api_auth_token)],
)
return agent.request(
method,
uri,
headers=headers,
bodyProducer=bodyProducer,
)
def APIConnect(self,method,params,uri,eid=None):
"""Encapsulates all the API encoding, starts the HTTP request, returns deferred
eid is used to pass Data along the chain to be used later
"""
encoded_string = ''
if params:
for key, value in sorted(params.iteritems(),key=lambda (key,value,) : key ):
encoded_string += str(key) + '=' + str(value) + '&'
encoded_string = encoded_string[:-1]
url = uri + '?' + encoded_string
else:
url = uri
self.nonce += 1
if method == 'POST':
md5_encoded_query_string = md5(encoded_string).hexdigest()
else:
md5_encoded_query_string = md5('').hexdigest()
hmac_data = method + '#' + url + '#' + self.api_key + '#' + str(self.nonce) + '#' + md5_encoded_query_string
hmac_signed = hmac_new(self.api_secret,digestmod=sha256, msg=hmac_data).hexdigest()
header = {'content-type':['application/x-www-form-urlencoded;charset=utf-8']}
header[b"X-API-KEY"] = [self.api_key]
header[b"X-API-NONCE"] = [b"%d"%self.nonce]
header[b"X-API-SIGNATURE"] = [hmac_signed]
h = Headers({})
for k,v in header.items():
h.setRawHeaders(k,v)
bodyProducer = None
if method == 'POST':
bodyProducer = StringProducer(encoded_string)
d = self.agent.request(method,url,headers=h,bodyProducer=bodyProducer)
d.addCallback(self.APIResponse,eid=eid)
def Error(result):
print "API-Connect-Error",result
d.addErrback(Error)
return d
def request(testCase, rootResource, method, uri, body=b"",
baseURI='http://localhost:8900/',
headers=None):
"""
Issue a request and return a synchronous response.
"""
# allow for relative or absolute URIs, since we're going to the same
# resource no matter what
if headers is not None:
headers_object = Headers()
for key, value in headers.items():
headers_object.setRawHeaders(key, value)
else:
headers_object = None
return (
RequestTraversalAgent(testCase, rootResource)
.request(method, str(URLPath.fromString(baseURI).click(uri)),
bodyProducer=SynchronousProducer(body),
headers=headers_object)
)
def request(self, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
contentType = headers.getRawHeaders('content-type', [""])[0]
date = headers.getRawHeaders('date',
[""])[0] or self._generateRequestDate(uri)
headers.setRawHeaders('date', [date])
uri_origin_form = URI.fromBytes(uri).originForm
contentMD5 = headers.getRawHeaders('content-md5', [""])[0]
if not contentMD5 and bodyProducer is not None:
r = getattr(self.agent, '_reactor') or reactor
bodyConsumer = StringConsumer(callLater=r.callLater)
yield bodyProducer.startProducing(bodyConsumer)
body = bodyConsumer.value()
bodyProducer = StringBodyProducer(body)
if body:
contentMD5 = binascii.b2a_base64(
hashlib.md5(body).digest()).strip()
headers.addRawHeader('content-md5', contentMD5)
sts = "\n".join([
method, contentType or "", contentMD5, date or "", uri_origin_form
])
mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest()
encodedMAC = binascii.b2a_base64(mac).strip()
auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC)
headers.addRawHeader('authorization', auth_header)
d = yield self.agent.request(method, uri, headers, bodyProducer)
self._handleResponseDate(uri, d)
defer.returnValue(d)
def request(self, method, url, **kwargs):
method = method.upper()
# Join parameters provided in the URL
# and the ones passed as argument.
params = kwargs.get('params')
if params:
url = _combine_query_params(url, params)
# Convert headers dictionary to
# twisted raw headers format.
headers = kwargs.get('headers')
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.iteritems():
if isinstance(v, str):
h.addRawHeader(k, v)
else:
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
# Here we choose a right producer
# based on the parameters passed in.
bodyProducer = None
data = kwargs.get('data')
files = kwargs.get('files')
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
# with files and/or large objects.
files = list(_convert_files(files))
boundary = uuid.uuid4()
headers.setRawHeaders(
'content-type', [
'multipart/form-data; boundary=%s' % (boundary,)])
if data:
data = _convert_params(data)
else:
data = []
bodyProducer = multipart.MultiPartProducer(
data + files, boundary=boundary)
elif data:
# Otherwise stick to x-www-form-urlencoded format
# as it's generally faster for smaller requests.
if isinstance(data, (dict, list, tuple)):
headers.setRawHeaders(
'content-type', ['application/x-www-form-urlencoded'])
data = urlencode(data, doseq=True)
bodyProducer = IBodyProducer(data)
d = self._agent.request(
method, url, headers=headers,
bodyProducer=bodyProducer)
return d
def request(self, method, url, **kwargs):
method = method.upper()
params = kwargs.get('params')
if params:
url = _combine_query_params(url, params)
headers = kwargs.get('headers')
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.iteritems():
if isinstance(v, str):
h.addRawHeader(k, v)
else:
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
data = kwargs.get('data')
bodyProducer = None
if data:
if isinstance(data, dict):
data = list(_flatten_param_dict(data))
if isinstance(data, list):
headers.setRawHeaders(
'content-type', ['application/x-www-form-urlencoded'])
data = urlencode(data)
bodyProducer = IBodyProducer(data)
d = self._agent.request(
method, url, headers=headers, bodyProducer=bodyProducer)
return d
def _request_headers(self, headers, stacklevel):
"""
Convert the *headers* argument to a :class:`Headers` instance
:returns:
:class:`twisted.web.http_headers.Headers`
"""
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.items():
if isinstance(v, (bytes, str)):
h.addRawHeader(k, v)
elif isinstance(v, list):
h.setRawHeaders(k, v)
else:
warnings.warn(
("The value of headers key {!r} has non-string type {}"
" and will be dropped."
" This will raise TypeError in the next treq release."
).format(k, type(v)),
DeprecationWarning,
stacklevel=stacklevel,
)
return h
if isinstance(headers, Headers):
return headers
if headers is None:
return Headers({})
warnings.warn(
("headers must be a dict, twisted.web.http_headers.Headers, or None,"
" but found {}, which will be ignored."
" This will raise TypeError in the next treq release.").format(
type(headers)),
DeprecationWarning,
stacklevel=stacklevel,
)
return Headers({})
def request(testCase,
rootResource,
method,
uri,
body=b"",
baseURI='http://localhost:8900/',
headers=None):
"""
Issue a request and return a synchronous response.
"""
# allow for relative or absolute URIs, since we're going to the same
# resource no matter what
if headers is not None:
headers_object = Headers()
for key, value in headers.items():
headers_object.setRawHeaders(key, value)
else:
headers_object = None
return (RequestTraversalAgent(testCase, rootResource).request(
method,
str(URLPath.fromString(baseURI).click(uri)),
bodyProducer=SynchronousProducer(body),
headers=headers_object))
def setup(self):
self.empty_resp = ''
if sys.version_info[0] >= 3:
self.empty_resp = b''
self.mock_redir = Mock(spec_set=VaultRedirector)
type(self.mock_redir).active_node_ip_port = 'mynode:1234'
type(self.mock_redir).last_poll_time = 'myPollTime'
type(self.mock_redir).log_enabled = True
type(self.mock_redir).consul_scheme = 'http'
type(self.mock_redir).consul_host_port = 'consul:5678'
self.cls = VaultRedirectorSite(self.mock_redir)
# mock client request
self.mock_request = Mock(spec_set=Request)
type(self.mock_request).method = 'GET'
client_addr = IPv4Address('TCP', '1.2.3.4', 12345)
type(self.mock_request).client = client_addr
type(self.mock_request).clientproto = 'HTTP/1.1'
headers = Headers()
headers.setRawHeaders('date', ['Mon, 11 Apr 2016 15:26:42 GMT'])
headers.setRawHeaders('server', ['TwistedWeb/16.1.0'])
type(self.mock_request).responseHeaders = headers
type(self.mock_request).queued = 0
def setup(self):
self.empty_resp = ''
if sys.version_info[0] >= 3:
self.empty_resp = b''
self.mock_redir = Mock(spec_set=VaultRedirector)
type(self.mock_redir).active_node_ip_port = 'mynode:1234'
type(self.mock_redir).last_poll_time = 'myPollTime'
type(self.mock_redir).log_enabled = True
type(self.mock_redir).consul_scheme = 'http'
type(self.mock_redir).consul_host_port = 'consul:5678'
self.cls = VaultRedirectorSite(self.mock_redir)
# mock client request
self.mock_request = Mock(spec_set=Request)
type(self.mock_request).method = 'GET'
client_addr = IPv4Address('TCP', '1.2.3.4', 12345)
type(self.mock_request).client = client_addr
type(self.mock_request).clientproto = 'HTTP/1.1'
headers = Headers()
headers.setRawHeaders('date', ['Mon, 11 Apr 2016 15:26:42 GMT'])
headers.setRawHeaders('server', ['TwistedWeb/16.1.0'])
type(self.mock_request).responseHeaders = headers
type(self.mock_request).queued = 0
def request(self, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
contentType = headers.getRawHeaders('content-type', [""])[0]
date = headers.getRawHeaders('date', [""])[0] or self._generateRequestDate(uri)
headers.setRawHeaders('date', [date])
uri_origin_form = URI.fromBytes(uri).originForm
contentMD5 = headers.getRawHeaders('content-md5', [""])[0]
if not contentMD5 and bodyProducer is not None:
r = getattr(self.agent, '_reactor') or reactor
bodyConsumer = StringConsumer(callLater=r.callLater)
yield bodyProducer.startProducing(bodyConsumer)
body = bodyConsumer.value()
bodyProducer = StringBodyProducer(body)
if body:
contentMD5 = binascii.b2a_base64(hashlib.md5(body).digest()).strip()
headers.addRawHeader('content-md5', contentMD5)
sts = "\n".join([method, contentType or "", contentMD5, date or "", uri_origin_form])
mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest()
encodedMAC = binascii.b2a_base64(mac).strip()
auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC)
headers.addRawHeader('authorization', auth_header)
d = yield self.agent.request(method, uri, headers, bodyProducer)
self._handleResponseDate(uri, d)
defer.returnValue(d)
def request(method, url, **kwargs):
oauth_tokens = settings.get("oauth_tokens")
if not oauth_tokens:
raise ValueError("OAuth token is unavailable")
if kwargs.get("params"):
encoded_params = urlencode(kwargs["params"])
url = "".join([url, "&" if "?" in url else "?", encoded_params])
del kwargs["params"]
client = oauth1.Client(
config.CONSUMER_KEY,
client_secret=config.CONSUMER_SECRET,
resource_owner_key=oauth_tokens["oauth_token"],
resource_owner_secret=oauth_tokens["oauth_token_secret"],
)
new_url, headers, data = client.sign(
url, method, body=kwargs.get("data"), headers=kwargs.get("headers"),
)
# Twisted doesn't support unicode...
new_url = new_url.encode("utf-8")
h = Headers({})
for k, v in headers.iteritems():
k = k.encode("utf-8")
if isinstance(v, basestring):
v = v.encode("utf-8")
h.addRawHeader(k, v)
else:
v = [x.encode("utf-8") for x in v]
h.setRawHeaders(k, v)
kwargs["headers"] = h
kwargs["data"] = data
defer = treq.request(method, new_url, **kwargs)
return defer.addCallback(treq.json_content)
def request(testCase, rootResource, method, uri, body=b"",
baseURI=b'http://localhost:8900/',
headers=None):
"""
Issue a request and return a synchronous response.
"""
# allow for relative or absolute URIs, since we're going to the same
# resource no matter what
if headers is not None:
headers_object = Headers()
for key, value in headers.items():
headers_object.setRawHeaders(key, value)
else:
headers_object = None
base = makeURLPath(baseURI)
if isinstance(uri, text_type):
uri = uri.encode("ascii")
relative = base.click(uri)
bytesURI = text_type(relative).encode("utf-8")
return (
RequestTraversalAgent(testCase, rootResource)
.request(method, bytesURI, bodyProducer=SynchronousProducer(body),
headers=headers_object)
)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Send a client request following HTTP redirects.
@see: L{Agent.request}.
"""
if self._parent_trace is None:
trace = Trace(method)
else:
trace = self._parent_trace.child(method)
if self._endpoint is not None:
trace.set_endpoint(self._endpoint)
if headers is None:
headers = Headers({})
# These headers are based on the headers used by finagle's tracing
# http Codec.
#
# https://github.com/twitter/finagle/blob/master/finagle-http/
#
# Currently not implemented are X-B3-Sampled and X-B3-Flags
# Tryfer's underlying Trace implementation has no notion of a Sampled
# trace and I haven't figured out what flags are for.
headers.setRawHeaders('X-B3-TraceId', [hex_str(trace.trace_id)])
headers.setRawHeaders('X-B3-SpanId', [hex_str(trace.span_id)])
if trace.parent_span_id is not None:
headers.setRawHeaders('X-B3-ParentSpanId',
[hex_str(trace.parent_span_id)])
# Similar to the headers above we use the annotation 'http.uri' for
# because that is the standard set forth in the finagle http Codec.
trace.record(Annotation.string('http.uri', uri))
trace.record(Annotation.client_send())
def _finished(resp):
# TODO: It may be advantageous here to return a wrapped response
# whose deliverBody can wrap it's protocol and record when the
# application has finished reading the contents.
trace.record(Annotation.string(
'http.responsecode',
'{0} {1}'.format(resp.code, resp.phrase)))
trace.record(Annotation.client_recv())
return resp
d = self._agent.request(method, uri, headers, bodyProducer)
d.addBoth(_finished)
return d
def test_headersComparison(self):
"""
A L{Headers} instance compares equal to itself and to another
L{Headers} instance with the same values.
"""
first = Headers()
first.setRawHeaders("foo", ["panda"])
second = Headers()
second.setRawHeaders("foo", ["panda"])
third = Headers()
third.setRawHeaders("foo", ["lemur", "panda"])
self.assertEqual(first, first)
self.assertEqual(first, second)
self.assertNotEqual(first, third)
def request(self, method, url, **kwargs):
method = method.upper()
# Join parameters provided in the URL
# and the ones passed as argument.
params = kwargs.get('params')
if params:
url = _combine_query_params(url, params)
# Convert headers dictionary to
# twisted raw headers format.
headers = kwargs.get('headers')
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.iteritems():
if isinstance(v, str):
h.addRawHeader(k, v)
else:
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
# Here we choose a right producer
# based on the parameters passed in.
bodyProducer = None
data = kwargs.get('data')
files = kwargs.get('files')
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
# with files and/or large objects.
files = list(_convert_files(files))
boundary = uuid.uuid4()
headers.setRawHeaders(
'content-type', [
'multipart/form-data; boundary=%s' % (boundary,)])
if data:
data = _convert_params(data)
else:
data = []
bodyProducer = multipart.MultiPartProducer(
data + files, boundary=boundary)
elif data:
# Otherwise stick to x-www-form-urlencoded format
# as it's generally faster for smaller requests.
if isinstance(data, (dict, list, tuple)):
headers.setRawHeaders(
'content-type', ['application/x-www-form-urlencoded'])
data = urlencode(data, doseq=True)
bodyProducer = IBodyProducer(data)
cookies = kwargs.get('cookies', {})
if not isinstance(cookies, CookieJar):
cookies = cookiejar_from_dict(cookies)
cookies = merge_cookies(self._cookiejar, cookies)
wrapped_agent = CookieAgent(self._agent, cookies)
if kwargs.get('allow_redirects', True):
wrapped_agent = RedirectAgent(wrapped_agent)
wrapped_agent = ContentDecoderAgent(wrapped_agent,
[('gzip', GzipDecoder)])
auth = kwargs.get('auth')
if auth:
wrapped_agent = add_auth(wrapped_agent, auth)
d = wrapped_agent.request(
method, url, headers=headers,
bodyProducer=bodyProducer)
timeout = kwargs.get('timeout')
if timeout:
delayedCall = default_reactor(kwargs.get('reactor')).callLater(
timeout, d.cancel)
def gotResult(result):
if delayedCall.active():
delayedCall.cancel()
return result
d.addBoth(gotResult)
if not kwargs.get('unbuffered', False):
d.addCallback(_BufferedResponse)
return d.addCallback(_Response, cookies)
