def _doPush( self, filename ):
print 'beginning push of file:', filename
producerDeferred = defer.Deferred()
producerDeferred.addCallback( self._finished )
producerDeferred.addErrback( self._error )
multiPartProducer = MultiPartProducer(
files={
'metrics': filename,
},
data={
'secret': agent_settings.PUSH_DETAILS['secret'],
},
callback=self._progress,
deferred=producerDeferred
)
headers = Headers()
headers.addRawHeader( 'Content-Type', 'multipart/form-data; boundary=%s' % multiPartProducer.boundary )
agent = client.Agent( reactor )
request = agent.request(
'POST',
agent_settings.PUSH_DETAILS['url'],
headers,
multiPartProducer
)
request.addCallback( self._responseReady, filename )
def http_get(reactor, url, accept=None):
agent = client.Agent(reactor)
headers = Headers()
if accept is not None:
headers.addRawHeader('Accept', str(accept))
d = agent.request(b'GET', str(url), headers=headers)
return _handle_agent_response(d)
def post(url, data=None, on_created=None, on_error=None):
errback = on_error or make_errback(frames_back=2)
try:
def handle_response(response):
if response.code == 200:
callback = partial(
_handle_post,
on_created=on_created,
on_error=errback)
response.deliverBody(JsonReceiver.create(callback, errback))
else:
errback('returned %s' % response.code)
agent = Agent(reactor)
headers = Headers(get_auth_headers())
headers.addRawHeader('Content-Type', 'application/json')
if not data:
data = {'dummy': 1}
d = agent.request(
'POST',
add_params(url, rid=uuid4().hex),
headers=headers,
bodyProducer=JsonProducer(data) if data else None)
d.addCallbacks(handle_response, errback)
except Exception as ex:
errback('error %s' % ex)
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders("test\u00E1", ["foo\u2603"])
i = h.copy()
# The copy contains the same value as the original
self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"])
self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"])
# Add a header to the original
h.addRawHeader("test\u00E1", "bar")
# Verify that the copy has not changed
self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"])
self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"])
# Add a header to the copy
i.addRawHeader("test\u00E1", b"baz")
# Verify that the orignal does not have it
self.assertEqual(h.getRawHeaders("test\u00E1"), ["foo\u2603", "bar"])
self.assertEqual(h.getRawHeaders(b"test\xe1"),
[b"foo\xe2\x98\x83", b"bar"])
def request(self, method, uri, headers=None, body_producer=None):
"""Issue a new request to the wrapped agent.
Args:
method (bytes): The HTTP method to use.
uri (bytes): The url to download from.
headers (t.w.h.Headers, optional): Any extra headers to send.
body_producer (t.w.i.IBodyProducer, optional): Request body data.
Returns:
Deferred: The filename of the of the downloaded file.
"""
if headers is None:
headers = Headers()
if not headers.hasHeader(b'User-Agent'):
version = get_version()
user_agent = 'Deluge/%s (https://deluge-torrent.org)' % version
headers.addRawHeader('User-Agent', user_agent)
d = self.agent.request(method=method,
uri=uri,
headers=headers,
bodyProducer=body_producer)
d.addCallback(self.request_callback)
return d
def _connectAndRequest(self,
method,
uri,
headers,
bodyProducer,
requestPath=None):
"""
Internal helper to make the request.
@param requestPath: If specified, the path to use for the request
instead of the path extracted from C{uri}.
"""
scheme, host, port, path = _parse(uri)
if requestPath is None:
requestPath = path
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(
Request(method, requestPath, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def encode_headers(headers):
twisted_headers = Headers()
for k, v in headers.items():
key = k.encode('ascii', 'ignore')
val = v.encode('ascii', 'ignore')
twisted_headers.addRawHeader(key, val)
return twisted_headers
def request(self, method, uri, headers=None, data=None):
if data is not None:
if headers is None:
headers = Headers()
headers.removeHeader('Content-Type')
headers.addRawHeader('Content-Type', 'application/json; charset=utf-8')
return BasicAgent.request(self, method, uri, headers, data)
def _make_request(self):
metrics = []
for x in xrange(HTTP_BATCH):
if not self._mq:
break
metrics.append(self._mq.popleft())
if not metrics:
return defer.succeed(None)
serialized_metrics = json.dumps({"metrics": metrics})
body_writer = StringProducer(serialized_metrics)
headers = Headers({
'User-Agent': ['Zenoss Metric Publisher: %s' % self._agent_suffix],
'Content-Type': ['application/json']})
if self._needsAuth and not self._authenticated:
log.info("Adding auth for metric http post %s", self._url)
headers.addRawHeader('Authorization',
basic_auth_string_content(self._username, self._password))
d = self._agent.request(
'POST', self._url, headers,
body_writer)
d.addCallbacks(self._metrics_published, errback=self._publish_failed,
callbackArgs = [len(metrics), len(self._mq)], errbackArgs = [metrics])
d.addCallbacks(self._response_finished, errback=self._publish_failed,
errbackArgs = [metrics])
return d
def encode_headers(headers):
twisted_headers = Headers()
for k, v in headers.items():
key = k.encode('ascii', 'ignore')
val = v.encode('ascii', 'ignore')
twisted_headers.addRawHeader(key, val)
return twisted_headers
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request to the wrapped L{Agent}.
Send a I{Cookie} header if a cookie for C{uri} is stored in
L{CookieAgent.cookieJar}. Cookies are automatically extracted and
stored from requests.
If a C{'cookie'} header appears in C{headers} it will override the
automatic cookie header obtained from the cookie jar.
@see: L{Agent.request}
"""
if headers is None:
headers = Headers()
lastRequest = _FakeUrllib2Request(uri)
# Setting a cookie header explicitly will disable automatic request
# cookies.
if not headers.hasHeader('cookie'):
self.cookieJar.add_cookie_header(lastRequest)
cookieHeader = lastRequest.get_header('Cookie', None)
if cookieHeader is not None:
headers = headers.copy()
headers.addRawHeader('cookie', cookieHeader)
d = self._agent.request(method, uri, headers, bodyProducer)
d.addCallback(self._extractCookies, lastRequest)
return d
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(u'test\u00E1', [u'foo\u2603'])
i = h.copy()
# The copy contains the same value as the original
self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603'])
self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83'])
# Add a header to the original
h.addRawHeader(u'test\u00E1', u'bar')
# Verify that the copy has not changed
self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603'])
self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83'])
# Add a header to the copy
i.addRawHeader(u'test\u00E1', b'baz')
# Verify that the orignal does not have it
self.assertEqual(
h.getRawHeaders(u'test\u00E1'), [u'foo\u2603', u'bar'])
self.assertEqual(
h.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83', b'bar'])
def request(self, method, url, **kwargs):
method = method.upper()
# Join parameters provided in the URL
# and the ones passed as argument.
params = kwargs.get('params')
if params:
url = _combine_query_params(url, params)
# Convert headers dictionary to
# twisted raw headers format.
headers = kwargs.get('headers')
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.iteritems():
if isinstance(v, str):
h.addRawHeader(k, v)
else:
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
# Here we choose a right producer
# based on the parameters passed in.
bodyProducer = None
data = kwargs.get('data')
files = kwargs.get('files')
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
# with files and/or large objects.
files = list(_convert_files(files))
boundary = uuid.uuid4()
headers.setRawHeaders(
'content-type', [
'multipart/form-data; boundary=%s' % (boundary,)])
if data:
data = _convert_params(data)
else:
data = []
bodyProducer = multipart.MultiPartProducer(
data + files, boundary=boundary)
elif data:
# Otherwise stick to x-www-form-urlencoded format
# as it's generally faster for smaller requests.
if isinstance(data, (dict, list, tuple)):
headers.setRawHeaders(
'content-type', ['application/x-www-form-urlencoded'])
data = urlencode(data, doseq=True)
bodyProducer = IBodyProducer(data)
d = self._agent.request(
method, url, headers=headers,
bodyProducer=bodyProducer)
return d
def test_request(self, mock_request):
self.patch_request.stop()
args = ('GET', self.url)
self.agent.request(*args)
mock_request.assert_called_once_with(self.agent, *args + (None, None))
mock_request.reset_mock()
args = ('GET', self.url)
data = 'foo'
self.agent.request(*args, data=data)
match_bodyProducer = Matcher(compare_bodyProducer, self.agent.bodyProducer(data))
mock_request.assert_called_once_with(self.agent, *args + (None, match_bodyProducer))
mock_request.reset_mock()
headers = Headers()
headers.addRawHeader('Content-Type', 'application/json')
args = ('GET', self.url, headers)
self.agent.request(*args, data='foo')
match_bodyProducer = Matcher(compare_bodyProducer, self.agent.bodyProducer(data))
mock_request.assert_called_once_with(self.agent, *args + (match_bodyProducer,))
mock_request.reset_mock()
headers = Headers()
headers.addRawHeader('Content-Type', 'application/json')
args = ('GET', self.url, headers, None)
self.agent.request(*args)
mock_request.assert_called_once_with(self.agent, *args)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request to the wrapped L{Agent}.
Send a I{Cookie} header if a cookie for C{uri} is stored in
L{CookieAgent.cookieJar}. Cookies are automatically extracted and
stored from requests.
If a C{'cookie'} header appears in C{headers} it will override the
automatic cookie header obtained from the cookie jar.
@see: L{Agent.request}
"""
if headers is None:
headers = Headers()
lastRequest = _FakeUrllib2Request(uri)
# Setting a cookie header explicitly will disable automatic request
# cookies.
if not headers.hasHeader('cookie'):
self.cookieJar.add_cookie_header(lastRequest)
cookieHeader = lastRequest.get_header('Cookie', None)
if cookieHeader is not None:
headers = headers.copy()
headers.addRawHeader('cookie', cookieHeader)
d = self._agent.request(method, uri, headers, bodyProducer)
d.addCallback(self._extractCookies, lastRequest)
return d
def http_get(reactor, url, accept=None):
agent = client.Agent(reactor)
headers = Headers()
if accept is not None:
headers.addRawHeader('Accept', str(accept))
d = agent.request('GET', url, headers=headers)
return _handle_agent_response(d)
def put(self, uri, data):
headers = Headers()
headers.addRawHeader("accept", "*/*")
return self.request("PUT",
uri,
headers=headers,
bodyProducer=StringProducer(data))
def post(url, data=None, on_created=None, on_error=None):
errback = on_error or make_errback(frames_back=2)
try:
def handle_response(response):
if response.code == 200:
callback = partial(_handle_post,
on_created=on_created,
on_error=errback)
response.deliverBody(JsonReceiver.create(callback, errback))
else:
errback('returned %s' % response.code)
agent = Agent(reactor)
headers = Headers(get_auth_headers())
headers.addRawHeader('Content-Type', 'application/json')
if not data:
data = {'dummy': 1}
d = agent.request('POST',
add_params(url, rid=uuid4().hex),
headers=headers,
bodyProducer=JsonProducer(data) if data else None)
d.addCallbacks(handle_response, errback)
except Exception as ex:
errback('error %s' % ex)
def request(self,
method,
uri,
headers=None,
bodyProducer=None,
parameters=None,
addAuthHeader=True):
"""Make a request, optionally signing it.
Any query string passed in `uri` will get clobbered by the urlencoded
version of `parameters`.
"""
if headers is None:
headers = Headers()
if parameters is None:
parameters = {}
if addAuthHeader:
req = oauth2.Request.from_consumer_and_token(self.consumer,
token=self.token,
http_method=method,
http_url=uri,
parameters=parameters)
req.sign_request(self.signatureMethod, self.consumer, self.token)
for header, value in req.to_header().iteritems():
# oauth2, for some bozotic reason, gives unicode header values
headers.addRawHeader(header, value.encode())
parsed = urlparse.urlparse(uri)
uri = urlparse.urlunparse(
parsed._replace(query=urllib.urlencode(parameters)))
return self.agent.request(method, uri, headers, bodyProducer)
def endpoint(endpoint, username='******', apikey=None, data=None):
global url, user_agent
def encode_body(data):
if (data):
if (isinstance(data, six.string_types)):
data = json_loads(data)
return ('POST', FileBodyProducer(six.StringIO(json_dumps(data))))
else:
return ('GET', None)
address = make_endpoint_address(endpoint, username)
headers = Headers({
'Accept': ['application/json,*/*;q=0.8'],
'Accept-Encoding': ['gzip,deflate,sdch'],
'Connection': ['keep-alive'],
'User-Agent': [user_agent],
'Content-Type': ['application/json'],
})
if (apikey):
headers.addRawHeader('Authorization', 'Basic ' + encode_apikey(str(apikey)))
method, body = encode_body(data)
agent = Agent(reactor)
response = yield agent.request(method, address.encode('utf-8'), headers, body)
if (response.code == 200):
finished = Deferred()
collector = WebBodyCollector(finished)
response.deliverBody(collector)
x = yield finished
returnValue(json_loads(collector.body))
else:
raise RequestError(response.phrase)
def headers(self) -> Headers:
if not self.result:
raise Exception("No result yet.")
h = Headers()
for i in self.result["headers"]:
h.addRawHeader(*i)
return h
def request(self, method, uri, headers=None, bodyProducer=None):
parsedURI = client._parse(uri)
host_addr = address.IPv4Address('TCP', parsedURI.host, parsedURI.port)
# ripped from _AgentBase._requestWithEndpoint
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader(
'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
parsedURI.port))
request = client.Request(method, parsedURI.path, headers, bodyProducer,
persistent=False)
c = ClientProtocol(request)
# ouch
self.root.putChild('', self.root)
server = Site(self.root).buildProtocol(self.addr)
loopbackAsync(server, c, host_addr, self.addr)
return c.response.addBoth(self._done, c)
def headers(self, rawHeaders):
# type: (RawHeaders) -> IMutableHTTPHeaders
headers = Headers()
for rawName, rawValue in rawHeaders:
headers.addRawHeader(rawName, rawValue)
return HTTPHeadersWrappingHeaders(headers=headers)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
if version >= Version('twisted', 13, 1, 0):
parsed_uri = _URI.getFromBytes(uri)
scheme = parsed_uri.scheme
host = parsed_uri.host
port = parsed_uri.port
else:
scheme, host, port, path = _parse(uri)
request_path = uri
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(
Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def _baseHeaders(self):
"""return a dictionary of the base headers needed in a peer request"""
headers = Headers()
headers.addRawHeader('Target',self.target)
headers.addRawHeader('Signature',self._signature)
return headers
def headers(self, rawHeaders):
# type: (RawHeaders) -> IMutableHTTPHeaders
headers = Headers()
for rawName, rawValue in rawHeaders:
headers.addRawHeader(rawName, rawValue)
return HTTPHeadersWrappingHeaders(headers=headers)
def _make_request(self):
metrics = []
for x in xrange(HTTP_BATCH):
if not self._mq:
break
metrics.append(self._mq.popleft())
if not metrics:
return defer.succeed(None)
serialized_metrics = json.dumps({"metrics": metrics})
body_writer = StringProducer(serialized_metrics)
headers = Headers({
'User-Agent': ['Zenoss Metric Publisher: %s' % self._agent_suffix],
'Content-Type': ['application/json']
})
if self._needsAuth and not self._authenticated:
log.info("Adding auth for metric http post %s", self._url)
headers.addRawHeader(
'Authorization',
basic_auth_string_content(self._username, self._password))
d = self._agent.request('POST', self._url, headers, body_writer)
d.addCallbacks(self._metrics_published,
errback=self._publish_failed,
callbackArgs=[len(metrics), len(self._mq)],
errbackArgs=[metrics])
d.addCallbacks(self._response_finished,
errback=self._publish_failed,
errbackArgs=[metrics])
return d
def request(self, method, uri, headers=None, bodyProducer=None,
parameters=None):
"""Make a request, optionally signing it.
Any query string passed in `uri` will get clobbered by the urlencoded
version of `parameters`.
"""
if headers is None:
headers = Headers()
if parameters is None:
parameters = {}
req = oauth2.Request.from_consumer_and_token(
self.consumer, token=self.token,
http_method=method, http_url=uri, parameters=parameters,
is_form_encoded=True)
req.sign_request(self.signatureMethod, self.consumer, self.token)
for header, value in req.to_header().iteritems():
# oauth2, for some bozotic reason, gives unicode header values
headers.addRawHeader(header, value.encode())
parsed = urlparse.urlparse(uri)
parameters = {k.encode('utf-8'): v.encode('utf-8')
for k, v in req.get_nonoauth_parameters().iteritems()}
uri = urlparse.urlunparse(
parsed._replace(query=urllib.urlencode(parameters)))
return self.agent.request(method, uri, headers, bodyProducer)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
scheme, host, port, path = _parse(uri)
request_path = uri
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(
Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def assertSanitized(testCase, components, expected):
"""
Assert that the components are sanitized to the expected value as
both a header name and value, across all of L{Header}'s setters
and getters.
@param testCase: A test case.
@param components: A sequence of values that contain linear
whitespace to use as header names and values; see
C{textLinearWhitespaceComponents} and
C{bytesLinearWhitespaceComponents}
@param expected: The expected sanitized form of the component for
both headers names and their values.
"""
for component in components:
headers = []
headers.append(Headers({component: [component]}))
added = Headers()
added.addRawHeader(component, component)
headers.append(added)
setHeader = Headers()
setHeader.setRawHeaders(component, [component])
headers.append(setHeader)
for header in headers:
testCase.assertEqual(list(header.getAllRawHeaders()),
[(expected, [expected])])
testCase.assertEqual(header.getRawHeaders(expected), [expected])
def test_request(self, mock_request):
self.patch_request.stop()
args = ('GET', self.url)
self.agent.request(*args)
mock_request.assert_called_once_with(self.agent, *args + (None, None))
mock_request.reset_mock()
args = ('GET', self.url)
data = 'foo'
self.agent.request(*args, data=data)
match_bodyProducer = Matcher(compare_bodyProducer,
self.agent.bodyProducer(data))
mock_request.assert_called_once_with(
self.agent, *args + (None, match_bodyProducer))
mock_request.reset_mock()
headers = Headers()
headers.addRawHeader('Content-Type', 'application/json')
args = ('GET', self.url, headers)
self.agent.request(*args, data='foo')
match_bodyProducer = Matcher(compare_bodyProducer,
self.agent.bodyProducer(data))
mock_request.assert_called_once_with(self.agent,
*args + (match_bodyProducer, ))
mock_request.reset_mock()
headers = Headers()
headers.addRawHeader('Content-Type', 'application/json')
args = ('GET', self.url, headers, None)
self.agent.request(*args)
mock_request.assert_called_once_with(self.agent, *args)
def url_upload_data_async(url, files={}, data={}, progressCallback=None, doneCallback=None):
def produce_finished(data):
print 'produce finished',data
def produce_error(error):
print 'produce error', error
producerDeferred = Deferred()
producerDeferred.addCallback(produce_finished)
producerDeferred.addErrback(produce_error)
def receive_finished(data):
print 'recieve finished', data
doneCallback(data)
def receive_error(error):
Logger.error('Problem uploading file')
print 'recieve error', error
receiverDeferred = Deferred()
receiverDeferred.addCallback(receive_finished)
receiverDeferred.addErrback(receive_error)
producer = MultiPartProducer(files, data, progressCallback, producerDeferred)
receiver = StringReceiver(receiverDeferred)
agent = Agent(reactor)
headers = Headers({'User-Agent': ['Twisted Web Client Example']})
headers.addRawHeader("Content-Type", "multipart/form-data; boundary=%s" % producer.boundary)
requestDeffered = agent.request('POST', url, headers, producer)
requestDeffered.addCallback(lambda response: response.deliverBody(receiver))
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers,
bodyProducer, requestPath):
"""
Issue a new request, given the endpoint and the path sent as part of
the request.
"""
# Create minimal headers, if necessary:
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
#headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us
headers.addRawHeader(
'host',
self._computeHostValue(parsedURI.scheme, parsedURI.host,
parsedURI.port))
d = self._pool.getConnection(key, endpoint)
def cbConnected(proto):
return proto.request(
Request(method,
requestPath,
headers,
bodyProducer,
persistent=self._pool.persistent))
d.addCallback(cbConnected)
return d
def obtain_auth_token(self):
"""Return a valid Microsoft Cognitive Services authentication
token, obtained with the current subscription key if necessary.
"""
start_time = time.time()
if self.auth_token is None or self.token_expiry < start_time:
headers = Headers()
headers.addRawHeader('Ocp-Apim-Subscription-Key',
self.subscription_key)
headers.addRawHeader('Content-Length', '0')
response = yield self.agent.request(
'POST', AUTH_URL, headers=headers)
if response.code != 200:
data = yield readBody(response)
self.log.error(
'Could not authenticate to Microsoft Cognitive '
'Services: {data}', data=data)
raise UserVisibleError(
'Could not authenticate to Microsoft Cognitive '
'Services. Try again later.')
# Coerce the access token to a byte string to avoid problems
# inside Twisted's header handling code down the line.
self.auth_token = (
(yield readBody(response)).strip().decode('ascii'))
self.token_expiry = start_time + AUTH_TOKEN_TTL
returnValue(self.auth_token)
def headers(self):
if not self.result:
raise Exception("No result yet.")
h = Headers()
for i in self.result["headers"]:
h.addRawHeader(*i)
return h
async def _do_connect(self, protocol_factory: IProtocolFactory) -> None:
first_exception = None
server_list = await self._resolve_server()
for server in server_list:
host = server.host
port = server.port
endpoint: IStreamClientEndpoint
try:
if self.https_proxy_endpoint and not proxy_bypass(host.decode()):
logger.debug(
"Connecting to %s:%i via %s",
host.decode("ascii"),
port,
self.https_proxy_endpoint,
)
connect_headers = Headers()
# Determine whether we need to set Proxy-Authorization headers
if self.https_proxy_creds:
# Set a Proxy-Authorization header
connect_headers.addRawHeader(
b"Proxy-Authorization",
self.https_proxy_creds.as_proxy_authorization_value(),
)
endpoint = HTTPConnectProxyEndpoint(
self._reactor,
self.https_proxy_endpoint,
host,
port,
headers=connect_headers,
)
else:
logger.debug("Connecting to %s:%i", host.decode("ascii"), port)
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, host, port)
if self._tls_options:
endpoint = wrapClientTLS(self._tls_options, endpoint)
result = await make_deferred_yieldable(
endpoint.connect(protocol_factory)
)
return result
except Exception as e:
logger.info(
"Failed to connect to %s:%i: %s", host.decode("ascii"), port, e
)
if not first_exception:
first_exception = e
# We return the first failure because that's probably the most interesting.
if first_exception:
raise first_exception
# This shouldn't happen as we should always have at least one host/port
# to try and if that doesn't work then we'll have an exception.
raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc,))
def request(self, method, uri, headers=None, data=None):
if data is not None:
if headers is None:
headers = Headers()
headers.removeHeader('Content-Type')
headers.addRawHeader('Content-Type',
'application/json; charset=utf-8')
return BasicAgent.request(self, method, uri, headers, data)
def request(self, method, uri, headers=None, bodyProducer=None):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
for name, value in self._header_list:
headers.addRawHeader(name, value)
return self._agent.request(method, uri, headers, bodyProducer)
def _create_headers(self, headers_dict):
headers = Headers()
for key, value in headers_dict.items():
if isinstance(value, list):
headers.setRawHeaders(key, value)
else:
headers.addRawHeader(key, value)
return headers
def request(self, method, uri, headers=None, bodyProducer=None):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
if not headers.hasHeader('user-agent'):
headers.addRawHeader('user-agent', self.user_agent)
return self.agent.request(method, uri, headers, bodyProducer)
def send_webhook(url, data):
if len(url):
agent = Agent(reactor)
headers = Headers()
headers.addRawHeader("content-type", "application/json")
agent.request(b"POST", url.encode(), headers,
BytesProducer(json.dumps(data).encode()))
def _respondToChallenge(self, challenge, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = Headers(dict(headers.getAllRawHeaders()))
for k, vs in challenge.response(uri, method, self._authinfo).iteritems():
for v in vs:
headers.addRawHeader(k, v)
return self._agent.request(method, uri, headers, bodyProducer)
def _get_headers(self, headers): # type: (Optional[Headers]) -> Headers
"""Return the basic headers to be used by default."""
if headers is None:
headers = Headers()
headers.addRawHeader(
"Authorization",
swissnum_auth_header(self._swissnum),
)
return headers
def _requestAvatarIdFromWeasyl(self, credentials, credKey):
headers = Headers()
headers.addRawHeader('x-weasyl-api-key', credentials.password)
d = self.agent.request('GET', self.weasylInfoAPI, headers)
d.addCallback(self._trapBadStatuses)
d.addCallback(receive, StringReceiver())
d.addCallback(json.loads)
d.addCallback(self._verifyUsername, credentials.username)
d.addBoth(self._gotResult, credKey)
def test_addRawHeader(self):
"""
L{Headers.addRawHeader} adds a new value for a given header.
"""
h = Headers()
h.addRawHeader(b"test", b"lemur")
self.assertEqual(h.getRawHeaders(b"test"), [b"lemur"])
h.addRawHeader(b"test", b"panda")
self.assertEqual(h.getRawHeaders(b"test"), [b"lemur", b"panda"])
def _respondToChallenge(self, challenge, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = Headers(dict(headers.getAllRawHeaders()))
for k, vs in challenge.response(uri, method, self._authinfo).iteritems():
for v in vs:
headers.addRawHeader(k, v)
return self._agent.request(method, uri, headers, bodyProducer)
def _get_token_dict_async(cls):
protocol, host, path, _, _ = urlparse.urlsplit(cls.ACQUIRE_URL)
headers = Headers()
for key, value in cls.ACQUIRE_HEADERS.iteritems():
headers.addRawHeader(key, value)
agent = Agent('%s://%s' % (protocol, host))
d = agent.GET(path, headers, retry=cls.ACQUIRE_RETRIES)
d.addCallback(JsonResponse.Get)
return d
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@see: U{http://docs.python.org/library/urllib2.html#request-objects}
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@type type: C{str}
@ivar type: The scheme of the URI.
@type host: C{str}
@ivar host: The host[:port] of the URI.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def get_host(self):
return self.host
def get_type(self):
return self.type
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
def test_addRawHeader(self):
"""
L{Headers.addRawHeader} adds a new value for a given header.
"""
h = Headers()
h.addRawHeader("test", "lemur")
self.assertEqual(h.getRawHeaders("test"), ["lemur"])
h.addRawHeader("test", "panda")
self.assertEqual(h.getRawHeaders("test"), ["lemur", "panda"])
def _get_url_and_headers(conn_info):
url = "{c.scheme}://{c.hostname}:{c.port}/wsman".format(c=conn_info)
headers = Headers(_CONTENT_TYPE)
headers.addRawHeader('Accept', '*/*')
if conn_info.auth_type == 'basic':
headers.addRawHeader(
'Authorization', _get_basic_auth_header(conn_info))
else:
raise Exception('unknown auth type: {0}'.format(conn_info.auth_type))
return (url, headers)
def _get_url_and_headers(conn_info):
url = "{c.scheme}://{c.hostname}:{c.port}/wsman".format(c=conn_info)
headers = Headers(_CONTENT_TYPE)
headers.addRawHeader('Accept', '*/*')
if conn_info.auth_type == 'basic':
headers.addRawHeader('Authorization',
_get_basic_auth_header(conn_info))
else:
raise Exception('unknown auth type: {0}'.format(conn_info.auth_type))
return (url, headers)
def request(self, method, uri, headers=None, bodyProducer=None):
if headers is None:
headers = Headers()
headers.addRawHeader(*self._auth_header)
return super(AuthenticatingAgent, self).request(
method,
uri,
headers,
bodyProducer,
)
class ContactRequest(DummyRequest):
"""Contact Dummy Request
"""
def __init__(self, postpath, params):
DummyRequest.__init__(self, postpath, None)
self.content = StringIO()
self.content.write(json.dumps(params))
self.content.seek(0, 0)
self.requestHeaders = Headers()
self.method = 'POST'
self.requestHeaders.addRawHeader('content-type', 'application/json')
def call_endpoint(self, operation, params=None):
"""Make a request to the Microsoft Translator API endpoint with
the given operation and parameters, and return the body of the
response."""
auth_token = yield self.obtain_auth_token()
headers = Headers()
headers.addRawHeader('Authorization', 'Bearer ' + auth_token)
url = TRANSLATOR_URL.format(operation)
if params is not None:
url += '?' + urllib.urlencode(sorted(params.iteritems()))
response = yield self.agent.request('GET', url, headers=headers)
returnValue(json.loads((yield readBody(response)).decode('utf-8-sig')))
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Send a client request which declares supporting compressed content.
@see: L{Agent.request}.
"""
if headers is None:
headers = Headers()
else:
headers = headers.copy()
headers.addRawHeader('accept-encoding', self._supported)
deferred = self._agent.request(method, uri, headers, bodyProducer)
return deferred.addCallback(self._handleResponse)
def http_request(reactor, url, method, body=None, accept=None, more_headers=None):
agent = client.Agent(reactor)
headers = Headers()
if accept is not None:
headers.addRawHeader('Accept', str(accept))
if more_headers:
for k, v in more_headers.iteritems():
headers.addRawHeader(str(k), str(v))
d = agent.request(
method=str(method),
uri=str(url),
headers=headers,
bodyProducer=client.FileBodyProducer(StringIO.StringIO(str(body))) if body else None)
return _handle_agent_response(d)
def test_copy(self):
"""
L{Headers.copy} creates a new independant copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b'test', [b'foo'])
i = h.copy()
self.assertEqual(i.getRawHeaders(b'test'), [b'foo'])
h.addRawHeader(b'test', b'bar')
self.assertEqual(i.getRawHeaders(b'test'), [b'foo'])
i.addRawHeader(b'test', b'baz')
self.assertEqual(h.getRawHeaders(b'test'), [b'foo', b'bar'])
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b"test", [b"foo"])
i = h.copy()
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
h.addRawHeader(b"test", b"bar")
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
i.addRawHeader(b"test", b"baz")
self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@see: U{http://docs.python.org/library/urllib2.html#request-objects}
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@type type: C{str}
@ivar type: The scheme of the URI.
@type host: C{str}
@ivar host: The host[:port] of the URI.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def get_host(self):
return self.host
def get_type(self):
return self.type
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
