def http_get(reactor, url, accept=None):
agent = client.Agent(reactor)
headers = Headers()
if accept is not None:
headers.addRawHeader('Accept', str(accept))
d = agent.request('GET', url, headers=headers)
return _handle_agent_response(d)
def _make_request(self):
metrics = []
for x in xrange(HTTP_BATCH):
if not self._mq:
break
metrics.append(self._mq.popleft())
if not metrics:
return defer.succeed(None)
serialized_metrics = json.dumps({"metrics": metrics})
body_writer = StringProducer(serialized_metrics)
headers = Headers({
'User-Agent': ['Zenoss Metric Publisher: %s' % self._agent_suffix],
'Content-Type': ['application/json']})
if self._needsAuth and not self._authenticated:
log.info("Adding auth for metric http post %s", self._url)
headers.addRawHeader('Authorization',
basic_auth_string_content(self._username, self._password))
d = self._agent.request(
'POST', self._url, headers,
body_writer)
d.addCallbacks(self._metrics_published, errback=self._publish_failed,
callbackArgs = [len(metrics), len(self._mq)], errbackArgs = [metrics])
d.addCallbacks(self._response_finished, errback=self._publish_failed,
errbackArgs = [metrics])
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request to the wrapped L{Agent}.
Send a I{Cookie} header if a cookie for C{uri} is stored in
L{CookieAgent.cookieJar}. Cookies are automatically extracted and
stored from requests.
If a C{'cookie'} header appears in C{headers} it will override the
automatic cookie header obtained from the cookie jar.
@see: L{Agent.request}
"""
if headers is None:
headers = Headers()
lastRequest = _FakeUrllib2Request(uri)
# Setting a cookie header explicitly will disable automatic request
# cookies.
if not headers.hasHeader('cookie'):
self.cookieJar.add_cookie_header(lastRequest)
cookieHeader = lastRequest.get_header('Cookie', None)
if cookieHeader is not None:
headers = headers.copy()
headers.addRawHeader('cookie', cookieHeader)
d = self._agent.request(method, uri, headers, bodyProducer)
d.addCallback(self._extractCookies, lastRequest)
return d
def encode_headers(headers):
twisted_headers = Headers()
for k, v in headers.items():
key = k.encode('ascii', 'ignore')
val = v.encode('ascii', 'ignore')
twisted_headers.addRawHeader(key, val)
return twisted_headers
def request(self, method, url, **kwargs):
method = method.upper()
# Join parameters provided in the URL
# and the ones passed as argument.
params = kwargs.get('params')
if params:
url = _combine_query_params(url, params)
# Convert headers dictionary to
# twisted raw headers format.
headers = kwargs.get('headers')
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.iteritems():
if isinstance(v, str):
h.addRawHeader(k, v)
else:
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
# Here we choose a right producer
# based on the parameters passed in.
bodyProducer = None
data = kwargs.get('data')
files = kwargs.get('files')
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
# with files and/or large objects.
files = list(_convert_files(files))
boundary = uuid.uuid4()
headers.setRawHeaders(
'content-type', [
'multipart/form-data; boundary=%s' % (boundary,)])
if data:
data = _convert_params(data)
else:
data = []
bodyProducer = multipart.MultiPartProducer(
data + files, boundary=boundary)
elif data:
# Otherwise stick to x-www-form-urlencoded format
# as it's generally faster for smaller requests.
if isinstance(data, (dict, list, tuple)):
headers.setRawHeaders(
'content-type', ['application/x-www-form-urlencoded'])
data = urlencode(data, doseq=True)
bodyProducer = IBodyProducer(data)
d = self._agent.request(
method, url, headers=headers,
bodyProducer=bodyProducer)
return d
def _baseHeaders(self):
"""return a dictionary of the base headers needed in a peer request"""
headers = Headers()
headers.addRawHeader('Target',self.target)
headers.addRawHeader('Signature',self._signature)
return headers
def request(self, method, uri, headers=None, bodyProducer=None):
parsedURI = client._parse(uri)
host_addr = address.IPv4Address('TCP', parsedURI.host, parsedURI.port)
# ripped from _AgentBase._requestWithEndpoint
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader(
'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
parsedURI.port))
request = client.Request(method, parsedURI.path, headers, bodyProducer,
persistent=False)
c = ClientProtocol(request)
# ouch
self.root.putChild('', self.root)
server = Site(self.root).buildProtocol(self.addr)
loopbackAsync(server, c, host_addr, self.addr)
return c.response.addBoth(self._done, c)
def headers(self, rawHeaders):
# type: (RawHeaders) -> IMutableHTTPHeaders
headers = Headers()
for rawName, rawValue in rawHeaders:
headers.addRawHeader(rawName, rawValue)
return HTTPHeadersWrappingHeaders(headers=headers)
def headers(self):
if not self.result:
raise Exception("No result yet.")
h = Headers()
for i in self.result["headers"]:
h.addRawHeader(*i)
return h
def post(url, data=None, on_created=None, on_error=None):
errback = on_error or make_errback(frames_back=2)
try:
def handle_response(response):
if response.code == 200:
callback = partial(
_handle_post,
on_created=on_created,
on_error=errback)
response.deliverBody(JsonReceiver.create(callback, errback))
else:
errback('returned %s' % response.code)
agent = Agent(reactor)
headers = Headers(get_auth_headers())
headers.addRawHeader('Content-Type', 'application/json')
if not data:
data = {'dummy': 1}
d = agent.request(
'POST',
add_params(url, rid=uuid4().hex),
headers=headers,
bodyProducer=JsonProducer(data) if data else None)
d.addCallbacks(handle_response, errback)
except Exception as ex:
errback('error %s' % ex)
def request(self, url, method='GET', data=None):
d = defer.Deferred()
_headers = {}
headers = None
if method == 'GET':
data = None
elif method == 'POST':
_headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
d.errback(Exception('Not Support Method:%s.' % method))
return
if _headers:
headers = Headers()
for k, v in _headers.iteritems():
headers.setRawHeaders(urllib.quote(k), [urllib.quote(v)])
if data:
data = StrBodyProducer(data)
#print 'Req url:', url, ' Head:', headers
self.agent.request(
method,
url,
headers,
data).addCallback(self.received, d).addErrback(
self.errback, d)
return d
def request(self, method, uri, headers=None, bodyProducer=None,
parameters=None):
"""Make a request, optionally signing it.
Any query string passed in `uri` will get clobbered by the urlencoded
version of `parameters`.
"""
if headers is None:
headers = Headers()
if parameters is None:
parameters = {}
req = oauth2.Request.from_consumer_and_token(
self.consumer, token=self.token,
http_method=method, http_url=uri, parameters=parameters,
is_form_encoded=True)
req.sign_request(self.signatureMethod, self.consumer, self.token)
for header, value in req.to_header().iteritems():
# oauth2, for some bozotic reason, gives unicode header values
headers.addRawHeader(header, value.encode())
parsed = urlparse.urlparse(uri)
parameters = {k.encode('utf-8'): v.encode('utf-8')
for k, v in req.get_nonoauth_parameters().iteritems()}
uri = urlparse.urlunparse(
parsed._replace(query=urllib.urlencode(parameters)))
return self.agent.request(method, uri, headers, bodyProducer)
def _doPush( self, filename ):
print 'beginning push of file:', filename
producerDeferred = defer.Deferred()
producerDeferred.addCallback( self._finished )
producerDeferred.addErrback( self._error )
multiPartProducer = MultiPartProducer(
files={
'metrics': filename,
},
data={
'secret': agent_settings.PUSH_DETAILS['secret'],
},
callback=self._progress,
deferred=producerDeferred
)
headers = Headers()
headers.addRawHeader( 'Content-Type', 'multipart/form-data; boundary=%s' % multiPartProducer.boundary )
agent = client.Agent( reactor )
request = agent.request(
'POST',
agent_settings.PUSH_DETAILS['url'],
headers,
multiPartProducer
)
request.addCallback( self._responseReady, filename )
def endpoint(endpoint, username='******', apikey=None, data=None):
global url, user_agent
def encode_body(data):
if (data):
if (isinstance(data, six.string_types)):
data = json_loads(data)
return ('POST', FileBodyProducer(six.StringIO(json_dumps(data))))
else:
return ('GET', None)
address = make_endpoint_address(endpoint, username)
headers = Headers({
'Accept': ['application/json,*/*;q=0.8'],
'Accept-Encoding': ['gzip,deflate,sdch'],
'Connection': ['keep-alive'],
'User-Agent': [user_agent],
'Content-Type': ['application/json'],
})
if (apikey):
headers.addRawHeader('Authorization', 'Basic ' + encode_apikey(str(apikey)))
method, body = encode_body(data)
agent = Agent(reactor)
response = yield agent.request(method, address.encode('utf-8'), headers, body)
if (response.code == 200):
finished = Deferred()
collector = WebBodyCollector(finished)
response.deliverBody(collector)
x = yield finished
returnValue(json_loads(collector.body))
else:
raise RequestError(response.phrase)
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}.
"""
h = Headers({b"Foo": [b"bar"]})
self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}.
"""
h = Headers({'Foo': ['bar']})
self.assertEqual(h.getRawHeaders('foo'), ['bar'])
def url_upload_data_async(url, files={}, data={}, progressCallback=None, doneCallback=None):
def produce_finished(data):
print 'produce finished',data
def produce_error(error):
print 'produce error', error
producerDeferred = Deferred()
producerDeferred.addCallback(produce_finished)
producerDeferred.addErrback(produce_error)
def receive_finished(data):
print 'recieve finished', data
doneCallback(data)
def receive_error(error):
Logger.error('Problem uploading file')
print 'recieve error', error
receiverDeferred = Deferred()
receiverDeferred.addCallback(receive_finished)
receiverDeferred.addErrback(receive_error)
producer = MultiPartProducer(files, data, progressCallback, producerDeferred)
receiver = StringReceiver(receiverDeferred)
agent = Agent(reactor)
headers = Headers({'User-Agent': ['Twisted Web Client Example']})
headers.addRawHeader("Content-Type", "multipart/form-data; boundary=%s" % producer.boundary)
requestDeffered = agent.request('POST', url, headers, producer)
requestDeffered.addCallback(lambda response: response.deliverBody(receiver))
def obtain_auth_token(self):
"""Return a valid Microsoft Cognitive Services authentication
token, obtained with the current subscription key if necessary.
"""
start_time = time.time()
if self.auth_token is None or self.token_expiry < start_time:
headers = Headers()
headers.addRawHeader('Ocp-Apim-Subscription-Key',
self.subscription_key)
headers.addRawHeader('Content-Length', '0')
response = yield self.agent.request(
'POST', AUTH_URL, headers=headers)
if response.code != 200:
data = yield readBody(response)
self.log.error(
'Could not authenticate to Microsoft Cognitive '
'Services: {data}', data=data)
raise UserVisibleError(
'Could not authenticate to Microsoft Cognitive '
'Services. Try again later.')
# Coerce the access token to a byte string to avoid problems
# inside Twisted's header handling code down the line.
self.auth_token = (
(yield readBody(response)).strip().decode('ascii'))
self.token_expiry = start_time + AUTH_TOKEN_TTL
returnValue(self.auth_token)
def request(self, method, uri, headers=None, data=None):
if data is not None:
if headers is None:
headers = Headers()
headers.removeHeader('Content-Type')
headers.addRawHeader('Content-Type', 'application/json; charset=utf-8')
return BasicAgent.request(self, method, uri, headers, data)
def request(rq):
headers = Headers(dict((k, [v]) for k, v in rq.headers.iteritems()))
scheme, host, port, path = _parse(rq.url)
headers.setRawHeaders('Host', [host])
return (rq.method, rq.url,
headers,
StringProducer(rq.data) if rq.data else None)
def test_getRawHeadersDefaultValue(self):
"""
L{Headers.getRawHeaders} returns the specified default value when no
header is found.
"""
h = Headers()
default = object()
self.assertIdentical(h.getRawHeaders("test", default), default)
def test_removeHeaderDoesntExist(self):
"""
L{Headers.removeHeader} is a no-operation when the specified header is
not found.
"""
h = Headers()
h.removeHeader("test")
self.assertEqual(list(h.getAllRawHeaders()), [])
def request(self, method, uri, headers=None, bodyProducer=None):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
for name, value in self._header_list:
headers.addRawHeader(name, value)
return self._agent.request(method, uri, headers, bodyProducer)
def request(self, method, uri, headers=None, bodyProducer=None):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
if not headers.hasHeader('user-agent'):
headers.addRawHeader('user-agent', self.user_agent)
return self.agent.request(method, uri, headers, bodyProducer)
def _respondToChallenge(self, challenge, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = Headers(dict(headers.getAllRawHeaders()))
for k, vs in challenge.response(uri, method, self._authinfo).iteritems():
for v in vs:
headers.addRawHeader(k, v)
return self._agent.request(method, uri, headers, bodyProducer)
def test_getRawHeaders(self):
"""
L{Headers.getRawHeaders} returns the values which have been set for a
given header.
"""
h = Headers()
h.setRawHeaders("test", ["lemur"])
self.assertEqual(h.getRawHeaders("test"), ["lemur"])
self.assertEqual(h.getRawHeaders("Test"), ["lemur"])
def test_rawHeadersValueEncoding(self):
"""
Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
ISO-8859-1 and values as UTF-8.
"""
h = Headers()
h.setRawHeaders(u"\u00E1", [u"\u2603", b"foo"])
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b'\xe2\x98\x83', b'foo'])
def test_hasHeaderTrue(self):
"""
Check that L{Headers.hasHeader} returns C{True} when the given header
is found.
"""
h = Headers()
h.setRawHeaders("test", ["lemur"])
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@see: U{http://docs.python.org/library/urllib2.html#request-objects}
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@type type: C{str}
@ivar type: The scheme of the URI.
@type host: C{str}
@ivar host: The host[:port] of the URI.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def get_host(self):
return self.host
def get_type(self):
return self.type
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
def _get_token_dict_async(cls):
protocol, host, path, _, _ = urlparse.urlsplit(cls.ACQUIRE_URL)
headers = Headers()
for key, value in cls.ACQUIRE_HEADERS.iteritems():
headers.addRawHeader(key, value)
agent = Agent('%s://%s' % (protocol, host))
d = agent.GET(path, headers, retry=cls.ACQUIRE_RETRIES)
d.addCallback(JsonResponse.Get)
return d
def _process_batch(self, batch_in):
#Map from JSON-RPC to future waiting for result
deferreds_map = dict()
#The outgoing JSON-RPC batch
batch_out = list()
#Set of unprocessed entries
unprocessed = set()
def process_batch_level_exception(exception):
"""Spread batch level exception to each of the batch entries"""
#On a batch level exception, forward the cought exception to the future for
# each of the commands that are part of the batch.
#pylint: disable=unused-variable
for key, entry_deferred in deferreds_map.items():
entry_deferred.errback(exception)
def process_json_parse_error(code, body):
"""Convert a json parse error and HTTP error code into appropriate exception type"""
if code > 499: #5xx server side error codes
process_batch_level_exception(HttpServerError(code, body))
else:
if code > 399: #4xx client side errors.
if code == 413 and self.maxbatch > 1:
self.maxbatch = 1
self.queue.json_rpcqueue_again(batch_in)
else:
process_batch_level_exception(
HttpClientError(code, body))
else:
# Non-error code in the 2xx and 3xx range.
process_batch_level_exception(
JsonRpcBatchError(code, body,
"Invalid JSON returned by server"))
def process_response(response):
code = response.code
def process_body(text_result):
"""Process JSON-RPC batch response body"""
#pylint: disable=broad-except, unused-variable
try:
#Parse the JSON content of the JSON-RPC batch response.
resp_obj = json.loads(text_result)
except json.decoder.JSONDecodeError as exception:
process_json_parse_error(code, text_result.decode())
resp_obj = None
if resp_obj:
#Assert the parsed JSON is a list.
if not isinstance(resp_obj, list):
process_batch_level_exception(
JsonRpcBatchError(
code, text_result.decode(),
"Non-batch JSON response from server."))
resp_obj = []
else:
#Process the individual command responses
for response in resp_obj:
#Get the id from the response to match with the apropriate reuest
if "id" in response and response[
"id"] in unprocessed:
query_id = response["id"]
#Maintain list of unprocessed id's
unprocessed.remove(query_id)
#Look up the proper command future to map this response to.
query_deferred = deferreds_map[query_id]
#Distinguish between responses and errors.
if "result" in response:
#Set future result
query_deferred.callback(response["result"])
if (not "result" in response) and "error" in response and \
"message" in response["error"] and \
"code" in response["error"] and \
"data" in response["error"]:
query_deferred.errback(
JsonRpcCommandError(
response["error"]["code"],
response["error"]["message"],
response["error"]["data"]))
if (not "result" in response) and (
not "error" in response):
query_deferred.errback(
JsonRpcCommandResponseError(
"Bad command response from server",
response))
#Work through any request item id not found in the response.
for no_valid_response_id in unprocessed:
query_deferred = deferreds_map[
no_valid_response_id]
query_deferred.errback(
JsonRpcCommandResponseError(
"Request command id not found in response.",
resp_obj))
self._fetch_batch()
#Get (text) content from the server response
body_deferred = readBody(response)
body_deferred.addCallbacks(process_body,
process_batch_level_exception)
body_deferred.addBoth(self._fetch_batch)
return body_deferred
def process_batch_level_exception_and_continue(failure):
try:
failure.raiseException()
except ResponseFailed as exception:
failure2 = exception.reasons[0]
try:
failure2.raiseException()
except VerificationError as exception2:
error = exception2.errors[0]
if isinstance(error, DNSMismatch):
process_batch_level_exception(
SSLNameMismatch(
"Problem with node certificate. Wrong DNS name."
))
else:
process_batch_level_exception(
SSLError("Problem with node certificate."))
except OpenSSL.SSL.Error as exception2:
process_batch_level_exception(
SSLError("Problem with node certificate."))
except Exception as exception2:
process_batch_level_exception(exception2)
except Exception as exception:
process_batch_level_exception(exception)
self._fetch_batch()
log.msg("Processing new batch for " + self.host_url.decode("utf8") +
" , " + str(len(batch_in)))
#Build the output batch and deferred map.
for cmd in batch_in:
self.cmd_id += 1
unprocessed.add(self.cmd_id)
newcmd = dict()
newcmd["id"] = self.cmd_id
newcmd["jsonrpc"] = "2.0"
newcmd["method"] = cmd["method"]
if cmd["params"]:
newcmd["params"] = cmd["params"]
deferreds_map[self.cmd_id] = cmd["deferred"]
batch_out.append(newcmd)
#Piggybag extra command onto single command batches. FIXME: this is a temporary workaround.
if len(batch_out) == 1 and self.maxbatch > 1:
newcmd = dict()
newcmd["id"] = 0
newcmd["jsonrpc"] = "2.0"
newcmd["method"] = "bogus_api.bogus_method"
newcmd["params"] = [
"Extra bogus API call for making sure the server supports batches of bigger than one."
]
batch_out.append(newcmd)
#Post the JSON-RPC batch request to the server and wait for response
log.msg("Posting batch to node " + self.host_url.decode("utf8"))
deferred_response = self.agent.request(
b'POST', self.host_url,
Headers({
b"User-Agent": [b'TxJsonRpcQueue v0.0.1'],
b"Content-Type": [b"application/json"]
}), _StringProducer(json.dumps(batch_out).encode("utf8")))
deferred_response.addCallbacks(
process_response, process_batch_level_exception_and_continue)
from twisted.web.client import Agent, readBody
from twisted.internet import reactor, defer
from twisted.internet.ssl import ClientContextFactory
from twisted.web.http_headers import Headers
from zope.interface import implementer
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from twisted.internet.protocol import Protocol
from pprint import pformat
import json
import time
from test.public_api.web import get_smzdm_datas, print_smzdm_result, end_crawl
headers = Headers({
'User-Agent': [
'MMozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0'
],
'content-type': ["application/json"]
})
class WebClientContextFactory(ClientContextFactory):
'''
用来实现https网页的访问验证
'''
def getContext(self, hostname, port):
return ClientContextFactory.getContext(self)
def cbRequest(response):
'''
print('Response version:', response.version)
def _request(self,
destination,
method,
path,
body_callback,
headers_dict={},
param_bytes=b"",
query_bytes=b"",
retry_on_dns_fail=True,
timeout=None,
long_retries=False,
ignore_backoff=False,
backoff_on_404=False):
""" Creates and sends a request to the given server
Args:
destination (str): The remote server to send the HTTP request to.
method (str): HTTP method
path (str): The HTTP path
ignore_backoff (bool): true to ignore the historical backoff data
and try the request anyway.
backoff_on_404 (bool): Back off if we get a 404
Returns:
Deferred: resolves with the http response object on success.
Fails with ``HTTPRequestException``: if we get an HTTP response
code >= 300.
Fails with ``NotRetryingDestination`` if we are not yet ready
to retry this server.
Fails with ``FederationDeniedError`` if this destination
is not on our federation whitelist
(May also fail with plenty of other Exceptions for things like DNS
failures, connection failures, SSL failures.)
"""
if (self.hs.config.federation_domain_whitelist and destination
not in self.hs.config.federation_domain_whitelist):
raise FederationDeniedError(destination)
limiter = yield synapse.util.retryutils.get_retry_limiter(
destination,
self.clock,
self._store,
backoff_on_404=backoff_on_404,
ignore_backoff=ignore_backoff,
)
destination = destination.encode("ascii")
path_bytes = path.encode("ascii")
with limiter:
headers_dict[b"User-Agent"] = [self.version_string]
headers_dict[b"Host"] = [destination]
url_bytes = self._create_url(destination, path_bytes, param_bytes,
query_bytes)
txn_id = "%s-O-%s" % (method, self._next_id)
self._next_id = (self._next_id + 1) % (sys.maxint - 1)
outbound_logger.info("{%s} [%s] Sending request: %s %s", txn_id,
destination, method, url_bytes)
# XXX: Would be much nicer to retry only at the transaction-layer
# (once we have reliable transactions in place)
if long_retries:
retries_left = MAX_LONG_RETRIES
else:
retries_left = MAX_SHORT_RETRIES
http_url_bytes = urlparse.urlunparse(
("", "", path_bytes, param_bytes, query_bytes, ""))
log_result = None
try:
while True:
producer = None
if body_callback:
producer = body_callback(method, http_url_bytes,
headers_dict)
try:
request_deferred = self.agent.request(
method, url_bytes, Headers(headers_dict), producer)
add_timeout_to_deferred(
request_deferred,
timeout / 1000. if timeout else 60,
self.hs.get_reactor(),
cancelled_to_request_timed_out_error,
)
response = yield make_deferred_yieldable(
request_deferred, )
log_result = "%d %s" % (
response.code,
response.phrase,
)
break
except Exception as e:
if not retry_on_dns_fail and isinstance(
e, DNSLookupError):
logger.warn("DNS Lookup failed to %s with %s",
destination, e)
log_result = "DNS Lookup failed to %s with %s" % (
destination, e)
raise
logger.warn(
"{%s} Sending request failed to %s: %s %s: %s",
txn_id,
destination,
method,
url_bytes,
_flatten_response_never_received(e),
)
log_result = _flatten_response_never_received(e)
if retries_left and not timeout:
if long_retries:
delay = 4**(MAX_LONG_RETRIES + 1 -
retries_left)
delay = min(delay, 60)
delay *= random.uniform(0.8, 1.4)
else:
delay = 0.5 * 2**(MAX_SHORT_RETRIES -
retries_left)
delay = min(delay, 2)
delay *= random.uniform(0.8, 1.4)
yield self.clock.sleep(delay)
retries_left -= 1
else:
raise
finally:
outbound_logger.info(
"{%s} [%s] Result: %s",
txn_id,
destination,
log_result,
)
if 200 <= response.code < 300:
pass
else:
# :'(
# Update transactions table?
with logcontext.PreserveLoggingContext():
body = yield readBody(response)
raise HttpResponseException(response.code, response.phrase,
body)
defer.returnValue(response)
from bytesprod import BytesProducer
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
agent = Agent(reactor)
body = BytesProducer(b"hello, world")
d = agent.request(
b"POST",
b"http://httpbin.org/post",
Headers({
"User-Agent": ["Twisted Web Client Example"],
"Content-Type": ["text/x-greeting"],
}),
body,
)
def cbResponse(ignored):
print("Response received")
d.addCallback(cbResponse)
def cbShutdown(ignored):
reactor.stop()
d.addBoth(cbShutdown)
def request(self, url, headers, method='GET'):
LOG(u'Will request URL %s with headers %s, method %s' %
(url, pformat(headers), method))
deferred = self.client.request(method, url, Headers(headers))
deferred.addCallbacks(self.on_success, self.on_failure)
def test_getRawHeadersNoDefault(self):
"""
L{Headers.getRawHeaders} returns C{None} if the header is not found and
no default is specified.
"""
self.assertIdentical(Headers().getRawHeaders("test"), None)
def do_test_payment(self, wc1, wc2, amt=1.1):
wallet_structures = [self.wallet_structure] * 2
wallet_cls = (wc1, wc2)
self.wallet_services = []
self.wallet_services.append(
make_wallets_to_list(
make_wallets(1,
wallet_structures=[wallet_structures[0]],
mean_amt=self.mean_amt,
wallet_cls=wallet_cls[0]))[0])
self.wallet_services.append(
make_wallets_to_list(
make_wallets(1,
wallet_structures=[wallet_structures[1]],
mean_amt=self.mean_amt,
wallet_cls=wallet_cls[1]))[0])
jm_single().bc_interface.tickchain()
sync_wallets(self.wallet_services)
# For accounting purposes, record the balances
# at the start.
self.rsb = getbals(self.wallet_services[0], 0)
self.ssb = getbals(self.wallet_services[1], 0)
self.cj_amount = int(amt * 10**8)
def cbStopListening():
return self.port.stopListening()
b78rm = JMBIP78ReceiverManager(self.wallet_services[0], 0,
self.cj_amount, 47083)
resource = DummyBIP78ReceiverResource(jmprint, cbStopListening, b78rm)
self.site = Site(resource)
self.site.displayTracebacks = False
# NB The connectivity aspects of the onion-based BIP78 setup
# are time heavy. This server is TCP only.
self.port = reactor.listenTCP(47083, self.site)
self.addCleanup(cbStopListening)
# setup of spender
bip78_btc_amount = amount_to_btc(amount_to_sat(self.cj_amount))
bip78_uri = encode_bip21_uri(str(b78rm.receiving_address), {
"amount": bip78_btc_amount,
"pj": b"http://127.0.0.1:47083"
},
safe=":/")
self.manager = parse_payjoin_setup(bip78_uri, self.wallet_services[1],
0)
self.manager.mode = "testing"
success, msg = make_payment_psbt(self.manager)
assert success, msg
params = make_payjoin_request_params(self.manager)
# avoiding backend daemon (testing only jmclient code here),
# we send the http request manually:
serv = b"http://127.0.0.1:47083"
agent = get_nontor_agent()
body = BytesProducer(
self.manager.initial_psbt.to_base64().encode("utf-8"))
url_parts = list(wrapped_urlparse(serv))
url_parts[4] = urlencode(params).encode("utf-8")
destination_url = urlparse.urlunparse(url_parts)
d = agent.request(b"POST",
destination_url,
Headers({"Content-Type": ["text/plain"]}),
bodyProducer=body)
d.addCallback(bip78_receiver_response, self.manager)
return d
def _send_request(
self,
request,
retry_on_dns_fail=True,
timeout=None,
long_retries=False,
ignore_backoff=False,
backoff_on_404=False,
):
"""
Sends a request to the given server.
Args:
request (MatrixFederationRequest): details of request to be sent
timeout (int|None): number of milliseconds to wait for the response headers
(including connecting to the server), *for each attempt*.
60s by default.
long_retries (bool): whether to use the long retry algorithm.
The regular retry algorithm makes 4 attempts, with intervals
[0.5s, 1s, 2s].
The long retry algorithm makes 11 attempts, with intervals
[4s, 16s, 60s, 60s, ...]
Both algorithms add -20%/+40% jitter to the retry intervals.
Note that the above intervals are *in addition* to the time spent
waiting for the request to complete (up to `timeout` ms).
NB: the long retry algorithm takes over 20 minutes to complete, with
a default timeout of 60s!
ignore_backoff (bool): true to ignore the historical backoff data
and try the request anyway.
backoff_on_404 (bool): Back off if we get a 404
Returns:
Deferred[twisted.web.client.Response]: resolves with the HTTP
response object on success.
Raises:
HttpResponseException: If we get an HTTP response code >= 300
(except 429).
NotRetryingDestination: If we are not yet ready to retry this
server.
FederationDeniedError: If this destination is not on our
federation whitelist
RequestSendFailed: If there were problems connecting to the
remote, due to e.g. DNS failures, connection timeouts etc.
"""
if timeout:
_sec_timeout = timeout / 1000
else:
_sec_timeout = self.default_timeout
if (self.hs.config.federation_domain_whitelist is not None
and request.destination
not in self.hs.config.federation_domain_whitelist):
raise FederationDeniedError(request.destination)
limiter = yield synapse.util.retryutils.get_retry_limiter(
request.destination,
self.clock,
self._store,
backoff_on_404=backoff_on_404,
ignore_backoff=ignore_backoff,
)
method_bytes = request.method.encode("ascii")
destination_bytes = request.destination.encode("ascii")
path_bytes = request.path.encode("ascii")
if request.query:
query_bytes = encode_query_args(request.query)
else:
query_bytes = b""
scope = start_active_span(
"outgoing-federation-request",
tags={
tags.SPAN_KIND: tags.SPAN_KIND_RPC_CLIENT,
tags.PEER_ADDRESS: request.destination,
tags.HTTP_METHOD: request.method,
tags.HTTP_URL: request.path,
},
finish_on_close=True,
)
# Inject the span into the headers
headers_dict = {}
inject_active_span_byte_dict(headers_dict, request.destination)
headers_dict[b"User-Agent"] = [self.version_string_bytes]
with limiter, scope:
# XXX: Would be much nicer to retry only at the transaction-layer
# (once we have reliable transactions in place)
if long_retries:
retries_left = MAX_LONG_RETRIES
else:
retries_left = MAX_SHORT_RETRIES
url_bytes = urllib.parse.urlunparse(
(b"matrix", destination_bytes, path_bytes, None, query_bytes,
b""))
url_str = url_bytes.decode("ascii")
url_to_sign_bytes = urllib.parse.urlunparse(
(b"", b"", path_bytes, None, query_bytes, b""))
while True:
try:
json = request.get_json()
if json:
headers_dict[b"Content-Type"] = [b"application/json"]
auth_headers = self.build_auth_headers(
destination_bytes, method_bytes, url_to_sign_bytes,
json)
data = encode_canonical_json(json)
producer = QuieterFileBodyProducer(
BytesIO(data), cooperator=self._cooperator)
else:
producer = None
auth_headers = self.build_auth_headers(
destination_bytes, method_bytes, url_to_sign_bytes)
headers_dict[b"Authorization"] = auth_headers
logger.info(
"{%s} [%s] Sending request: %s %s; timeout %fs",
request.txn_id,
request.destination,
request.method,
url_str,
_sec_timeout,
)
try:
with Measure(self.clock, "outbound_request"):
# we don't want all the fancy cookie and redirect handling
# that treq.request gives: just use the raw Agent.
request_deferred = self.agent.request(
method_bytes,
url_bytes,
headers=Headers(headers_dict),
bodyProducer=producer,
)
request_deferred = timeout_deferred(
request_deferred,
timeout=_sec_timeout,
reactor=self.reactor,
)
response = yield request_deferred
except DNSLookupError as e:
raise_from(
RequestSendFailed(e, can_retry=retry_on_dns_fail),
e)
except Exception as e:
logger.info("Failed to send request: %s", e)
raise_from(RequestSendFailed(e, can_retry=True), e)
logger.info(
"{%s} [%s] Got response headers: %d %s",
request.txn_id,
request.destination,
response.code,
response.phrase.decode("ascii", errors="replace"),
)
set_tag(tags.HTTP_STATUS_CODE, response.code)
if 200 <= response.code < 300:
pass
else:
# :'(
# Update transactions table?
d = treq.content(response)
d = timeout_deferred(d,
timeout=_sec_timeout,
reactor=self.reactor)
try:
body = yield make_deferred_yieldable(d)
except Exception as e:
# Eh, we're already going to raise an exception so lets
# ignore if this fails.
logger.warn(
"{%s} [%s] Failed to get error response: %s %s: %s",
request.txn_id,
request.destination,
request.method,
url_str,
_flatten_response_never_received(e),
)
body = None
e = HttpResponseException(response.code,
response.phrase, body)
# Retry if the error is a 429 (Too Many Requests),
# otherwise just raise a standard HttpResponseException
if response.code == 429:
raise_from(RequestSendFailed(e, can_retry=True), e)
else:
raise e
break
except RequestSendFailed as e:
logger.warn(
"{%s} [%s] Request failed: %s %s: %s",
request.txn_id,
request.destination,
request.method,
url_str,
_flatten_response_never_received(e.inner_exception),
)
if not e.can_retry:
raise
if retries_left and not timeout:
if long_retries:
delay = 4**(MAX_LONG_RETRIES + 1 - retries_left)
delay = min(delay, 60)
delay *= random.uniform(0.8, 1.4)
else:
delay = 0.5 * 2**(MAX_SHORT_RETRIES - retries_left)
delay = min(delay, 2)
delay *= random.uniform(0.8, 1.4)
logger.debug(
"{%s} [%s] Waiting %ss before re-sending...",
request.txn_id,
request.destination,
delay,
)
yield self.clock.sleep(delay)
retries_left -= 1
else:
raise
except Exception as e:
logger.warn(
"{%s} [%s] Request failed: %s %s: %s",
request.txn_id,
request.destination,
request.method,
url_str,
_flatten_response_never_received(e),
)
raise
return response
from email.mime.text import MIMEText
from datetime import date
from datetime import datetime
from datetime import timedelta
from functools import partial
from twisted.python import log
from twisted.internet import defer
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.client import readBody
from twisted.web.http_headers import Headers
TRAC_BUILDBOT_URL = 'http://trac.buildbot.net'
TRAC_BUILDBOT_TICKET_URL = TRAC_BUILDBOT_URL + '/ticket/%(ticket)s'
GITHUB_API_URL = 'https://api.github.com'
HTTP_HEADERS = Headers({'User-Agent': ['buildbot.net weekly summary']})
FROM = '*****@*****.**'
RECIPIENTS = ['*****@*****.**', '*****@*****.**']
WEEKLY_MEETING_TEXT = textwrap.dedent("""
<p>Buildbot has weekly meetings via irc, held at 16:30 UTC on Tuesdays.
That is about 90 minutes from now!
<p>Meetings are in #buildbot on Freenode, open to any and all participants.
They generally focus on organizational, rather than technical issues, but are open to anything Buildbot-related.
To raise a topic, add it to "All Other Business" in the <a href="https://titanpad.com/buildbot-agenda">agenda</a>, or just speak up during the meeting.
<p>Meeting minutes are available <a href="https://supybot.buildbot.net/meetings/">here</a>.
""")
def requestMock(path,
method=b"GET",
host=b"localhost",
port=8080,
isSecure=False,
body=None,
headers=None):
if not headers:
headers = {}
if not body:
body = b''
request = server.Request(DummyChannel(), False)
request.site = Mock(server.Site)
request.gotLength(len(body))
request.content = BytesIO()
request.content.write(body)
request.content.seek(0)
request.requestHeaders = Headers(headers)
request.setHost(host, port, isSecure)
request.uri = path
request.prepath = []
request.postpath = path.split(b'/')[1:]
request.method = method
request.clientproto = b'HTTP/1.1'
request.setHeader = Mock(wraps=request.setHeader)
request.setResponseCode = Mock(wraps=request.setResponseCode)
request._written = BytesIO()
request.finishCount = 0
request.writeCount = 0
def registerProducer(producer, streaming):
request.producer = producer
for x in range(2):
if request.producer:
request.producer.resumeProducing()
def unregisterProducer():
request.producer = None
def finish():
request.finishCount += 1
if not request.startedWriting:
request.write(b'')
if not request.finished:
request.finished = True
request._cleanup()
def write(data):
request.writeCount += 1
request.startedWriting = True
if not request.finished:
request._written.write(data)
else:
raise RuntimeError('Request.write called on a request after '
'Request.finish was called.')
def getWrittenData():
return request._written.getvalue()
request.finish = finish
request.write = write
request.getWrittenData = getWrittenData
request.registerProducer = registerProducer
request.unregisterProducer = unregisterProducer
request.processingFailed = Mock(wraps=request.processingFailed)
return request
def test_rawHeadersTypeChecking(self):
"""
L{Headers.setRawHeaders} requires values to be of type list.
"""
h = Headers()
self.assertRaises(TypeError, h.setRawHeaders, u'key', {u'Foo': u'bar'})
def test_headersComparison(self):
"""
A L{Headers} instance compares equal to itself and to another
L{Headers} instance with the same values.
"""
first = Headers()
first.setRawHeaders(b"foo", [b"panda"])
second = Headers()
second.setRawHeaders(b"foo", [b"panda"])
third = Headers()
third.setRawHeaders(b"foo", [b"lemur", b"panda"])
self.assertEqual(first, first)
self.assertEqual(first, second)
self.assertNotEqual(first, third)
def test_multiple_proxys(self):
headers = Headers({
b'X-Forwarded-For': [b'10.1.2.3, 10.1.2.4'],
})
self.assertEqual(parse_x_forwarded_for(headers), ['10.1.2.3', 0])
def request(
self,
method: bytes,
uri: bytes,
headers: Optional[Headers] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> defer.Deferred:
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method: The request method to use, such as `GET`, `POST`, etc
uri: The location of the resource to request.
headers: Extra headers to send with the request
bodyProducer: An object which can generate bytes to make up the body of
this request (for example, the properly encoded contents of a file for
a file upload). Or, None if the request is to have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
Can fail with:
SchemeNotSupported: if the uri is not http or https
twisted.internet.error.TimeoutError if the server we are connecting
to (proxy or destination) does not accept a connection before
connectTimeout.
... other things too.
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError(f"Invalid URI {uri!r}")
parsed_uri = URI.fromBytes(uri)
pool_key = f"{parsed_uri.scheme!r}{parsed_uri.host!r}{parsed_uri.port}"
request_path = parsed_uri.originForm
should_skip_proxy = False
if self.no_proxy is not None:
should_skip_proxy = proxy_bypass_environment(
parsed_uri.host.decode(),
proxies={"no": self.no_proxy},
)
if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint
and not should_skip_proxy):
# Determine whether we need to set Proxy-Authorization headers
if self.http_proxy_creds:
# Set a Proxy-Authorization header
if headers is None:
headers = Headers()
headers.addRawHeader(
b"Proxy-Authorization",
self.http_proxy_creds.as_proxy_authorization_value(),
)
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = "http-proxy"
endpoint = self.http_proxy_endpoint
request_path = uri
elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint
and not should_skip_proxy):
endpoint = HTTPConnectProxyEndpoint(
self.proxy_reactor,
self.https_proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
self.https_proxy_creds,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
def request(self, method, url, **kwargs):
"""
See :func:`treq.request()`.
"""
method = method.encode('ascii').upper()
if isinstance(url, DecodedURL):
parsed_url = url
elif isinstance(url, EncodedURL):
parsed_url = DecodedURL(url)
elif isinstance(url, six.text_type):
parsed_url = DecodedURL.from_text(url)
else:
parsed_url = DecodedURL.from_text(url.decode('ascii'))
# Join parameters provided in the URL
# and the ones passed as argument.
params = kwargs.get('params')
if params:
parsed_url = parsed_url.replace(
query=parsed_url.query + tuple(_coerced_query_params(params)))
url = parsed_url.to_uri().to_text().encode('ascii')
# Convert headers dictionary to
# twisted raw headers format.
headers = kwargs.get('headers')
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.items():
if isinstance(v, (bytes, six.text_type)):
h.addRawHeader(k, v)
elif isinstance(v, list):
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
# Here we choose a right producer
# based on the parameters passed in.
bodyProducer = None
data = kwargs.get('data')
files = kwargs.get('files')
# since json=None needs to be serialized as 'null', we need to
# explicitly check kwargs for this key
has_json = 'json' in kwargs
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
# with files and/or large objects.
files = list(_convert_files(files))
boundary = str(uuid.uuid4()).encode('ascii')
headers.setRawHeaders(
b'content-type',
[b'multipart/form-data; boundary=' + boundary])
if data:
data = _convert_params(data)
else:
data = []
bodyProducer = multipart.MultiPartProducer(data + files,
boundary=boundary)
elif data:
# Otherwise stick to x-www-form-urlencoded format
# as it's generally faster for smaller requests.
if isinstance(data, (dict, list, tuple)):
headers.setRawHeaders(b'content-type',
[b'application/x-www-form-urlencoded'])
data = urlencode(data, doseq=True)
bodyProducer = self._data_to_body_producer(data)
elif has_json:
# If data is sent as json, set Content-Type as 'application/json'
headers.setRawHeaders(b'content-type',
[b'application/json; charset=UTF-8'])
content = kwargs['json']
json = json_dumps(content, separators=(u',', u':')).encode('utf-8')
bodyProducer = self._data_to_body_producer(json)
cookies = kwargs.get('cookies', {})
if not isinstance(cookies, CookieJar):
cookies = cookiejar_from_dict(cookies)
cookies = merge_cookies(self._cookiejar, cookies)
wrapped_agent = CookieAgent(self._agent, cookies)
if kwargs.get('allow_redirects', True):
if kwargs.get('browser_like_redirects', False):
wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
else:
wrapped_agent = RedirectAgent(wrapped_agent)
wrapped_agent = ContentDecoderAgent(wrapped_agent,
[(b'gzip', GzipDecoder)])
auth = kwargs.get('auth')
if auth:
wrapped_agent = add_auth(wrapped_agent, auth)
d = wrapped_agent.request(method,
url,
headers=headers,
bodyProducer=bodyProducer)
timeout = kwargs.get('timeout')
if timeout:
delayedCall = default_reactor(kwargs.get('reactor')).callLater(
timeout, d.cancel)
def gotResult(result):
if delayedCall.active():
delayedCall.cancel()
return result
d.addBoth(gotResult)
if not kwargs.get('unbuffered', False):
d.addCallback(_BufferedResponse)
return d.addCallback(_Response, cookies)
def request(
self,
method: bytes,
uri: bytes,
headers: Optional[Headers] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> Generator[defer.Deferred, Any, defer.Deferred]:
"""
Args:
method: HTTP method: GET/POST/etc
uri: Absolute URI to be retrieved
headers:
HTTP headers to send with the request, or None to send no extra headers.
bodyProducer:
An object which can generate bytes to make up the
body of this request (for example, the properly encoded contents of
a file for a file upload). Or None if the request is to have
no body.
Returns:
Deferred[twisted.web.iweb.IResponse]:
fires when the header of the response has been received (regardless of the
response status code). Fails if there is any problem which prevents that
response from being received (including problems that prevent the request
from being sent).
"""
# We use urlparse as that will set `port` to None if there is no
# explicit port.
parsed_uri = urllib.parse.urlparse(uri)
# There must be a valid hostname.
assert parsed_uri.hostname
# If this is a matrix:// URI check if the server has delegated matrix
# traffic using well-known delegation.
#
# We have to do this here and not in the endpoint as we need to rewrite
# the host header with the delegated server name.
delegated_server = None
if (
parsed_uri.scheme == b"matrix"
and not _is_ip_literal(parsed_uri.hostname)
and not parsed_uri.port
):
well_known_result = yield defer.ensureDeferred(
self._well_known_resolver.get_well_known(parsed_uri.hostname)
)
delegated_server = well_known_result.delegated_server
if delegated_server:
# Ok, the server has delegated matrix traffic to somewhere else, so
# lets rewrite the URL to replace the server with the delegated
# server name.
uri = urllib.parse.urlunparse(
(
parsed_uri.scheme,
delegated_server,
parsed_uri.path,
parsed_uri.params,
parsed_uri.query,
parsed_uri.fragment,
)
)
parsed_uri = urllib.parse.urlparse(uri)
# We need to make sure the host header is set to the netloc of the
# server and that a user-agent is provided.
if headers is None:
request_headers = Headers()
else:
request_headers = headers.copy()
if not request_headers.hasHeader(b"host"):
request_headers.addRawHeader(b"host", parsed_uri.netloc)
if not request_headers.hasHeader(b"user-agent"):
request_headers.addRawHeader(b"user-agent", self.user_agent)
res = yield make_deferred_yieldable(
self._agent.request(method, uri, request_headers, bodyProducer)
)
return res
async def _make_well_known_request(self, server_name: bytes,
retry: bool) -> Tuple[IResponse, bytes]:
"""Make the well known request.
This will retry the request if requested and it fails (with unable
to connect or receives a 5xx error).
Args:
server_name: name of the server, from the requested url
retry: Whether to retry the request if it fails.
Raises:
_FetchWellKnownFailure if we fail to lookup a result
Returns:
Returns the response object and body. Response may be a non-200 response.
"""
uri = b"https://%s/.well-known/matrix/server" % (server_name, )
uri_str = uri.decode("ascii")
headers = {
b"User-Agent": [self.user_agent],
}
i = 0
while True:
i += 1
logger.info("Fetching %s", uri_str)
try:
response = await make_deferred_yieldable(
self._well_known_agent.request(b"GET",
uri,
headers=Headers(headers)))
body_stream = BytesIO()
await make_deferred_yieldable(
read_body_with_max_size(response, body_stream,
WELL_KNOWN_MAX_SIZE))
body = body_stream.getvalue()
if 500 <= response.code < 600:
raise Exception("Non-200 response %s" % (response.code, ))
return response, body
except defer.CancelledError:
# Bail if we've been cancelled
raise
except BodyExceededMaxSize:
# If the well-known file was too large, do not keep attempting
# to download it, but consider it a temporary error.
logger.warning(
"Requested .well-known file for %s is too large > %r bytes",
server_name.decode("ascii"),
WELL_KNOWN_MAX_SIZE,
)
raise _FetchWellKnownFailure(temporary=True)
except Exception as e:
if not retry or i >= WELL_KNOWN_RETRY_ATTEMPTS:
logger.info("Error fetching %s: %s", uri_str, e)
raise _FetchWellKnownFailure(temporary=True)
logger.info("Error fetching %s: %s. Retrying", uri_str, e)
# Sleep briefly in the hopes that they come back up
await self._clock.sleep(0.5)
def test_no_original(self):
headers = Headers({})
self.assertIsNone(parse_x_forwarded_for(headers))
class DummyRequest(object):
"""
Represents a dummy or fake request. See L{twisted.web.server.Request}.
@ivar _finishedDeferreds: L{None} or a C{list} of L{Deferreds} which will
be called back with L{None} when C{finish} is called or which will be
errbacked if C{processingFailed} is called.
@type requestheaders: C{Headers}
@ivar requestheaders: A Headers instance that stores values for all request
headers.
@type responseHeaders: C{Headers}
@ivar responseHeaders: A Headers instance that stores values for all
response headers.
@type responseCode: C{int}
@ivar responseCode: The response code which was passed to
C{setResponseCode}.
@type written: C{list} of C{bytes}
@ivar written: The bytes which have been written to the request.
"""
uri = b'http://dummy/'
method = b'GET'
client = None
def registerProducer(self, prod, s):
"""
Call an L{IPullProducer}'s C{resumeProducing} method in a
loop until it unregisters itself.
@param prod: The producer.
@type prod: L{IPullProducer}
@param s: Whether or not the producer is streaming.
"""
# XXX: Handle IPushProducers
self.go = 1
while self.go:
prod.resumeProducing()
def unregisterProducer(self):
self.go = 0
def __init__(self, postpath, session=None, client=None):
self.sitepath = []
self.written = []
self.finished = 0
self.postpath = postpath
self.prepath = []
self.session = None
self.protoSession = session or Session(0, self)
self.args = {}
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self.responseCode = None
self._finishedDeferreds = []
self._serverName = b"dummy"
self.clientproto = b"HTTP/1.0"
def getAllHeaders(self):
"""
Return dictionary mapping the names of all received headers to the last
value received for each.
Since this method does not return all header information,
C{self.requestHeaders.getAllRawHeaders()} may be preferred.
NOTE: This function is a direct copy of
C{twisted.web.http.Request.getAllRawHeaders}.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def getHeader(self, name):
"""
Retrieve the value of a request header.
@type name: C{bytes}
@param name: The name of the request header for which to retrieve the
value. Header names are compared case-insensitively.
@rtype: C{bytes} or L{None}
@return: The value of the specified request header.
"""
return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]
def setHeader(self, name, value):
"""TODO: make this assert on write() if the header is content-length
"""
self.responseHeaders.addRawHeader(name, value)
def getSession(self):
if self.session:
return self.session
assert not self.written, "Session cannot be requested after data has been written."
self.session = self.protoSession
return self.session
def render(self, resource):
"""
Render the given resource as a response to this request.
This implementation only handles a few of the most common behaviors of
resources. It can handle a render method that returns a string or
C{NOT_DONE_YET}. It doesn't know anything about the semantics of
request methods (eg HEAD) nor how to set any particular headers.
Basically, it's largely broken, but sufficient for some tests at least.
It should B{not} be expanded to do all the same stuff L{Request} does.
Instead, L{DummyRequest} should be phased out and L{Request} (or some
other real code factored in a different way) used.
"""
result = resource.render(self)
if result is NOT_DONE_YET:
return
self.write(result)
self.finish()
def write(self, data):
if not isinstance(data, bytes):
raise TypeError("write() only accepts bytes")
self.written.append(data)
def notifyFinish(self):
"""
Return a L{Deferred} which is called back with L{None} when the request
is finished. This will probably only work if you haven't called
C{finish} yet.
"""
finished = Deferred()
self._finishedDeferreds.append(finished)
return finished
def finish(self):
"""
Record that the request is finished and callback and L{Deferred}s
waiting for notification of this.
"""
self.finished = self.finished + 1
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.callback(None)
def processingFailed(self, reason):
"""
Errback and L{Deferreds} waiting for finish notification.
"""
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.errback(reason)
def addArg(self, name, value):
self.args[name] = [value]
def setResponseCode(self, code, message=None):
"""
Set the HTTP status response code, but takes care that this is called
before any data is written.
"""
assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(
self.written)
self.responseCode = code
self.responseMessage = message
def setLastModified(self, when):
assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(
self.written)
def setETag(self, tag):
assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(
self.written)
def getClientIP(self):
"""
Return the IPv4 address of the client which made this request, if there
is one, otherwise L{None}.
"""
if isinstance(self.client, (IPv4Address, IPv6Address)):
return self.client.host
return None
def getClientAddress(self):
"""
Return the L{IAddress} of the client that made this request.
@return: an address.
@rtype: an L{IAddress} provider.
"""
if self.client is None:
return NullAddress()
return self.client
def getRequestHostname(self):
"""
Get a dummy hostname associated to the HTTP request.
@rtype: C{bytes}
@returns: a dummy hostname
"""
return self._serverName
def getHost(self):
"""
Get a dummy transport's host.
@rtype: C{IPv4Address}
@returns: a dummy transport's host
"""
return IPv4Address('TCP', '127.0.0.1', 80)
def setHost(self, host, port, ssl=0):
"""
Change the host and port the request thinks it's using.
@type host: C{bytes}
@param host: The value to which to change the host header.
@type ssl: C{bool}
@param ssl: A flag which, if C{True}, indicates that the request is
considered secure (if C{True}, L{isSecure} will return C{True}).
"""
self._forceSSL = ssl # set first so isSecure will work
if self.isSecure():
default = 443
else:
default = 80
if port == default:
hostHeader = host
else:
hostHeader = host + b":" + intToBytes(port)
self.requestHeaders.addRawHeader(b"host", hostHeader)
def redirect(self, url):
"""
Utility function that does a redirect.
The request should have finish() called after this.
"""
self.setResponseCode(FOUND)
self.setHeader(b"location", url)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Args:
method (bytes): HTTP method: GET/POST/etc
uri (bytes): Absolute URI to be retrieved
headers (twisted.web.http_headers.Headers|None):
HTTP headers to send with the request, or None to
send no extra headers.
bodyProducer (twisted.web.iweb.IBodyProducer|None):
An object which can generate bytes to make up the
body of this request (for example, the properly encoded contents of
a file for a file upload). Or None if the request is to have
no body.
Returns:
Deferred[twisted.web.iweb.IResponse]:
fires when the header of the response has been received (regardless of the
response status code). Fails if there is any problem which prevents that
response from being received (including problems that prevent the request
from being sent).
"""
parsed_uri = URI.fromBytes(uri, defaultPort=-1)
res = yield self._route_matrix_uri(parsed_uri)
# set up the TLS connection params
#
# XXX disabling TLS is really only supported here for the benefit of the
# unit tests. We should make the UTs cope with TLS rather than having to make
# the code support the unit tests.
if self._tls_client_options_factory is None:
tls_options = None
else:
tls_options = self._tls_client_options_factory.get_options(
res.tls_server_name.decode("ascii"))
# make sure that the Host header is set correctly
if headers is None:
headers = Headers()
else:
headers = headers.copy()
if not headers.hasHeader(b"host"):
headers.addRawHeader(b"host", res.host_header)
class EndpointFactory(object):
@staticmethod
def endpointForURI(_uri):
ep = LoggingHostnameEndpoint(self._reactor, res.target_host,
res.target_port)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(),
self._pool)
res = yield make_deferred_yieldable(
agent.request(method, uri, headers, bodyProducer))
defer.returnValue(res)
def _headers(self):
return Headers({
'Content-Type': ['application/json'],
'Accept' : ['application/json'],
'User-Agent' : [self.appName]
})
def test_hasHeaderFalse(self):
"""
L{Headers.hasHeader} returns C{False} when the given header is not
found.
"""
self.assertFalse(Headers().hasHeader(b"test"))
def getResponse(twisted_agent, url):
try:
response = yield twisted_agent.request(
method=METHOD, uri=url, headers=Headers(HEADERS)).addErrback(error)
except Exception, e:
print(e)
def returnToSio(self, x, url, orig_env=None, tid=None, count=0):
if isinstance(x, Failure):
assert orig_env
env = orig_env
log.failure('Returning with error', x, LogLevel.warn)
else:
env = x
if not tid:
tid = env['group_id']
bodygen, hdr = encode.multipart_encode({'data': json.dumps(env)})
body = ''.join(bodygen)
headers = Headers({'User-Agent': ['sioworkersd']})
for k, v in six.iteritems(hdr):
headers.addRawHeader(k, v)
def do_return():
# This looks a bit too complicated for just POSTing a string,
# but there seems to be no other way. Blame Twisted.
# agent.request() will add content-length based on length
# from FileBodyProducer. If we have another in headers,
# there will be a duplicate, so remove it.
headers.removeHeader('content-length')
producer = client.FileBodyProducer(StringIO(body))
d = self.agent.request('POST', url.encode('utf-8'), headers,
producer)
@defer.inlineCallbacks
def _response(r):
if r.code != 200:
log.error(
'return error: server responded with status" \
"code {r.code}, response body follows...', r)
bodyD = yield client.readBody(r)
log.debug(bodyD)
raise RuntimeError('Failed to return task')
d.addCallback(_response)
return d
ret = do_return()
def _updateCount(x, n):
self.database.update(tid, {'retry_cnt': n}, sync=False)
# No db sync here, because we are allowing more attempts
# of retrying returning job result for better performance.
# It should be synced soon with other task
# or `self.database` itself.
return x # Transparent callback
def retry(err, retry_cnt):
if retry_cnt >= MAX_RETRIES_OF_RESULT_RETURNING:
log.error('Failed to return {tid} {count} times, giving up.',
tid=tid,
count=retry_cnt)
return
log.warn('Returning {tid} to url {url} failed, retrying[{n}]...',
tid=tid,
url=url,
n=retry_cnt)
log.failure('error was:', err, LogLevel.info)
d = deferLater(reactor, RETRY_DELAY_OF_RESULT_RETURNING[retry_cnt],
do_return)
d.addBoth(_updateCount, n=retry_cnt)
d.addErrback(retry, retry_cnt + 1)
return d
ret.addErrback(retry, retry_cnt=count)
ret.addBoth(self._returnDone, tid=tid)
return ret
def test_original(self):
headers = Headers({})
self.assertEqual(
parse_x_forwarded_for(headers, original=['127.0.0.1', 80]),
['127.0.0.1', 80])
def send_payjoin(manager, accept_callback=None, info_callback=None):
""" Given a JMPayjoinManager object `manager`, initialised with the
payment request data from the server, use its wallet_service to construct
a payment transaction, with coins sourced from mixdepth `mixdepth`,
then wait for the server response, parse the PSBT, perform checks and complete sign.
The info and accept callbacks are to ask the user to confirm the creation of
the original payment transaction (None defaults to terminal/CLI processing),
and are as defined in `taker_utils.direct_send`.
Returns:
(True, None) in case of payment setup successful (response will be delivered
asynchronously) - the `manager` object can be inspected for more detail.
(False, errormsg) in case of failure.
"""
# wallet should already be synced before calling here;
# we can create a standard payment, but have it returned as a PSBT.
assert isinstance(manager, JMPayjoinManager)
assert manager.wallet_service.synced
payment_psbt = direct_send(manager.wallet_service,
manager.amount,
manager.mixdepth,
str(manager.destination),
accept_callback=accept_callback,
info_callback=info_callback,
with_final_psbt=True)
if not payment_psbt:
return (False, "could not create non-payjoin payment")
# TLS whitelist is for regtest testing, it is treated as hostnames for
# which tls certificate verification is ignored.
tls_whitelist = None
if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface):
tls_whitelist = [b"127.0.0.1"]
manager.set_payment_tx_and_psbt(payment_psbt)
# add delayed call to broadcast this after 1 minute
manager.timeout_fallback_dc = reactor.callLater(
60, fallback_nonpayjoin_broadcast, manager, b"timeout")
# Now we send the request to the server, with the encoded
# payment PSBT
# First we create a twisted web Agent object:
# TODO genericize/move out/use library function:
def is_hs_uri(s):
x = urlparse.urlparse(s)
if x.hostname.endswith(".onion"):
return (x.scheme, x.hostname, x.port)
return False
tor_url_data = is_hs_uri(manager.server)
if tor_url_data:
# note the return value is currently unused here
socks5_host = jm_single().config.get("PAYJOIN", "onion_socks5_host")
socks5_port = int(jm_single().config.get("PAYJOIN",
"onion_socks5_port"))
# note: SSL not supported at the moment:
torEndpoint = TCP4ClientEndpoint(reactor, socks5_host, socks5_port)
agent = tor_agent(reactor, torEndpoint)
else:
if not tls_whitelist:
agent = Agent(reactor)
else:
agent = Agent(
reactor, contextFactory=WhitelistContextFactory(tls_whitelist))
body = BytesProducer(payment_psbt.to_base64().encode("utf-8"))
#Set the query parameters for the request:
# construct the URI from the given parameters
pj_version = jm_single().config.getint("PAYJOIN", "payjoin_version")
params = {"v": pj_version}
disable_output_substitution = "false"
if manager.disable_output_substitution:
disable_output_substitution = "true"
else:
if jm_single().config.getint("PAYJOIN",
"disable_output_substitution") == 1:
disable_output_substitution = "true"
params["disableoutputsubstitution"] = disable_output_substitution
# to determine the additionalfeeoutputindex in cases where we have
# change and we are allowing fee bump, we examine the initial tx:
if manager.change_out:
params["additionalfeeoutputindex"] = manager.change_out_index
params["maxadditionalfeecontribution"] = \
get_max_additional_fee_contribution(manager)
min_fee_rate = float(jm_single().config.get("PAYJOIN", "min_fee_rate"))
params["minfeerate"] = min_fee_rate
destination_url = manager.server.encode("utf-8")
url_parts = list(urlparse.urlparse(destination_url))
url_parts[4] = urlencode(params).encode("utf-8")
destination_url = urlparse.urlunparse(url_parts)
# TODO what to use as user agent?
d = agent.request(b"POST",
destination_url,
Headers({
"User-Agent": ["Twisted Web Client Example"],
"Content-Type": ["text/plain"]
}),
bodyProducer=body)
d.addCallback(receive_payjoin_proposal_from_server, manager)
# note that the errback (here "noResponse") is *not* triggered
# by a server rejection (which is accompanied by a non-200
# status code returned), but by failure to communicate.
def noResponse(failure):
failure.trap(ResponseFailed, ConnectionRefusedError,
HostUnreachableError)
log.error(failure.value)
fallback_nonpayjoin_broadcast(manager, b"connection refused")
d.addErrback(noResponse)
return (True, None)
def test_canonicalNameCaps(self):
"""
L{Headers._canonicalNameCaps} returns the canonical capitalization for
the given header.
"""
h = Headers()
self.assertEqual(h._canonicalNameCaps(b"test"), b"Test")
self.assertEqual(h._canonicalNameCaps(b"test-stuff"), b"Test-Stuff")
self.assertEqual(h._canonicalNameCaps(b"content-md5"), b"Content-MD5")
self.assertEqual(h._canonicalNameCaps(b"dnt"), b"DNT")
self.assertEqual(h._canonicalNameCaps(b"etag"), b"ETag")
self.assertEqual(h._canonicalNameCaps(b"p3p"), b"P3P")
self.assertEqual(h._canonicalNameCaps(b"te"), b"TE")
self.assertEqual(h._canonicalNameCaps(b"www-authenticate"),
b"WWW-Authenticate")
self.assertEqual(h._canonicalNameCaps(b"x-xss-protection"),
b"X-XSS-Protection")
def request(path,
method=b"GET",
args=[],
isSecure=False,
headers={},
body=b'',
host=b"localhost",
port=8000,
reactor=None):
"""
A fake `server.Request` which implements just enough for our tests.
"""
if reactor is None:
from twisted.internet import reactor
channel = DummyChannel()
site = Mock(server.Site)
req = server.Request(channel=channel, queued=False)
req.site = site
req.method = method
req.uri = path
req.path = path
req.prepath = []
req.postpath = path.split(b'/')[1:]
req.clientProto = b"HTTP/1.1"
req.args = args
# Set the headers we've got, as setHost writes to them
req.requestHeaders = Headers(headers)
# Put in a bogus date of no real significance, but one that will stay the
# same
req.requestHeaders.setRawHeaders(b"Date",
[b"Sun, 1 Jan 2013 15:21:01 GMT"])
# Set the host we are, and the client we're talking to
req.setHost(host, port, isSecure)
req.client = IPv4Address("TCP", "127.0.0.1", 8000)
_written_data = BytesIO()
# Writing
def _write(data):
assert not req.finished
req.startedWriting = True
_written_data.write(data)
req.write = _write
# Finishing
def _finish():
if not req.startedWriting:
req.write(b"")
if not req.finished:
req.finished = True
req._cleanup()
req.finish = _finish
# Getting what was wrote
def _get_written_data():
return _written_data.getvalue()
req.get_written_data = _get_written_data
# We have content now!
req.content = BytesIO()
req.content.write(body)
req.content.seek(0)
return req
def test_removeHeader(self):
"""
Check that L{Headers.removeHeader} removes the given header.
"""
h = Headers()
h.setRawHeaders(b"foo", [b"lemur"])
self.assertTrue(h.hasHeader(b"foo"))
h.removeHeader(b"foo")
self.assertFalse(h.hasHeader(b"foo"))
h.setRawHeaders(b"bar", [b"panda"])
self.assertTrue(h.hasHeader(b"bar"))
h.removeHeader(b"Bar")
self.assertFalse(h.hasHeader(b"bar"))
def test_v6_address(self):
headers = Headers({
b'X-Forwarded-For': [b'1043::a321:0001, 10.0.5.6'],
})
self.assertEqual(parse_x_forwarded_for(headers),
['1043::a321:0001', 0])
