Example #1
0
def http_get(reactor, url, accept=None):
    agent = client.Agent(reactor)
    headers = Headers()
    if accept is not None:
        headers.addRawHeader('Accept', str(accept))
    d = agent.request('GET', url, headers=headers)
    return _handle_agent_response(d)
Example #2
0
    def _make_request(self):
        metrics = []
        for x in xrange(HTTP_BATCH):
            if not self._mq:
                break
            metrics.append(self._mq.popleft())
        if not metrics:
            return defer.succeed(None)

        serialized_metrics = json.dumps({"metrics": metrics})
        body_writer = StringProducer(serialized_metrics)

        headers = Headers({
            'User-Agent': ['Zenoss Metric Publisher: %s' % self._agent_suffix],
            'Content-Type': ['application/json']})

        if self._needsAuth and not self._authenticated:
            log.info("Adding auth for metric http post %s", self._url)
            headers.addRawHeader('Authorization',
                                 basic_auth_string_content(self._username, self._password))

        d = self._agent.request(
            'POST', self._url, headers,
            body_writer)

        d.addCallbacks(self._metrics_published, errback=self._publish_failed,
        callbackArgs = [len(metrics), len(self._mq)], errbackArgs = [metrics])
        d.addCallbacks(self._response_finished, errback=self._publish_failed,
                       errbackArgs = [metrics])

        return d
Example #3
0
    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Issue a new request to the wrapped L{Agent}.

        Send a I{Cookie} header if a cookie for C{uri} is stored in
        L{CookieAgent.cookieJar}. Cookies are automatically extracted and
        stored from requests.

        If a C{'cookie'} header appears in C{headers} it will override the
        automatic cookie header obtained from the cookie jar.

        @see: L{Agent.request}
        """
        if headers is None:
            headers = Headers()
        lastRequest = _FakeUrllib2Request(uri)
        # Setting a cookie header explicitly will disable automatic request
        # cookies.
        if not headers.hasHeader('cookie'):
            self.cookieJar.add_cookie_header(lastRequest)
            cookieHeader = lastRequest.get_header('Cookie', None)
            if cookieHeader is not None:
                headers = headers.copy()
                headers.addRawHeader('cookie', cookieHeader)

        d = self._agent.request(method, uri, headers, bodyProducer)
        d.addCallback(self._extractCookies, lastRequest)
        return d
Example #4
0
def encode_headers(headers):
    twisted_headers = Headers()
    for k, v in headers.items():
        key = k.encode('ascii', 'ignore')
        val = v.encode('ascii', 'ignore')
        twisted_headers.addRawHeader(key, val)
    return twisted_headers
Example #5
0
    def request(self, method, url, **kwargs):
        method = method.upper()

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.get('params')
        if params:
            url = _combine_query_params(url, params)

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.get('headers')
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.iteritems():
                    if isinstance(v, str):
                        h.addRawHeader(k, v)
                    else:
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        # Here we choose a right producer
        # based on the parameters passed in.
        bodyProducer = None
        data = kwargs.get('data')
        files = kwargs.get('files')
        if files:
            # If the files keyword is present we will issue a
            # multipart/form-data request as it suits better for cases
            # with files and/or large objects.
            files = list(_convert_files(files))
            boundary = uuid.uuid4()
            headers.setRawHeaders(
                'content-type', [
                    'multipart/form-data; boundary=%s' % (boundary,)])
            if data:
                data = _convert_params(data)
            else:
                data = []

            bodyProducer = multipart.MultiPartProducer(
                data + files, boundary=boundary)
        elif data:
            # Otherwise stick to x-www-form-urlencoded format
            # as it's generally faster for smaller requests.
            if isinstance(data, (dict, list, tuple)):
                headers.setRawHeaders(
                    'content-type', ['application/x-www-form-urlencoded'])
                data = urlencode(data, doseq=True)
            bodyProducer = IBodyProducer(data)

        d = self._agent.request(
            method, url, headers=headers,
            bodyProducer=bodyProducer)

        return d
Example #6
0
	def _baseHeaders(self):
		"""return a dictionary of the base headers needed in a peer request"""
		headers = Headers()
		headers.addRawHeader('Target',self.target)
		headers.addRawHeader('Signature',self._signature)

		return headers
Example #7
0
    def request(self, method, uri, headers=None, bodyProducer=None):
        parsedURI = client._parse(uri)
        
        host_addr = address.IPv4Address('TCP', parsedURI.host, parsedURI.port)
        
        
        # ripped from _AgentBase._requestWithEndpoint
        if headers is None:
            headers = Headers()
        if not headers.hasHeader('host'):
            headers = headers.copy()
            headers.addRawHeader(
                'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
                                               parsedURI.port))
        request = client.Request(method, parsedURI.path, headers, bodyProducer,
                                 persistent=False)

        c = ClientProtocol(request)
        
        # ouch
        self.root.putChild('', self.root)
        
        server = Site(self.root).buildProtocol(self.addr)
        loopbackAsync(server, c, host_addr, self.addr)
        return c.response.addBoth(self._done, c)
Example #8
0
    def headers(self, rawHeaders):
        # type: (RawHeaders) -> IMutableHTTPHeaders
        headers = Headers()
        for rawName, rawValue in rawHeaders:
            headers.addRawHeader(rawName, rawValue)

        return HTTPHeadersWrappingHeaders(headers=headers)
Example #9
0
 def headers(self):
     if not self.result:
         raise Exception("No result yet.")
     h = Headers()
     for i in self.result["headers"]:
         h.addRawHeader(*i)
     return h
Example #10
0
def post(url, data=None, on_created=None, on_error=None):
    errback = on_error or make_errback(frames_back=2)
    try:
        def handle_response(response):
            if response.code == 200:
                callback = partial(
                    _handle_post,
                    on_created=on_created,
                    on_error=errback)
                response.deliverBody(JsonReceiver.create(callback, errback))
            else:
                errback('returned %s' % response.code)

        agent = Agent(reactor)
        headers = Headers(get_auth_headers())
        headers.addRawHeader('Content-Type', 'application/json')
        if not data:
            data = {'dummy': 1}
        d = agent.request(
            'POST',
            add_params(url, rid=uuid4().hex),
            headers=headers,
            bodyProducer=JsonProducer(data) if data else None)
        d.addCallbacks(handle_response, errback)
    except Exception as ex:
        errback('error %s' % ex)
Example #11
0
    def request(self, url, method='GET', data=None):
        d = defer.Deferred()
        _headers = {}
        headers = None

        if method == 'GET':
            data = None
        elif method == 'POST':
            _headers['Content-Type'] = 'application/x-www-form-urlencoded'
        else:
            d.errback(Exception('Not Support Method:%s.' % method))
            return

        if _headers:
            headers = Headers()
            for k, v in _headers.iteritems():
                headers.setRawHeaders(urllib.quote(k), [urllib.quote(v)])

        if data:
            data = StrBodyProducer(data)
        #print 'Req url:', url, ' Head:', headers
        self.agent.request(
                method,
                url,
                headers,
                data).addCallback(self.received, d).addErrback(
                    self.errback, d)

        return d
Example #12
0
    def request(self, method, uri, headers=None, bodyProducer=None,
                parameters=None):
        """Make a request, optionally signing it.

        Any query string passed in `uri` will get clobbered by the urlencoded
        version of `parameters`.
        """
        if headers is None:
            headers = Headers()
        if parameters is None:
            parameters = {}
        req = oauth2.Request.from_consumer_and_token(
            self.consumer, token=self.token,
            http_method=method, http_url=uri, parameters=parameters,
            is_form_encoded=True)
        req.sign_request(self.signatureMethod, self.consumer, self.token)
        for header, value in req.to_header().iteritems():
            # oauth2, for some bozotic reason, gives unicode header values
            headers.addRawHeader(header, value.encode())
        parsed = urlparse.urlparse(uri)
        parameters = {k.encode('utf-8'): v.encode('utf-8')
                      for k, v in req.get_nonoauth_parameters().iteritems()}
        uri = urlparse.urlunparse(
            parsed._replace(query=urllib.urlencode(parameters)))
        return self.agent.request(method, uri, headers, bodyProducer)
Example #13
0
	def _doPush( self, filename ):
		print 'beginning push of file:', filename
		producerDeferred = defer.Deferred()
		producerDeferred.addCallback( self._finished )
		producerDeferred.addErrback( self._error )
		
		multiPartProducer = MultiPartProducer(
			files={
				'metrics': filename,
			},
			data={
				'secret': agent_settings.PUSH_DETAILS['secret'],
			},
			callback=self._progress,
			deferred=producerDeferred
		)
	
		headers = Headers()
		headers.addRawHeader( 'Content-Type', 'multipart/form-data; boundary=%s' % multiPartProducer.boundary )
	
		agent = client.Agent( reactor )
		request = agent.request(
			'POST',
			agent_settings.PUSH_DETAILS['url'],
			headers,
			multiPartProducer
		)
		request.addCallback( self._responseReady, filename )
Example #14
0
def endpoint(endpoint, username='******', apikey=None, data=None):
    global url, user_agent
    def encode_body(data):
        if (data):
            if (isinstance(data, six.string_types)):
                data = json_loads(data)
            return ('POST', FileBodyProducer(six.StringIO(json_dumps(data))))
        else:
            return ('GET', None)
    address = make_endpoint_address(endpoint, username)
    headers = Headers({
        'Accept': ['application/json,*/*;q=0.8'],
        'Accept-Encoding': ['gzip,deflate,sdch'],
        'Connection': ['keep-alive'],
        'User-Agent': [user_agent],
        'Content-Type': ['application/json'],
    })
    if (apikey):
        headers.addRawHeader('Authorization', 'Basic ' + encode_apikey(str(apikey)))
    method, body = encode_body(data)
    agent = Agent(reactor)
    response = yield agent.request(method, address.encode('utf-8'), headers, body)
    if (response.code == 200):
        finished = Deferred()
        collector = WebBodyCollector(finished)
        response.deliverBody(collector)
        x = yield finished
        returnValue(json_loads(collector.body))
    else:
        raise RequestError(response.phrase)
Example #15
0
 def test_initializer(self):
     """
     The header values passed to L{Headers.__init__} can be retrieved via
     L{Headers.getRawHeaders}.
     """
     h = Headers({b"Foo": [b"bar"]})
     self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])
 def test_initializer(self):
     """
     The header values passed to L{Headers.__init__} can be retrieved via
     L{Headers.getRawHeaders}.
     """
     h = Headers({'Foo': ['bar']})
     self.assertEqual(h.getRawHeaders('foo'), ['bar'])
Example #17
0
def url_upload_data_async(url, files={}, data={}, progressCallback=None, doneCallback=None):

    def produce_finished(data):
        print 'produce finished',data
    def produce_error(error):
        print 'produce error', error

    producerDeferred = Deferred()
    producerDeferred.addCallback(produce_finished)
    producerDeferred.addErrback(produce_error)

    def receive_finished(data):
        print 'recieve finished', data
        doneCallback(data)

    def receive_error(error):
        Logger.error('Problem uploading file')
        print 'recieve error', error

    receiverDeferred = Deferred()
    receiverDeferred.addCallback(receive_finished)
    receiverDeferred.addErrback(receive_error)

    producer = MultiPartProducer(files, data, progressCallback, producerDeferred)
    receiver = StringReceiver(receiverDeferred)

    agent = Agent(reactor)
    headers = Headers({'User-Agent': ['Twisted Web Client Example']})
    headers.addRawHeader("Content-Type", "multipart/form-data; boundary=%s" % producer.boundary)

    requestDeffered = agent.request('POST', url, headers, producer)
    requestDeffered.addCallback(lambda response: response.deliverBody(receiver))
Example #18
0
 def obtain_auth_token(self):
     """Return a valid Microsoft Cognitive Services authentication
     token, obtained with the current subscription key if necessary.
     """
     start_time = time.time()
     if self.auth_token is None or self.token_expiry < start_time:
         headers = Headers()
         headers.addRawHeader('Ocp-Apim-Subscription-Key',
                              self.subscription_key)
         headers.addRawHeader('Content-Length', '0')
         response = yield self.agent.request(
             'POST', AUTH_URL, headers=headers)
         if response.code != 200:
             data = yield readBody(response)
             self.log.error(
                 'Could not authenticate to Microsoft Cognitive '
                 'Services: {data}', data=data)
             raise UserVisibleError(
                 'Could not authenticate to Microsoft Cognitive '
                 'Services. Try again later.')
         # Coerce the access token to a byte string to avoid problems
         # inside Twisted's header handling code down the line.
         self.auth_token = (
             (yield readBody(response)).strip().decode('ascii'))
         self.token_expiry = start_time + AUTH_TOKEN_TTL
     returnValue(self.auth_token)
Example #19
0
 def request(self, method, uri, headers=None, data=None):
     if data is not None:
         if headers is None:
             headers = Headers()
         headers.removeHeader('Content-Type')
         headers.addRawHeader('Content-Type', 'application/json; charset=utf-8')
     return BasicAgent.request(self, method, uri, headers, data)
Example #20
0
def request(rq):
    headers = Headers(dict((k, [v]) for k, v in rq.headers.iteritems()))
    scheme, host, port, path = _parse(rq.url)
    headers.setRawHeaders('Host', [host])
    return (rq.method, rq.url,
            headers,
            StringProducer(rq.data) if rq.data else None)
 def test_getRawHeadersDefaultValue(self):
     """
     L{Headers.getRawHeaders} returns the specified default value when no
     header is found.
     """
     h = Headers()
     default = object()
     self.assertIdentical(h.getRawHeaders("test", default), default)
 def test_removeHeaderDoesntExist(self):
     """
     L{Headers.removeHeader} is a no-operation when the specified header is
     not found.
     """
     h = Headers()
     h.removeHeader("test")
     self.assertEqual(list(h.getAllRawHeaders()), [])
Example #23
0
 def request(self, method, uri, headers=None, bodyProducer=None):
     if headers is None:
         headers = Headers()
     else:
         headers = headers.copy()
     for name, value in self._header_list:
         headers.addRawHeader(name, value)
     return self._agent.request(method, uri, headers, bodyProducer)
Example #24
0
 def request(self, method, uri, headers=None, bodyProducer=None):
     if headers is None:
         headers = Headers()
     else:
         headers = headers.copy()
     if not headers.hasHeader('user-agent'):
         headers.addRawHeader('user-agent', self.user_agent)
     return self.agent.request(method, uri, headers, bodyProducer)
Example #25
0
 def _respondToChallenge(self, challenge, method, uri, headers, bodyProducer):
     if headers is None:
         headers = Headers()
     else:
         headers = Headers(dict(headers.getAllRawHeaders()))
     for k, vs in challenge.response(uri, method, self._authinfo).iteritems():
         for v in vs:
             headers.addRawHeader(k, v)
     return self._agent.request(method, uri, headers, bodyProducer)
 def test_getRawHeaders(self):
     """
     L{Headers.getRawHeaders} returns the values which have been set for a
     given header.
     """
     h = Headers()
     h.setRawHeaders("test", ["lemur"])
     self.assertEqual(h.getRawHeaders("test"), ["lemur"])
     self.assertEqual(h.getRawHeaders("Test"), ["lemur"])
 def test_rawHeadersValueEncoding(self):
     """
     Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
     ISO-8859-1 and values as UTF-8.
     """
     h = Headers()
     h.setRawHeaders(u"\u00E1", [u"\u2603", b"foo"])
     self.assertTrue(h.hasHeader(b"\xe1"))
     self.assertEqual(h.getRawHeaders(b"\xe1"), [b'\xe2\x98\x83', b'foo'])
 def test_hasHeaderTrue(self):
     """
     Check that L{Headers.hasHeader} returns C{True} when the given header
     is found.
     """
     h = Headers()
     h.setRawHeaders("test", ["lemur"])
     self.assertTrue(h.hasHeader("test"))
     self.assertTrue(h.hasHeader("Test"))
Example #29
0
class _FakeUrllib2Request(object):
    """
    A fake C{urllib2.Request} object for C{cookielib} to work with.

    @see: U{http://docs.python.org/library/urllib2.html#request-objects}

    @type uri: C{str}
    @ivar uri: Request URI.

    @type headers: L{twisted.web.http_headers.Headers}
    @ivar headers: Request headers.

    @type type: C{str}
    @ivar type: The scheme of the URI.

    @type host: C{str}
    @ivar host: The host[:port] of the URI.

    @since: 11.1
    """
    def __init__(self, uri):
        self.uri = uri
        self.headers = Headers()
        self.type, rest = splittype(self.uri)
        self.host, rest = splithost(rest)


    def has_header(self, header):
        return self.headers.hasHeader(header)


    def add_unredirected_header(self, name, value):
        self.headers.addRawHeader(name, value)


    def get_full_url(self):
        return self.uri


    def get_header(self, name, default=None):
        headers = self.headers.getRawHeaders(name, default)
        if headers is not None:
            return headers[0]
        return None


    def get_host(self):
        return self.host


    def get_type(self):
        return self.type


    def is_unverifiable(self):
        # In theory this shouldn't be hardcoded.
        return False
Example #30
0
 def _get_token_dict_async(cls):
   protocol, host, path, _, _ = urlparse.urlsplit(cls.ACQUIRE_URL)
   headers = Headers()
   for key, value in cls.ACQUIRE_HEADERS.iteritems():
     headers.addRawHeader(key, value)
   agent = Agent('%s://%s' % (protocol, host))
   d = agent.GET(path, headers, retry=cls.ACQUIRE_RETRIES)
   d.addCallback(JsonResponse.Get)
   return d
    def _process_batch(self, batch_in):
        #Map from JSON-RPC to future waiting for result
        deferreds_map = dict()
        #The outgoing JSON-RPC batch
        batch_out = list()
        #Set of unprocessed entries
        unprocessed = set()

        def process_batch_level_exception(exception):
            """Spread batch level exception to each of the batch entries"""
            #On a batch level exception, forward the cought exception to the future for
            # each of the commands that are part of the batch.
            #pylint: disable=unused-variable
            for key, entry_deferred in deferreds_map.items():
                entry_deferred.errback(exception)

        def process_json_parse_error(code, body):
            """Convert a json parse error and HTTP error code into appropriate exception type"""
            if code > 499:  #5xx server side error codes
                process_batch_level_exception(HttpServerError(code, body))
            else:
                if code > 399:  #4xx client side errors.
                    if code == 413 and self.maxbatch > 1:
                        self.maxbatch = 1
                        self.queue.json_rpcqueue_again(batch_in)
                    else:
                        process_batch_level_exception(
                            HttpClientError(code, body))
                else:
                    # Non-error code in the 2xx and 3xx range.
                    process_batch_level_exception(
                        JsonRpcBatchError(code, body,
                                          "Invalid JSON returned by server"))

        def process_response(response):
            code = response.code

            def process_body(text_result):
                """Process JSON-RPC batch response body"""
                #pylint: disable=broad-except, unused-variable
                try:
                    #Parse the JSON content of the JSON-RPC batch response.
                    resp_obj = json.loads(text_result)
                except json.decoder.JSONDecodeError as exception:
                    process_json_parse_error(code, text_result.decode())
                    resp_obj = None
                if resp_obj:
                    #Assert the parsed JSON is a list.
                    if not isinstance(resp_obj, list):
                        process_batch_level_exception(
                            JsonRpcBatchError(
                                code, text_result.decode(),
                                "Non-batch JSON response from server."))
                        resp_obj = []
                    else:
                        #Process the individual command responses
                        for response in resp_obj:
                            #Get the id from the response to match with the apropriate reuest
                            if "id" in response and response[
                                    "id"] in unprocessed:
                                query_id = response["id"]
                                #Maintain list of unprocessed id's
                                unprocessed.remove(query_id)
                                #Look up the proper command future to map this response to.
                                query_deferred = deferreds_map[query_id]
                                #Distinguish between responses and errors.
                                if "result" in response:
                                    #Set future result
                                    query_deferred.callback(response["result"])
                                if (not "result" in response) and "error" in response and \
                                        "message" in response["error"] and \
                                        "code" in response["error"] and \
                                        "data" in response["error"]:
                                    query_deferred.errback(
                                        JsonRpcCommandError(
                                            response["error"]["code"],
                                            response["error"]["message"],
                                            response["error"]["data"]))
                                if (not "result" in response) and (
                                        not "error" in response):
                                    query_deferred.errback(
                                        JsonRpcCommandResponseError(
                                            "Bad command response from server",
                                            response))
                        #Work through any request item id not found in the response.
                        for no_valid_response_id in unprocessed:
                            query_deferred = deferreds_map[
                                no_valid_response_id]
                            query_deferred.errback(
                                JsonRpcCommandResponseError(
                                    "Request command id not found in response.",
                                    resp_obj))
                self._fetch_batch()

            #Get (text) content from the server response
            body_deferred = readBody(response)
            body_deferred.addCallbacks(process_body,
                                       process_batch_level_exception)
            body_deferred.addBoth(self._fetch_batch)
            return body_deferred

        def process_batch_level_exception_and_continue(failure):
            try:
                failure.raiseException()
            except ResponseFailed as exception:
                failure2 = exception.reasons[0]
                try:
                    failure2.raiseException()
                except VerificationError as exception2:
                    error = exception2.errors[0]
                    if isinstance(error, DNSMismatch):
                        process_batch_level_exception(
                            SSLNameMismatch(
                                "Problem with node certificate. Wrong DNS name."
                            ))
                    else:
                        process_batch_level_exception(
                            SSLError("Problem with node certificate."))
                except OpenSSL.SSL.Error as exception2:
                    process_batch_level_exception(
                        SSLError("Problem with node certificate."))
                except Exception as exception2:
                    process_batch_level_exception(exception2)
            except Exception as exception:
                process_batch_level_exception(exception)
            self._fetch_batch()

        log.msg("Processing new batch for " + self.host_url.decode("utf8") +
                " , " + str(len(batch_in)))
        #Build the output batch and deferred map.
        for cmd in batch_in:
            self.cmd_id += 1
            unprocessed.add(self.cmd_id)
            newcmd = dict()
            newcmd["id"] = self.cmd_id
            newcmd["jsonrpc"] = "2.0"
            newcmd["method"] = cmd["method"]
            if cmd["params"]:
                newcmd["params"] = cmd["params"]
            deferreds_map[self.cmd_id] = cmd["deferred"]
            batch_out.append(newcmd)
        #Piggybag extra command onto single command batches. FIXME: this is a temporary workaround.
        if len(batch_out) == 1 and self.maxbatch > 1:
            newcmd = dict()
            newcmd["id"] = 0
            newcmd["jsonrpc"] = "2.0"
            newcmd["method"] = "bogus_api.bogus_method"
            newcmd["params"] = [
                "Extra bogus API call for making sure the server supports batches of bigger than one."
            ]
            batch_out.append(newcmd)
        #Post the JSON-RPC batch request to the server and wait for response
        log.msg("Posting batch to node " + self.host_url.decode("utf8"))
        deferred_response = self.agent.request(
            b'POST', self.host_url,
            Headers({
                b"User-Agent": [b'TxJsonRpcQueue v0.0.1'],
                b"Content-Type": [b"application/json"]
            }), _StringProducer(json.dumps(batch_out).encode("utf8")))
        deferred_response.addCallbacks(
            process_response, process_batch_level_exception_and_continue)
from twisted.web.client import Agent, readBody
from twisted.internet import reactor, defer
from twisted.internet.ssl import ClientContextFactory
from twisted.web.http_headers import Headers
from zope.interface import implementer
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from twisted.internet.protocol import Protocol
from pprint import pformat

import json
import time
from test.public_api.web import get_smzdm_datas, print_smzdm_result, end_crawl
headers = Headers({
    'User-Agent': [
        'MMozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0'
    ],
    'content-type': ["application/json"]
})


class WebClientContextFactory(ClientContextFactory):
    '''
    用来实现https网页的访问验证
    '''
    def getContext(self, hostname, port):
        return ClientContextFactory.getContext(self)


def cbRequest(response):
    '''
    print('Response version:', response.version)
    def _request(self,
                 destination,
                 method,
                 path,
                 body_callback,
                 headers_dict={},
                 param_bytes=b"",
                 query_bytes=b"",
                 retry_on_dns_fail=True,
                 timeout=None,
                 long_retries=False,
                 ignore_backoff=False,
                 backoff_on_404=False):
        """ Creates and sends a request to the given server
        Args:
            destination (str): The remote server to send the HTTP request to.
            method (str): HTTP method
            path (str): The HTTP path
            ignore_backoff (bool): true to ignore the historical backoff data
                and try the request anyway.
            backoff_on_404 (bool): Back off if we get a 404

        Returns:
            Deferred: resolves with the http response object on success.

            Fails with ``HTTPRequestException``: if we get an HTTP response
                code >= 300.

            Fails with ``NotRetryingDestination`` if we are not yet ready
                to retry this server.

            Fails with ``FederationDeniedError`` if this destination
                is not on our federation whitelist

            (May also fail with plenty of other Exceptions for things like DNS
                failures, connection failures, SSL failures.)
        """
        if (self.hs.config.federation_domain_whitelist and destination
                not in self.hs.config.federation_domain_whitelist):
            raise FederationDeniedError(destination)

        limiter = yield synapse.util.retryutils.get_retry_limiter(
            destination,
            self.clock,
            self._store,
            backoff_on_404=backoff_on_404,
            ignore_backoff=ignore_backoff,
        )

        destination = destination.encode("ascii")
        path_bytes = path.encode("ascii")
        with limiter:
            headers_dict[b"User-Agent"] = [self.version_string]
            headers_dict[b"Host"] = [destination]

            url_bytes = self._create_url(destination, path_bytes, param_bytes,
                                         query_bytes)

            txn_id = "%s-O-%s" % (method, self._next_id)
            self._next_id = (self._next_id + 1) % (sys.maxint - 1)

            outbound_logger.info("{%s} [%s] Sending request: %s %s", txn_id,
                                 destination, method, url_bytes)

            # XXX: Would be much nicer to retry only at the transaction-layer
            # (once we have reliable transactions in place)
            if long_retries:
                retries_left = MAX_LONG_RETRIES
            else:
                retries_left = MAX_SHORT_RETRIES

            http_url_bytes = urlparse.urlunparse(
                ("", "", path_bytes, param_bytes, query_bytes, ""))

            log_result = None
            try:
                while True:
                    producer = None
                    if body_callback:
                        producer = body_callback(method, http_url_bytes,
                                                 headers_dict)

                    try:
                        request_deferred = self.agent.request(
                            method, url_bytes, Headers(headers_dict), producer)
                        add_timeout_to_deferred(
                            request_deferred,
                            timeout / 1000. if timeout else 60,
                            self.hs.get_reactor(),
                            cancelled_to_request_timed_out_error,
                        )
                        response = yield make_deferred_yieldable(
                            request_deferred, )

                        log_result = "%d %s" % (
                            response.code,
                            response.phrase,
                        )
                        break
                    except Exception as e:
                        if not retry_on_dns_fail and isinstance(
                                e, DNSLookupError):
                            logger.warn("DNS Lookup failed to %s with %s",
                                        destination, e)
                            log_result = "DNS Lookup failed to %s with %s" % (
                                destination, e)
                            raise

                        logger.warn(
                            "{%s} Sending request failed to %s: %s %s: %s",
                            txn_id,
                            destination,
                            method,
                            url_bytes,
                            _flatten_response_never_received(e),
                        )

                        log_result = _flatten_response_never_received(e)

                        if retries_left and not timeout:
                            if long_retries:
                                delay = 4**(MAX_LONG_RETRIES + 1 -
                                            retries_left)
                                delay = min(delay, 60)
                                delay *= random.uniform(0.8, 1.4)
                            else:
                                delay = 0.5 * 2**(MAX_SHORT_RETRIES -
                                                  retries_left)
                                delay = min(delay, 2)
                                delay *= random.uniform(0.8, 1.4)

                            yield self.clock.sleep(delay)
                            retries_left -= 1
                        else:
                            raise
            finally:
                outbound_logger.info(
                    "{%s} [%s] Result: %s",
                    txn_id,
                    destination,
                    log_result,
                )

            if 200 <= response.code < 300:
                pass
            else:
                # :'(
                # Update transactions table?
                with logcontext.PreserveLoggingContext():
                    body = yield readBody(response)
                raise HttpResponseException(response.code, response.phrase,
                                            body)

            defer.returnValue(response)
Example #34
0
from bytesprod import BytesProducer

from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

agent = Agent(reactor)
body = BytesProducer(b"hello, world")
d = agent.request(
    b"POST",
    b"http://httpbin.org/post",
    Headers({
        "User-Agent": ["Twisted Web Client Example"],
        "Content-Type": ["text/x-greeting"],
    }),
    body,
)


def cbResponse(ignored):
    print("Response received")


d.addCallback(cbResponse)


def cbShutdown(ignored):
    reactor.stop()


d.addBoth(cbShutdown)
Example #35
0
 def request(self, url, headers, method='GET'):
     LOG(u'Will request URL %s with headers %s, method %s' %
         (url, pformat(headers), method))
     deferred = self.client.request(method, url, Headers(headers))
     deferred.addCallbacks(self.on_success, self.on_failure)
Example #36
0
 def test_getRawHeadersNoDefault(self):
     """
     L{Headers.getRawHeaders} returns C{None} if the header is not found and
     no default is specified.
     """
     self.assertIdentical(Headers().getRawHeaders("test"), None)
    def do_test_payment(self, wc1, wc2, amt=1.1):
        wallet_structures = [self.wallet_structure] * 2
        wallet_cls = (wc1, wc2)
        self.wallet_services = []
        self.wallet_services.append(
            make_wallets_to_list(
                make_wallets(1,
                             wallet_structures=[wallet_structures[0]],
                             mean_amt=self.mean_amt,
                             wallet_cls=wallet_cls[0]))[0])
        self.wallet_services.append(
            make_wallets_to_list(
                make_wallets(1,
                             wallet_structures=[wallet_structures[1]],
                             mean_amt=self.mean_amt,
                             wallet_cls=wallet_cls[1]))[0])
        jm_single().bc_interface.tickchain()
        sync_wallets(self.wallet_services)

        # For accounting purposes, record the balances
        # at the start.
        self.rsb = getbals(self.wallet_services[0], 0)
        self.ssb = getbals(self.wallet_services[1], 0)

        self.cj_amount = int(amt * 10**8)

        def cbStopListening():
            return self.port.stopListening()

        b78rm = JMBIP78ReceiverManager(self.wallet_services[0], 0,
                                       self.cj_amount, 47083)
        resource = DummyBIP78ReceiverResource(jmprint, cbStopListening, b78rm)
        self.site = Site(resource)
        self.site.displayTracebacks = False
        # NB The connectivity aspects of the onion-based BIP78 setup
        # are time heavy. This server is TCP only.
        self.port = reactor.listenTCP(47083, self.site)
        self.addCleanup(cbStopListening)

        # setup of spender
        bip78_btc_amount = amount_to_btc(amount_to_sat(self.cj_amount))
        bip78_uri = encode_bip21_uri(str(b78rm.receiving_address), {
            "amount": bip78_btc_amount,
            "pj": b"http://127.0.0.1:47083"
        },
                                     safe=":/")
        self.manager = parse_payjoin_setup(bip78_uri, self.wallet_services[1],
                                           0)
        self.manager.mode = "testing"
        success, msg = make_payment_psbt(self.manager)
        assert success, msg
        params = make_payjoin_request_params(self.manager)
        # avoiding backend daemon (testing only jmclient code here),
        # we send the http request manually:
        serv = b"http://127.0.0.1:47083"
        agent = get_nontor_agent()
        body = BytesProducer(
            self.manager.initial_psbt.to_base64().encode("utf-8"))
        url_parts = list(wrapped_urlparse(serv))
        url_parts[4] = urlencode(params).encode("utf-8")
        destination_url = urlparse.urlunparse(url_parts)
        d = agent.request(b"POST",
                          destination_url,
                          Headers({"Content-Type": ["text/plain"]}),
                          bodyProducer=body)
        d.addCallback(bip78_receiver_response, self.manager)
        return d
Example #38
0
    def _send_request(
        self,
        request,
        retry_on_dns_fail=True,
        timeout=None,
        long_retries=False,
        ignore_backoff=False,
        backoff_on_404=False,
    ):
        """
        Sends a request to the given server.

        Args:
            request (MatrixFederationRequest): details of request to be sent

            timeout (int|None): number of milliseconds to wait for the response headers
                (including connecting to the server), *for each attempt*.
                60s by default.

            long_retries (bool): whether to use the long retry algorithm.

                The regular retry algorithm makes 4 attempts, with intervals
                [0.5s, 1s, 2s].

                The long retry algorithm makes 11 attempts, with intervals
                [4s, 16s, 60s, 60s, ...]

                Both algorithms add -20%/+40% jitter to the retry intervals.

                Note that the above intervals are *in addition* to the time spent
                waiting for the request to complete (up to `timeout` ms).

                NB: the long retry algorithm takes over 20 minutes to complete, with
                a default timeout of 60s!

            ignore_backoff (bool): true to ignore the historical backoff data
                and try the request anyway.

            backoff_on_404 (bool): Back off if we get a 404

        Returns:
            Deferred[twisted.web.client.Response]: resolves with the HTTP
            response object on success.

        Raises:
            HttpResponseException: If we get an HTTP response code >= 300
                (except 429).
            NotRetryingDestination: If we are not yet ready to retry this
                server.
            FederationDeniedError: If this destination  is not on our
                federation whitelist
            RequestSendFailed: If there were problems connecting to the
                remote, due to e.g. DNS failures, connection timeouts etc.
        """
        if timeout:
            _sec_timeout = timeout / 1000
        else:
            _sec_timeout = self.default_timeout

        if (self.hs.config.federation_domain_whitelist is not None
                and request.destination
                not in self.hs.config.federation_domain_whitelist):
            raise FederationDeniedError(request.destination)

        limiter = yield synapse.util.retryutils.get_retry_limiter(
            request.destination,
            self.clock,
            self._store,
            backoff_on_404=backoff_on_404,
            ignore_backoff=ignore_backoff,
        )

        method_bytes = request.method.encode("ascii")
        destination_bytes = request.destination.encode("ascii")
        path_bytes = request.path.encode("ascii")
        if request.query:
            query_bytes = encode_query_args(request.query)
        else:
            query_bytes = b""

        scope = start_active_span(
            "outgoing-federation-request",
            tags={
                tags.SPAN_KIND: tags.SPAN_KIND_RPC_CLIENT,
                tags.PEER_ADDRESS: request.destination,
                tags.HTTP_METHOD: request.method,
                tags.HTTP_URL: request.path,
            },
            finish_on_close=True,
        )

        # Inject the span into the headers
        headers_dict = {}
        inject_active_span_byte_dict(headers_dict, request.destination)

        headers_dict[b"User-Agent"] = [self.version_string_bytes]

        with limiter, scope:
            # XXX: Would be much nicer to retry only at the transaction-layer
            # (once we have reliable transactions in place)
            if long_retries:
                retries_left = MAX_LONG_RETRIES
            else:
                retries_left = MAX_SHORT_RETRIES

            url_bytes = urllib.parse.urlunparse(
                (b"matrix", destination_bytes, path_bytes, None, query_bytes,
                 b""))
            url_str = url_bytes.decode("ascii")

            url_to_sign_bytes = urllib.parse.urlunparse(
                (b"", b"", path_bytes, None, query_bytes, b""))

            while True:
                try:
                    json = request.get_json()
                    if json:
                        headers_dict[b"Content-Type"] = [b"application/json"]
                        auth_headers = self.build_auth_headers(
                            destination_bytes, method_bytes, url_to_sign_bytes,
                            json)
                        data = encode_canonical_json(json)
                        producer = QuieterFileBodyProducer(
                            BytesIO(data), cooperator=self._cooperator)
                    else:
                        producer = None
                        auth_headers = self.build_auth_headers(
                            destination_bytes, method_bytes, url_to_sign_bytes)

                    headers_dict[b"Authorization"] = auth_headers

                    logger.info(
                        "{%s} [%s] Sending request: %s %s; timeout %fs",
                        request.txn_id,
                        request.destination,
                        request.method,
                        url_str,
                        _sec_timeout,
                    )

                    try:
                        with Measure(self.clock, "outbound_request"):
                            # we don't want all the fancy cookie and redirect handling
                            # that treq.request gives: just use the raw Agent.
                            request_deferred = self.agent.request(
                                method_bytes,
                                url_bytes,
                                headers=Headers(headers_dict),
                                bodyProducer=producer,
                            )

                            request_deferred = timeout_deferred(
                                request_deferred,
                                timeout=_sec_timeout,
                                reactor=self.reactor,
                            )

                            response = yield request_deferred
                    except DNSLookupError as e:
                        raise_from(
                            RequestSendFailed(e, can_retry=retry_on_dns_fail),
                            e)
                    except Exception as e:
                        logger.info("Failed to send request: %s", e)
                        raise_from(RequestSendFailed(e, can_retry=True), e)

                    logger.info(
                        "{%s} [%s] Got response headers: %d %s",
                        request.txn_id,
                        request.destination,
                        response.code,
                        response.phrase.decode("ascii", errors="replace"),
                    )

                    set_tag(tags.HTTP_STATUS_CODE, response.code)

                    if 200 <= response.code < 300:
                        pass
                    else:
                        # :'(
                        # Update transactions table?
                        d = treq.content(response)
                        d = timeout_deferred(d,
                                             timeout=_sec_timeout,
                                             reactor=self.reactor)

                        try:
                            body = yield make_deferred_yieldable(d)
                        except Exception as e:
                            # Eh, we're already going to raise an exception so lets
                            # ignore if this fails.
                            logger.warn(
                                "{%s} [%s] Failed to get error response: %s %s: %s",
                                request.txn_id,
                                request.destination,
                                request.method,
                                url_str,
                                _flatten_response_never_received(e),
                            )
                            body = None

                        e = HttpResponseException(response.code,
                                                  response.phrase, body)

                        # Retry if the error is a 429 (Too Many Requests),
                        # otherwise just raise a standard HttpResponseException
                        if response.code == 429:
                            raise_from(RequestSendFailed(e, can_retry=True), e)
                        else:
                            raise e

                    break
                except RequestSendFailed as e:
                    logger.warn(
                        "{%s} [%s] Request failed: %s %s: %s",
                        request.txn_id,
                        request.destination,
                        request.method,
                        url_str,
                        _flatten_response_never_received(e.inner_exception),
                    )

                    if not e.can_retry:
                        raise

                    if retries_left and not timeout:
                        if long_retries:
                            delay = 4**(MAX_LONG_RETRIES + 1 - retries_left)
                            delay = min(delay, 60)
                            delay *= random.uniform(0.8, 1.4)
                        else:
                            delay = 0.5 * 2**(MAX_SHORT_RETRIES - retries_left)
                            delay = min(delay, 2)
                            delay *= random.uniform(0.8, 1.4)

                        logger.debug(
                            "{%s} [%s] Waiting %ss before re-sending...",
                            request.txn_id,
                            request.destination,
                            delay,
                        )

                        yield self.clock.sleep(delay)
                        retries_left -= 1
                    else:
                        raise

                except Exception as e:
                    logger.warn(
                        "{%s} [%s] Request failed: %s %s: %s",
                        request.txn_id,
                        request.destination,
                        request.method,
                        url_str,
                        _flatten_response_never_received(e),
                    )
                    raise
        return response
Example #39
0
from email.mime.text import MIMEText
from datetime import date
from datetime import datetime
from datetime import timedelta
from functools import partial
from twisted.python import log
from twisted.internet import defer
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.client import readBody
from twisted.web.http_headers import Headers

TRAC_BUILDBOT_URL = 'http://trac.buildbot.net'
TRAC_BUILDBOT_TICKET_URL = TRAC_BUILDBOT_URL + '/ticket/%(ticket)s'
GITHUB_API_URL = 'https://api.github.com'
HTTP_HEADERS = Headers({'User-Agent': ['buildbot.net weekly summary']})
FROM = '*****@*****.**'
RECIPIENTS = ['*****@*****.**', '*****@*****.**']

WEEKLY_MEETING_TEXT = textwrap.dedent("""

    <p>Buildbot has weekly meetings via irc, held at 16:30 UTC on Tuesdays.
    That is about 90 minutes from now!

    <p>Meetings are in #buildbot on Freenode, open to any and all participants.
    They generally focus on organizational, rather than technical issues, but are open to anything Buildbot-related.
    To raise a topic, add it to "All Other Business" in the <a href="https://titanpad.com/buildbot-agenda">agenda</a>, or just speak up during the meeting.

    <p>Meeting minutes are available <a href="https://supybot.buildbot.net/meetings/">here</a>.
    """)
Example #40
0
def requestMock(path,
                method=b"GET",
                host=b"localhost",
                port=8080,
                isSecure=False,
                body=None,
                headers=None):
    if not headers:
        headers = {}

    if not body:
        body = b''

    request = server.Request(DummyChannel(), False)
    request.site = Mock(server.Site)
    request.gotLength(len(body))
    request.content = BytesIO()
    request.content.write(body)
    request.content.seek(0)
    request.requestHeaders = Headers(headers)
    request.setHost(host, port, isSecure)
    request.uri = path
    request.prepath = []
    request.postpath = path.split(b'/')[1:]
    request.method = method
    request.clientproto = b'HTTP/1.1'

    request.setHeader = Mock(wraps=request.setHeader)
    request.setResponseCode = Mock(wraps=request.setResponseCode)

    request._written = BytesIO()
    request.finishCount = 0
    request.writeCount = 0

    def registerProducer(producer, streaming):
        request.producer = producer
        for x in range(2):
            if request.producer:
                request.producer.resumeProducing()

    def unregisterProducer():
        request.producer = None

    def finish():
        request.finishCount += 1

        if not request.startedWriting:
            request.write(b'')

        if not request.finished:
            request.finished = True
            request._cleanup()

    def write(data):
        request.writeCount += 1
        request.startedWriting = True

        if not request.finished:
            request._written.write(data)
        else:
            raise RuntimeError('Request.write called on a request after '
                               'Request.finish was called.')

    def getWrittenData():
        return request._written.getvalue()

    request.finish = finish
    request.write = write
    request.getWrittenData = getWrittenData

    request.registerProducer = registerProducer
    request.unregisterProducer = unregisterProducer

    request.processingFailed = Mock(wraps=request.processingFailed)

    return request
Example #41
0
 def test_rawHeadersTypeChecking(self):
     """
     L{Headers.setRawHeaders} requires values to be of type list.
     """
     h = Headers()
     self.assertRaises(TypeError, h.setRawHeaders, u'key', {u'Foo': u'bar'})
Example #42
0
 def test_headersComparison(self):
     """
     A L{Headers} instance compares equal to itself and to another
     L{Headers} instance with the same values.
     """
     first = Headers()
     first.setRawHeaders(b"foo", [b"panda"])
     second = Headers()
     second.setRawHeaders(b"foo", [b"panda"])
     third = Headers()
     third.setRawHeaders(b"foo", [b"lemur", b"panda"])
     self.assertEqual(first, first)
     self.assertEqual(first, second)
     self.assertNotEqual(first, third)
Example #43
0
 def test_multiple_proxys(self):
     headers = Headers({
         b'X-Forwarded-For': [b'10.1.2.3, 10.1.2.4'],
     })
     self.assertEqual(parse_x_forwarded_for(headers), ['10.1.2.3', 0])
Example #44
0
    def request(
        self,
        method: bytes,
        uri: bytes,
        headers: Optional[Headers] = None,
        bodyProducer: Optional[IBodyProducer] = None,
    ) -> defer.Deferred:
        """
        Issue a request to the server indicated by the given uri.

        Supports `http` and `https` schemes.

        An existing connection from the connection pool may be used or a new one may be
        created.

        See also: twisted.web.iweb.IAgent.request

        Args:
            method: The request method to use, such as `GET`, `POST`, etc

            uri: The location of the resource to request.

            headers: Extra headers to send with the request

            bodyProducer: An object which can generate bytes to make up the body of
                this request (for example, the properly encoded contents of a file for
                a file upload). Or, None if the request is to have no body.

        Returns:
            Deferred[IResponse]: completes when the header of the response has
                 been received (regardless of the response status code).

                 Can fail with:
                    SchemeNotSupported: if the uri is not http or https

                    twisted.internet.error.TimeoutError if the server we are connecting
                        to (proxy or destination) does not accept a connection before
                        connectTimeout.

                    ... other things too.
        """
        uri = uri.strip()
        if not _VALID_URI.match(uri):
            raise ValueError(f"Invalid URI {uri!r}")

        parsed_uri = URI.fromBytes(uri)
        pool_key = f"{parsed_uri.scheme!r}{parsed_uri.host!r}{parsed_uri.port}"
        request_path = parsed_uri.originForm

        should_skip_proxy = False
        if self.no_proxy is not None:
            should_skip_proxy = proxy_bypass_environment(
                parsed_uri.host.decode(),
                proxies={"no": self.no_proxy},
            )

        if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint
                and not should_skip_proxy):
            # Determine whether we need to set Proxy-Authorization headers
            if self.http_proxy_creds:
                # Set a Proxy-Authorization header
                if headers is None:
                    headers = Headers()
                headers.addRawHeader(
                    b"Proxy-Authorization",
                    self.http_proxy_creds.as_proxy_authorization_value(),
                )
            # Cache *all* connections under the same key, since we are only
            # connecting to a single destination, the proxy:
            pool_key = "http-proxy"
            endpoint = self.http_proxy_endpoint
            request_path = uri
        elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint
              and not should_skip_proxy):
            endpoint = HTTPConnectProxyEndpoint(
                self.proxy_reactor,
                self.https_proxy_endpoint,
                parsed_uri.host,
                parsed_uri.port,
                self.https_proxy_creds,
            )
        else:
            # not using a proxy
            endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
                                        parsed_uri.port,
                                        **self._endpoint_kwargs)

        logger.debug("Requesting %s via %s", uri, endpoint)

        if parsed_uri.scheme == b"https":
            tls_connection_creator = self._policy_for_https.creatorForNetloc(
                parsed_uri.host, parsed_uri.port)
            endpoint = wrapClientTLS(tls_connection_creator, endpoint)
        elif parsed_uri.scheme == b"http":
            pass
        else:
            return defer.fail(
                Failure(
                    SchemeNotSupported("Unsupported scheme: %r" %
                                       (parsed_uri.scheme, ))))

        return self._requestWithEndpoint(pool_key, endpoint, method,
                                         parsed_uri, headers, bodyProducer,
                                         request_path)
Example #45
0
    def request(self, method, url, **kwargs):
        """
        See :func:`treq.request()`.
        """
        method = method.encode('ascii').upper()

        if isinstance(url, DecodedURL):
            parsed_url = url
        elif isinstance(url, EncodedURL):
            parsed_url = DecodedURL(url)
        elif isinstance(url, six.text_type):
            parsed_url = DecodedURL.from_text(url)
        else:
            parsed_url = DecodedURL.from_text(url.decode('ascii'))

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.get('params')
        if params:
            parsed_url = parsed_url.replace(
                query=parsed_url.query + tuple(_coerced_query_params(params)))

        url = parsed_url.to_uri().to_text().encode('ascii')

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.get('headers')
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.items():
                    if isinstance(v, (bytes, six.text_type)):
                        h.addRawHeader(k, v)
                    elif isinstance(v, list):
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        # Here we choose a right producer
        # based on the parameters passed in.
        bodyProducer = None
        data = kwargs.get('data')
        files = kwargs.get('files')
        # since json=None needs to be serialized as 'null', we need to
        # explicitly check kwargs for this key
        has_json = 'json' in kwargs

        if files:
            # If the files keyword is present we will issue a
            # multipart/form-data request as it suits better for cases
            # with files and/or large objects.
            files = list(_convert_files(files))
            boundary = str(uuid.uuid4()).encode('ascii')
            headers.setRawHeaders(
                b'content-type',
                [b'multipart/form-data; boundary=' + boundary])
            if data:
                data = _convert_params(data)
            else:
                data = []

            bodyProducer = multipart.MultiPartProducer(data + files,
                                                       boundary=boundary)
        elif data:
            # Otherwise stick to x-www-form-urlencoded format
            # as it's generally faster for smaller requests.
            if isinstance(data, (dict, list, tuple)):
                headers.setRawHeaders(b'content-type',
                                      [b'application/x-www-form-urlencoded'])
                data = urlencode(data, doseq=True)
            bodyProducer = self._data_to_body_producer(data)
        elif has_json:
            # If data is sent as json, set Content-Type as 'application/json'
            headers.setRawHeaders(b'content-type',
                                  [b'application/json; charset=UTF-8'])
            content = kwargs['json']
            json = json_dumps(content, separators=(u',', u':')).encode('utf-8')
            bodyProducer = self._data_to_body_producer(json)

        cookies = kwargs.get('cookies', {})

        if not isinstance(cookies, CookieJar):
            cookies = cookiejar_from_dict(cookies)

        cookies = merge_cookies(self._cookiejar, cookies)
        wrapped_agent = CookieAgent(self._agent, cookies)

        if kwargs.get('allow_redirects', True):
            if kwargs.get('browser_like_redirects', False):
                wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
            else:
                wrapped_agent = RedirectAgent(wrapped_agent)

        wrapped_agent = ContentDecoderAgent(wrapped_agent,
                                            [(b'gzip', GzipDecoder)])

        auth = kwargs.get('auth')
        if auth:
            wrapped_agent = add_auth(wrapped_agent, auth)

        d = wrapped_agent.request(method,
                                  url,
                                  headers=headers,
                                  bodyProducer=bodyProducer)

        timeout = kwargs.get('timeout')
        if timeout:
            delayedCall = default_reactor(kwargs.get('reactor')).callLater(
                timeout, d.cancel)

            def gotResult(result):
                if delayedCall.active():
                    delayedCall.cancel()
                return result

            d.addBoth(gotResult)

        if not kwargs.get('unbuffered', False):
            d.addCallback(_BufferedResponse)

        return d.addCallback(_Response, cookies)
    def request(
        self,
        method: bytes,
        uri: bytes,
        headers: Optional[Headers] = None,
        bodyProducer: Optional[IBodyProducer] = None,
    ) -> Generator[defer.Deferred, Any, defer.Deferred]:
        """
        Args:
            method: HTTP method: GET/POST/etc
            uri: Absolute URI to be retrieved
            headers:
                HTTP headers to send with the request, or None to send no extra headers.
            bodyProducer:
                An object which can generate bytes to make up the
                body of this request (for example, the properly encoded contents of
                a file for a file upload).  Or None if the request is to have
                no body.
        Returns:
            Deferred[twisted.web.iweb.IResponse]:
                fires when the header of the response has been received (regardless of the
                response status code). Fails if there is any problem which prevents that
                response from being received (including problems that prevent the request
                from being sent).
        """
        # We use urlparse as that will set `port` to None if there is no
        # explicit port.
        parsed_uri = urllib.parse.urlparse(uri)

        # There must be a valid hostname.
        assert parsed_uri.hostname

        # If this is a matrix:// URI check if the server has delegated matrix
        # traffic using well-known delegation.
        #
        # We have to do this here and not in the endpoint as we need to rewrite
        # the host header with the delegated server name.
        delegated_server = None
        if (
            parsed_uri.scheme == b"matrix"
            and not _is_ip_literal(parsed_uri.hostname)
            and not parsed_uri.port
        ):
            well_known_result = yield defer.ensureDeferred(
                self._well_known_resolver.get_well_known(parsed_uri.hostname)
            )
            delegated_server = well_known_result.delegated_server

        if delegated_server:
            # Ok, the server has delegated matrix traffic to somewhere else, so
            # lets rewrite the URL to replace the server with the delegated
            # server name.
            uri = urllib.parse.urlunparse(
                (
                    parsed_uri.scheme,
                    delegated_server,
                    parsed_uri.path,
                    parsed_uri.params,
                    parsed_uri.query,
                    parsed_uri.fragment,
                )
            )
            parsed_uri = urllib.parse.urlparse(uri)

        # We need to make sure the host header is set to the netloc of the
        # server and that a user-agent is provided.
        if headers is None:
            request_headers = Headers()
        else:
            request_headers = headers.copy()

        if not request_headers.hasHeader(b"host"):
            request_headers.addRawHeader(b"host", parsed_uri.netloc)
        if not request_headers.hasHeader(b"user-agent"):
            request_headers.addRawHeader(b"user-agent", self.user_agent)

        res = yield make_deferred_yieldable(
            self._agent.request(method, uri, request_headers, bodyProducer)
        )

        return res
Example #47
0
    async def _make_well_known_request(self, server_name: bytes,
                                       retry: bool) -> Tuple[IResponse, bytes]:
        """Make the well known request.

        This will retry the request if requested and it fails (with unable
        to connect or receives a 5xx error).

        Args:
            server_name: name of the server, from the requested url
            retry: Whether to retry the request if it fails.

        Raises:
            _FetchWellKnownFailure if we fail to lookup a result

        Returns:
            Returns the response object and body. Response may be a non-200 response.
        """
        uri = b"https://%s/.well-known/matrix/server" % (server_name, )
        uri_str = uri.decode("ascii")

        headers = {
            b"User-Agent": [self.user_agent],
        }

        i = 0
        while True:
            i += 1

            logger.info("Fetching %s", uri_str)
            try:
                response = await make_deferred_yieldable(
                    self._well_known_agent.request(b"GET",
                                                   uri,
                                                   headers=Headers(headers)))
                body_stream = BytesIO()
                await make_deferred_yieldable(
                    read_body_with_max_size(response, body_stream,
                                            WELL_KNOWN_MAX_SIZE))
                body = body_stream.getvalue()

                if 500 <= response.code < 600:
                    raise Exception("Non-200 response %s" % (response.code, ))

                return response, body
            except defer.CancelledError:
                # Bail if we've been cancelled
                raise
            except BodyExceededMaxSize:
                # If the well-known file was too large, do not keep attempting
                # to download it, but consider it a temporary error.
                logger.warning(
                    "Requested .well-known file for %s is too large > %r bytes",
                    server_name.decode("ascii"),
                    WELL_KNOWN_MAX_SIZE,
                )
                raise _FetchWellKnownFailure(temporary=True)
            except Exception as e:
                if not retry or i >= WELL_KNOWN_RETRY_ATTEMPTS:
                    logger.info("Error fetching %s: %s", uri_str, e)
                    raise _FetchWellKnownFailure(temporary=True)

                logger.info("Error fetching %s: %s. Retrying", uri_str, e)

            # Sleep briefly in the hopes that they come back up
            await self._clock.sleep(0.5)
Example #48
0
 def test_no_original(self):
     headers = Headers({})
     self.assertIsNone(parse_x_forwarded_for(headers))
Example #49
0
class DummyRequest(object):
    """
    Represents a dummy or fake request. See L{twisted.web.server.Request}.

    @ivar _finishedDeferreds: L{None} or a C{list} of L{Deferreds} which will
        be called back with L{None} when C{finish} is called or which will be
        errbacked if C{processingFailed} is called.

    @type requestheaders: C{Headers}
    @ivar requestheaders: A Headers instance that stores values for all request
        headers.

    @type responseHeaders: C{Headers}
    @ivar responseHeaders: A Headers instance that stores values for all
        response headers.

    @type responseCode: C{int}
    @ivar responseCode: The response code which was passed to
        C{setResponseCode}.

    @type written: C{list} of C{bytes}
    @ivar written: The bytes which have been written to the request.
    """
    uri = b'http://dummy/'
    method = b'GET'
    client = None

    def registerProducer(self, prod, s):
        """
        Call an L{IPullProducer}'s C{resumeProducing} method in a
        loop until it unregisters itself.

        @param prod: The producer.
        @type prod: L{IPullProducer}

        @param s: Whether or not the producer is streaming.
        """
        # XXX: Handle IPushProducers
        self.go = 1
        while self.go:
            prod.resumeProducing()

    def unregisterProducer(self):
        self.go = 0

    def __init__(self, postpath, session=None, client=None):
        self.sitepath = []
        self.written = []
        self.finished = 0
        self.postpath = postpath
        self.prepath = []
        self.session = None
        self.protoSession = session or Session(0, self)
        self.args = {}
        self.requestHeaders = Headers()
        self.responseHeaders = Headers()
        self.responseCode = None
        self._finishedDeferreds = []
        self._serverName = b"dummy"
        self.clientproto = b"HTTP/1.0"

    def getAllHeaders(self):
        """
        Return dictionary mapping the names of all received headers to the last
        value received for each.

        Since this method does not return all header information,
        C{self.requestHeaders.getAllRawHeaders()} may be preferred.

        NOTE: This function is a direct copy of
        C{twisted.web.http.Request.getAllRawHeaders}.
        """
        headers = {}
        for k, v in self.requestHeaders.getAllRawHeaders():
            headers[k.lower()] = v[-1]
        return headers

    def getHeader(self, name):
        """
        Retrieve the value of a request header.

        @type name: C{bytes}
        @param name: The name of the request header for which to retrieve the
            value.  Header names are compared case-insensitively.

        @rtype: C{bytes} or L{None}
        @return: The value of the specified request header.
        """
        return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]

    def setHeader(self, name, value):
        """TODO: make this assert on write() if the header is content-length
        """
        self.responseHeaders.addRawHeader(name, value)

    def getSession(self):
        if self.session:
            return self.session
        assert not self.written, "Session cannot be requested after data has been written."
        self.session = self.protoSession
        return self.session

    def render(self, resource):
        """
        Render the given resource as a response to this request.

        This implementation only handles a few of the most common behaviors of
        resources.  It can handle a render method that returns a string or
        C{NOT_DONE_YET}.  It doesn't know anything about the semantics of
        request methods (eg HEAD) nor how to set any particular headers.
        Basically, it's largely broken, but sufficient for some tests at least.
        It should B{not} be expanded to do all the same stuff L{Request} does.
        Instead, L{DummyRequest} should be phased out and L{Request} (or some
        other real code factored in a different way) used.
        """
        result = resource.render(self)
        if result is NOT_DONE_YET:
            return
        self.write(result)
        self.finish()

    def write(self, data):
        if not isinstance(data, bytes):
            raise TypeError("write() only accepts bytes")
        self.written.append(data)

    def notifyFinish(self):
        """
        Return a L{Deferred} which is called back with L{None} when the request
        is finished.  This will probably only work if you haven't called
        C{finish} yet.
        """
        finished = Deferred()
        self._finishedDeferreds.append(finished)
        return finished

    def finish(self):
        """
        Record that the request is finished and callback and L{Deferred}s
        waiting for notification of this.
        """
        self.finished = self.finished + 1
        if self._finishedDeferreds is not None:
            observers = self._finishedDeferreds
            self._finishedDeferreds = None
            for obs in observers:
                obs.callback(None)

    def processingFailed(self, reason):
        """
        Errback and L{Deferreds} waiting for finish notification.
        """
        if self._finishedDeferreds is not None:
            observers = self._finishedDeferreds
            self._finishedDeferreds = None
            for obs in observers:
                obs.errback(reason)

    def addArg(self, name, value):
        self.args[name] = [value]

    def setResponseCode(self, code, message=None):
        """
        Set the HTTP status response code, but takes care that this is called
        before any data is written.
        """
        assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(
            self.written)
        self.responseCode = code
        self.responseMessage = message

    def setLastModified(self, when):
        assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(
            self.written)

    def setETag(self, tag):
        assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(
            self.written)

    def getClientIP(self):
        """
        Return the IPv4 address of the client which made this request, if there
        is one, otherwise L{None}.
        """
        if isinstance(self.client, (IPv4Address, IPv6Address)):
            return self.client.host
        return None

    def getClientAddress(self):
        """
        Return the L{IAddress} of the client that made this request.

        @return: an address.
        @rtype: an L{IAddress} provider.
        """
        if self.client is None:
            return NullAddress()
        return self.client

    def getRequestHostname(self):
        """
        Get a dummy hostname associated to the HTTP request.

        @rtype: C{bytes}
        @returns: a dummy hostname
        """
        return self._serverName

    def getHost(self):
        """
        Get a dummy transport's host.

        @rtype: C{IPv4Address}
        @returns: a dummy transport's host
        """
        return IPv4Address('TCP', '127.0.0.1', 80)

    def setHost(self, host, port, ssl=0):
        """
        Change the host and port the request thinks it's using.

        @type host: C{bytes}
        @param host: The value to which to change the host header.

        @type ssl: C{bool}
        @param ssl: A flag which, if C{True}, indicates that the request is
            considered secure (if C{True}, L{isSecure} will return C{True}).
        """
        self._forceSSL = ssl  # set first so isSecure will work
        if self.isSecure():
            default = 443
        else:
            default = 80
        if port == default:
            hostHeader = host
        else:
            hostHeader = host + b":" + intToBytes(port)
        self.requestHeaders.addRawHeader(b"host", hostHeader)

    def redirect(self, url):
        """
        Utility function that does a redirect.

        The request should have finish() called after this.
        """
        self.setResponseCode(FOUND)
        self.setHeader(b"location", url)
Example #50
0
    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Args:
            method (bytes): HTTP method: GET/POST/etc

            uri (bytes): Absolute URI to be retrieved

            headers (twisted.web.http_headers.Headers|None):
                HTTP headers to send with the request, or None to
                send no extra headers.

            bodyProducer (twisted.web.iweb.IBodyProducer|None):
                An object which can generate bytes to make up the
                body of this request (for example, the properly encoded contents of
                a file for a file upload).  Or None if the request is to have
                no body.

        Returns:
            Deferred[twisted.web.iweb.IResponse]:
                fires when the header of the response has been received (regardless of the
                response status code). Fails if there is any problem which prevents that
                response from being received (including problems that prevent the request
                from being sent).
        """
        parsed_uri = URI.fromBytes(uri, defaultPort=-1)
        res = yield self._route_matrix_uri(parsed_uri)

        # set up the TLS connection params
        #
        # XXX disabling TLS is really only supported here for the benefit of the
        # unit tests. We should make the UTs cope with TLS rather than having to make
        # the code support the unit tests.
        if self._tls_client_options_factory is None:
            tls_options = None
        else:
            tls_options = self._tls_client_options_factory.get_options(
                res.tls_server_name.decode("ascii"))

        # make sure that the Host header is set correctly
        if headers is None:
            headers = Headers()
        else:
            headers = headers.copy()

        if not headers.hasHeader(b"host"):
            headers.addRawHeader(b"host", res.host_header)

        class EndpointFactory(object):
            @staticmethod
            def endpointForURI(_uri):
                ep = LoggingHostnameEndpoint(self._reactor, res.target_host,
                                             res.target_port)
                if tls_options is not None:
                    ep = wrapClientTLS(tls_options, ep)
                return ep

        agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(),
                                           self._pool)
        res = yield make_deferred_yieldable(
            agent.request(method, uri, headers, bodyProducer))
        defer.returnValue(res)
Example #51
0
 def _headers(self):
     return Headers({
         'Content-Type': ['application/json'],
         'Accept'      : ['application/json'],
         'User-Agent'  : [self.appName]
     })
Example #52
0
 def test_hasHeaderFalse(self):
     """
     L{Headers.hasHeader} returns C{False} when the given header is not
     found.
     """
     self.assertFalse(Headers().hasHeader(b"test"))
def getResponse(twisted_agent, url):
    try:
        response = yield twisted_agent.request(
            method=METHOD, uri=url, headers=Headers(HEADERS)).addErrback(error)
    except Exception, e:
        print(e)
Example #54
0
    def returnToSio(self, x, url, orig_env=None, tid=None, count=0):
        if isinstance(x, Failure):
            assert orig_env
            env = orig_env
            log.failure('Returning with error', x, LogLevel.warn)
        else:
            env = x

        if not tid:
            tid = env['group_id']

        bodygen, hdr = encode.multipart_encode({'data': json.dumps(env)})
        body = ''.join(bodygen)

        headers = Headers({'User-Agent': ['sioworkersd']})
        for k, v in six.iteritems(hdr):
            headers.addRawHeader(k, v)

        def do_return():
            # This looks a bit too complicated for just POSTing a string,
            # but there seems to be no other way. Blame Twisted.

            # agent.request() will add content-length based on length
            # from FileBodyProducer. If we have another in headers,
            # there will be a duplicate, so remove it.
            headers.removeHeader('content-length')

            producer = client.FileBodyProducer(StringIO(body))
            d = self.agent.request('POST', url.encode('utf-8'), headers,
                                   producer)

            @defer.inlineCallbacks
            def _response(r):
                if r.code != 200:
                    log.error(
                        'return error: server responded with status" \
                            "code {r.code}, response body follows...', r)
                    bodyD = yield client.readBody(r)
                    log.debug(bodyD)
                    raise RuntimeError('Failed to return task')

            d.addCallback(_response)
            return d

        ret = do_return()

        def _updateCount(x, n):
            self.database.update(tid, {'retry_cnt': n}, sync=False)
            # No db sync here, because we are allowing more attempts
            # of retrying returning job result for better performance.
            # It should be synced soon with other task
            # or `self.database` itself.
            return x  # Transparent callback

        def retry(err, retry_cnt):
            if retry_cnt >= MAX_RETRIES_OF_RESULT_RETURNING:
                log.error('Failed to return {tid} {count} times, giving up.',
                          tid=tid,
                          count=retry_cnt)
                return
            log.warn('Returning {tid} to url {url} failed, retrying[{n}]...',
                     tid=tid,
                     url=url,
                     n=retry_cnt)
            log.failure('error was:', err, LogLevel.info)
            d = deferLater(reactor, RETRY_DELAY_OF_RESULT_RETURNING[retry_cnt],
                           do_return)
            d.addBoth(_updateCount, n=retry_cnt)
            d.addErrback(retry, retry_cnt + 1)
            return d

        ret.addErrback(retry, retry_cnt=count)
        ret.addBoth(self._returnDone, tid=tid)
        return ret
Example #55
0
 def test_original(self):
     headers = Headers({})
     self.assertEqual(
         parse_x_forwarded_for(headers, original=['127.0.0.1', 80]),
         ['127.0.0.1', 80])
def send_payjoin(manager, accept_callback=None, info_callback=None):
    """ Given a JMPayjoinManager object `manager`, initialised with the
    payment request data from the server, use its wallet_service to construct
    a payment transaction, with coins sourced from mixdepth `mixdepth`,
    then wait for the server response, parse the PSBT, perform checks and complete sign.
    The info and accept callbacks are to ask the user to confirm the creation of
    the original payment transaction (None defaults to terminal/CLI processing),
    and are as defined in `taker_utils.direct_send`.

    Returns:
    (True, None) in case of payment setup successful (response will be delivered
     asynchronously) - the `manager` object can be inspected for more detail.
    (False, errormsg) in case of failure.
    """

    # wallet should already be synced before calling here;
    # we can create a standard payment, but have it returned as a PSBT.
    assert isinstance(manager, JMPayjoinManager)
    assert manager.wallet_service.synced
    payment_psbt = direct_send(manager.wallet_service,
                               manager.amount,
                               manager.mixdepth,
                               str(manager.destination),
                               accept_callback=accept_callback,
                               info_callback=info_callback,
                               with_final_psbt=True)
    if not payment_psbt:
        return (False, "could not create non-payjoin payment")

    # TLS whitelist is for regtest testing, it is treated as hostnames for
    # which tls certificate verification is ignored.
    tls_whitelist = None
    if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface):
        tls_whitelist = [b"127.0.0.1"]

    manager.set_payment_tx_and_psbt(payment_psbt)

    # add delayed call to broadcast this after 1 minute
    manager.timeout_fallback_dc = reactor.callLater(
        60, fallback_nonpayjoin_broadcast, manager, b"timeout")

    # Now we send the request to the server, with the encoded
    # payment PSBT

    # First we create a twisted web Agent object:

    # TODO genericize/move out/use library function:
    def is_hs_uri(s):
        x = urlparse.urlparse(s)
        if x.hostname.endswith(".onion"):
            return (x.scheme, x.hostname, x.port)
        return False

    tor_url_data = is_hs_uri(manager.server)
    if tor_url_data:
        # note the return value is currently unused here
        socks5_host = jm_single().config.get("PAYJOIN", "onion_socks5_host")
        socks5_port = int(jm_single().config.get("PAYJOIN",
                                                 "onion_socks5_port"))
        # note: SSL not supported at the moment:
        torEndpoint = TCP4ClientEndpoint(reactor, socks5_host, socks5_port)
        agent = tor_agent(reactor, torEndpoint)
    else:
        if not tls_whitelist:
            agent = Agent(reactor)
        else:
            agent = Agent(
                reactor, contextFactory=WhitelistContextFactory(tls_whitelist))

    body = BytesProducer(payment_psbt.to_base64().encode("utf-8"))

    #Set the query parameters for the request:

    # construct the URI from the given parameters
    pj_version = jm_single().config.getint("PAYJOIN", "payjoin_version")
    params = {"v": pj_version}

    disable_output_substitution = "false"
    if manager.disable_output_substitution:
        disable_output_substitution = "true"
    else:
        if jm_single().config.getint("PAYJOIN",
                                     "disable_output_substitution") == 1:
            disable_output_substitution = "true"
    params["disableoutputsubstitution"] = disable_output_substitution

    # to determine the additionalfeeoutputindex in cases where we have
    # change and we are allowing fee bump, we examine the initial tx:
    if manager.change_out:
        params["additionalfeeoutputindex"] = manager.change_out_index
        params["maxadditionalfeecontribution"] = \
            get_max_additional_fee_contribution(manager)

    min_fee_rate = float(jm_single().config.get("PAYJOIN", "min_fee_rate"))
    params["minfeerate"] = min_fee_rate

    destination_url = manager.server.encode("utf-8")
    url_parts = list(urlparse.urlparse(destination_url))
    url_parts[4] = urlencode(params).encode("utf-8")
    destination_url = urlparse.urlunparse(url_parts)
    # TODO what to use as user agent?
    d = agent.request(b"POST",
                      destination_url,
                      Headers({
                          "User-Agent": ["Twisted Web Client Example"],
                          "Content-Type": ["text/plain"]
                      }),
                      bodyProducer=body)

    d.addCallback(receive_payjoin_proposal_from_server, manager)

    # note that the errback (here "noResponse") is *not* triggered
    # by a server rejection (which is accompanied by a non-200
    # status code returned), but by failure to communicate.
    def noResponse(failure):
        failure.trap(ResponseFailed, ConnectionRefusedError,
                     HostUnreachableError)
        log.error(failure.value)
        fallback_nonpayjoin_broadcast(manager, b"connection refused")

    d.addErrback(noResponse)
    return (True, None)
Example #57
0
 def test_canonicalNameCaps(self):
     """
     L{Headers._canonicalNameCaps} returns the canonical capitalization for
     the given header.
     """
     h = Headers()
     self.assertEqual(h._canonicalNameCaps(b"test"), b"Test")
     self.assertEqual(h._canonicalNameCaps(b"test-stuff"), b"Test-Stuff")
     self.assertEqual(h._canonicalNameCaps(b"content-md5"), b"Content-MD5")
     self.assertEqual(h._canonicalNameCaps(b"dnt"), b"DNT")
     self.assertEqual(h._canonicalNameCaps(b"etag"), b"ETag")
     self.assertEqual(h._canonicalNameCaps(b"p3p"), b"P3P")
     self.assertEqual(h._canonicalNameCaps(b"te"), b"TE")
     self.assertEqual(h._canonicalNameCaps(b"www-authenticate"),
                      b"WWW-Authenticate")
     self.assertEqual(h._canonicalNameCaps(b"x-xss-protection"),
                      b"X-XSS-Protection")
Example #58
0
def request(path,
            method=b"GET",
            args=[],
            isSecure=False,
            headers={},
            body=b'',
            host=b"localhost",
            port=8000,
            reactor=None):
    """
    A fake `server.Request` which implements just enough for our tests.
    """

    if reactor is None:
        from twisted.internet import reactor

    channel = DummyChannel()
    site = Mock(server.Site)

    req = server.Request(channel=channel, queued=False)
    req.site = site

    req.method = method
    req.uri = path
    req.path = path
    req.prepath = []
    req.postpath = path.split(b'/')[1:]
    req.clientProto = b"HTTP/1.1"

    req.args = args

    # Set the headers we've got, as setHost writes to them
    req.requestHeaders = Headers(headers)

    # Put in a bogus date of no real significance, but one that will stay the
    # same
    req.requestHeaders.setRawHeaders(b"Date",
                                     [b"Sun, 1 Jan 2013 15:21:01 GMT"])

    # Set the host we are, and the client we're talking to
    req.setHost(host, port, isSecure)
    req.client = IPv4Address("TCP", "127.0.0.1", 8000)

    _written_data = BytesIO()

    # Writing
    def _write(data):
        assert not req.finished
        req.startedWriting = True
        _written_data.write(data)

    req.write = _write

    # Finishing
    def _finish():

        if not req.startedWriting:
            req.write(b"")

        if not req.finished:
            req.finished = True
            req._cleanup()

    req.finish = _finish

    # Getting what was wrote

    def _get_written_data():
        return _written_data.getvalue()

    req.get_written_data = _get_written_data

    # We have content now!
    req.content = BytesIO()
    req.content.write(body)
    req.content.seek(0)

    return req
Example #59
0
    def test_removeHeader(self):
        """
        Check that L{Headers.removeHeader} removes the given header.
        """
        h = Headers()

        h.setRawHeaders(b"foo", [b"lemur"])
        self.assertTrue(h.hasHeader(b"foo"))
        h.removeHeader(b"foo")
        self.assertFalse(h.hasHeader(b"foo"))

        h.setRawHeaders(b"bar", [b"panda"])
        self.assertTrue(h.hasHeader(b"bar"))
        h.removeHeader(b"Bar")
        self.assertFalse(h.hasHeader(b"bar"))
Example #60
0
 def test_v6_address(self):
     headers = Headers({
         b'X-Forwarded-For': [b'1043::a321:0001, 10.0.5.6'],
     })
     self.assertEqual(parse_x_forwarded_for(headers),
                      ['1043::a321:0001', 0])