def download_request(self, request): from twisted.internet import reactor timeout = request.meta.get('download_timeout') or self._connectTimeout agent = self._get_agent(request, timeout) # request details url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b'Proxy-Authorization') if request.body: bodyproducer = _RequestBodyProducer(request.body) else: bodyproducer = None start_time = time() d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed d.addCallback(self._cb_bodyready, request) d.addCallback(self._cb_bodydone, request, url) # check download timeout self._timeout_cl = reactor.callLater(timeout, d.cancel) d.addBoth(self._cb_timeout, request, url, timeout) return d
def download_request(self, request): timeout = request.meta.get('download_timeout') or self._connectTimeout url = urldefrag(request.url[len('browser+'):])[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) if '_browser_stream' in request.meta: d = defer.Deferred() request.meta['_browser_stream'].await_response(d) else: agent = self._get_agent(request, timeout) if request.body: bodyproducer = _RequestBodyProducer(request.body) else: bodyproducer = _RequestBodyProducer(b'') d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer) d.addCallback(self._cb_bodyready, request) d.addCallback(self._cb_bodydone) self._timeout_cl = reactor.callLater(timeout, d.cancel) d.addBoth(self._cb_timeout, url, timeout) return d
def download_request(self, request): timeout = request.meta.get('download_timeout') or self._connectTimeout url = urldefrag(request.url)[0] method = request.method headers = TxHeaders(request.headers) bodyproducer = _RequestBodyProducer( request.body) if request.body else None agent = self._get_agent(request, timeout) start_time = time() d = agent.request(method, url, headers, bodyproducer) d.addBoth(self._both_cb, request, start_time, url, timeout) d.addCallback(self._downloaded, request) self._timeout_cl = reactor.callLater(timeout, d.cancel) return d
def download_request(self, request): agent = self._get_agent(request) # request details url = urldefrag(request.url)[0] method = request.method headers = TxHeaders(request.headers) body_producer = _RequestBodyProducer(request.body) if request.body else None start_time = time() d = agent.request(method, url, headers, body_producer) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed d.addCallback(self._cb_body_ready) d.addCallback(self._cb_body_done, request, url) # check download timeout self._timeout_cl = reactor.callLater(self.timeout, d.cancel) d.addBoth(self._cb_timeout, request, url, self.timeout) return d
def download_request(self, request): from twisted.internet import reactor timeout = request.meta.get('download_timeout') or self._connectTimeout agent = self._get_agent(request, timeout) # request details url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b'Proxy-Authorization') if request.body: bodyproducer = _RequestBodyProducer(request.body) elif method == b'POST': # Setting Content-Length: 0 even for POST requests is not a # MUST per HTTP RFCs, but it's common behavior, and some # servers require this, otherwise returning HTTP 411 Length required # # RFC 7230#section-3.3.2: # "a Content-Length header field is normally sent in a POST # request even when the value is 0 (indicating an empty payload body)." # # Twisted < 17 will not add "Content-Length: 0" by itself; # Twisted >= 17 fixes this; # Using a producer with an empty-string sends `0` as Content-Length # for all versions of Twisted. bodyproducer = _RequestBodyProducer(b'') else: bodyproducer = None start_time = time() d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed d.addCallback(self._cb_bodyready, request) d.addCallback(self._cb_bodydone, request, url) # check download timeout self._timeout_cl = reactor.callLater(timeout, d.cancel) d.addBoth(self._cb_timeout, request, url, timeout) return d