Ejemplo n.º 1
0
    def _handle_exception(
        self,
        typ: "Optional[Type[BaseException]]",
        value: Optional[BaseException],
        tb: Optional[TracebackType],
    ) -> bool:
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPStreamClosedError("Stream closed")
                else:
                    value = value.real_error
            self._run_callback(
                HTTPResponse(
                    self.request,
                    599,
                    error=value,
                    request_time=self.io_loop.time() - self.start_time,
                    start_time=self.start_wall_time,
                ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)
Ejemplo n.º 2
0
 def finish(self) -> None:
     assert self.code is not None
     data = b"".join(self.chunks)
     self._remove_timeout()
     original_request = getattr(self.request, "original_request",
                                self.request)
     if self._should_follow_redirect():
         assert isinstance(self.request, _RequestProxy)
         new_request = copy.copy(self.request.request)
         new_request.url = urllib.parse.urljoin(self.request.url,
                                                self.headers["Location"])
         new_request.max_redirects = self.request.max_redirects - 1
         del new_request.headers["Host"]
         # https://tools.ietf.org/html/rfc7231#section-6.4
         #
         # The original HTTP spec said that after a 301 or 302
         # redirect, the request method should be preserved.
         # However, browsers implemented this by changing the
         # method to GET, and the behavior stuck. 303 redirects
         # always specified this POST-to-GET behavior, arguably
         # for *all* methods, but libcurl < 7.70 only does this
         # for POST, while libcurl >= 7.70 does it for other methods.
         if (self.code == 303 and self.request.method != "HEAD") or (
                 self.code in (301, 302) and self.request.method == "POST"):
             new_request.method = "GET"
             new_request.body = None
             for h in [
                     "Content-Length",
                     "Content-Type",
                     "Content-Encoding",
                     "Transfer-Encoding",
             ]:
                 try:
                     del self.request.headers[h]
                 except KeyError:
                     pass
         new_request.original_request = original_request
         final_callback = self.final_callback
         self.final_callback = None
         self._release()
         fut = self.client.fetch(new_request, raise_error=False)
         fut.add_done_callback(lambda f: final_callback(f.result()))
         self._on_end_request()
         return
     if self.request.streaming_callback:
         buffer = BytesIO()
     else:
         buffer = BytesIO(data)  # TODO: don't require one big string?
     response = HTTPResponse(
         original_request,
         self.code,
         reason=getattr(self, "reason", None),
         headers=self.headers,
         request_time=self.io_loop.time() - self.start_time,
         start_time=self.start_wall_time,
         buffer=buffer,
         effective_url=self.request.url,
     )
     self._run_callback(response)
     self._on_end_request()
Ejemplo n.º 3
0
 def _finish(
     self,
     curl: pycurl.Curl,
     curl_error: Optional[int] = None,
     curl_message: Optional[str] = None,
 ) -> None:
     info = curl.info  # type: ignore
     curl.info = None  # type: ignore
     self._multi.remove_handle(curl)
     self._free_list.append(curl)
     buffer = info["buffer"]
     if curl_error:
         assert curl_message is not None
         error = CurlError(curl_error,
                           curl_message)  # type: Optional[CurlError]
         assert error is not None
         code = error.code
         effective_url = None
         buffer.close()
         buffer = None
     else:
         error = None
         code = curl.getinfo(pycurl.HTTP_CODE)
         effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
         buffer.seek(0)
     # the various curl timings are documented at
     # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
     time_info = dict(
         queue=info["curl_start_ioloop_time"] - info["queue_start_time"],
         namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME),
         connect=curl.getinfo(pycurl.CONNECT_TIME),
         appconnect=curl.getinfo(pycurl.APPCONNECT_TIME),
         pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME),
         starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME),
         total=curl.getinfo(pycurl.TOTAL_TIME),
         redirect=curl.getinfo(pycurl.REDIRECT_TIME),
     )
     try:
         info["callback"](HTTPResponse(
             request=info["request"],
             code=code,
             headers=info["headers"],
             buffer=buffer,
             effective_url=effective_url,
             error=error,
             reason=info["headers"].get("X-Http-Reason", None),
             request_time=self.io_loop.time() -
             info["curl_start_ioloop_time"],
             start_time=info["curl_start_time"],
             time_info=time_info,
         ))
     except Exception:
         self.handle_callback_exception(info["callback"])
Ejemplo n.º 4
0
    def _on_timeout(self, key: object, info: Optional[str] = None) -> None:
        """Timeout callback of request.

        Construct a timeout HTTPResponse when a timeout occurs.

        :arg object key: A simple object to mark the request.
        :info string key: More detailed timeout information.
        """
        request, callback, timeout_handle = self.waiting[key]
        self.queue.remove((key, request, callback))

        error_message = "Timeout {0}".format(info) if info else "Timeout"
        timeout_response = HTTPResponse(
            request,
            599,
            error=HTTPTimeoutError(error_message),
            request_time=self.io_loop.time() - request.start_time,
        )
        self.io_loop.add_callback(callback, timeout_response)
        del self.waiting[key]
Ejemplo n.º 5
0
    def _process_queue(self) -> None:
        while True:
            started = 0
            while self._free_list and self._requests:
                started += 1
                curl = self._free_list.pop()
                (request, callback,
                 queue_start_time) = self._requests.popleft()
                # TODO: Don't smuggle extra data on an attribute of the Curl object.
                curl.info = {  # type: ignore
                    "headers": httputil.HTTPHeaders(),
                    "buffer": BytesIO(),
                    "request": request,
                    "callback": callback,
                    "queue_start_time": queue_start_time,
                    "curl_start_time": time.time(),
                    "curl_start_ioloop_time": self.io_loop.current().time(),
                }
                try:
                    self._curl_setup_request(
                        curl,
                        request,
                        curl.info["buffer"],  # type: ignore
                        curl.info["headers"],  # type: ignore
                    )
                except Exception as e:
                    # If there was an error in setup, pass it on
                    # to the callback. Note that allowing the
                    # error to escape here will appear to work
                    # most of the time since we are still in the
                    # caller's original stack frame, but when
                    # _process_queue() is called from
                    # _finish_pending_requests the exceptions have
                    # nowhere to go.
                    self._free_list.append(curl)
                    callback(HTTPResponse(request=request, code=599, error=e))
                else:
                    self._multi.add_handle(curl)

            if not started:
                break