def relay(self, upstream: HTTPResponse): # add upstream response headers after sanitization headers = sanitize_headers(upstream.getheaders()) # transport level headers total_length = response_length(upstream) down_stream = self.wfile if total_length is None: headers.append(('Transfer-Encoding', 'chunked')) down_stream = ChunkedEncoder(self.wfile) else: headers.append(('Content-Length', str(total_length))) # send headers self.log_message("got upstream response. relaying to client...") if upstream.getcode() == 206 and 'Range' not in self.headers: # translate 206 -> 200 if the downstream is not a range request self.send_response(200) else: self.send_response(upstream.getcode()) for key, val in headers: self.send_header(key, val) self.end_headers() # relay data while True: # read try: data = upstream.read(self.read_size) except (TimeoutError, IncompleteRead) as e: self.log_error("upstream closed prematurely: %s" % e) self.close_connection = True break if len(data) == 0: # EOF if isinstance(down_stream, ChunkedEncoder): down_stream.finish() break # send try: down_stream.write(data) except ConnectionError as e: self.log_error("client closed prematurely: %s" % e.strerror) self.close_connection = True break self.log_message("done!")
def response_length(resp: HTTPResponse): """In case of chunked encoding, calc length from content-range. TODO""" headers = dict((key.lower(), val) for key, val in resp.getheaders()) if resp.getcode() == 206 and 'content-range' in headers: match = re.match(r' *bytes *(\d+) *- *(\d+)', headers['content-range']) if match is None: raise RuntimeError("unexpected content-range: %s" % headers['content-range']) start_byte, end_byte = match.groups() return int(end_byte) - int(start_byte) + 1 else: return resp.length
def from_http_client_response(obj: HTTPResponse) -> Response: body = obj.read() res = ResponseBuilder.from_dict( dict( statusCode=obj.getcode(), headers={k: v for k, v in obj.getheaders()}, body=body if isinstance(body, str) else body.decode("utf8") if isinstance(body, bytes) else None, )) ResponseBuilder.validate(res) return res
def _analyse(fetch_request: HTTPResponse, future_nzo: NzbObject): """Analyze response of indexer returns fetch_request|None, error-message|None, retry, wait-seconds, data """ data = None if not fetch_request or fetch_request.getcode() != 200: if fetch_request: msg = fetch_request.msg else: msg = "" # Increasing wait-time in steps for standard errors when = DEF_TIMEOUT * (future_nzo.url_tries + 1) logging.debug("No usable response from indexer, retry after %s sec", when) return None, msg, True, when, data return fetch_request, fetch_request.msg, False, 0, data
async def _asyncio_send(loop, req, *, timeout=10, max_redirects=10): """A rudimentary HTTP client using :mod:`asyncio`""" if not any(h.lower() == "user-agent" for h in req.headers): req = req.with_headers({"User-Agent": _ASYNCIO_USER_AGENT}) url = urllib.parse.urlsplit( req.url + "?" + urllib.parse.urlencode(req.params) ) open_ = partial(asyncio.open_connection, url.hostname, loop=loop) connect = open_(443, ssl=True) if url.scheme == "https" else open_(80) reader, writer = await connect try: headers = "\r\n".join( [ "{} {} HTTP/1.1".format( req.method, url.path + "?" + url.query ), "Host: " + url.hostname, "Connection: close", "Content-Length: {}".format(len(req.content or b"")), "\r\n".join(starmap("{}: {}".format, req.headers.items())), ] ) writer.write( b"\r\n".join([headers.encode("latin-1"), b"", req.content or b""]) ) response_bytes = BytesIO( await asyncio.wait_for(reader.read(), timeout=timeout) ) finally: writer.close() resp = HTTPResponse( _SocketAdaptor(response_bytes), method=req.method, url=req.url ) resp.begin() status = resp.getcode() if 300 <= status < 400 and "Location" in resp.headers and max_redirects: new_url = urllib.parse.urljoin(req.url, resp.headers["Location"]) return await _asyncio_send( loop, req.replace(url=new_url), timeout=timeout, max_redirects=max_redirects - 1, ) return Response(status, content=resp.read(), headers=resp.headers)
def onRecv(self, task, sock, file_no): if sock.fileno() == file_no: t_id = self._write_timer.pop(file_no, None) if t_id: self.cancel_timer(t_id) try: # INFO_MSG('------------- onRecv ------------ {} {}'.format(task[0], task[2])) resp = HTTPResponse(sock) if task and task[2]: resp.begin() if resp.getcode() / 100 == 2: decode = resp.read().decode('utf-8') task[2](decode) else: task[2](None) except: self.logsError() finally: KBEngine.deregisterReadFileDescriptor(file_no) if not sock._closed: sock.close()
def _asyncio_send(loop, req, *, timeout=10, max_redirects=10): """A rudimentary HTTP client using :mod:`asyncio`""" if not any(h.lower() == 'user-agent' for h in req.headers): req = req.with_headers({'User-Agent': _ASYNCIO_USER_AGENT}) url = urllib.parse.urlsplit(req.url + '?' + urllib.parse.urlencode(req.params)) open_ = partial(asyncio.open_connection, url.hostname, loop=loop) connect = open_(443, ssl=True) if url.scheme == 'https' else open_(80) reader, writer = yield from connect try: headers = '\r\n'.join([ '{} {} HTTP/1.1'.format(req.method, url.path + '?' + url.query), 'Host: ' + url.hostname, 'Connection: close', 'Content-Length: {}'.format(len(req.content or b'')), '\r\n'.join(starmap('{}: {}'.format, req.headers.items())), ]) writer.write(b'\r\n'.join( [headers.encode('latin-1'), b'', req.content or b''])) response_bytes = BytesIO((yield from asyncio.wait_for(reader.read(), timeout=timeout))) finally: writer.close() resp = HTTPResponse(_SocketAdaptor(response_bytes), method=req.method, url=req.url) resp.begin() status = resp.getcode() if 300 <= status < 400 and 'Location' in resp.headers and max_redirects: new_url = urllib.parse.urljoin(req.url, resp.headers['Location']) return (yield from _asyncio_send(loop, req.replace(url=new_url), timeout=timeout, max_redirects=max_redirects - 1)) return Response(status, content=resp.read(), headers=resp.headers)
def get_headers_str(res: HTTPResponse) -> str: result: str = "HTTP " + str(res.getcode()) + '\n' headers: list = res.getheaders() for t in headers: result += (t[0] + ': ' + t[1] + '\n') return result
def _validate_response(res: HTTPResponse) -> Dict[str, Any]: if (code := res.getcode()) != HTTPStatus.OK: raise RuntimeError(f'Requets failed with status code {code}.')