def relay(self, upstream: HTTPResponse):
# add upstream response headers after sanitization
headers = sanitize_headers(upstream.getheaders())
# transport level headers
total_length = response_length(upstream)
down_stream = self.wfile
if total_length is None:
headers.append(('Transfer-Encoding', 'chunked'))
down_stream = ChunkedEncoder(self.wfile)
else:
headers.append(('Content-Length', str(total_length)))
# send headers
self.log_message("got upstream response. relaying to client...")
if upstream.getcode() == 206 and 'Range' not in self.headers:
# translate 206 -> 200 if the downstream is not a range request
self.send_response(200)
else:
self.send_response(upstream.getcode())
for key, val in headers:
self.send_header(key, val)
self.end_headers()
# relay data
while True:
# read
try:
data = upstream.read(self.read_size)
except (TimeoutError, IncompleteRead) as e:
self.log_error("upstream closed prematurely: %s" % e)
self.close_connection = True
break
if len(data) == 0: # EOF
if isinstance(down_stream, ChunkedEncoder):
down_stream.finish()
break
# send
try:
down_stream.write(data)
except ConnectionError as e:
self.log_error("client closed prematurely: %s" % e.strerror)
self.close_connection = True
break
self.log_message("done!")
def response_length(resp: HTTPResponse):
"""In case of chunked encoding, calc length from content-range. TODO"""
headers = dict((key.lower(), val) for key, val in resp.getheaders())
if resp.getcode() == 206 and 'content-range' in headers:
match = re.match(r' *bytes *(\d+) *- *(\d+)', headers['content-range'])
if match is None:
raise RuntimeError("unexpected content-range: %s" % headers['content-range'])
start_byte, end_byte = match.groups()
return int(end_byte) - int(start_byte) + 1
else:
return resp.length
def from_http_client_response(obj: HTTPResponse) -> Response:
body = obj.read()
res = ResponseBuilder.from_dict(
dict(
statusCode=obj.getcode(),
headers={k: v
for k, v in obj.getheaders()},
body=body if isinstance(body, str) else
body.decode("utf8") if isinstance(body, bytes) else None,
))
ResponseBuilder.validate(res)
return res
def _analyse(fetch_request: HTTPResponse, future_nzo: NzbObject):
"""Analyze response of indexer
returns fetch_request|None, error-message|None, retry, wait-seconds, data
"""
data = None
if not fetch_request or fetch_request.getcode() != 200:
if fetch_request:
msg = fetch_request.msg
else:
msg = ""
# Increasing wait-time in steps for standard errors
when = DEF_TIMEOUT * (future_nzo.url_tries + 1)
logging.debug("No usable response from indexer, retry after %s sec",
when)
return None, msg, True, when, data
return fetch_request, fetch_request.msg, False, 0, data
async def _asyncio_send(loop, req, *, timeout=10, max_redirects=10):
"""A rudimentary HTTP client using :mod:`asyncio`"""
if not any(h.lower() == "user-agent" for h in req.headers):
req = req.with_headers({"User-Agent": _ASYNCIO_USER_AGENT})
url = urllib.parse.urlsplit(
req.url + "?" + urllib.parse.urlencode(req.params)
)
open_ = partial(asyncio.open_connection, url.hostname, loop=loop)
connect = open_(443, ssl=True) if url.scheme == "https" else open_(80)
reader, writer = await connect
try:
headers = "\r\n".join(
[
"{} {} HTTP/1.1".format(
req.method, url.path + "?" + url.query
),
"Host: " + url.hostname,
"Connection: close",
"Content-Length: {}".format(len(req.content or b"")),
"\r\n".join(starmap("{}: {}".format, req.headers.items())),
]
)
writer.write(
b"\r\n".join([headers.encode("latin-1"), b"", req.content or b""])
)
response_bytes = BytesIO(
await asyncio.wait_for(reader.read(), timeout=timeout)
)
finally:
writer.close()
resp = HTTPResponse(
_SocketAdaptor(response_bytes), method=req.method, url=req.url
)
resp.begin()
status = resp.getcode()
if 300 <= status < 400 and "Location" in resp.headers and max_redirects:
new_url = urllib.parse.urljoin(req.url, resp.headers["Location"])
return await _asyncio_send(
loop,
req.replace(url=new_url),
timeout=timeout,
max_redirects=max_redirects - 1,
)
return Response(status, content=resp.read(), headers=resp.headers)
def onRecv(self, task, sock, file_no):
if sock.fileno() == file_no:
t_id = self._write_timer.pop(file_no, None)
if t_id:
self.cancel_timer(t_id)
try:
# INFO_MSG('------------- onRecv ------------ {} {}'.format(task[0], task[2]))
resp = HTTPResponse(sock)
if task and task[2]:
resp.begin()
if resp.getcode() / 100 == 2:
decode = resp.read().decode('utf-8')
task[2](decode)
else:
task[2](None)
except:
self.logsError()
finally:
KBEngine.deregisterReadFileDescriptor(file_no)
if not sock._closed:
sock.close()
def _asyncio_send(loop, req, *, timeout=10, max_redirects=10):
"""A rudimentary HTTP client using :mod:`asyncio`"""
if not any(h.lower() == 'user-agent' for h in req.headers):
req = req.with_headers({'User-Agent': _ASYNCIO_USER_AGENT})
url = urllib.parse.urlsplit(req.url + '?' +
urllib.parse.urlencode(req.params))
open_ = partial(asyncio.open_connection, url.hostname, loop=loop)
connect = open_(443, ssl=True) if url.scheme == 'https' else open_(80)
reader, writer = yield from connect
try:
headers = '\r\n'.join([
'{} {} HTTP/1.1'.format(req.method, url.path + '?' + url.query),
'Host: ' + url.hostname,
'Connection: close',
'Content-Length: {}'.format(len(req.content or b'')),
'\r\n'.join(starmap('{}: {}'.format, req.headers.items())),
])
writer.write(b'\r\n'.join(
[headers.encode('latin-1'), b'', req.content or b'']))
response_bytes = BytesIO((yield from
asyncio.wait_for(reader.read(),
timeout=timeout)))
finally:
writer.close()
resp = HTTPResponse(_SocketAdaptor(response_bytes),
method=req.method,
url=req.url)
resp.begin()
status = resp.getcode()
if 300 <= status < 400 and 'Location' in resp.headers and max_redirects:
new_url = urllib.parse.urljoin(req.url, resp.headers['Location'])
return (yield from _asyncio_send(loop,
req.replace(url=new_url),
timeout=timeout,
max_redirects=max_redirects - 1))
return Response(status, content=resp.read(), headers=resp.headers)
def get_headers_str(res: HTTPResponse) -> str:
result: str = "HTTP " + str(res.getcode()) + '\n'
headers: list = res.getheaders()
for t in headers:
result += (t[0] + ': ' + t[1] + '\n')
return result
def _validate_response(res: HTTPResponse) -> Dict[str, Any]:
if (code := res.getcode()) != HTTPStatus.OK:
raise RuntimeError(f'Requets failed with status code {code}.')
