async def _request(request):
request.validate()
client = tornado.httpclient.AsyncHTTPClient()
url = request.url
if request.get_params:
url = "%s?%s" % (url, urllib.parse.urlencode(request.get_params))
t_request = tornado.httpclient.HTTPRequest(
request.url,
connect_timeout=2,
request_timeout=2,
method=request.method,
body=request.get_body(),
headers=request.get_headers(),
follow_redirects=request.allow_redirects,
)
try:
response = await client.fetch(t_request)
except:
raise RequestManyException("request_many failed for %s" % url)
headers = utils.CaseInsensitiveDict(dict(response.headers))
responses[request.id] = Response(response.code, response.body, "utf8",
headers, {})
def request(url: str, method: str="GET", get_params: dict={},
post_data: typing.Any=None, headers: dict={},
json_data: typing.Any=None, code: bool=False, json: bool=False,
soup: bool=False, parser: str="lxml", fallback_encoding: str="utf8",
allow_redirects: bool=True
) -> Response:
if not urllib.parse.urlparse(url).scheme:
url = "http://%s" % url
if not "Accept-Language" in headers:
headers["Accept-Language"] = "en-GB"
if not "User-Agent" in headers:
headers["User-Agent"] = USER_AGENT
signal.signal(signal.SIGALRM, lambda _1, _2: throw_timeout())
signal.alarm(5)
try:
response = requests.request(
method.upper(),
url,
headers=headers,
params=get_params,
data=post_data,
json=json_data,
allow_redirects=allow_redirects,
stream=True
)
response_content = response.raw.read(RESPONSE_MAX, decode_content=True)
except TimeoutError:
raise HTTPTimeoutException()
finally:
signal.signal(signal.SIGALRM, signal.SIG_IGN)
response_headers = utils.CaseInsensitiveDict(dict(response.headers))
content_type = response.headers["Content-Type"].split(";", 1)[0]
def _decode_data():
return response_content.decode(response.encoding or fallback_encoding)
if soup:
if content_type in SOUP_CONTENT_TYPES:
soup = bs4.BeautifulSoup(_decode_data(), parser)
return Response(response.status_code, soup, response_headers)
else:
raise HTTPWrongContentTypeException(
"Tried to soup non-html/non-xml data")
data = _decode_data()
if json and data:
try:
return Response(response.status_code, _json.loads(data),
response_headers)
except _json.decoder.JSONDecodeError as e:
raise HTTPParsingException(str(e))
return Response(response.status_code, data, response_headers)
async def _request(url):
client = tornado.httpclient.AsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(url, method="GET",
connect_timeout=2, request_timeout=2)
response = await client.fetch(request)
headers = utils.CaseInsensitiveDict(dict(response.headers))
data = response.body.decode("utf8")
responses[url] = Response(response.code, data, headers)
def _handle(self, method):
path, endpoint, args = self._path_data()
headers = utils.CaseInsensitiveDict(dict(self.headers.items()))
params = self._url_params()
data = self._body()
response = ""
code = 404
content_type = "text/plain"
hooks = _events.on("api").on(method).on(endpoint).get_hooks()
if hooks:
hook = hooks[0]
authenticated = hook.get_kwarg("authenticated", True)
key = params.get("key", None)
key_setting = _bot.get_setting("api-key-%s" % key, {})
permissions = key_setting.get("permissions", [])
if key_setting:
_log.debug("[HTTP] %s from API key %s (%s)",
[method, key, key_setting["comment"]])
if not authenticated or path in permissions or "*" in permissions:
if path.startswith("/api/"):
event_response = None
try:
event_response = _bot.trigger(lambda:
_events.on("api").on(method).on(
endpoint).call_unsafe_for_result(params=params,
path=args, data=data, headers=headers))
except Exception as e:
_log.error("failed to call API endpoint \"%s\"",
[path], exc_info=True)
code = 500
if not event_response == None:
content_type = "application/json"
if _bot.get_setting("rest-api-minify", False):
response = json.dumps(event_response,
sort_keys=True, separators=(",", ":"))
else:
response = json.dumps(event_response,
sort_keys=True, indent=4)
code = 200
else:
code = 401
headers = {
"Content-type": content_type
}
self._respond(code, headers, response)
def _handle(self, method, path, data="", params={}):
_, _, endpoint = path[1:].partition("/")
endpoint, _, args = endpoint.partition("/")
args = list(filter(None, args.split("/")))
headers = utils.CaseInsensitiveDict(dict(self.headers.items()))
response = ""
code = 404
hooks = _events.on("api").on(method).on(endpoint).get_hooks()
if hooks:
hook = hooks[0]
authenticated = hook.get_kwarg("authenticated", True)
key = params.get("key", None)
key_setting = _bot.get_setting("api-key-%s" % key, {})
permissions = key_setting.get("permissions", [])
if not authenticated or path in permissions or "*" in permissions:
if path.startswith("/api/"):
event_response = None
try:
event_response = _bot.trigger(
lambda: _events.on("api").on(method).on(endpoint).
call_unsafe_for_result(params=params,
path=args,
data=data,
headers=headers))
except Exception as e:
_log.error("failed to call API endpoint \"%s\"",
[path],
exc_info=True)
code = 500
if not event_response == None:
if _bot.get_setting("rest-api-minify", False):
response = json.dumps(event_response,
sort_keys=True,
separators=(",", ":"))
else:
response = json.dumps(event_response,
sort_keys=True,
indent=4)
code = 200
else:
code = 401
self.send_response(code)
self.send_header("Content-type", "application/json")
self.end_headers()
self.wfile.write(response.encode("utf8"))
def _wrap() -> Response:
headers = request_obj.get_headers()
redirect = 0
current_url = request_obj.url
session = requests.Session()
if not request_obj.bindhost is None:
new_source = source.SourceAddressAdapter(request_obj.bindhost)
session.mount('http://', new_source)
session.mount('https://', new_source)
while True:
if request_obj.check_hostname:
_assert_allowed(current_url)
response = session.request(request_obj.method,
current_url,
headers=headers,
params=request_obj.get_params,
data=request_obj.get_body(),
allow_redirects=False,
stream=True,
cookies=request_obj.cookies)
if response.status_code in [301, 302]:
redirect += 1
if redirect == 5:
raise TooManyRedirectionsError(f"{redirect} redirects")
else:
current_url = response.headers["location"]
continue
response_content = response.raw.read(RESPONSE_MAX,
decode_content=True)
if not response.raw.read(1) == b"":
raise ValueError("Response too large")
break
session.close()
headers = utils.CaseInsensitiveDict(dict(response.headers))
our_response = Response(response.status_code,
response_content,
encoding=response.encoding,
headers=headers,
cookies=response.cookies.get_dict())
return our_response
def _handle(self, method, path, endpoint, args):
headers = utils.CaseInsensitiveDict(dict(self.headers.items()))
params = self._url_params()
data = self._body()
response = Response(compact=self._minify_setting())
response.code = 404
hooks = _events.on("api").on(method).on(endpoint).get_hooks()
if hooks:
response.code = 200
hook = hooks[0]
authenticated = hook.get_kwarg("authenticated", True)
key = params.get("key", None)
key_setting = self._key_settings(key)
permissions = key_setting.get("permissions", [])
if key_setting:
_log.debug("[HTTP] %s from API key %s (%s)",
[method, key, key_setting["comment"]])
if not authenticated or path in permissions or "*" in permissions:
if path.startswith("/api/"):
event_response = None
try:
event_response = _events.on("api").on(method).on(
endpoint).call_for_result_unsafe(
params=params,
args=args,
data=data,
headers=headers,
response=response,
url_for=self._url_for(headers))
except Exception as e:
_log.error("failed to call API endpoint \"%s\"",
[path],
exc_info=True)
response.code = 500
if not event_response == None:
response.write_json(event_response)
response.content_type = "application/json"
else:
response.code = 401
return response
def _wrap():
response = requests.request(
request_obj.method,
request_obj.url,
headers=headers,
params=request_obj.get_params,
data=request_obj.get_body(),
allow_redirects=request_obj.allow_redirects,
stream=True)
response_content = response.raw.read(RESPONSE_MAX, decode_content=True)
if not response.raw.read(1) == b"":
raise ValueError("Response too large")
our_response = Response(response.status_code,
response_content,
headers=utils.CaseInsensitiveDict(
dict(response.headers)),
encoding=response.encoding)
return our_response
