async def _request(request): request.validate() client = tornado.httpclient.AsyncHTTPClient() url = request.url if request.get_params: url = "%s?%s" % (url, urllib.parse.urlencode(request.get_params)) t_request = tornado.httpclient.HTTPRequest( request.url, connect_timeout=2, request_timeout=2, method=request.method, body=request.get_body(), headers=request.get_headers(), follow_redirects=request.allow_redirects, ) try: response = await client.fetch(t_request) except: raise RequestManyException("request_many failed for %s" % url) headers = utils.CaseInsensitiveDict(dict(response.headers)) responses[request.id] = Response(response.code, response.body, "utf8", headers, {})
def request(url: str, method: str="GET", get_params: dict={}, post_data: typing.Any=None, headers: dict={}, json_data: typing.Any=None, code: bool=False, json: bool=False, soup: bool=False, parser: str="lxml", fallback_encoding: str="utf8", allow_redirects: bool=True ) -> Response: if not urllib.parse.urlparse(url).scheme: url = "http://%s" % url if not "Accept-Language" in headers: headers["Accept-Language"] = "en-GB" if not "User-Agent" in headers: headers["User-Agent"] = USER_AGENT signal.signal(signal.SIGALRM, lambda _1, _2: throw_timeout()) signal.alarm(5) try: response = requests.request( method.upper(), url, headers=headers, params=get_params, data=post_data, json=json_data, allow_redirects=allow_redirects, stream=True ) response_content = response.raw.read(RESPONSE_MAX, decode_content=True) except TimeoutError: raise HTTPTimeoutException() finally: signal.signal(signal.SIGALRM, signal.SIG_IGN) response_headers = utils.CaseInsensitiveDict(dict(response.headers)) content_type = response.headers["Content-Type"].split(";", 1)[0] def _decode_data(): return response_content.decode(response.encoding or fallback_encoding) if soup: if content_type in SOUP_CONTENT_TYPES: soup = bs4.BeautifulSoup(_decode_data(), parser) return Response(response.status_code, soup, response_headers) else: raise HTTPWrongContentTypeException( "Tried to soup non-html/non-xml data") data = _decode_data() if json and data: try: return Response(response.status_code, _json.loads(data), response_headers) except _json.decoder.JSONDecodeError as e: raise HTTPParsingException(str(e)) return Response(response.status_code, data, response_headers)
async def _request(url): client = tornado.httpclient.AsyncHTTPClient() request = tornado.httpclient.HTTPRequest(url, method="GET", connect_timeout=2, request_timeout=2) response = await client.fetch(request) headers = utils.CaseInsensitiveDict(dict(response.headers)) data = response.body.decode("utf8") responses[url] = Response(response.code, data, headers)
def _handle(self, method): path, endpoint, args = self._path_data() headers = utils.CaseInsensitiveDict(dict(self.headers.items())) params = self._url_params() data = self._body() response = "" code = 404 content_type = "text/plain" hooks = _events.on("api").on(method).on(endpoint).get_hooks() if hooks: hook = hooks[0] authenticated = hook.get_kwarg("authenticated", True) key = params.get("key", None) key_setting = _bot.get_setting("api-key-%s" % key, {}) permissions = key_setting.get("permissions", []) if key_setting: _log.debug("[HTTP] %s from API key %s (%s)", [method, key, key_setting["comment"]]) if not authenticated or path in permissions or "*" in permissions: if path.startswith("/api/"): event_response = None try: event_response = _bot.trigger(lambda: _events.on("api").on(method).on( endpoint).call_unsafe_for_result(params=params, path=args, data=data, headers=headers)) except Exception as e: _log.error("failed to call API endpoint \"%s\"", [path], exc_info=True) code = 500 if not event_response == None: content_type = "application/json" if _bot.get_setting("rest-api-minify", False): response = json.dumps(event_response, sort_keys=True, separators=(",", ":")) else: response = json.dumps(event_response, sort_keys=True, indent=4) code = 200 else: code = 401 headers = { "Content-type": content_type } self._respond(code, headers, response)
def _handle(self, method, path, data="", params={}): _, _, endpoint = path[1:].partition("/") endpoint, _, args = endpoint.partition("/") args = list(filter(None, args.split("/"))) headers = utils.CaseInsensitiveDict(dict(self.headers.items())) response = "" code = 404 hooks = _events.on("api").on(method).on(endpoint).get_hooks() if hooks: hook = hooks[0] authenticated = hook.get_kwarg("authenticated", True) key = params.get("key", None) key_setting = _bot.get_setting("api-key-%s" % key, {}) permissions = key_setting.get("permissions", []) if not authenticated or path in permissions or "*" in permissions: if path.startswith("/api/"): event_response = None try: event_response = _bot.trigger( lambda: _events.on("api").on(method).on(endpoint). call_unsafe_for_result(params=params, path=args, data=data, headers=headers)) except Exception as e: _log.error("failed to call API endpoint \"%s\"", [path], exc_info=True) code = 500 if not event_response == None: if _bot.get_setting("rest-api-minify", False): response = json.dumps(event_response, sort_keys=True, separators=(",", ":")) else: response = json.dumps(event_response, sort_keys=True, indent=4) code = 200 else: code = 401 self.send_response(code) self.send_header("Content-type", "application/json") self.end_headers() self.wfile.write(response.encode("utf8"))
def _wrap() -> Response: headers = request_obj.get_headers() redirect = 0 current_url = request_obj.url session = requests.Session() if not request_obj.bindhost is None: new_source = source.SourceAddressAdapter(request_obj.bindhost) session.mount('http://', new_source) session.mount('https://', new_source) while True: if request_obj.check_hostname: _assert_allowed(current_url) response = session.request(request_obj.method, current_url, headers=headers, params=request_obj.get_params, data=request_obj.get_body(), allow_redirects=False, stream=True, cookies=request_obj.cookies) if response.status_code in [301, 302]: redirect += 1 if redirect == 5: raise TooManyRedirectionsError(f"{redirect} redirects") else: current_url = response.headers["location"] continue response_content = response.raw.read(RESPONSE_MAX, decode_content=True) if not response.raw.read(1) == b"": raise ValueError("Response too large") break session.close() headers = utils.CaseInsensitiveDict(dict(response.headers)) our_response = Response(response.status_code, response_content, encoding=response.encoding, headers=headers, cookies=response.cookies.get_dict()) return our_response
def _handle(self, method, path, endpoint, args): headers = utils.CaseInsensitiveDict(dict(self.headers.items())) params = self._url_params() data = self._body() response = Response(compact=self._minify_setting()) response.code = 404 hooks = _events.on("api").on(method).on(endpoint).get_hooks() if hooks: response.code = 200 hook = hooks[0] authenticated = hook.get_kwarg("authenticated", True) key = params.get("key", None) key_setting = self._key_settings(key) permissions = key_setting.get("permissions", []) if key_setting: _log.debug("[HTTP] %s from API key %s (%s)", [method, key, key_setting["comment"]]) if not authenticated or path in permissions or "*" in permissions: if path.startswith("/api/"): event_response = None try: event_response = _events.on("api").on(method).on( endpoint).call_for_result_unsafe( params=params, args=args, data=data, headers=headers, response=response, url_for=self._url_for(headers)) except Exception as e: _log.error("failed to call API endpoint \"%s\"", [path], exc_info=True) response.code = 500 if not event_response == None: response.write_json(event_response) response.content_type = "application/json" else: response.code = 401 return response
def _wrap(): response = requests.request( request_obj.method, request_obj.url, headers=headers, params=request_obj.get_params, data=request_obj.get_body(), allow_redirects=request_obj.allow_redirects, stream=True) response_content = response.raw.read(RESPONSE_MAX, decode_content=True) if not response.raw.read(1) == b"": raise ValueError("Response too large") our_response = Response(response.status_code, response_content, headers=utils.CaseInsensitiveDict( dict(response.headers)), encoding=response.encoding) return our_response