class KafkaSchemaReader(Thread): def __init__(self, config, master_coordinator=None): Thread.__init__(self) self.master_coordinator = master_coordinator self.log = logging.getLogger("KafkaSchemaReader") self.timeout_ms = 200 self.config = config self.subjects = {} self.schemas: Dict[int, TypedSchema] = {} self.global_schema_id = 0 self.offset = 0 self.admin_client = None self.schema_topic = None self.topic_replication_factor = self.config["replication_factor"] self.consumer = None self.queue = Queue() self.ready = False self.running = True self.id_lock = Lock() sentry_config = config.get("sentry", {"dsn": None}).copy() if "tags" not in sentry_config: sentry_config["tags"] = {} self.stats = StatsClient(sentry_config=sentry_config) def init_consumer(self): # Group not set on purpose, all consumers read the same data session_timeout_ms = self.config["session_timeout_ms"] request_timeout_ms = max( session_timeout_ms, KafkaConsumer.DEFAULT_CONFIG["request_timeout_ms"]) self.consumer = KafkaConsumer( self.config["topic_name"], enable_auto_commit=False, api_version=(1, 0, 0), bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], auto_offset_reset="earliest", session_timeout_ms=session_timeout_ms, request_timeout_ms=request_timeout_ms, kafka_client=KarapaceKafkaClient, metadata_max_age_ms=self.config["metadata_max_age_ms"], ) def init_admin_client(self): try: self.admin_client = KafkaAdminClient( api_version_auto_timeout_ms=constants. API_VERSION_AUTO_TIMEOUT_MS, bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], ) return True except (NodeNotReadyError, NoBrokersAvailable, AssertionError): self.log.warning( "No Brokers available yet, retrying init_admin_client()") time.sleep(2.0) except: # pylint: disable=bare-except self.log.exception( "Failed to initialize admin client, retrying init_admin_client()" ) time.sleep(2.0) return False @staticmethod def get_new_schema_topic(config): return NewTopic(name=config["topic_name"], num_partitions=constants.SCHEMA_TOPIC_NUM_PARTITIONS, replication_factor=config["replication_factor"], topic_configs={"cleanup.policy": "compact"}) def create_schema_topic(self): schema_topic = self.get_new_schema_topic(self.config) try: self.log.info("Creating topic: %r", schema_topic) self.admin_client.create_topics( [schema_topic], timeout_ms=constants.TOPIC_CREATION_TIMEOUT_MS) self.log.info("Topic: %r created successfully", self.config["topic_name"]) self.schema_topic = schema_topic return True except TopicAlreadyExistsError: self.log.warning("Topic: %r already exists", self.config["topic_name"]) self.schema_topic = schema_topic return True except: # pylint: disable=bare-except self.log.exception( "Failed to create topic: %r, retrying create_schema_topic()", self.config["topic_name"]) time.sleep(5) return False def get_schema_id(self, new_schema): with self.id_lock: schemas = self.schemas.items() for schema_id, schema in schemas: if schema == new_schema: return schema_id with self.id_lock: self.global_schema_id += 1 return self.global_schema_id def close(self): self.log.info("Closing schema_reader") self.running = False def run(self): while self.running: try: if not self.admin_client: if self.init_admin_client() is False: continue if not self.schema_topic: if self.create_schema_topic() is False: continue if not self.consumer: self.init_consumer() self.handle_messages() except Exception as e: # pylint: disable=broad-except if self.stats: self.stats.unexpected_exception(ex=e, where="schema_reader_loop") self.log.exception( "Unexpected exception in schema reader loop") try: if self.admin_client: self.admin_client.close() if self.consumer: self.consumer.close() except Exception as e: # pylint: disable=broad-except if self.stats: self.stats.unexpected_exception(ex=e, where="schema_reader_exit") self.log.exception("Unexpected exception closing schema reader") def handle_messages(self): raw_msgs = self.consumer.poll(timeout_ms=self.timeout_ms) if self.ready is False and raw_msgs == {}: self.ready = True add_offsets = False if self.master_coordinator is not None: master, _ = self.master_coordinator.get_master_info() # keep old behavior for True. When master is False, then we are a follower, so we should not accept direct # writes anyway. When master is None, then this particular node is waiting for a stable value, so any # messages off the topic are writes performed by another node if master is True: add_offsets = True for _, msgs in raw_msgs.items(): for msg in msgs: try: key = json.loads(msg.key.decode("utf8")) except json.JSONDecodeError: self.log.exception( "Invalid JSON in msg.key: %r, value: %r", msg.key, msg.value) continue value = None if msg.value: try: value = json.loads(msg.value.decode("utf8")) except json.JSONDecodeError: self.log.exception( "Invalid JSON in msg.value: %r, key: %r", msg.value, msg.key) continue self.log.info( "Read new record: key: %r, value: %r, offset: %r", key, value, msg.offset) self.handle_msg(key, value) self.offset = msg.offset self.log.info("Handled message, current offset: %r", self.offset) if self.ready and add_offsets: self.queue.put(self.offset) def handle_msg(self, key: dict, value: dict): if key["keytype"] == "CONFIG": if "subject" in key and key["subject"] is not None: if not value: self.log.info( "Deleting compatibility config completely for subject: %r", key["subject"]) self.subjects[key["subject"]].pop("compatibility", None) return self.log.info("Setting subject: %r config to: %r, value: %r", key["subject"], value["compatibilityLevel"], value) if not key["subject"] in self.subjects: self.log.info( "Adding first version of subject: %r with no schemas", key["subject"]) self.subjects[key["subject"]] = {"schemas": {}} subject_data = self.subjects.get(key["subject"]) subject_data["compatibility"] = value["compatibilityLevel"] else: self.log.info("Setting global config to: %r, value: %r", value["compatibilityLevel"], value) self.config["compatibility"] = value["compatibilityLevel"] elif key["keytype"] == "SCHEMA": if not value: subject, version = key["subject"], key["version"] self.log.info("Deleting subject: %r version: %r completely", subject, version) if subject not in self.subjects: self.log.error("Subject %s did not exist, should have", subject) elif version not in self.subjects[subject]["schemas"]: self.log.error( "Version %d for subject %s did not exist, should have", version, subject) else: self.subjects[subject]["schemas"].pop(version, None) return schema_type = value.get("schemaType", "AVRO") schema_str = value["schema"] try: typed_schema = TypedSchema.parse( schema_type=SchemaType(schema_type), schema_str=schema_str) except InvalidSchema: try: schema_json = json.loads(schema_str) typed_schema = TypedSchema( schema_type=SchemaType(schema_type), schema=schema_json, schema_str=schema_str) except JSONDecodeError: self.log.error("Invalid json: %s", value["schema"]) return self.log.debug("Got typed schema %r", typed_schema) subject = value["subject"] if subject not in self.subjects: self.log.info("Adding first version of subject: %r, value: %r", subject, value) self.subjects[subject] = { "schemas": { value["version"]: { "schema": typed_schema, "version": value["version"], "id": value["id"], "deleted": value.get("deleted", False), } } } self.log.info("Setting schema_id: %r with schema: %r", value["id"], typed_schema) self.schemas[value["id"]] = typed_schema if value["id"] > self.global_schema_id: # Not an existing schema self.global_schema_id = value["id"] elif value.get("deleted", False) is True: self.log.info("Deleting subject: %r, version: %r", subject, value["version"]) if not value["version"] in self.subjects[subject]["schemas"]: self.schemas[value["id"]] = typed_schema else: self.subjects[subject]["schemas"][ value["version"]]["deleted"] = True elif value.get("deleted", False) is False: self.log.info("Adding new version of subject: %r, value: %r", subject, value) self.subjects[subject]["schemas"][value["version"]] = { "schema": typed_schema, "version": value["version"], "id": value["id"], "deleted": value.get("deleted", False), } self.log.info("Setting schema_id: %r with schema: %r", value["id"], value["schema"]) with self.id_lock: self.schemas[value["id"]] = typed_schema if value["id"] > self.global_schema_id: # Not an existing schema self.global_schema_id = value["id"] elif key["keytype"] == "DELETE_SUBJECT": self.log.info("Deleting subject: %r, value: %r", value["subject"], value) if not value["subject"] in self.subjects: self.log.error("Subject: %r did not exist, should have", value["subject"]) else: updated_schemas = { key: self._delete_schema_below_version(schema, value["version"]) for key, schema in self.subjects[ value["subject"]]["schemas"].items() } self.subjects[value["subject"]]["schemas"] = updated_schemas elif key["keytype"] == "NOOP": # for spec completeness pass @staticmethod def _delete_schema_below_version(schema, version): if schema["version"] <= version: schema["deleted"] = True return schema def get_schemas(self, subject): non_deleted_schemas = { key: val for key, val in self.subjects[subject]["schemas"].items() if val.get("deleted", False) is False } return non_deleted_schemas
class RestApp: def __init__(self, *, app_name, sentry_config): self.app_name = app_name self.app_request_metric = "{}_request".format(app_name) self.app = aiohttp.web.Application() self.app.on_startup.append(self.create_http_client) self.app.on_cleanup.append(self.cleanup_http_client) self.http_client_v = None self.http_client_no_v = None self.log = logging.getLogger(self.app_name) self.stats = StatsClient(sentry_config=sentry_config) self.raven_client = self.stats.raven_client self.app.on_cleanup.append(self.cleanup_stats_client) async def cleanup_stats_client(self, app): # pylint: disable=unused-argument self.stats.close() async def create_http_client(self, app): # pylint: disable=unused-argument no_v_conn = aiohttp.TCPConnector(ssl=False) self.http_client_no_v = aiohttp.ClientSession(connector=no_v_conn, headers={"User-Agent": SERVER_NAME}) self.http_client_v = aiohttp.ClientSession(headers={"User-Agent": SERVER_NAME}) async def cleanup_http_client(self, app): # pylint: disable=unused-argument if self.http_client_no_v: await self.http_client_no_v.close() if self.http_client_v: await self.http_client_v.close() @staticmethod def cors_and_server_headers_for_request(*, request, origin="*"): # pylint: disable=unused-argument return { "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Methods": "DELETE, GET, OPTIONS, POST, PUT", "Access-Control-Allow-Headers": "Authorization, Content-Type", "Server": SERVER_NAME, } def check_rest_headers(self, request: HTTPRequest) -> dict: # pylint:disable=inconsistent-return-statements method = request.method default_content = "application/vnd.kafka.json.v2+json" default_accept = "*/*" result: dict = {"content_type": default_content} content_matcher = REST_CONTENT_TYPE_RE.search( cgi.parse_header(request.get_header("Content-Type", default_content))[0] ) accept_matcher = REST_ACCEPT_RE.search(cgi.parse_header(request.get_header("Accept", default_accept))[0]) if method in {"POST", "PUT"}: if not content_matcher: http_error( message="HTTP 415 Unsupported Media Type", content_type=result["content_type"], code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE, ) if content_matcher and accept_matcher: header_info = content_matcher.groupdict() header_info["embedded_format"] = header_info.get("embedded_format") or "binary" result["requests"] = header_info result["accepts"] = accept_matcher.groupdict() return result self.log.error("Not acceptable: %r", request.get_header("accept")) http_error( message="HTTP 406 Not Acceptable", content_type=result["content_type"], code=HTTPStatus.NOT_ACCEPTABLE, ) def check_schema_headers(self, request: HTTPRequest): method = request.method response_default_content_type = "application/vnd.schemaregistry.v1+json" content_type = request.get_header("Content-Type", JSON_CONTENT_TYPE) if method in {"POST", "PUT"} and cgi.parse_header(content_type)[0] not in SCHEMA_CONTENT_TYPES: http_error( message="HTTP 415 Unsupported Media Type", content_type=response_default_content_type, code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE, ) accept_val = request.get_header("Accept") if accept_val: if accept_val in ("*/*", "*") or accept_val.startswith("*/"): return response_default_content_type content_type_match = get_best_match(accept_val, SCHEMA_ACCEPT_VALUES) if not content_type_match: self.log.debug("Unexpected Accept value: %r", accept_val) http_error( message="HTTP 406 Not Acceptable", content_type=response_default_content_type, code=HTTPStatus.NOT_ACCEPTABLE, ) return content_type_match return response_default_content_type async def _handle_request( self, *, request, path_for_stats, callback, schema_request=False, callback_with_request=False, json_request=False, rest_request=False ): start_time = time.monotonic() resp = None rapu_request = HTTPRequest( headers=request.headers, query=request.query, method=request.method, url=request.url, path_for_stats=path_for_stats, ) try: if request.method == "OPTIONS": origin = request.headers.get("Origin") if not origin: raise HTTPResponse(body="OPTIONS missing Origin", status=HTTPStatus.BAD_REQUEST) headers = self.cors_and_server_headers_for_request(request=rapu_request, origin=origin) raise HTTPResponse(body=b"", status=HTTPStatus.OK, headers=headers) body = await request.read() if json_request: if not body: raise HTTPResponse(body="Missing request JSON body", status=HTTPStatus.BAD_REQUEST) try: _, options = cgi.parse_header(rapu_request.get_header("Content-Type")) charset = options.get("charset", "utf-8") body_string = body.decode(charset) rapu_request.json = jsonlib.loads(body_string) except jsonlib.decoder.JSONDecodeError: raise HTTPResponse(body="Invalid request JSON body", status=HTTPStatus.BAD_REQUEST) except UnicodeDecodeError: raise HTTPResponse(body=f"Request body is not valid {charset}", status=HTTPStatus.BAD_REQUEST) except LookupError: raise HTTPResponse(body=f"Unknown charset {charset}", status=HTTPStatus.BAD_REQUEST) else: if body not in {b"", b"{}"}: raise HTTPResponse(body="No request body allowed for this operation", status=HTTPStatus.BAD_REQUEST) callback_kwargs = dict(request.match_info) if callback_with_request: callback_kwargs["request"] = rapu_request if rest_request: params = self.check_rest_headers(rapu_request) if "requests" in params: rapu_request.content_type = params["requests"] params.pop("requests") if "accepts" in params: rapu_request.accepts = params["accepts"] params.pop("accepts") callback_kwargs.update(params) if schema_request: content_type = self.check_schema_headers(rapu_request) callback_kwargs["content_type"] = content_type try: data = await callback(**callback_kwargs) status = HTTPStatus.OK headers = {} except HTTPResponse as ex: data = ex.body status = ex.status headers = ex.headers except: # pylint: disable=bare-except self.log.exception("Internal server error") data = {"error_code": HTTPStatus.INTERNAL_SERVER_ERROR.value, "message": "Internal server error"} status = HTTPStatus.INTERNAL_SERVER_ERROR headers = {} headers.update(self.cors_and_server_headers_for_request(request=rapu_request)) if isinstance(data, (dict, list)): resp_bytes = json_encode(data, binary=True, sort_keys=True, compact=True) elif isinstance(data, str): if "Content-Type" not in headers: headers["Content-Type"] = "text/plain; charset=utf-8" resp_bytes = data.encode("utf-8") else: resp_bytes = data # On 204 - NO CONTENT there is no point of calculating cache headers if is_success(status): if resp_bytes: etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest()) else: etag = '""' if_none_match = request.headers.get("if-none-match") if if_none_match and if_none_match.replace("W/", "") == etag: status = HTTPStatus.NOT_MODIFIED resp_bytes = b"" headers["access-control-expose-headers"] = "etag" headers["etag"] = etag resp = aiohttp.web.Response(body=resp_bytes, status=status.value, headers=headers) except HTTPResponse as ex: if isinstance(ex.body, str): resp = aiohttp.web.Response(text=ex.body, status=ex.status.value, headers=ex.headers) else: resp = aiohttp.web.Response(body=ex.body, status=ex.status.value, headers=ex.headers) except asyncio.CancelledError: self.log.debug("Client closed connection") raise except Exception as ex: # pylint: disable=broad-except self.stats.unexpected_exception(ex=ex, where="rapu_wrapped_callback") self.log.exception("Unexpected error handling user request: %s %s", request.method, request.url) resp = aiohttp.web.Response(text="Internal Server Error", status=HTTPStatus.INTERNAL_SERVER_ERROR.value) finally: self.stats.timing( self.app_request_metric, time.monotonic() - start_time, tags={ "path": path_for_stats, # no `resp` means that we had a failure in exception handler "result": resp.status if resp else 0, "method": request.method, } ) return resp def route(self, path, *, callback, method, schema_request=False, with_request=None, json_body=None, rest_request=False): # pretty path for statsd reporting path_for_stats = re.sub(r"<[\w:]+>", "x", path) # bottle compatible routing aio_route = path aio_route = re.sub(r"<(\w+):path>", r"{\1:.+}", aio_route) aio_route = re.sub(r"<(\w+)>", r"{\1}", aio_route) if (method in {"POST", "PUT"}) and with_request is None: with_request = True if with_request and json_body is None: json_body = True async def wrapped_callback(request): return await self._handle_request( request=request, path_for_stats=path_for_stats, callback=callback, schema_request=schema_request, callback_with_request=with_request, json_request=json_body, rest_request=rest_request ) async def wrapped_cors(request): return await self._handle_request( request=request, path_for_stats=path_for_stats, callback=None, ) if not aio_route.endswith("/"): self.app.router.add_route(method, aio_route + "/", wrapped_callback) self.app.router.add_route(method, aio_route, wrapped_callback) else: self.app.router.add_route(method, aio_route, wrapped_callback) self.app.router.add_route(method, aio_route[:-1], wrapped_callback) try: self.app.router.add_route("OPTIONS", aio_route, wrapped_cors) except RuntimeError as ex: if "Added route will never be executed, method OPTIONS is already registered" not in str(ex): raise async def http_request(self, url, *, method="GET", json=None, timeout=10.0, verify=True, proxy=None): close_session = False if isinstance(verify, str): sslcontext = ssl.create_default_context(cadata=verify) else: sslcontext = None if proxy: connector = aiohttp_socks.SocksConnector( socks_ver=aiohttp_socks.SocksVer.SOCKS5, host=proxy["host"], port=proxy["port"], username=proxy["username"], password=proxy["password"], rdns=False, verify_ssl=verify, ssl_context=sslcontext, ) session = aiohttp.ClientSession(connector=connector) close_session = True elif sslcontext: conn = aiohttp.TCPConnector(ssl_context=sslcontext) session = aiohttp.ClientSession(connector=conn) close_session = True elif verify is True: session = self.http_client_v elif verify is False: session = self.http_client_no_v else: raise ValueError("invalid arguments to http_request") func = getattr(session, method.lower()) try: with async_timeout.timeout(timeout): async with func(url, json=json) as response: if response.headers.get("content-type", "").startswith(JSON_CONTENT_TYPE): resp_content = await response.json() else: resp_content = await response.text() result = HTTPResponse(body=resp_content, status=HTTPStatus(response.status)) finally: if close_session: await session.close() return result def run(self, *, host, port): aiohttp.web.run_app( app=self.app, host=host, port=port, access_log_format='%Tfs %{x-client-ip}i "%r" %s "%{user-agent}i" response=%bb request_body=%{content-length}ib', ) def add_routes(self): pass # Override in sub-classes
class RestApp: def __init__(self, *, app_name, sentry_config): self.app_name = app_name self.app_request_metric = "{}_request".format(app_name) self.app = aiohttp.web.Application() self.app.on_startup.append(self.create_http_client) self.app.on_cleanup.append(self.cleanup_http_client) self.http_client_v = None self.http_client_no_v = None self.log = logging.getLogger(self.app_name) self.stats = StatsClient(sentry_config=sentry_config) self.raven_client = self.stats.raven_client self.app.on_cleanup.append(self.cleanup_stats_client) async def cleanup_stats_client(self, app): # pylint: disable=unused-argument self.stats.close() async def create_http_client(self, app): # pylint: disable=unused-argument no_v_conn = aiohttp.TCPConnector(verify_ssl=False) self.http_client_no_v = aiohttp.ClientSession( connector=no_v_conn, headers={"User-Agent": SERVER_NAME}) self.http_client_v = aiohttp.ClientSession( headers={"User-Agent": SERVER_NAME}) async def cleanup_http_client(self, app): # pylint: disable=unused-argument if self.http_client_no_v: await self.http_client_no_v.close() if self.http_client_v: await self.http_client_v.close() @staticmethod def cors_and_server_headers_for_request(*, request, origin="*"): # pylint: disable=unused-argument return { "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Methods": "DELETE, GET, OPTIONS, POST, PUT", "Access-Control-Allow-Headers": "Authorization, Content-Type", "Server": SERVER_NAME, } def check_schema_headers(self, request): method = request.method headers = request.headers content_type = "application/vnd.schemaregistry.v1+json" if method in { "POST", "PUT" } and headers["Content-Type"] not in ACCEPTED_SCHEMA_CONTENT_TYPES: raise HTTPResponse( body=json_encode( { "error_code": 415, "message": "HTTP 415 Unsupported Media Type", }, binary=True), headers={"Content-Type": content_type}, status=415, ) if "Accept" in headers: if headers["Accept"] == "*/*" or headers["Accept"].startswith( "*/"): return "application/vnd.schemaregistry.v1+json" content_type_match = get_best_match(headers["Accept"], ACCEPTED_SCHEMA_CONTENT_TYPES) if not content_type_match: self.log.debug("Unexpected Accept value: %r", headers["Accept"]) raise HTTPResponse( body=json_encode( { "error_code": 406, "message": "HTTP 406 Not Acceptable", }, binary=True), headers={"Content-Type": content_type}, status=406, ) return content_type_match return content_type async def _handle_request(self, *, request, path_for_stats, callback, schema_request=False, callback_with_request=False, json_request=False): start_time = time.monotonic() resp = None rapu_request = HTTPRequest( headers=request.headers, query=request.query, method=request.method, url=request.url, path_for_stats=path_for_stats, ) try: if request.method == "OPTIONS": origin = request.headers.get("Origin") if not origin: raise HTTPResponse(body="OPTIONS missing Origin", status=400) headers = self.cors_and_server_headers_for_request( request=rapu_request, origin=origin) raise HTTPResponse(body=b"", status=200, headers=headers) body = await request.read() if json_request: if not body: raise HTTPResponse(body="Missing request JSON body", status=400) if request.charset and request.charset.lower( ) != "utf-8" and request.charset.lower() != "utf8": raise HTTPResponse( body="Request character set must be UTF-8", status=400) try: body_string = body.decode("utf-8") rapu_request.json = jsonlib.loads(body_string) except jsonlib.decoder.JSONDecodeError: raise HTTPResponse(body="Invalid request JSON body", status=400) except UnicodeDecodeError: raise HTTPResponse(body="Request body is not valid UTF-8", status=400) else: if body not in {b"", b"{}"}: raise HTTPResponse( body="No request body allowed for this operation", status=400) callback_kwargs = dict(request.match_info) if callback_with_request: callback_kwargs["request"] = rapu_request if schema_request: content_type = self.check_schema_headers(request) callback_kwargs["content_type"] = content_type try: data = await callback(**callback_kwargs) status = 200 headers = {} except HTTPResponse as ex: data = ex.body status = ex.status headers = ex.headers headers.update( self.cors_and_server_headers_for_request(request=rapu_request)) if isinstance(data, (dict, list)): resp_bytes = json_encode(data, binary=True, sort_keys=True, compact=True) elif isinstance(data, str): if "Content-Type" not in headers: headers["Content-Type"] = "text/plain; charset=utf-8" resp_bytes = data.encode("utf-8") else: resp_bytes = data # On 204 - NO CONTENT there is no point of calculating cache headers if 200 >= status <= 299: if resp_bytes: etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest()) else: etag = '""' if_none_match = request.headers.get("if-none-match") if if_none_match and if_none_match.replace("W/", "") == etag: status = 304 resp_bytes = b"" headers["access-control-expose-headers"] = "etag" headers["etag"] = etag resp = aiohttp.web.Response(body=resp_bytes, status=status, headers=headers) except HTTPResponse as ex: if isinstance(ex.body, str): resp = aiohttp.web.Response(text=ex.body, status=ex.status, headers=ex.headers) else: resp = aiohttp.web.Response(body=ex.body, status=ex.status, headers=ex.headers) except asyncio.CancelledError: self.log.debug("Client closed connection") raise except Exception as ex: # pylint: disable=broad-except self.stats.unexpected_exception(ex=ex, where="rapu_wrapped_callback") self.log.exception("Unexpected error handling user request: %s %s", request.method, request.url) resp = aiohttp.web.Response(text="Internal Server Error", status=500) finally: self.stats.timing( self.app_request_metric, time.monotonic() - start_time, tags={ "path": path_for_stats, # no `resp` means that we had a failure in exception handler "result": resp.status if resp else 0, "method": request.method, }) return resp def route(self, path, *, callback, method, schema_request=False, with_request=None, json_body=None): # pretty path for statsd reporting path_for_stats = re.sub(r"<[\w:]+>", "x", path) # bottle compatible routing aio_route = path aio_route = re.sub(r"<(\w+):path>", r"{\1:.+}", aio_route) aio_route = re.sub(r"<(\w+)>", r"{\1}", aio_route) if (method in {"POST", "PUT"}) and with_request is None: with_request = True if with_request and json_body is None: json_body = True async def wrapped_callback(request): return await self._handle_request( request=request, path_for_stats=path_for_stats, callback=callback, schema_request=schema_request, callback_with_request=with_request, json_request=json_body, ) async def wrapped_cors(request): return await self._handle_request( request=request, path_for_stats=path_for_stats, callback=None, ) self.app.router.add_route(method, aio_route, wrapped_callback) try: self.app.router.add_route("OPTIONS", aio_route, wrapped_cors) except RuntimeError as ex: if "Added route will never be executed, method OPTIONS is already registered" not in str( ex): raise async def http_request(self, url, *, method="GET", json=None, timeout=10.0, verify=True, proxy=None): close_session = False if isinstance(verify, str): sslcontext = ssl.create_default_context(cadata=verify) else: sslcontext = None if proxy: connector = aiohttp_socks.SocksConnector( socks_ver=aiohttp_socks.SocksVer.SOCKS5, host=proxy["host"], port=proxy["port"], username=proxy["username"], password=proxy["password"], rdns=False, verify_ssl=verify, ssl_context=sslcontext, ) session = aiohttp.ClientSession(connector=connector) close_session = True elif sslcontext: conn = aiohttp.TCPConnector(ssl_context=sslcontext) session = aiohttp.ClientSession(connector=conn) close_session = True elif verify is True: session = self.http_client_v elif verify is False: session = self.http_client_no_v else: raise ValueError("invalid arguments to http_request") func = getattr(session, method.lower()) try: with async_timeout.timeout(timeout): async with func(url, json=json) as response: if response.headers.get("content-type", "").startswith("application/json"): resp_content = await response.json() else: resp_content = await response.text() result = HTTPResponse(body=resp_content, status=response.status) finally: if close_session: await session.close() return result def run(self, *, host, port): aiohttp.web.run_app( app=self.app, host=host, port=port, access_log_format= '%Tfs %{x-client-ip}i "%r" %s "%{user-agent}i" response=%bb request_body=%{content-length}ib', ) def add_routes(self): pass # Override in sub-classes