def __init__(self, *, app_name, sentry_config):
self.app_name = app_name
self.app_request_metric = "{}_request".format(app_name)
self.app = aiohttp.web.Application()
self.app.on_startup.append(self.create_http_client)
self.app.on_cleanup.append(self.cleanup_http_client)
self.http_client_v = None
self.http_client_no_v = None
self.log = logging.getLogger(self.app_name)
self.stats = StatsClient(sentry_config=sentry_config)
self.raven_client = self.stats.raven_client
self.app.on_cleanup.append(self.cleanup_stats_client)
def __init__(self, config, master_coordinator=None):
Thread.__init__(self)
self.master_coordinator = master_coordinator
self.log = logging.getLogger("KafkaSchemaReader")
self.timeout_ms = 200
self.config = config
self.subjects = {}
self.schemas: Dict[int, TypedSchema] = {}
self.global_schema_id = 0
self.offset = 0
self.admin_client = None
self.schema_topic = None
self.topic_replication_factor = self.config["replication_factor"]
self.consumer = None
self.queue = Queue()
self.ready = False
self.running = True
self.id_lock = Lock()
sentry_config = config.get("sentry", {"dsn": None}).copy()
if "tags" not in sentry_config:
sentry_config["tags"] = {}
self.stats = StatsClient(sentry_config=sentry_config)
class RestApp:
def __init__(self, *, app_name, sentry_config):
self.app_name = app_name
self.app_request_metric = "{}_request".format(app_name)
self.app = aiohttp.web.Application()
self.app.on_startup.append(self.create_http_client)
self.app.on_cleanup.append(self.cleanup_http_client)
self.http_client_v = None
self.http_client_no_v = None
self.log = logging.getLogger(self.app_name)
self.stats = StatsClient(sentry_config=sentry_config)
self.raven_client = self.stats.raven_client
self.app.on_cleanup.append(self.cleanup_stats_client)
async def cleanup_stats_client(self, app): # pylint: disable=unused-argument
self.stats.close()
async def create_http_client(self, app): # pylint: disable=unused-argument
no_v_conn = aiohttp.TCPConnector(ssl=False)
self.http_client_no_v = aiohttp.ClientSession(connector=no_v_conn, headers={"User-Agent": SERVER_NAME})
self.http_client_v = aiohttp.ClientSession(headers={"User-Agent": SERVER_NAME})
async def cleanup_http_client(self, app): # pylint: disable=unused-argument
if self.http_client_no_v:
await self.http_client_no_v.close()
if self.http_client_v:
await self.http_client_v.close()
@staticmethod
def cors_and_server_headers_for_request(*, request, origin="*"): # pylint: disable=unused-argument
return {
"Access-Control-Allow-Origin": origin,
"Access-Control-Allow-Methods": "DELETE, GET, OPTIONS, POST, PUT",
"Access-Control-Allow-Headers": "Authorization, Content-Type",
"Server": SERVER_NAME,
}
def check_rest_headers(self, request: HTTPRequest) -> dict: # pylint:disable=inconsistent-return-statements
method = request.method
default_content = "application/vnd.kafka.json.v2+json"
default_accept = "*/*"
result: dict = {"content_type": default_content}
content_matcher = REST_CONTENT_TYPE_RE.search(
cgi.parse_header(request.get_header("Content-Type", default_content))[0]
)
accept_matcher = REST_ACCEPT_RE.search(cgi.parse_header(request.get_header("Accept", default_accept))[0])
if method in {"POST", "PUT"}:
if not content_matcher:
http_error(
message="HTTP 415 Unsupported Media Type",
content_type=result["content_type"],
code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
)
if content_matcher and accept_matcher:
header_info = content_matcher.groupdict()
header_info["embedded_format"] = header_info.get("embedded_format") or "binary"
result["requests"] = header_info
result["accepts"] = accept_matcher.groupdict()
return result
self.log.error("Not acceptable: %r", request.get_header("accept"))
http_error(
message="HTTP 406 Not Acceptable",
content_type=result["content_type"],
code=HTTPStatus.NOT_ACCEPTABLE,
)
def check_schema_headers(self, request: HTTPRequest):
method = request.method
response_default_content_type = "application/vnd.schemaregistry.v1+json"
content_type = request.get_header("Content-Type", JSON_CONTENT_TYPE)
if method in {"POST", "PUT"} and cgi.parse_header(content_type)[0] not in SCHEMA_CONTENT_TYPES:
http_error(
message="HTTP 415 Unsupported Media Type",
content_type=response_default_content_type,
code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
)
accept_val = request.get_header("Accept")
if accept_val:
if accept_val in ("*/*", "*") or accept_val.startswith("*/"):
return response_default_content_type
content_type_match = get_best_match(accept_val, SCHEMA_ACCEPT_VALUES)
if not content_type_match:
self.log.debug("Unexpected Accept value: %r", accept_val)
http_error(
message="HTTP 406 Not Acceptable",
content_type=response_default_content_type,
code=HTTPStatus.NOT_ACCEPTABLE,
)
return content_type_match
return response_default_content_type
async def _handle_request(
self,
*,
request,
path_for_stats,
callback,
schema_request=False,
callback_with_request=False,
json_request=False,
rest_request=False
):
start_time = time.monotonic()
resp = None
rapu_request = HTTPRequest(
headers=request.headers,
query=request.query,
method=request.method,
url=request.url,
path_for_stats=path_for_stats,
)
try:
if request.method == "OPTIONS":
origin = request.headers.get("Origin")
if not origin:
raise HTTPResponse(body="OPTIONS missing Origin", status=HTTPStatus.BAD_REQUEST)
headers = self.cors_and_server_headers_for_request(request=rapu_request, origin=origin)
raise HTTPResponse(body=b"", status=HTTPStatus.OK, headers=headers)
body = await request.read()
if json_request:
if not body:
raise HTTPResponse(body="Missing request JSON body", status=HTTPStatus.BAD_REQUEST)
try:
_, options = cgi.parse_header(rapu_request.get_header("Content-Type"))
charset = options.get("charset", "utf-8")
body_string = body.decode(charset)
rapu_request.json = jsonlib.loads(body_string)
except jsonlib.decoder.JSONDecodeError:
raise HTTPResponse(body="Invalid request JSON body", status=HTTPStatus.BAD_REQUEST)
except UnicodeDecodeError:
raise HTTPResponse(body=f"Request body is not valid {charset}", status=HTTPStatus.BAD_REQUEST)
except LookupError:
raise HTTPResponse(body=f"Unknown charset {charset}", status=HTTPStatus.BAD_REQUEST)
else:
if body not in {b"", b"{}"}:
raise HTTPResponse(body="No request body allowed for this operation", status=HTTPStatus.BAD_REQUEST)
callback_kwargs = dict(request.match_info)
if callback_with_request:
callback_kwargs["request"] = rapu_request
if rest_request:
params = self.check_rest_headers(rapu_request)
if "requests" in params:
rapu_request.content_type = params["requests"]
params.pop("requests")
if "accepts" in params:
rapu_request.accepts = params["accepts"]
params.pop("accepts")
callback_kwargs.update(params)
if schema_request:
content_type = self.check_schema_headers(rapu_request)
callback_kwargs["content_type"] = content_type
try:
data = await callback(**callback_kwargs)
status = HTTPStatus.OK
headers = {}
except HTTPResponse as ex:
data = ex.body
status = ex.status
headers = ex.headers
except: # pylint: disable=bare-except
self.log.exception("Internal server error")
data = {"error_code": HTTPStatus.INTERNAL_SERVER_ERROR.value, "message": "Internal server error"}
status = HTTPStatus.INTERNAL_SERVER_ERROR
headers = {}
headers.update(self.cors_and_server_headers_for_request(request=rapu_request))
if isinstance(data, (dict, list)):
resp_bytes = json_encode(data, binary=True, sort_keys=True, compact=True)
elif isinstance(data, str):
if "Content-Type" not in headers:
headers["Content-Type"] = "text/plain; charset=utf-8"
resp_bytes = data.encode("utf-8")
else:
resp_bytes = data
# On 204 - NO CONTENT there is no point of calculating cache headers
if is_success(status):
if resp_bytes:
etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest())
else:
etag = '""'
if_none_match = request.headers.get("if-none-match")
if if_none_match and if_none_match.replace("W/", "") == etag:
status = HTTPStatus.NOT_MODIFIED
resp_bytes = b""
headers["access-control-expose-headers"] = "etag"
headers["etag"] = etag
resp = aiohttp.web.Response(body=resp_bytes, status=status.value, headers=headers)
except HTTPResponse as ex:
if isinstance(ex.body, str):
resp = aiohttp.web.Response(text=ex.body, status=ex.status.value, headers=ex.headers)
else:
resp = aiohttp.web.Response(body=ex.body, status=ex.status.value, headers=ex.headers)
except asyncio.CancelledError:
self.log.debug("Client closed connection")
raise
except Exception as ex: # pylint: disable=broad-except
self.stats.unexpected_exception(ex=ex, where="rapu_wrapped_callback")
self.log.exception("Unexpected error handling user request: %s %s", request.method, request.url)
resp = aiohttp.web.Response(text="Internal Server Error", status=HTTPStatus.INTERNAL_SERVER_ERROR.value)
finally:
self.stats.timing(
self.app_request_metric,
time.monotonic() - start_time,
tags={
"path": path_for_stats,
# no `resp` means that we had a failure in exception handler
"result": resp.status if resp else 0,
"method": request.method,
}
)
return resp
def route(self, path, *, callback, method, schema_request=False, with_request=None, json_body=None, rest_request=False):
# pretty path for statsd reporting
path_for_stats = re.sub(r"<[\w:]+>", "x", path)
# bottle compatible routing
aio_route = path
aio_route = re.sub(r"<(\w+):path>", r"{\1:.+}", aio_route)
aio_route = re.sub(r"<(\w+)>", r"{\1}", aio_route)
if (method in {"POST", "PUT"}) and with_request is None:
with_request = True
if with_request and json_body is None:
json_body = True
async def wrapped_callback(request):
return await self._handle_request(
request=request,
path_for_stats=path_for_stats,
callback=callback,
schema_request=schema_request,
callback_with_request=with_request,
json_request=json_body,
rest_request=rest_request
)
async def wrapped_cors(request):
return await self._handle_request(
request=request,
path_for_stats=path_for_stats,
callback=None,
)
if not aio_route.endswith("/"):
self.app.router.add_route(method, aio_route + "/", wrapped_callback)
self.app.router.add_route(method, aio_route, wrapped_callback)
else:
self.app.router.add_route(method, aio_route, wrapped_callback)
self.app.router.add_route(method, aio_route[:-1], wrapped_callback)
try:
self.app.router.add_route("OPTIONS", aio_route, wrapped_cors)
except RuntimeError as ex:
if "Added route will never be executed, method OPTIONS is already registered" not in str(ex):
raise
async def http_request(self, url, *, method="GET", json=None, timeout=10.0, verify=True, proxy=None):
close_session = False
if isinstance(verify, str):
sslcontext = ssl.create_default_context(cadata=verify)
else:
sslcontext = None
if proxy:
connector = aiohttp_socks.SocksConnector(
socks_ver=aiohttp_socks.SocksVer.SOCKS5,
host=proxy["host"],
port=proxy["port"],
username=proxy["username"],
password=proxy["password"],
rdns=False,
verify_ssl=verify,
ssl_context=sslcontext,
)
session = aiohttp.ClientSession(connector=connector)
close_session = True
elif sslcontext:
conn = aiohttp.TCPConnector(ssl_context=sslcontext)
session = aiohttp.ClientSession(connector=conn)
close_session = True
elif verify is True:
session = self.http_client_v
elif verify is False:
session = self.http_client_no_v
else:
raise ValueError("invalid arguments to http_request")
func = getattr(session, method.lower())
try:
with async_timeout.timeout(timeout):
async with func(url, json=json) as response:
if response.headers.get("content-type", "").startswith(JSON_CONTENT_TYPE):
resp_content = await response.json()
else:
resp_content = await response.text()
result = HTTPResponse(body=resp_content, status=HTTPStatus(response.status))
finally:
if close_session:
await session.close()
return result
def run(self, *, host, port):
aiohttp.web.run_app(
app=self.app,
host=host,
port=port,
access_log_format='%Tfs %{x-client-ip}i "%r" %s "%{user-agent}i" response=%bb request_body=%{content-length}ib',
)
def add_routes(self):
pass # Override in sub-classes
class KafkaSchemaReader(Thread):
def __init__(self, config, master_coordinator=None):
Thread.__init__(self)
self.master_coordinator = master_coordinator
self.log = logging.getLogger("KafkaSchemaReader")
self.timeout_ms = 200
self.config = config
self.subjects = {}
self.schemas: Dict[int, TypedSchema] = {}
self.global_schema_id = 0
self.offset = 0
self.admin_client = None
self.schema_topic = None
self.topic_replication_factor = self.config["replication_factor"]
self.consumer = None
self.queue = Queue()
self.ready = False
self.running = True
self.id_lock = Lock()
sentry_config = config.get("sentry", {"dsn": None}).copy()
if "tags" not in sentry_config:
sentry_config["tags"] = {}
self.stats = StatsClient(sentry_config=sentry_config)
def init_consumer(self):
# Group not set on purpose, all consumers read the same data
session_timeout_ms = self.config["session_timeout_ms"]
request_timeout_ms = max(
session_timeout_ms,
KafkaConsumer.DEFAULT_CONFIG["request_timeout_ms"])
self.consumer = KafkaConsumer(
self.config["topic_name"],
enable_auto_commit=False,
api_version=(1, 0, 0),
bootstrap_servers=self.config["bootstrap_uri"],
client_id=self.config["client_id"],
security_protocol=self.config["security_protocol"],
ssl_cafile=self.config["ssl_cafile"],
ssl_certfile=self.config["ssl_certfile"],
ssl_keyfile=self.config["ssl_keyfile"],
auto_offset_reset="earliest",
session_timeout_ms=session_timeout_ms,
request_timeout_ms=request_timeout_ms,
kafka_client=KarapaceKafkaClient,
metadata_max_age_ms=self.config["metadata_max_age_ms"],
)
def init_admin_client(self):
try:
self.admin_client = KafkaAdminClient(
api_version_auto_timeout_ms=constants.
API_VERSION_AUTO_TIMEOUT_MS,
bootstrap_servers=self.config["bootstrap_uri"],
client_id=self.config["client_id"],
security_protocol=self.config["security_protocol"],
ssl_cafile=self.config["ssl_cafile"],
ssl_certfile=self.config["ssl_certfile"],
ssl_keyfile=self.config["ssl_keyfile"],
)
return True
except (NodeNotReadyError, NoBrokersAvailable, AssertionError):
self.log.warning(
"No Brokers available yet, retrying init_admin_client()")
time.sleep(2.0)
except: # pylint: disable=bare-except
self.log.exception(
"Failed to initialize admin client, retrying init_admin_client()"
)
time.sleep(2.0)
return False
@staticmethod
def get_new_schema_topic(config):
return NewTopic(name=config["topic_name"],
num_partitions=constants.SCHEMA_TOPIC_NUM_PARTITIONS,
replication_factor=config["replication_factor"],
topic_configs={"cleanup.policy": "compact"})
def create_schema_topic(self):
schema_topic = self.get_new_schema_topic(self.config)
try:
self.log.info("Creating topic: %r", schema_topic)
self.admin_client.create_topics(
[schema_topic], timeout_ms=constants.TOPIC_CREATION_TIMEOUT_MS)
self.log.info("Topic: %r created successfully",
self.config["topic_name"])
self.schema_topic = schema_topic
return True
except TopicAlreadyExistsError:
self.log.warning("Topic: %r already exists",
self.config["topic_name"])
self.schema_topic = schema_topic
return True
except: # pylint: disable=bare-except
self.log.exception(
"Failed to create topic: %r, retrying create_schema_topic()",
self.config["topic_name"])
time.sleep(5)
return False
def get_schema_id(self, new_schema):
with self.id_lock:
schemas = self.schemas.items()
for schema_id, schema in schemas:
if schema == new_schema:
return schema_id
with self.id_lock:
self.global_schema_id += 1
return self.global_schema_id
def close(self):
self.log.info("Closing schema_reader")
self.running = False
def run(self):
while self.running:
try:
if not self.admin_client:
if self.init_admin_client() is False:
continue
if not self.schema_topic:
if self.create_schema_topic() is False:
continue
if not self.consumer:
self.init_consumer()
self.handle_messages()
except Exception as e: # pylint: disable=broad-except
if self.stats:
self.stats.unexpected_exception(ex=e,
where="schema_reader_loop")
self.log.exception(
"Unexpected exception in schema reader loop")
try:
if self.admin_client:
self.admin_client.close()
if self.consumer:
self.consumer.close()
except Exception as e: # pylint: disable=broad-except
if self.stats:
self.stats.unexpected_exception(ex=e,
where="schema_reader_exit")
self.log.exception("Unexpected exception closing schema reader")
def handle_messages(self):
raw_msgs = self.consumer.poll(timeout_ms=self.timeout_ms)
if self.ready is False and raw_msgs == {}:
self.ready = True
add_offsets = False
if self.master_coordinator is not None:
master, _ = self.master_coordinator.get_master_info()
# keep old behavior for True. When master is False, then we are a follower, so we should not accept direct
# writes anyway. When master is None, then this particular node is waiting for a stable value, so any
# messages off the topic are writes performed by another node
if master is True:
add_offsets = True
for _, msgs in raw_msgs.items():
for msg in msgs:
try:
key = json.loads(msg.key.decode("utf8"))
except json.JSONDecodeError:
self.log.exception(
"Invalid JSON in msg.key: %r, value: %r", msg.key,
msg.value)
continue
value = None
if msg.value:
try:
value = json.loads(msg.value.decode("utf8"))
except json.JSONDecodeError:
self.log.exception(
"Invalid JSON in msg.value: %r, key: %r",
msg.value, msg.key)
continue
self.log.info(
"Read new record: key: %r, value: %r, offset: %r", key,
value, msg.offset)
self.handle_msg(key, value)
self.offset = msg.offset
self.log.info("Handled message, current offset: %r",
self.offset)
if self.ready and add_offsets:
self.queue.put(self.offset)
def handle_msg(self, key: dict, value: dict):
if key["keytype"] == "CONFIG":
if "subject" in key and key["subject"] is not None:
if not value:
self.log.info(
"Deleting compatibility config completely for subject: %r",
key["subject"])
self.subjects[key["subject"]].pop("compatibility", None)
return
self.log.info("Setting subject: %r config to: %r, value: %r",
key["subject"], value["compatibilityLevel"],
value)
if not key["subject"] in self.subjects:
self.log.info(
"Adding first version of subject: %r with no schemas",
key["subject"])
self.subjects[key["subject"]] = {"schemas": {}}
subject_data = self.subjects.get(key["subject"])
subject_data["compatibility"] = value["compatibilityLevel"]
else:
self.log.info("Setting global config to: %r, value: %r",
value["compatibilityLevel"], value)
self.config["compatibility"] = value["compatibilityLevel"]
elif key["keytype"] == "SCHEMA":
if not value:
subject, version = key["subject"], key["version"]
self.log.info("Deleting subject: %r version: %r completely",
subject, version)
if subject not in self.subjects:
self.log.error("Subject %s did not exist, should have",
subject)
elif version not in self.subjects[subject]["schemas"]:
self.log.error(
"Version %d for subject %s did not exist, should have",
version, subject)
else:
self.subjects[subject]["schemas"].pop(version, None)
return
schema_type = value.get("schemaType", "AVRO")
schema_str = value["schema"]
try:
typed_schema = TypedSchema.parse(
schema_type=SchemaType(schema_type), schema_str=schema_str)
except InvalidSchema:
try:
schema_json = json.loads(schema_str)
typed_schema = TypedSchema(
schema_type=SchemaType(schema_type),
schema=schema_json,
schema_str=schema_str)
except JSONDecodeError:
self.log.error("Invalid json: %s", value["schema"])
return
self.log.debug("Got typed schema %r", typed_schema)
subject = value["subject"]
if subject not in self.subjects:
self.log.info("Adding first version of subject: %r, value: %r",
subject, value)
self.subjects[subject] = {
"schemas": {
value["version"]: {
"schema": typed_schema,
"version": value["version"],
"id": value["id"],
"deleted": value.get("deleted", False),
}
}
}
self.log.info("Setting schema_id: %r with schema: %r",
value["id"], typed_schema)
self.schemas[value["id"]] = typed_schema
if value["id"] > self.global_schema_id: # Not an existing schema
self.global_schema_id = value["id"]
elif value.get("deleted", False) is True:
self.log.info("Deleting subject: %r, version: %r", subject,
value["version"])
if not value["version"] in self.subjects[subject]["schemas"]:
self.schemas[value["id"]] = typed_schema
else:
self.subjects[subject]["schemas"][
value["version"]]["deleted"] = True
elif value.get("deleted", False) is False:
self.log.info("Adding new version of subject: %r, value: %r",
subject, value)
self.subjects[subject]["schemas"][value["version"]] = {
"schema": typed_schema,
"version": value["version"],
"id": value["id"],
"deleted": value.get("deleted", False),
}
self.log.info("Setting schema_id: %r with schema: %r",
value["id"], value["schema"])
with self.id_lock:
self.schemas[value["id"]] = typed_schema
if value["id"] > self.global_schema_id: # Not an existing schema
self.global_schema_id = value["id"]
elif key["keytype"] == "DELETE_SUBJECT":
self.log.info("Deleting subject: %r, value: %r", value["subject"],
value)
if not value["subject"] in self.subjects:
self.log.error("Subject: %r did not exist, should have",
value["subject"])
else:
updated_schemas = {
key:
self._delete_schema_below_version(schema, value["version"])
for key, schema in self.subjects[
value["subject"]]["schemas"].items()
}
self.subjects[value["subject"]]["schemas"] = updated_schemas
elif key["keytype"] == "NOOP": # for spec completeness
pass
@staticmethod
def _delete_schema_below_version(schema, version):
if schema["version"] <= version:
schema["deleted"] = True
return schema
def get_schemas(self, subject):
non_deleted_schemas = {
key: val
for key, val in self.subjects[subject]["schemas"].items()
if val.get("deleted", False) is False
}
return non_deleted_schemas
class RestApp:
def __init__(self, *, app_name, sentry_config):
self.app_name = app_name
self.app_request_metric = "{}_request".format(app_name)
self.app = aiohttp.web.Application()
self.app.on_startup.append(self.create_http_client)
self.app.on_cleanup.append(self.cleanup_http_client)
self.http_client_v = None
self.http_client_no_v = None
self.log = logging.getLogger(self.app_name)
self.stats = StatsClient(sentry_config=sentry_config)
self.raven_client = self.stats.raven_client
self.app.on_cleanup.append(self.cleanup_stats_client)
async def cleanup_stats_client(self, app): # pylint: disable=unused-argument
self.stats.close()
async def create_http_client(self, app): # pylint: disable=unused-argument
no_v_conn = aiohttp.TCPConnector(verify_ssl=False)
self.http_client_no_v = aiohttp.ClientSession(
connector=no_v_conn, headers={"User-Agent": SERVER_NAME})
self.http_client_v = aiohttp.ClientSession(
headers={"User-Agent": SERVER_NAME})
async def cleanup_http_client(self, app): # pylint: disable=unused-argument
if self.http_client_no_v:
await self.http_client_no_v.close()
if self.http_client_v:
await self.http_client_v.close()
@staticmethod
def cors_and_server_headers_for_request(*, request, origin="*"): # pylint: disable=unused-argument
return {
"Access-Control-Allow-Origin": origin,
"Access-Control-Allow-Methods": "DELETE, GET, OPTIONS, POST, PUT",
"Access-Control-Allow-Headers": "Authorization, Content-Type",
"Server": SERVER_NAME,
}
def check_schema_headers(self, request):
method = request.method
headers = request.headers
content_type = "application/vnd.schemaregistry.v1+json"
if method in {
"POST", "PUT"
} and headers["Content-Type"] not in ACCEPTED_SCHEMA_CONTENT_TYPES:
raise HTTPResponse(
body=json_encode(
{
"error_code": 415,
"message": "HTTP 415 Unsupported Media Type",
},
binary=True),
headers={"Content-Type": content_type},
status=415,
)
if "Accept" in headers:
if headers["Accept"] == "*/*" or headers["Accept"].startswith(
"*/"):
return "application/vnd.schemaregistry.v1+json"
content_type_match = get_best_match(headers["Accept"],
ACCEPTED_SCHEMA_CONTENT_TYPES)
if not content_type_match:
self.log.debug("Unexpected Accept value: %r",
headers["Accept"])
raise HTTPResponse(
body=json_encode(
{
"error_code": 406,
"message": "HTTP 406 Not Acceptable",
},
binary=True),
headers={"Content-Type": content_type},
status=406,
)
return content_type_match
return content_type
async def _handle_request(self,
*,
request,
path_for_stats,
callback,
schema_request=False,
callback_with_request=False,
json_request=False):
start_time = time.monotonic()
resp = None
rapu_request = HTTPRequest(
headers=request.headers,
query=request.query,
method=request.method,
url=request.url,
path_for_stats=path_for_stats,
)
try:
if request.method == "OPTIONS":
origin = request.headers.get("Origin")
if not origin:
raise HTTPResponse(body="OPTIONS missing Origin",
status=400)
headers = self.cors_and_server_headers_for_request(
request=rapu_request, origin=origin)
raise HTTPResponse(body=b"", status=200, headers=headers)
body = await request.read()
if json_request:
if not body:
raise HTTPResponse(body="Missing request JSON body",
status=400)
if request.charset and request.charset.lower(
) != "utf-8" and request.charset.lower() != "utf8":
raise HTTPResponse(
body="Request character set must be UTF-8", status=400)
try:
body_string = body.decode("utf-8")
rapu_request.json = jsonlib.loads(body_string)
except jsonlib.decoder.JSONDecodeError:
raise HTTPResponse(body="Invalid request JSON body",
status=400)
except UnicodeDecodeError:
raise HTTPResponse(body="Request body is not valid UTF-8",
status=400)
else:
if body not in {b"", b"{}"}:
raise HTTPResponse(
body="No request body allowed for this operation",
status=400)
callback_kwargs = dict(request.match_info)
if callback_with_request:
callback_kwargs["request"] = rapu_request
if schema_request:
content_type = self.check_schema_headers(request)
callback_kwargs["content_type"] = content_type
try:
data = await callback(**callback_kwargs)
status = 200
headers = {}
except HTTPResponse as ex:
data = ex.body
status = ex.status
headers = ex.headers
headers.update(
self.cors_and_server_headers_for_request(request=rapu_request))
if isinstance(data, (dict, list)):
resp_bytes = json_encode(data,
binary=True,
sort_keys=True,
compact=True)
elif isinstance(data, str):
if "Content-Type" not in headers:
headers["Content-Type"] = "text/plain; charset=utf-8"
resp_bytes = data.encode("utf-8")
else:
resp_bytes = data
# On 204 - NO CONTENT there is no point of calculating cache headers
if 200 >= status <= 299:
if resp_bytes:
etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest())
else:
etag = '""'
if_none_match = request.headers.get("if-none-match")
if if_none_match and if_none_match.replace("W/", "") == etag:
status = 304
resp_bytes = b""
headers["access-control-expose-headers"] = "etag"
headers["etag"] = etag
resp = aiohttp.web.Response(body=resp_bytes,
status=status,
headers=headers)
except HTTPResponse as ex:
if isinstance(ex.body, str):
resp = aiohttp.web.Response(text=ex.body,
status=ex.status,
headers=ex.headers)
else:
resp = aiohttp.web.Response(body=ex.body,
status=ex.status,
headers=ex.headers)
except asyncio.CancelledError:
self.log.debug("Client closed connection")
raise
except Exception as ex: # pylint: disable=broad-except
self.stats.unexpected_exception(ex=ex,
where="rapu_wrapped_callback")
self.log.exception("Unexpected error handling user request: %s %s",
request.method, request.url)
resp = aiohttp.web.Response(text="Internal Server Error",
status=500)
finally:
self.stats.timing(
self.app_request_metric,
time.monotonic() - start_time,
tags={
"path": path_for_stats,
# no `resp` means that we had a failure in exception handler
"result": resp.status if resp else 0,
"method": request.method,
})
return resp
def route(self,
path,
*,
callback,
method,
schema_request=False,
with_request=None,
json_body=None):
# pretty path for statsd reporting
path_for_stats = re.sub(r"<[\w:]+>", "x", path)
# bottle compatible routing
aio_route = path
aio_route = re.sub(r"<(\w+):path>", r"{\1:.+}", aio_route)
aio_route = re.sub(r"<(\w+)>", r"{\1}", aio_route)
if (method in {"POST", "PUT"}) and with_request is None:
with_request = True
if with_request and json_body is None:
json_body = True
async def wrapped_callback(request):
return await self._handle_request(
request=request,
path_for_stats=path_for_stats,
callback=callback,
schema_request=schema_request,
callback_with_request=with_request,
json_request=json_body,
)
async def wrapped_cors(request):
return await self._handle_request(
request=request,
path_for_stats=path_for_stats,
callback=None,
)
self.app.router.add_route(method, aio_route, wrapped_callback)
try:
self.app.router.add_route("OPTIONS", aio_route, wrapped_cors)
except RuntimeError as ex:
if "Added route will never be executed, method OPTIONS is already registered" not in str(
ex):
raise
async def http_request(self,
url,
*,
method="GET",
json=None,
timeout=10.0,
verify=True,
proxy=None):
close_session = False
if isinstance(verify, str):
sslcontext = ssl.create_default_context(cadata=verify)
else:
sslcontext = None
if proxy:
connector = aiohttp_socks.SocksConnector(
socks_ver=aiohttp_socks.SocksVer.SOCKS5,
host=proxy["host"],
port=proxy["port"],
username=proxy["username"],
password=proxy["password"],
rdns=False,
verify_ssl=verify,
ssl_context=sslcontext,
)
session = aiohttp.ClientSession(connector=connector)
close_session = True
elif sslcontext:
conn = aiohttp.TCPConnector(ssl_context=sslcontext)
session = aiohttp.ClientSession(connector=conn)
close_session = True
elif verify is True:
session = self.http_client_v
elif verify is False:
session = self.http_client_no_v
else:
raise ValueError("invalid arguments to http_request")
func = getattr(session, method.lower())
try:
with async_timeout.timeout(timeout):
async with func(url, json=json) as response:
if response.headers.get("content-type",
"").startswith("application/json"):
resp_content = await response.json()
else:
resp_content = await response.text()
result = HTTPResponse(body=resp_content,
status=response.status)
finally:
if close_session:
await session.close()
return result
def run(self, *, host, port):
aiohttp.web.run_app(
app=self.app,
host=host,
port=port,
access_log_format=
'%Tfs %{x-client-ip}i "%r" %s "%{user-agent}i" response=%bb request_body=%{content-length}ib',
)
def add_routes(self):
pass # Override in sub-classes