def send_schema_message(self, subject, parsed_schema_json, schema_id,
version, deleted):
key = f'{{"subject":"{subject}","version":{version},"magic":1,"keytype":"SCHEMA"}}'
value = {
"subject": subject,
"version": version,
"id": schema_id,
"schema": json_encode(parsed_schema_json, compact=True),
"deleted": deleted
}
return self.send_kafka_message(key, json_encode(value, compact=True))
def send_schema_message(
self,
*,
subject: str,
schema: Optional[TypedSchema],
schema_id: int,
version: int,
deleted: bool,
):
key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format(
subject, version)
if schema:
valuedict = {
"subject": subject,
"version": version,
"id": schema_id,
"schema": schema.schema_str,
"deleted": deleted
}
if schema.schema_type is not SchemaType.AVRO:
valuedict["schemaType"] = schema.schema_type
value = json_encode(valuedict, compact=True)
else:
value = ""
return self.send_kafka_message(key, value)
def get_schema_id(self, new_schema):
new_schema_encoded = json_encode(new_schema.to_json(), compact=True)
for schema_id, schema in self.schemas.items():
if schema == new_schema_encoded:
return schema_id
self.global_schema_id += 1
return self.global_schema_id
def http_error(message, content_type: str, code: HTTPStatus) -> NoReturn:
raise HTTPResponse(
body=json_encode({
"error_code": code,
"message": message,
}, binary=True),
headers={"Content-Type": content_type},
status=code,
)
async def post_new_schema(self, subject: str, schema: TypedSchema) -> int:
payload = {
"schema": json_encode(schema.to_json()),
"schemaType": schema.schema_type.value
}
result = await self.client.post(f"subjects/{quote(subject)}/versions",
json=payload)
if not result.ok:
raise SchemaRetrievalError(result.json())
return result.json()["id"]
def write_value(schema: TypedSchema, bio: io.BytesIO, value: dict):
if schema.schema_type is SchemaType.AVRO:
writer = DatumWriter(schema.schema)
writer.write(value, BinaryEncoder(bio))
elif schema.schema_type is SchemaType.JSONSCHEMA:
try:
schema.schema.validate(value)
except ValidationError as e:
raise InvalidPayload from e
bio.write(json_encode(value, binary=True))
else:
raise ValueError("Unknown schema type")
def send_schema_message(
self,
*,
subject: str,
schema: TypedSchema,
schema_id: int,
version: int,
deleted: bool,
):
key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format(
subject, version)
value = {
"subject": subject,
"version": version,
"id": schema_id,
"schema": json_encode(schema.to_json(), compact=True),
"deleted": deleted
}
if schema.schema_type is not SchemaType.AVRO:
value["schemaType"] = schema.schema_type
return self.send_kafka_message(key, json_encode(value, compact=True))
def check_schema_headers(self, request):
method = request.method
headers = request.headers
content_type = "application/vnd.schemaregistry.v1+json"
if method in {
"POST", "PUT"
} and headers["Content-Type"] not in ACCEPTED_SCHEMA_CONTENT_TYPES:
raise HTTPResponse(
body=json_encode(
{
"error_code": 415,
"message": "HTTP 415 Unsupported Media Type",
},
binary=True),
headers={"Content-Type": content_type},
status=415,
)
if "Accept" in headers:
if headers["Accept"] == "*/*" or headers["Accept"].startswith(
"*/"):
return "application/vnd.schemaregistry.v1+json"
content_type_match = get_best_match(headers["Accept"],
ACCEPTED_SCHEMA_CONTENT_TYPES)
if not content_type_match:
self.log.debug("Unexpected Accept value: %r",
headers["Accept"])
raise HTTPResponse(
body=json_encode(
{
"error_code": 406,
"message": "HTTP 406 Not Acceptable",
},
binary=True),
headers={"Content-Type": content_type},
status=406,
)
return content_type_match
return content_type
async def subject_version_get(self,
*,
subject,
version,
return_dict=False):
if version != "latest" and int(version) < 1:
self.r(
{
"error_code":
42202,
"message":
'The specified version is not a valid version id. '
'Allowed values are between [1, 2^31-1] and the string "latest"'
},
status=422)
subject_data = self.ksr.subjects.get(subject)
if not subject_data:
self.r({"error_code": 40401, "message": "no subject"}, status=404)
max_version = max(subject_data["schemas"])
if version == "latest":
schema = subject_data["schemas"][max(subject_data["schemas"])]
version = max(subject_data["schemas"])
elif int(version) <= max_version:
schema = subject_data["schemas"].get(int(version))
else:
self.r({
"error_code": 40402,
"message": "Version not found."
},
status=404)
schema_string = schema["schema"]
schema_id = schema["id"]
ret = {
"subject": subject,
"version": int(version),
"id": schema_id,
"schema": json_encode(schema_string, compact=True)
}
if return_dict:
# Return also compatibility information to compatibility check
if subject_data.get("compatibility"):
ret["compatibility"] = subject_data.get("compatibility")
return ret
self.r(ret)
async def subjects_schema_post(self, content_type, *, subject, request):
body = request.json
self._validate_schema_request_body(content_type, body)
subject_data = self._subject_get(subject, content_type)
if "schema" not in body:
self.r({
"error_code": 500,
"message": "Internal Server Error"
},
content_type,
status=500)
try:
new_schema = avro.schema.Parse(body["schema"])
except avro.schema.SchemaParseException:
self.r(body={
"error_code":
500,
"message":
f"Error while looking up schema under subject {subject}"
},
content_type=content_type,
status=500)
new_schema_encoded = json_encode(new_schema.to_json(), compact=True)
for schema in subject_data["schemas"].values():
if schema["schema"] == new_schema_encoded:
ret = {
"subject": subject,
"version": schema["version"],
"id": schema["id"],
"schema": schema["schema"],
}
self.r(ret, content_type)
self.r({
"error_code": 40403,
"message": "Schema not found"
},
content_type,
status=404)
async def _handle_request(
self,
*,
request,
path_for_stats,
callback,
schema_request=False,
callback_with_request=False,
json_request=False,
rest_request=False
):
start_time = time.monotonic()
resp = None
rapu_request = HTTPRequest(
headers=request.headers,
query=request.query,
method=request.method,
url=request.url,
path_for_stats=path_for_stats,
)
try:
if request.method == "OPTIONS":
origin = request.headers.get("Origin")
if not origin:
raise HTTPResponse(body="OPTIONS missing Origin", status=HTTPStatus.BAD_REQUEST)
headers = self.cors_and_server_headers_for_request(request=rapu_request, origin=origin)
raise HTTPResponse(body=b"", status=HTTPStatus.OK, headers=headers)
body = await request.read()
if json_request:
if not body:
raise HTTPResponse(body="Missing request JSON body", status=HTTPStatus.BAD_REQUEST)
try:
_, options = cgi.parse_header(rapu_request.get_header("Content-Type"))
charset = options.get("charset", "utf-8")
body_string = body.decode(charset)
rapu_request.json = jsonlib.loads(body_string)
except jsonlib.decoder.JSONDecodeError:
raise HTTPResponse(body="Invalid request JSON body", status=HTTPStatus.BAD_REQUEST)
except UnicodeDecodeError:
raise HTTPResponse(body=f"Request body is not valid {charset}", status=HTTPStatus.BAD_REQUEST)
except LookupError:
raise HTTPResponse(body=f"Unknown charset {charset}", status=HTTPStatus.BAD_REQUEST)
else:
if body not in {b"", b"{}"}:
raise HTTPResponse(body="No request body allowed for this operation", status=HTTPStatus.BAD_REQUEST)
callback_kwargs = dict(request.match_info)
if callback_with_request:
callback_kwargs["request"] = rapu_request
if rest_request:
params = self.check_rest_headers(rapu_request)
if "requests" in params:
rapu_request.content_type = params["requests"]
params.pop("requests")
if "accepts" in params:
rapu_request.accepts = params["accepts"]
params.pop("accepts")
callback_kwargs.update(params)
if schema_request:
content_type = self.check_schema_headers(rapu_request)
callback_kwargs["content_type"] = content_type
try:
data = await callback(**callback_kwargs)
status = HTTPStatus.OK
headers = {}
except HTTPResponse as ex:
data = ex.body
status = ex.status
headers = ex.headers
except: # pylint: disable=bare-except
self.log.exception("Internal server error")
data = {"error_code": HTTPStatus.INTERNAL_SERVER_ERROR.value, "message": "Internal server error"}
status = HTTPStatus.INTERNAL_SERVER_ERROR
headers = {}
headers.update(self.cors_and_server_headers_for_request(request=rapu_request))
if isinstance(data, (dict, list)):
resp_bytes = json_encode(data, binary=True, sort_keys=True, compact=True)
elif isinstance(data, str):
if "Content-Type" not in headers:
headers["Content-Type"] = "text/plain; charset=utf-8"
resp_bytes = data.encode("utf-8")
else:
resp_bytes = data
# On 204 - NO CONTENT there is no point of calculating cache headers
if is_success(status):
if resp_bytes:
etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest())
else:
etag = '""'
if_none_match = request.headers.get("if-none-match")
if if_none_match and if_none_match.replace("W/", "") == etag:
status = HTTPStatus.NOT_MODIFIED
resp_bytes = b""
headers["access-control-expose-headers"] = "etag"
headers["etag"] = etag
resp = aiohttp.web.Response(body=resp_bytes, status=status.value, headers=headers)
except HTTPResponse as ex:
if isinstance(ex.body, str):
resp = aiohttp.web.Response(text=ex.body, status=ex.status.value, headers=ex.headers)
else:
resp = aiohttp.web.Response(body=ex.body, status=ex.status.value, headers=ex.headers)
except asyncio.CancelledError:
self.log.debug("Client closed connection")
raise
except Exception as ex: # pylint: disable=broad-except
self.stats.unexpected_exception(ex=ex, where="rapu_wrapped_callback")
self.log.exception("Unexpected error handling user request: %s %s", request.method, request.url)
resp = aiohttp.web.Response(text="Internal Server Error", status=HTTPStatus.INTERNAL_SERVER_ERROR.value)
finally:
self.stats.timing(
self.app_request_metric,
time.monotonic() - start_time,
tags={
"path": path_for_stats,
# no `resp` means that we had a failure in exception handler
"result": resp.status if resp else 0,
"method": request.method,
}
)
return resp
def __str__(self) -> str:
return json_encode(self.to_json(), compact=True)
def encode_value(value):
if value == "null":
return None
if isinstance(value, str):
return value.encode("utf8")
return json_encode(value, sort_keys=False, binary=True)
async def _handle_request(self,
*,
request,
path_for_stats,
callback,
callback_with_request=False,
json_request=False):
start_time = time.monotonic()
resp = None
rapu_request = HTTPRequest(
headers=request.headers,
query=request.query,
method=request.method,
url=request.url,
path_for_stats=path_for_stats,
)
try:
if request.method == "OPTIONS":
origin = request.headers.get("Origin")
if not origin:
raise HTTPResponse(body="OPTIONS missing Origin",
status=400)
headers = self.cors_and_server_headers_for_request(
request=rapu_request, origin=origin)
raise HTTPResponse(body=b"", status=200, headers=headers)
body = await request.read()
if json_request:
if not body:
raise HTTPResponse(body="Missing request JSON body",
status=400)
if request.charset and request.charset.lower(
) != "utf-8" and request.charset.lower() != "utf8":
raise HTTPResponse(
body="Request character set must be UTF-8", status=400)
try:
body_string = body.decode("utf-8")
rapu_request.json = jsonlib.loads(body_string)
except jsonlib.decoder.JSONDecodeError:
raise HTTPResponse(body="Invalid request JSON body",
status=400)
except UnicodeDecodeError:
raise HTTPResponse(body="Request body is not valid UTF-8",
status=400)
else:
if body not in {b"", b"{}"}:
raise HTTPResponse(
body="No request body allowed for this operation",
status=400)
callback_kwargs = dict(request.match_info)
if callback_with_request:
callback_kwargs["request"] = rapu_request
try:
data = await callback(**callback_kwargs)
status = 200
headers = {}
except HTTPResponse as ex:
data = ex.body
status = ex.status
headers = ex.headers
headers.update(
self.cors_and_server_headers_for_request(request=rapu_request))
if isinstance(data, (dict, list)):
resp_bytes = json_encode(data,
binary=True,
sort_keys=True,
compact=True)
elif isinstance(data, str):
if "Content-Type" not in headers:
headers["Content-Type"] = "text/plain; charset=utf-8"
resp_bytes = data.encode("utf-8")
else:
resp_bytes = data
# On 204 - NO CONTENT there is no point of calculating cache headers
if 200 >= status <= 299:
if resp_bytes:
etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest())
else:
etag = '""'
if_none_match = request.headers.get("if-none-match")
if if_none_match and if_none_match.replace("W/", "") == etag:
status = 304
resp_bytes = b""
headers["access-control-expose-headers"] = "etag"
headers["etag"] = etag
resp = aiohttp.web.Response(body=resp_bytes,
status=status,
headers=headers)
except HTTPResponse as ex:
if isinstance(ex.body, str):
resp = aiohttp.web.Response(text=ex.body,
status=ex.status,
headers=ex.headers)
else:
resp = aiohttp.web.Response(body=ex.body,
status=ex.status,
headers=ex.headers)
except asyncio.CancelledError:
self.log.debug("Client closed connection")
raise
except Exception as ex: # pylint: disable=broad-except
self.stats.unexpected_exception(ex=ex,
where="rapu_wrapped_callback")
self.log.exception("Unexpected error handling user request: %s %s",
request.method, request.url)
resp = aiohttp.web.Response(text="Internal Server Error",
status=500)
finally:
self.stats.timing(
self.app_request_metric,
time.monotonic() - start_time,
tags={
"path": path_for_stats,
# no `resp` means that we had a failure in exception handler
"result": resp.status if resp else 0,
"method": request.method,
})
return resp
def __str__(self) -> str:
if isinstance(self.schema, ProtobufSchema):
return str(self.schema)
return json_encode(self.to_json(), compact=True)
