def send_schema_message(self, subject, parsed_schema_json, schema_id, version, deleted): key = f'{{"subject":"{subject}","version":{version},"magic":1,"keytype":"SCHEMA"}}' value = { "subject": subject, "version": version, "id": schema_id, "schema": json_encode(parsed_schema_json, compact=True), "deleted": deleted } return self.send_kafka_message(key, json_encode(value, compact=True))
def send_schema_message( self, *, subject: str, schema: Optional[TypedSchema], schema_id: int, version: int, deleted: bool, ): key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format( subject, version) if schema: valuedict = { "subject": subject, "version": version, "id": schema_id, "schema": schema.schema_str, "deleted": deleted } if schema.schema_type is not SchemaType.AVRO: valuedict["schemaType"] = schema.schema_type value = json_encode(valuedict, compact=True) else: value = "" return self.send_kafka_message(key, value)
def get_schema_id(self, new_schema): new_schema_encoded = json_encode(new_schema.to_json(), compact=True) for schema_id, schema in self.schemas.items(): if schema == new_schema_encoded: return schema_id self.global_schema_id += 1 return self.global_schema_id
def http_error(message, content_type: str, code: HTTPStatus) -> NoReturn: raise HTTPResponse( body=json_encode({ "error_code": code, "message": message, }, binary=True), headers={"Content-Type": content_type}, status=code, )
async def post_new_schema(self, subject: str, schema: TypedSchema) -> int: payload = { "schema": json_encode(schema.to_json()), "schemaType": schema.schema_type.value } result = await self.client.post(f"subjects/{quote(subject)}/versions", json=payload) if not result.ok: raise SchemaRetrievalError(result.json()) return result.json()["id"]
def write_value(schema: TypedSchema, bio: io.BytesIO, value: dict): if schema.schema_type is SchemaType.AVRO: writer = DatumWriter(schema.schema) writer.write(value, BinaryEncoder(bio)) elif schema.schema_type is SchemaType.JSONSCHEMA: try: schema.schema.validate(value) except ValidationError as e: raise InvalidPayload from e bio.write(json_encode(value, binary=True)) else: raise ValueError("Unknown schema type")
def send_schema_message( self, *, subject: str, schema: TypedSchema, schema_id: int, version: int, deleted: bool, ): key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format( subject, version) value = { "subject": subject, "version": version, "id": schema_id, "schema": json_encode(schema.to_json(), compact=True), "deleted": deleted } if schema.schema_type is not SchemaType.AVRO: value["schemaType"] = schema.schema_type return self.send_kafka_message(key, json_encode(value, compact=True))
def check_schema_headers(self, request): method = request.method headers = request.headers content_type = "application/vnd.schemaregistry.v1+json" if method in { "POST", "PUT" } and headers["Content-Type"] not in ACCEPTED_SCHEMA_CONTENT_TYPES: raise HTTPResponse( body=json_encode( { "error_code": 415, "message": "HTTP 415 Unsupported Media Type", }, binary=True), headers={"Content-Type": content_type}, status=415, ) if "Accept" in headers: if headers["Accept"] == "*/*" or headers["Accept"].startswith( "*/"): return "application/vnd.schemaregistry.v1+json" content_type_match = get_best_match(headers["Accept"], ACCEPTED_SCHEMA_CONTENT_TYPES) if not content_type_match: self.log.debug("Unexpected Accept value: %r", headers["Accept"]) raise HTTPResponse( body=json_encode( { "error_code": 406, "message": "HTTP 406 Not Acceptable", }, binary=True), headers={"Content-Type": content_type}, status=406, ) return content_type_match return content_type
async def subject_version_get(self, *, subject, version, return_dict=False): if version != "latest" and int(version) < 1: self.r( { "error_code": 42202, "message": 'The specified version is not a valid version id. ' 'Allowed values are between [1, 2^31-1] and the string "latest"' }, status=422) subject_data = self.ksr.subjects.get(subject) if not subject_data: self.r({"error_code": 40401, "message": "no subject"}, status=404) max_version = max(subject_data["schemas"]) if version == "latest": schema = subject_data["schemas"][max(subject_data["schemas"])] version = max(subject_data["schemas"]) elif int(version) <= max_version: schema = subject_data["schemas"].get(int(version)) else: self.r({ "error_code": 40402, "message": "Version not found." }, status=404) schema_string = schema["schema"] schema_id = schema["id"] ret = { "subject": subject, "version": int(version), "id": schema_id, "schema": json_encode(schema_string, compact=True) } if return_dict: # Return also compatibility information to compatibility check if subject_data.get("compatibility"): ret["compatibility"] = subject_data.get("compatibility") return ret self.r(ret)
async def subjects_schema_post(self, content_type, *, subject, request): body = request.json self._validate_schema_request_body(content_type, body) subject_data = self._subject_get(subject, content_type) if "schema" not in body: self.r({ "error_code": 500, "message": "Internal Server Error" }, content_type, status=500) try: new_schema = avro.schema.Parse(body["schema"]) except avro.schema.SchemaParseException: self.r(body={ "error_code": 500, "message": f"Error while looking up schema under subject {subject}" }, content_type=content_type, status=500) new_schema_encoded = json_encode(new_schema.to_json(), compact=True) for schema in subject_data["schemas"].values(): if schema["schema"] == new_schema_encoded: ret = { "subject": subject, "version": schema["version"], "id": schema["id"], "schema": schema["schema"], } self.r(ret, content_type) self.r({ "error_code": 40403, "message": "Schema not found" }, content_type, status=404)
async def _handle_request( self, *, request, path_for_stats, callback, schema_request=False, callback_with_request=False, json_request=False, rest_request=False ): start_time = time.monotonic() resp = None rapu_request = HTTPRequest( headers=request.headers, query=request.query, method=request.method, url=request.url, path_for_stats=path_for_stats, ) try: if request.method == "OPTIONS": origin = request.headers.get("Origin") if not origin: raise HTTPResponse(body="OPTIONS missing Origin", status=HTTPStatus.BAD_REQUEST) headers = self.cors_and_server_headers_for_request(request=rapu_request, origin=origin) raise HTTPResponse(body=b"", status=HTTPStatus.OK, headers=headers) body = await request.read() if json_request: if not body: raise HTTPResponse(body="Missing request JSON body", status=HTTPStatus.BAD_REQUEST) try: _, options = cgi.parse_header(rapu_request.get_header("Content-Type")) charset = options.get("charset", "utf-8") body_string = body.decode(charset) rapu_request.json = jsonlib.loads(body_string) except jsonlib.decoder.JSONDecodeError: raise HTTPResponse(body="Invalid request JSON body", status=HTTPStatus.BAD_REQUEST) except UnicodeDecodeError: raise HTTPResponse(body=f"Request body is not valid {charset}", status=HTTPStatus.BAD_REQUEST) except LookupError: raise HTTPResponse(body=f"Unknown charset {charset}", status=HTTPStatus.BAD_REQUEST) else: if body not in {b"", b"{}"}: raise HTTPResponse(body="No request body allowed for this operation", status=HTTPStatus.BAD_REQUEST) callback_kwargs = dict(request.match_info) if callback_with_request: callback_kwargs["request"] = rapu_request if rest_request: params = self.check_rest_headers(rapu_request) if "requests" in params: rapu_request.content_type = params["requests"] params.pop("requests") if "accepts" in params: rapu_request.accepts = params["accepts"] params.pop("accepts") callback_kwargs.update(params) if schema_request: content_type = self.check_schema_headers(rapu_request) callback_kwargs["content_type"] = content_type try: data = await callback(**callback_kwargs) status = HTTPStatus.OK headers = {} except HTTPResponse as ex: data = ex.body status = ex.status headers = ex.headers except: # pylint: disable=bare-except self.log.exception("Internal server error") data = {"error_code": HTTPStatus.INTERNAL_SERVER_ERROR.value, "message": "Internal server error"} status = HTTPStatus.INTERNAL_SERVER_ERROR headers = {} headers.update(self.cors_and_server_headers_for_request(request=rapu_request)) if isinstance(data, (dict, list)): resp_bytes = json_encode(data, binary=True, sort_keys=True, compact=True) elif isinstance(data, str): if "Content-Type" not in headers: headers["Content-Type"] = "text/plain; charset=utf-8" resp_bytes = data.encode("utf-8") else: resp_bytes = data # On 204 - NO CONTENT there is no point of calculating cache headers if is_success(status): if resp_bytes: etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest()) else: etag = '""' if_none_match = request.headers.get("if-none-match") if if_none_match and if_none_match.replace("W/", "") == etag: status = HTTPStatus.NOT_MODIFIED resp_bytes = b"" headers["access-control-expose-headers"] = "etag" headers["etag"] = etag resp = aiohttp.web.Response(body=resp_bytes, status=status.value, headers=headers) except HTTPResponse as ex: if isinstance(ex.body, str): resp = aiohttp.web.Response(text=ex.body, status=ex.status.value, headers=ex.headers) else: resp = aiohttp.web.Response(body=ex.body, status=ex.status.value, headers=ex.headers) except asyncio.CancelledError: self.log.debug("Client closed connection") raise except Exception as ex: # pylint: disable=broad-except self.stats.unexpected_exception(ex=ex, where="rapu_wrapped_callback") self.log.exception("Unexpected error handling user request: %s %s", request.method, request.url) resp = aiohttp.web.Response(text="Internal Server Error", status=HTTPStatus.INTERNAL_SERVER_ERROR.value) finally: self.stats.timing( self.app_request_metric, time.monotonic() - start_time, tags={ "path": path_for_stats, # no `resp` means that we had a failure in exception handler "result": resp.status if resp else 0, "method": request.method, } ) return resp
def __str__(self) -> str: return json_encode(self.to_json(), compact=True)
def encode_value(value): if value == "null": return None if isinstance(value, str): return value.encode("utf8") return json_encode(value, sort_keys=False, binary=True)
async def _handle_request(self, *, request, path_for_stats, callback, callback_with_request=False, json_request=False): start_time = time.monotonic() resp = None rapu_request = HTTPRequest( headers=request.headers, query=request.query, method=request.method, url=request.url, path_for_stats=path_for_stats, ) try: if request.method == "OPTIONS": origin = request.headers.get("Origin") if not origin: raise HTTPResponse(body="OPTIONS missing Origin", status=400) headers = self.cors_and_server_headers_for_request( request=rapu_request, origin=origin) raise HTTPResponse(body=b"", status=200, headers=headers) body = await request.read() if json_request: if not body: raise HTTPResponse(body="Missing request JSON body", status=400) if request.charset and request.charset.lower( ) != "utf-8" and request.charset.lower() != "utf8": raise HTTPResponse( body="Request character set must be UTF-8", status=400) try: body_string = body.decode("utf-8") rapu_request.json = jsonlib.loads(body_string) except jsonlib.decoder.JSONDecodeError: raise HTTPResponse(body="Invalid request JSON body", status=400) except UnicodeDecodeError: raise HTTPResponse(body="Request body is not valid UTF-8", status=400) else: if body not in {b"", b"{}"}: raise HTTPResponse( body="No request body allowed for this operation", status=400) callback_kwargs = dict(request.match_info) if callback_with_request: callback_kwargs["request"] = rapu_request try: data = await callback(**callback_kwargs) status = 200 headers = {} except HTTPResponse as ex: data = ex.body status = ex.status headers = ex.headers headers.update( self.cors_and_server_headers_for_request(request=rapu_request)) if isinstance(data, (dict, list)): resp_bytes = json_encode(data, binary=True, sort_keys=True, compact=True) elif isinstance(data, str): if "Content-Type" not in headers: headers["Content-Type"] = "text/plain; charset=utf-8" resp_bytes = data.encode("utf-8") else: resp_bytes = data # On 204 - NO CONTENT there is no point of calculating cache headers if 200 >= status <= 299: if resp_bytes: etag = '"{}"'.format(hashlib.md5(resp_bytes).hexdigest()) else: etag = '""' if_none_match = request.headers.get("if-none-match") if if_none_match and if_none_match.replace("W/", "") == etag: status = 304 resp_bytes = b"" headers["access-control-expose-headers"] = "etag" headers["etag"] = etag resp = aiohttp.web.Response(body=resp_bytes, status=status, headers=headers) except HTTPResponse as ex: if isinstance(ex.body, str): resp = aiohttp.web.Response(text=ex.body, status=ex.status, headers=ex.headers) else: resp = aiohttp.web.Response(body=ex.body, status=ex.status, headers=ex.headers) except asyncio.CancelledError: self.log.debug("Client closed connection") raise except Exception as ex: # pylint: disable=broad-except self.stats.unexpected_exception(ex=ex, where="rapu_wrapped_callback") self.log.exception("Unexpected error handling user request: %s %s", request.method, request.url) resp = aiohttp.web.Response(text="Internal Server Error", status=500) finally: self.stats.timing( self.app_request_metric, time.monotonic() - start_time, tags={ "path": path_for_stats, # no `resp` means that we had a failure in exception handler "result": resp.status if resp else 0, "method": request.method, }) return resp
def __str__(self) -> str: if isinstance(self.schema, ProtobufSchema): return str(self.schema) return json_encode(self.to_json(), compact=True)