class KarapaceSchemaRegistry(KarapaceBase):
# pylint: disable=attribute-defined-outside-init
def __init__(self, config):
super().__init__(config)
self._add_routes()
self._init()
def _init(self):
self.ksr = None
self.producer = None
self.producer = self._create_producer()
self._create_master_coordinator()
self._create_schema_reader()
def _add_routes(self):
self.route(
"/compatibility/subjects/<subject:path>/versions/<version:path>",
callback=self.compatibility_check,
method="POST",
schema_request=True)
self.route(
"/config/<subject:path>",
callback=self.config_subject_get,
method="GET",
schema_request=True,
with_request=True,
json_body=False,
)
self.route("/config/<subject:path>",
callback=self.config_subject_set,
method="PUT",
schema_request=True)
self.route("/config",
callback=self.config_get,
method="GET",
schema_request=True)
self.route("/config",
callback=self.config_set,
method="PUT",
schema_request=True)
self.route("/schemas/ids/<schema_id:path>",
callback=self.schemas_get,
method="GET",
schema_request=True)
self.route("/subjects",
callback=self.subjects_list,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>/versions",
callback=self.subject_post,
method="POST",
schema_request=True)
self.route("/subjects/<subject:path>",
callback=self.subjects_schema_post,
method="POST",
schema_request=True)
self.route("/subjects/<subject:path>/versions",
callback=self.subject_versions_list,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>/versions/<version>",
callback=self.subject_version_get,
method="GET",
schema_request=True)
self.route(
"/subjects/<subject:path>/versions/<version:path>", # needs
callback=self.subject_version_delete,
method="DELETE",
schema_request=True)
self.route("/subjects/<subject:path>/versions/<version>/schema",
callback=self.subject_version_schema_get,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>",
callback=self.subject_delete,
method="DELETE",
schema_request=True) # needs
def close(self):
super().close()
self.log.info("Shutting down all auxiliary threads")
if self.mc:
self.mc.close()
if self.ksr:
self.ksr.close()
if self.producer:
self.producer.close()
def _create_schema_reader(self):
self.ksr = KafkaSchemaReader(config=self.config,
master_coordinator=self.mc)
self.ksr.start()
def _create_master_coordinator(self):
self.mc = MasterCoordinator(config=self.config)
self.mc.start()
def _subject_get(self, subject, content_type):
subject_data = self.ksr.subjects.get(subject)
if not subject_data:
self.r({
"error_code": 40401,
"message": "Subject not found."
},
content_type,
status=404)
schemas = self.ksr.get_schemas(subject)
if not schemas:
self.r({
"error_code": 40401,
"message": "Subject not found."
},
content_type,
status=404)
subject_data["schemas"] = schemas
return subject_data
def _validate_version(self, content_type, version): # pylint: disable=inconsistent-return-statements
try:
version_number = int(version)
if version_number > 0:
return version
except ValueError:
if version == "latest":
return version
self.r(body={
"error_code":
42202,
"message":
"The specified version is not a valid version id. "
"Allowed values are between [1, 2^31-1] and the string \"latest\""
},
content_type=content_type,
status=422)
def get_offset_from_queue(self, sent_offset):
start_time = time.monotonic()
while True:
self.log.info("Starting to wait for offset: %r from ksr queue",
sent_offset)
offset = self.ksr.queue.get()
if offset == sent_offset:
self.log.info(
"We've consumed back produced offset: %r message back, everything is in sync, took: %.4f",
offset,
time.monotonic() - start_time)
break
self.log.warning(
"Put the offset: %r back to queue, someone else is waiting for this?",
offset)
self.ksr.queue.put(offset)
def send_kafka_message(self, key, value):
if isinstance(key, str):
key = key.encode("utf8")
if isinstance(value, str):
value = value.encode("utf8")
future = self.producer.send(self.config["topic_name"],
key=key,
value=value)
self.producer.flush(timeout=self.kafka_timeout)
msg = future.get(self.kafka_timeout)
self.log.debug("Sent kafka msg key: %r, value: %r, offset: %r", key,
value, msg.offset)
self.get_offset_from_queue(msg.offset)
return future
def send_schema_message(
self,
*,
subject: str,
schema: TypedSchema,
schema_id: int,
version: int,
deleted: bool,
):
key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format(
subject, version)
value = {
"subject": subject,
"version": version,
"id": schema_id,
"schema": json_encode(schema.to_json(), compact=True),
"deleted": deleted
}
if schema.schema_type is not SchemaType.AVRO:
value["schemaType"] = schema.schema_type
return self.send_kafka_message(key, json_encode(value, compact=True))
def send_config_message(self, compatibility_level, subject=None):
if subject is not None:
key = '{{"subject":"{}","magic":0,"keytype":"CONFIG"}}'.format(
subject)
else:
key = '{"subject":null,"magic":0,"keytype":"CONFIG"}'
value = '{{"compatibilityLevel":"{}"}}'.format(compatibility_level)
return self.send_kafka_message(key, value)
def send_delete_subject_message(self, subject, version):
key = '{{"subject":"{}","magic":0,"keytype":"DELETE_SUBJECT"}}'.format(
subject)
value = '{{"subject":"{}","version":{}}}'.format(subject, version)
return self.send_kafka_message(key, value)
async def compatibility_check(self, content_type, *, subject, version,
request):
"""Check for schema compatibility"""
body = request.json
self.log.info(
"Got request to check subject: %r, version_id: %r compatibility",
subject, version)
old = await self.subject_version_get(content_type=content_type,
subject=subject,
version=version,
return_dict=True)
self.log.info("Existing schema: %r, new_schema: %r", old["schema"],
body["schema"])
try:
schema_type = SchemaType(body.get("schemaType", "AVRO"))
new = TypedSchema.parse(schema_type, body["schema"])
except InvalidSchema:
self.log.warning("Invalid schema: %r", body["schema"])
self.r(body={
"error_code": 44201,
"message": "Invalid Avro schema"
},
content_type=content_type,
status=422)
try:
old_schema_type = SchemaType(old.get("schemaType", "AVRO"))
old_schema = TypedSchema.parse(old_schema_type, old["schema"])
except InvalidSchema:
self.log.warning("Invalid existing schema: %r", old["schema"])
self.r(body={
"error_code": 44201,
"message": "Invalid Avro schema"
},
content_type=content_type,
status=422)
compat = Compatibility(source=old_schema,
target=new,
compatibility=old.get(
"compatibility",
self.ksr.config["compatibility"]))
try:
compat.check()
except IncompatibleSchema as ex:
self.log.warning(
"Invalid schema %s found by compatibility check: old: %s new: %s",
ex, old_schema, new)
self.r({"is_compatible": False}, content_type)
self.r({"is_compatible": True}, content_type)
async def schemas_get(self, content_type, *, schema_id):
try:
schema_id_int = int(schema_id)
except ValueError:
self.r({
"error_code": 404,
"message": "HTTP 404 Not Found"
},
content_type,
status=404)
with self.ksr.id_lock:
schema = self.ksr.schemas.get(schema_id_int)
if not schema:
self.log.warning("Schema: %r that was requested, not found",
int(schema_id))
self.r(body={
"error_code": 40403,
"message": "Schema not found"
},
content_type=content_type,
status=404)
response_body = {"schema": str(schema)}
if schema.schema_type is not SchemaType.AVRO:
response_body["schemaType"] = schema.schema_type
self.r(response_body, content_type)
async def config_get(self, content_type):
self.r({"compatibilityLevel": self.ksr.config["compatibility"]},
content_type)
async def config_set(self, content_type, *, request):
body = request.json
if "compatibility" in request.json and request.json[
"compatibility"] in COMPATIBILITY_MODES:
compatibility_level = request.json["compatibility"]
are_we_master, master_url = await self.get_master()
if are_we_master:
self.send_config_message(
compatibility_level=compatibility_level, subject=None)
elif are_we_master is None:
self.no_master_error(content_type)
else:
url = f"{master_url}/config"
await self.forward_request_remote(body=body,
url=url,
content_type=content_type,
method="PUT")
else:
self.r(body={
"error_code":
42203,
"message":
"Invalid compatibility level. Valid values are none, backward, forward and full"
},
content_type=content_type,
status=422)
self.r({"compatibility": self.ksr.config["compatibility"]},
content_type)
async def config_subject_get(self, content_type, subject: str, *,
request: HTTPRequest):
# Config for a subject can exist without schemas so no need to check for their existence
subject_data = self.ksr.subjects.get(subject, {})
if subject_data:
default_to_global = request.query.get("defaultToGlobal",
"false").lower() == "true"
compatibility = subject_data.get("compatibility")
if not compatibility and default_to_global:
compatibility = self.ksr.config["compatibility"]
if compatibility:
self.r({"compatibilityLevel": compatibility}, content_type)
self.r({
"error_code": 40401,
"message": "Subject not found."
},
content_type,
status=404)
async def config_subject_set(self, content_type, *, request, subject):
if "compatibility" in request.json and request.json[
"compatibility"] in COMPATIBILITY_MODES:
are_we_master, master_url = await self.get_master()
if are_we_master:
self.send_config_message(
compatibility_level=request.json["compatibility"],
subject=subject)
elif are_we_master is None:
self.no_master_error(content_type)
else:
url = f"{master_url}/config/{subject}"
await self.forward_request_remote(body=request.json,
url=url,
content_type=content_type,
method="PUT")
else:
self.r(body={
"error_code": 42203,
"message": "Invalid compatibility level"
},
content_type=content_type,
status=422)
self.r({"compatibility": request.json["compatibility"]}, content_type)
async def subjects_list(self, content_type):
subjects_list = [
key for key, val in self.ksr.subjects.items()
if self.ksr.get_schemas(key)
]
self.r(subjects_list, content_type, status=200)
async def subject_delete(self, content_type, *, subject):
self._subject_get(subject, content_type)
version_list = list(self.ksr.get_schemas(subject))
if version_list:
latest_schema_id = version_list[-1]
else:
latest_schema_id = 0
are_we_master, master_url = await self.get_master()
if are_we_master:
self.send_delete_subject_message(subject, latest_schema_id)
self.r(version_list, content_type, status=200)
elif are_we_master is None:
self.no_master_error(content_type)
else:
url = f"{master_url}/subjects/{subject}"
await self.forward_request_remote(body={},
url=url,
content_type=content_type,
method="DELETE")
async def subject_version_get(self,
content_type,
*,
subject,
version,
return_dict=False):
self._validate_version(content_type, version)
subject_data = self._subject_get(subject, content_type)
schema_data = None
max_version = max(subject_data["schemas"])
if version == "latest":
version = max(subject_data["schemas"])
schema_data = subject_data["schemas"][version]
elif int(version) <= max_version:
schema_data = subject_data["schemas"].get(int(version))
else:
self.r({
"error_code": 40402,
"message": "Version not found."
},
content_type,
status=404)
if not schema_data:
self.r({
"error_code": 40402,
"message": "Version not found."
},
content_type,
status=404)
schema_id = schema_data["id"]
schema = schema_data["schema"]
ret = {
"subject": subject,
"version": int(version),
"id": schema_id,
"schema": str(schema),
}
if schema.schema_type is not SchemaType.AVRO:
ret["schemaType"] = schema.schema_type
if return_dict:
# Return also compatibility information to compatibility check
if subject_data.get("compatibility"):
ret["compatibility"] = subject_data.get("compatibility")
return ret
self.r(ret, content_type)
async def subject_version_delete(self, content_type, *, subject, version):
version = int(version)
subject_data = self._subject_get(subject, content_type)
subject_schema_data = subject_data["schemas"].get(version, None)
if not subject_schema_data:
self.r({
"error_code": 40402,
"message": "Version not found."
},
content_type,
status=404)
schema_id = subject_schema_data["id"]
schema = subject_schema_data["schema"]
are_we_master, master_url = await self.get_master()
if are_we_master:
self.send_schema_message(subject=subject,
schema=schema,
schema_id=schema_id,
version=version,
deleted=True)
self.r(str(version), content_type, status=200)
elif are_we_master is None:
self.no_master_error(content_type)
else:
url = f"{master_url}/subjects/{subject}/versions/{version}"
await self.forward_request_remote(body={},
url=url,
content_type=content_type,
method="DELETE")
async def subject_version_schema_get(self, content_type, *, subject,
version):
self._validate_version(content_type, version)
subject_data = self._subject_get(subject, content_type)
max_version = max(subject_data["schemas"])
if version == "latest":
schema_data = subject_data["schemas"][max_version]
elif int(version) <= max_version:
schema_data = subject_data["schemas"].get(int(version))
else:
self.r({
"error_code": 40402,
"message": "Version not found."
},
content_type,
status=404)
self.r(str(schema_data["schema"]), content_type)
async def subject_versions_list(self, content_type, *, subject):
subject_data = self._subject_get(subject, content_type)
self.r(list(subject_data["schemas"]), content_type, status=200)
async def get_master(self):
async with self.master_lock:
while True:
master, master_url = self.mc.get_master_info()
if master is None:
self.log.info("No master set: %r, url: %r", master,
master_url)
elif self.ksr.ready is False:
self.log.info("Schema reader isn't ready yet: %r",
self.ksr.ready)
else:
return master, master_url
await asyncio.sleep(1.0)
def _validate_schema_request_body(self, content_type, body):
if not isinstance(body, dict):
self.r({
"error_code": 500,
"message": "Internal Server Error"
},
content_type,
status=500)
for field in body:
if field not in {"schema", "schemaType"}:
self.r(body={
"error_code": 422,
"message": f"Unrecognized field: {field}",
},
content_type=content_type,
status=422)
def _validate_schema_type(self, content_type, body):
schema_type = SchemaType(body.get("schemaType", SchemaType.AVRO.value))
if schema_type not in {SchemaType.JSONSCHEMA, SchemaType.AVRO}:
self.r(body={
"error_code": 422,
"message": f"unrecognized schemaType: {schema_type}"
},
content_type=content_type,
status=422)
async def subjects_schema_post(self, content_type, *, subject, request):
body = request.json
self._validate_schema_request_body(content_type, body)
subject_data = self._subject_get(subject, content_type)
new_schema = None
if "schema" not in body:
self.r({
"error_code": 500,
"message": "Internal Server Error"
},
content_type,
status=500)
schema_str = body["schema"]
schema_type = SchemaType(body.get("schemaType", "AVRO"))
try:
new_schema = TypedSchema.parse(schema_type, schema_str)
except InvalidSchema:
self.log.exception("No proper parser found")
self.r(
{
"error_code":
500,
"message":
f"Error while looking up schema under subject {subject}"
},
content_type,
status=500)
for schema in subject_data["schemas"].values():
typed_schema = schema["schema"]
if typed_schema == new_schema:
ret = {
"subject": subject,
"version": schema["version"],
"id": schema["id"],
"schema": str(typed_schema),
}
if schema_type is not SchemaType.AVRO:
ret["schemaType"] = schema_type
self.r(ret, content_type)
else:
self.log.debug("Schema %r did not match %r", schema,
typed_schema)
self.r({
"error_code": 40403,
"message": "Schema not found"
},
content_type,
status=404)
async def subject_post(self, content_type, *, subject, request):
body = request.json
self.log.debug("POST with subject: %r, request: %r", subject, body)
self._validate_schema_request_body(content_type, body)
self._validate_schema_type(content_type, body)
if "schema" not in body:
self.r({
"error_code": 500,
"message": "Internal Server Error"
},
content_type,
status=500)
are_we_master, master_url = await self.get_master()
if are_we_master:
self.write_new_schema_local(subject, body, content_type)
elif are_we_master is None:
self.no_master_error(content_type)
else:
url = f"{master_url}/subjects/{subject}/versions"
await self.forward_request_remote(body=body,
url=url,
content_type=content_type,
method="POST")
def write_new_schema_local(self, subject, body, content_type):
"""Since we're the master we get to write the new schema"""
self.log.info("Writing new schema locally since we're the master")
schema_type = SchemaType(body.get("schemaType", SchemaType.AVRO))
try:
new_schema = TypedSchema.parse(schema_type=schema_type,
schema_str=body["schema"])
except (InvalidSchema, InvalidSchemaType):
self.log.warning("Invalid schema: %r",
body["schema"],
exc_info=True)
self.r(body={
"error_code": 44201,
"message": f"Invalid {schema_type} schema"
},
content_type=content_type,
status=422)
if subject not in self.ksr.subjects or not self.ksr.subjects.get(
subject)["schemas"]:
schema_id = self.ksr.get_schema_id(new_schema)
version = 1
self.log.info(
"Registering new subject: %r with version: %r to schema %r, schema_id: %r",
subject, version, new_schema.to_json(), schema_id)
else:
# First check if any of the existing schemas for the subject match
subject_data = self.ksr.subjects[subject]
schemas = self.ksr.get_schemas(subject)
if not schemas: # Previous ones have been deleted by the user.
version = max(self.ksr.subjects[subject]["schemas"]) + 1
schema_id = self.ksr.get_schema_id(new_schema)
self.log.info(
"Registering subject: %r, id: %r new version: %r with schema %r, schema_id: %r",
subject, schema_id, version, new_schema.to_json(),
schema_id)
self.send_schema_message(
subject=subject,
schema=new_schema,
schema_id=schema_id,
version=version,
deleted=False,
)
self.r({"id": schema_id}, content_type)
schema_versions = sorted(list(schemas))
# Go through these in version order
for version in schema_versions:
schema = subject_data["schemas"][version]
if schema["schema"] == new_schema:
self.r({"id": schema["id"]}, content_type)
else:
self.log.debug("schema: %s did not match with: %s", schema,
new_schema)
compatibility = subject_data.get("compatibility",
self.ksr.config["compatibility"])
# Run a compatibility check between on file schema(s) and the one being submitted now
# the check is either towards the latest one or against all previous ones in case of
# transitive mode
if compatibility in TRANSITIVE_MODES:
check_against = schema_versions
else:
check_against = [schema_versions[-1]]
for old_version in check_against:
old_schema = subject_data["schemas"][old_version]["schema"]
compat = Compatibility(old_schema,
new_schema,
compatibility=compatibility)
try:
compat.check()
except IncompatibleSchema as ex:
self.log.warning("Incompatible schema: %s", ex)
self.r(body={
"error_code":
409,
"message":
"Schema being registered is incompatible with an earlier schema"
},
content_type=content_type,
status=409)
# We didn't find an existing schema and the schema is compatible so go and create one
schema_id = self.ksr.get_schema_id(new_schema)
version = max(self.ksr.subjects[subject]["schemas"]) + 1
self.log.info(
"Registering subject: %r, id: %r new version: %r with schema %r, schema_id: %r",
subject, schema_id, version, new_schema.to_json(), schema_id)
self.send_schema_message(
subject=subject,
schema=new_schema,
schema_id=schema_id,
version=version,
deleted=False,
)
self.r({"id": schema_id}, content_type)
async def forward_request_remote(self,
*,
body,
url,
content_type,
method="POST"):
self.log.info(
"Writing new schema to remote url: %r since we're not the master",
url)
response = await self.http_request(url=url,
method=method,
json=body,
timeout=60.0)
self.r(body=response.body,
content_type=content_type,
status=response.status)
def no_master_error(self, content_type):
self.r(
{
"error_code": 50003,
"message": "Error while forwarding the request to the master."
},
content_type,
status=500)
class KarapaceSchemaRegistry(KarapaceBase):
# pylint: disable=attribute-defined-outside-init
def __init__(self, config: dict) -> None:
super().__init__(config=config)
self._add_schema_registry_routes()
self._init_schema_registry(config=config)
def _init_schema_registry(self, config: dict) -> None: # pylint: disable=unused-argument
self.ksr = None
self.producer = None
self.producer = self._create_producer()
self._create_master_coordinator()
self._create_schema_reader()
self.schema_lock = asyncio.Lock()
def _add_schema_registry_routes(self) -> None:
self.route(
"/compatibility/subjects/<subject:path>/versions/<version:path>",
callback=self.compatibility_check,
method="POST",
schema_request=True)
self.route(
"/config/<subject:path>",
callback=self.config_subject_get,
method="GET",
schema_request=True,
with_request=True,
json_body=False,
)
self.route("/config/<subject:path>",
callback=self.config_subject_set,
method="PUT",
schema_request=True)
self.route("/config",
callback=self.config_get,
method="GET",
schema_request=True)
self.route("/config",
callback=self.config_set,
method="PUT",
schema_request=True)
self.route("/schemas/ids/<schema_id:path>/versions",
callback=self.schemas_get_versions,
method="GET",
schema_request=True)
self.route("/schemas/ids/<schema_id:path>",
callback=self.schemas_get,
method="GET",
schema_request=True)
self.route("/schemas/types",
callback=self.schemas_types,
method="GET",
schema_request=True)
self.route("/subjects",
callback=self.subjects_list,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>/versions",
callback=self.subject_post,
method="POST",
schema_request=True)
self.route("/subjects/<subject:path>",
callback=self.subjects_schema_post,
method="POST",
schema_request=True)
self.route("/subjects/<subject:path>/versions",
callback=self.subject_versions_list,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>/versions/<version>",
callback=self.subject_version_get,
method="GET",
schema_request=True)
self.route(
"/subjects/<subject:path>/versions/<version:path>", # needs
callback=self.subject_version_delete,
method="DELETE",
schema_request=True,
with_request=True,
json_body=False,
)
self.route("/subjects/<subject:path>/versions/<version>/schema",
callback=self.subject_version_schema_get,
method="GET",
schema_request=True)
self.route(
"/subjects/<subject:path>",
callback=self.subject_delete,
method="DELETE",
schema_request=True,
with_request=True,
json_body=False,
)
async def close(self) -> None:
await super().close()
self.log.info("Shutting down all auxiliary threads")
if self.mc:
self.mc.close()
if self.ksr:
self.ksr.close()
if self.producer:
self.producer.close()
def _create_schema_reader(self):
self.ksr = KafkaSchemaReader(config=self.config,
master_coordinator=self.mc)
self.ksr.start()
def _create_master_coordinator(self):
self.mc = MasterCoordinator(config=self.config)
self.mc.start()
def _subject_get(self,
subject,
content_type,
include_deleted=False) -> Dict[str, Any]:
subject_data = self.ksr.subjects.get(subject)
if not subject_data:
self.r(
body={
"error_code":
SchemaErrorCodes.SUBJECT_NOT_FOUND.value,
"message":
SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(
subject=subject),
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
schemas = self.ksr.get_schemas(subject,
include_deleted=include_deleted)
if not schemas:
self.r(
body={
"error_code":
SchemaErrorCodes.SUBJECT_NOT_FOUND.value,
"message":
SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(
subject=subject),
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
subject_data = subject_data.copy()
subject_data["schemas"] = schemas
return subject_data
def _validate_version(self, content_type, version): # pylint: disable=inconsistent-return-statements
try:
version_number = int(version)
if version_number > 0:
return version
except ValueError:
if version == "latest":
return version
self.r(
body={
"error_code":
SchemaErrorCodes.INVALID_VERSION_ID.value,
"message":
(f"The specified version '{version}' is not a valid version id. "
"Allowed values are between [1, 2^31-1] and the string \"latest\""
),
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
def _get_compatibility_mode(self, subject,
content_type) -> CompatibilityModes:
compatibility = subject.get("compatibility",
self.ksr.config["compatibility"])
try:
compatibility_mode = CompatibilityModes(compatibility)
except ValueError:
# Using INTERNAL_SERVER_ERROR because the subject and configuration
# should have been validated before.
self.r(
body={
"error_code":
SchemaErrorCodes.HTTP_INTERNAL_SERVER_ERROR.value,
"message": f"Unknown compatibility mode {compatibility}",
},
content_type=content_type,
status=HTTPStatus.INTERNAL_SERVER_ERROR,
)
return compatibility_mode
def get_offset_from_queue(self, sent_offset):
start_time = time.monotonic()
while True:
self.log.info("Starting to wait for offset: %r from ksr queue",
sent_offset)
offset = self.ksr.queue.get()
if offset == sent_offset:
self.log.info(
"We've consumed back produced offset: %r message back, everything is in sync, took: %.4f",
offset,
time.monotonic() - start_time)
break
self.log.warning(
"Put the offset: %r back to queue, someone else is waiting for this?",
offset)
self.ksr.queue.put(offset)
def send_kafka_message(self, key, value):
if isinstance(key, str):
key = key.encode("utf8")
if isinstance(value, str):
value = value.encode("utf8")
future = self.producer.send(self.config["topic_name"],
key=key,
value=value)
self.producer.flush(timeout=self.kafka_timeout)
msg = future.get(self.kafka_timeout)
self.log.debug("Sent kafka msg key: %r, value: %r, offset: %r", key,
value, msg.offset)
self.get_offset_from_queue(msg.offset)
return future
def send_schema_message(
self,
*,
subject: str,
schema: Optional[TypedSchema],
schema_id: int,
version: int,
deleted: bool,
):
key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format(
subject, version)
if schema:
valuedict = {
"subject": subject,
"version": version,
"id": schema_id,
"schema": schema.schema_str,
"deleted": deleted
}
if schema.schema_type is not SchemaType.AVRO:
valuedict["schemaType"] = schema.schema_type
value = json_encode(valuedict, compact=True)
else:
value = ""
return self.send_kafka_message(key, value)
def send_config_message(self,
compatibility_level: CompatibilityModes,
subject=None):
if subject is not None:
key = '{{"subject":"{}","magic":0,"keytype":"CONFIG"}}'.format(
subject)
else:
key = '{"subject":null,"magic":0,"keytype":"CONFIG"}'
value = '{{"compatibilityLevel":"{}"}}'.format(
compatibility_level.value)
return self.send_kafka_message(key, value)
def send_delete_subject_message(self, subject, version):
key = '{{"subject":"{}","magic":0,"keytype":"DELETE_SUBJECT"}}'.format(
subject)
value = '{{"subject":"{}","version":{}}}'.format(subject, version)
return self.send_kafka_message(key, value)
async def compatibility_check(self, content_type, *, subject, version,
request):
"""Check for schema compatibility"""
body = request.json
self.log.info(
"Got request to check subject: %r, version_id: %r compatibility",
subject, version)
old = await self.subject_version_get(content_type=content_type,
subject=subject,
version=version,
return_dict=True)
self.log.info("Existing schema: %r, new_schema: %r", old["schema"],
body["schema"])
try:
schema_type = SchemaType(body.get("schemaType", "AVRO"))
new_schema = TypedSchema.parse(schema_type, body["schema"])
except InvalidSchema:
self.log.warning("Invalid schema: %r", body["schema"])
self.r(
body={
"error_code": SchemaErrorCodes.INVALID_AVRO_SCHEMA.value,
"message": "Invalid Avro schema",
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
try:
old_schema_type = SchemaType(old.get("schemaType", "AVRO"))
old_schema = TypedSchema.parse(old_schema_type, old["schema"])
except InvalidSchema:
self.log.warning("Invalid existing schema: %r", old["schema"])
self.r(
body={
"error_code": SchemaErrorCodes.INVALID_AVRO_SCHEMA.value,
"message": "Invalid Avro schema",
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
compatibility_mode = self._get_compatibility_mode(
subject=old, content_type=content_type)
result = check_compatibility(
old_schema=old_schema,
new_schema=new_schema,
compatibility_mode=compatibility_mode,
)
if is_incompatible(result):
self.log.warning(
"Invalid schema %s found by compatibility check: old: %s new: %s",
result, old_schema, new_schema)
self.r({"is_compatible": False}, content_type)
self.r({"is_compatible": True}, content_type)
async def schemas_get(self, content_type, *, schema_id):
try:
schema_id_int = int(schema_id)
except ValueError:
self.r(
body={
"error_code": SchemaErrorCodes.HTTP_NOT_FOUND.value,
"message": "HTTP 404 Not Found",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
with self.ksr.id_lock:
schema = self.ksr.schemas.get(schema_id_int)
if not schema:
self.log.warning("Schema: %r that was requested, not found",
int(schema_id))
self.r(
body={
"error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value,
"message": "Schema not found",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
response_body = {"schema": schema.schema_str}
if schema.schema_type is not SchemaType.AVRO:
response_body["schemaType"] = schema.schema_type
self.r(response_body, content_type)
async def schemas_get_versions(self, content_type, *, schema_id):
try:
schema_id_int = int(schema_id)
except ValueError:
self.r(
body={
"error_code": SchemaErrorCodes.HTTP_NOT_FOUND.value,
"message": "HTTP 404 Not Found",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
subject_versions = []
with self.ksr.id_lock:
for subject, val in self.ksr.subjects.items():
if self.ksr.get_schemas(subject) and "schemas" in val:
schemas = val["schemas"]
for version, schema in schemas.items():
if int(schema["id"]
) == schema_id_int and not schema["deleted"]:
subject_versions.append({
"subject": subject,
"version": int(version)
})
subject_versions = sorted(subject_versions,
key=lambda s: (s["subject"], s["version"]))
self.r(subject_versions, content_type)
async def schemas_types(self, content_type):
self.r(["JSON", "AVRO"], content_type)
async def config_get(self, content_type):
# Note: The format sent by the user differs from the return value, this
# is for compatibility reasons.
self.r({"compatibilityLevel": self.ksr.config["compatibility"]},
content_type)
async def config_set(self, content_type, *, request):
body = request.json
try:
compatibility_level = CompatibilityModes(
request.json["compatibility"])
except (ValueError, KeyError):
self.r(
body={
"error_code":
SchemaErrorCodes.INVALID_COMPATIBILITY_LEVEL.value,
"message":
SchemaErrorMessages.INVALID_COMPATIBILITY_LEVEL.value,
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
are_we_master, master_url = await self.get_master()
if are_we_master:
self.send_config_message(compatibility_level=compatibility_level,
subject=None)
elif not master_url:
self.no_master_error(content_type)
else:
url = f"{master_url}/config"
await self.forward_request_remote(body=body,
url=url,
content_type=content_type,
method="PUT")
self.r({"compatibility": self.ksr.config["compatibility"]},
content_type)
async def config_subject_get(self, content_type, subject: str, *,
request: HTTPRequest):
# Config for a subject can exist without schemas so no need to check for their existence
assert self.ksr, "KarapaceSchemaRegistry not initialized. Missing call to _init"
subject_data = self.ksr.subjects.get(subject, {})
if subject_data:
default_to_global = request.query.get("defaultToGlobal",
"false").lower() == "true"
compatibility = subject_data.get("compatibility")
if not compatibility and default_to_global:
compatibility = self.ksr.config["compatibility"]
if compatibility:
# Note: The format sent by the user differs from the return
# value, this is for compatibility reasons.
self.r(
{"compatibilityLevel": compatibility},
content_type,
)
self.r(
body={
"error_code":
SchemaErrorCodes.SUBJECT_NOT_FOUND.value,
"message":
SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(
subject=subject),
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
async def config_subject_set(self, content_type, *, request, subject):
try:
compatibility_level = CompatibilityModes(
request.json["compatibility"])
except (ValueError, KeyError):
self.r(
body={
"error_code":
SchemaErrorCodes.INVALID_COMPATIBILITY_LEVEL.value,
"message": "Invalid compatibility level",
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
are_we_master, master_url = await self.get_master()
if are_we_master:
self.send_config_message(compatibility_level=compatibility_level,
subject=subject)
elif not master_url:
self.no_master_error(content_type)
else:
url = f"{master_url}/config/{subject}"
await self.forward_request_remote(body=request.json,
url=url,
content_type=content_type,
method="PUT")
self.r({"compatibility": compatibility_level.value}, content_type)
async def subjects_list(self, content_type):
subjects_list = [
key for key, val in self.ksr.subjects.items()
if self.ksr.get_schemas(key)
]
self.r(subjects_list, content_type, status=HTTPStatus.OK)
async def _subject_delete_local(self, content_type: str, subject: str,
permanent: bool):
subject_data = self._subject_get(subject,
content_type,
include_deleted=permanent)
if permanent and [
version for version, value in subject_data["schemas"].items()
if not value.get("deleted", False)
]:
self.r(
body={
"error_code":
SchemaErrorCodes.SUBJECT_NOT_SOFT_DELETED.value,
"message":
f"Subject '{subject}' was not deleted first before being permanently deleted",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
version_list = list(subject_data["schemas"])
if version_list:
latest_schema_id = version_list[-1]
else:
latest_schema_id = 0
if permanent:
for version, value in list(subject_data["schemas"].items()):
schema_id = value.get("id")
self.log.info(
"Permanently deleting subject '%s' version %s (schema id=%s)",
subject, version, schema_id)
self.send_schema_message(subject=subject,
schema=None,
schema_id=schema_id,
version=version,
deleted=True)
else:
self.send_delete_subject_message(subject, latest_schema_id)
self.r(version_list, content_type, status=HTTPStatus.OK)
async def subject_delete(self, content_type, *, subject,
request: HTTPRequest):
permanent = request.query.get("permanent", "false").lower() == "true"
are_we_master, master_url = await self.get_master()
if are_we_master:
async with self.schema_lock:
await self._subject_delete_local(content_type, subject,
permanent)
elif not master_url:
self.no_master_error(content_type)
else:
url = f"{master_url}/subjects/{subject}?permanent={permanent}"
await self.forward_request_remote(body={},
url=url,
content_type=content_type,
method="DELETE")
async def subject_version_get(self,
content_type,
*,
subject,
version,
return_dict=False):
self._validate_version(content_type, version)
subject_data = self._subject_get(subject, content_type)
schema_data = None
max_version = max(subject_data["schemas"])
if version == "latest":
version = max(subject_data["schemas"])
schema_data = subject_data["schemas"][version]
elif int(version) <= max_version:
schema_data = subject_data["schemas"].get(int(version))
else:
self.r(
body={
"error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value,
"message": f"Version {version} not found.",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
if not schema_data:
self.r(
body={
"error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value,
"message": f"Version {version} not found.",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
schema_id = schema_data["id"]
schema = schema_data["schema"]
ret = {
"subject": subject,
"version": int(version),
"id": schema_id,
"schema": schema.schema_str,
}
if schema.schema_type is not SchemaType.AVRO:
ret["schemaType"] = schema.schema_type
if return_dict:
# Return also compatibility information to compatibility check
if subject_data.get("compatibility"):
ret["compatibility"] = subject_data.get("compatibility")
return ret
self.r(ret, content_type)
async def _subject_version_delete_local(self, content_type: str,
subject: str, version: int,
permanent: bool):
subject_data = self._subject_get(subject,
content_type,
include_deleted=True)
subject_schema_data = subject_data["schemas"].get(version, None)
if not subject_schema_data:
self.r(
body={
"error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value,
"message": f"Version {version} not found.",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
if subject_schema_data.get("deleted", False) and not permanent:
self.r(
body={
"error_code":
SchemaErrorCodes.SCHEMAVERSION_SOFT_DELETED.value,
"message":
f"Subject '{subject}' Version 1 was soft deleted.Set permanent=true to delete permanently",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
# Cannot directly hard delete
if permanent and not subject_schema_data.get("deleted", False):
self.r(
body={
"error_code":
SchemaErrorCodes.SCHEMAVERSION_NOT_SOFT_DELETED.value,
"message":
(f"Subject '{subject}' Version {version} was not deleted "
"first before being permanently deleted"),
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
schema_id = subject_schema_data["id"]
schema = subject_schema_data["schema"]
self.send_schema_message(subject=subject,
schema=None if permanent else schema,
schema_id=schema_id,
version=version,
deleted=True)
self.r(str(version), content_type, status=HTTPStatus.OK)
async def subject_version_delete(self, content_type, *, subject, version,
request: HTTPRequest):
version = int(version)
permanent = request.query.get("permanent", "false").lower() == "true"
are_we_master, master_url = await self.get_master()
if are_we_master:
async with self.schema_lock:
await self._subject_version_delete_local(
content_type, subject, version, permanent)
elif not master_url:
self.no_master_error(content_type)
else:
url = f"{master_url}/subjects/{subject}/versions/{version}?permanent={permanent}"
await self.forward_request_remote(body={},
url=url,
content_type=content_type,
method="DELETE")
async def subject_version_schema_get(self, content_type, *, subject,
version):
self._validate_version(content_type, version)
subject_data = self._subject_get(subject, content_type)
max_version = max(subject_data["schemas"])
if version == "latest":
schema_data = subject_data["schemas"][max_version]
elif int(version) <= max_version:
schema_data = subject_data["schemas"].get(int(version))
else:
self.r(
body={
"error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value,
"message": f"Version {version} not found.",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
self.r(schema_data["schema"].schema_str, content_type)
async def subject_versions_list(self, content_type, *, subject):
subject_data = self._subject_get(subject, content_type)
self.r(list(subject_data["schemas"]),
content_type,
status=HTTPStatus.OK)
async def get_master(self) -> Tuple[bool, Optional[str]]:
async with self.master_lock:
while True:
are_we_master, master_url = self.mc.get_master_info()
if are_we_master is None:
self.log.info("No master set: %r, url: %r", are_we_master,
master_url)
elif self.ksr.ready is False:
self.log.info("Schema reader isn't ready yet: %r",
self.ksr.ready)
else:
return are_we_master, master_url
await asyncio.sleep(1.0)
def _validate_schema_request_body(self, content_type, body) -> None:
if not isinstance(body, dict):
self.r(
body={
"error_code":
SchemaErrorCodes.HTTP_INTERNAL_SERVER_ERROR.value,
"message": "Internal Server Error",
},
content_type=content_type,
status=HTTPStatus.INTERNAL_SERVER_ERROR,
)
for field in body:
if field not in {"schema", "schemaType"}:
self.r(
body={
"error_code":
SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value,
"message": f"Unrecognized field: {field}",
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
def _validate_schema_type(self, content_type, body) -> None:
schema_type = SchemaType(body.get("schemaType", SchemaType.AVRO.value))
if schema_type not in {SchemaType.JSONSCHEMA, SchemaType.AVRO}:
self.r(
body={
"error_code":
SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value,
"message": f"unrecognized schemaType: {schema_type}",
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
def _validate_schema_key(self, content_type, body) -> None:
if "schema" not in body:
self.r(
body={
"error_code": SchemaErrorCodes.EMPTY_SCHEMA.value,
"message": "Empty schema",
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
async def subjects_schema_post(self, content_type, *, subject, request):
body = request.json
self._validate_schema_request_body(content_type, body)
subject_data = self._subject_get(subject, content_type)
new_schema = None
if "schema" not in body:
self.r(
body={
"error_code":
SchemaErrorCodes.HTTP_INTERNAL_SERVER_ERROR.value,
"message":
f"Error while looking up schema under subject {subject}",
},
content_type=content_type,
status=HTTPStatus.INTERNAL_SERVER_ERROR,
)
schema_str = body["schema"]
schema_type = SchemaType(body.get("schemaType", "AVRO"))
try:
new_schema = TypedSchema.parse(schema_type, schema_str)
except InvalidSchema:
self.log.exception("No proper parser found")
self.r(
body={
"error_code":
SchemaErrorCodes.HTTP_INTERNAL_SERVER_ERROR.value,
"message":
f"Error while looking up schema under subject {subject}",
},
content_type=content_type,
status=HTTPStatus.INTERNAL_SERVER_ERROR,
)
for schema in subject_data["schemas"].values():
typed_schema = schema["schema"]
if typed_schema == new_schema:
ret = {
"subject": subject,
"version": schema["version"],
"id": schema["id"],
"schema": typed_schema.schema_str,
}
if schema_type is not SchemaType.AVRO:
ret["schemaType"] = schema_type
self.r(ret, content_type)
else:
self.log.debug("Schema %r did not match %r", schema,
typed_schema)
self.r(
body={
"error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value,
"message": "Schema not found",
},
content_type=content_type,
status=HTTPStatus.NOT_FOUND,
)
async def subject_post(self, content_type, *, subject, request):
body = request.json
self.log.debug("POST with subject: %r, request: %r", subject, body)
self._validate_schema_request_body(content_type, body)
self._validate_schema_type(content_type, body)
self._validate_schema_key(content_type, body)
are_we_master, master_url = await self.get_master()
if are_we_master:
async with self.schema_lock:
await self.write_new_schema_local(subject, body, content_type)
elif not master_url:
self.no_master_error(content_type)
else:
url = f"{master_url}/subjects/{subject}/versions"
await self.forward_request_remote(body=body,
url=url,
content_type=content_type,
method="POST")
def write_new_schema_local(self, subject, body, content_type):
"""Since we're the master we get to write the new schema"""
self.log.info("Writing new schema locally since we're the master")
schema_type = SchemaType(body.get("schemaType", SchemaType.AVRO))
try:
new_schema = TypedSchema.parse(schema_type=schema_type,
schema_str=body["schema"])
except (InvalidSchema, InvalidSchemaType) as e:
self.log.warning("Invalid schema: %r",
body["schema"],
exc_info=True)
if isinstance(e.__cause__,
(SchemaParseException, JSONDecodeError)):
human_error = f"{e.__cause__.args[0]}" # pylint: disable=no-member
else:
human_error = "Provided schema is not valid"
self.r(
body={
"error_code":
SchemaErrorCodes.INVALID_AVRO_SCHEMA.value,
"message":
f"Invalid {schema_type} schema. Error: {human_error}",
},
content_type=content_type,
status=HTTPStatus.UNPROCESSABLE_ENTITY,
)
if subject not in self.ksr.subjects or not self.ksr.subjects.get(
subject)["schemas"]:
schema_id = self.ksr.get_schema_id(new_schema)
version = 1
self.log.info(
"Registering new subject: %r with version: %r to schema %r, schema_id: %r",
subject, version, new_schema.schema_str, schema_id)
else:
# First check if any of the existing schemas for the subject match
subject_data = self.ksr.subjects[subject]
schemas = self.ksr.get_schemas(subject)
if not schemas: # Previous ones have been deleted by the user.
version = max(self.ksr.subjects[subject]["schemas"]) + 1
schema_id = self.ksr.get_schema_id(new_schema)
self.log.info(
"Registering subject: %r, id: %r new version: %r with schema %r, schema_id: %r",
subject, schema_id, version, new_schema.schema_str,
schema_id)
self.send_schema_message(
subject=subject,
schema=new_schema,
schema_id=schema_id,
version=version,
deleted=False,
)
self.r({"id": schema_id}, content_type)
schema_versions = sorted(list(schemas))
# Go through these in version order
for version in schema_versions:
schema = subject_data["schemas"][version]
if schema["schema"] == new_schema:
self.r({"id": schema["id"]}, content_type)
else:
self.log.debug("schema: %s did not match with: %s", schema,
new_schema)
compatibility_mode = self._get_compatibility_mode(
subject=subject_data, content_type=content_type)
# Run a compatibility check between on file schema(s) and the one being submitted now
# the check is either towards the latest one or against all previous ones in case of
# transitive mode
if compatibility_mode.is_transitive():
check_against = schema_versions
else:
check_against = [schema_versions[-1]]
for old_version in check_against:
old_schema = subject_data["schemas"][old_version]["schema"]
result = check_compatibility(
old_schema=old_schema,
new_schema=new_schema,
compatibility_mode=compatibility_mode,
)
if is_incompatible(result):
message = set(
result.messages).pop() if result.messages else ""
self.log.warning("Incompatible schema: %s", result)
self.r(
body={
"error_code":
SchemaErrorCodes.HTTP_CONFLICT.value,
"message":
f"Incompatible schema, compatibility_mode={compatibility_mode.value} {message}",
},
content_type=content_type,
status=HTTPStatus.CONFLICT,
)
# We didn't find an existing schema and the schema is compatible so go and create one
schema_id = self.ksr.get_schema_id(new_schema)
version = max(self.ksr.subjects[subject]["schemas"]) + 1
self.log.info(
"Registering subject: %r, id: %r new version: %r with schema %r, schema_id: %r",
subject, schema_id, version, new_schema.to_json(), schema_id)
self.send_schema_message(
subject=subject,
schema=new_schema,
schema_id=schema_id,
version=version,
deleted=False,
)
self.r({"id": schema_id}, content_type)
async def forward_request_remote(self,
*,
body,
url,
content_type,
method="POST"):
self.log.info(
"Writing new schema to remote url: %r since we're not the master",
url)
response = await self.http_request(url=url,
method=method,
json=body,
timeout=60.0)
self.r(body=response.body,
content_type=content_type,
status=HTTPStatus(response.status))
def no_master_error(self, content_type):
self.r(
body={
"error_code": SchemaErrorCodes.NO_MASTER_ERROR.value,
"message": "Error while forwarding the request to the master.",
},
content_type=content_type,
status=HTTPStatus.INTERNAL_SERVER_ERROR,
)
class Karapace(RestApp):
def __init__(self, config_path):
self.config = {}
self.config_path = config_path
self.config = self.read_config(self.config_path)
self._sentry_config = self.config.get("sentry", {"dsn": None}).copy()
if os.environ.get("SENTRY_DSN"):
self._sentry_config["dsn"] = os.environ["SENTRY_DSN"]
if "tags" not in self._sentry_config:
self._sentry_config["tags"] = {}
self._sentry_config["tags"]["app"] = "Karapace"
super().__init__(app_name="Karapace",
sentry_config=self._sentry_config)
self.log = logging.getLogger("Karapace")
self.kafka_timeout = 10
self.route(
"/compatibility/subjects/<subject:path>/versions/<version:path>",
callback=self.compatibility_check,
method="POST",
schema_request=True)
self.route("/", callback=self.root_get, method="GET")
self.route("/config/<subject:path>",
callback=self.config_subject_get,
method="GET",
schema_request=True)
self.route("/config/<subject:path>",
callback=self.config_subject_set,
method="PUT",
schema_request=True)
self.route("/config",
callback=self.config_get,
method="GET",
schema_request=True)
self.route("/config",
callback=self.config_set,
method="PUT",
schema_request=True)
self.route("/schemas/ids/<schema_id:path>",
callback=self.schemas_get,
method="GET",
schema_request=True)
self.route("/subjects",
callback=self.subjects_list,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>/versions",
callback=self.subject_post,
method="POST",
schema_request=True)
self.route("/subjects/<subject:path>",
callback=self.subjects_schema_post,
method="POST",
schema_request=True)
self.route("/subjects/<subject:path>/versions",
callback=self.subject_versions_list,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>/versions/<version>",
callback=self.subject_version_get,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>/versions/<version:path>",
callback=self.subject_version_delete,
method="DELETE",
schema_request=True)
self.route("/subjects/<subject:path>/versions/<version>/schema",
callback=self.subject_version_schema_get,
method="GET",
schema_request=True)
self.route("/subjects/<subject:path>",
callback=self.subject_delete,
method="DELETE",
schema_request=True)
self.ksr = None
self._set_log_level()
self._create_producer()
self._create_schema_reader()
self._create_master_coordinator()
self.app.on_startup.append(self.create_http_client)
self.master_lock = asyncio.Lock()
self.log.info("Karapace initialized")
def close(self):
self.log.info("Shutting down all auxiliary threads")
if self.mc:
self.mc.close()
if self.ksr:
self.ksr.close()
if self.producer:
self.producer.close()
@staticmethod
def read_config(config_path):
if os.path.exists(config_path):
try:
config = json.loads(open(config_path, "r").read())
config = set_config_defaults(config)
return config
except Exception as ex:
raise InvalidConfiguration(ex)
else:
raise InvalidConfiguration()
def _set_log_level(self):
try:
logging.getLogger().setLevel(self.config["log_level"])
except ValueError:
self.log.excption("Problem with log_level: %r",
self.config["log_level"])
def _create_schema_reader(self):
self.ksr = KafkaSchemaReader(config=self.config, )
self.ksr.start()
def _create_master_coordinator(self):
self.mc = MasterCoordinator(config=self.config)
self.mc.start()
def _create_producer(self):
self.producer = KafkaProducer(
bootstrap_servers=self.config["bootstrap_uri"],
security_protocol=self.config["security_protocol"],
ssl_cafile=self.config["ssl_cafile"],
ssl_certfile=self.config["ssl_certfile"],
ssl_keyfile=self.config["ssl_keyfile"],
api_version=(1, 0, 0),
metadata_max_age_ms=self.config["metadata_max_age_ms"],
)
def _subject_get(self, subject, content_type):
subject_data = self.ksr.subjects.get(subject)
if not subject_data:
self.r({
"error_code": 40401,
"message": "Subject not found."
},
content_type,
status=404)
schemas = self.ksr.get_schemas(subject)
if not schemas:
self.r({
"error_code": 40401,
"message": "Subject not found."
},
content_type,
status=404)
subject_data["schemas"] = schemas
return subject_data
def _validate_version(self, content_type, version): # pylint: disable=inconsistent-return-statements
try:
version_number = int(version)
if version_number > 0:
return version
except ValueError:
if version == "latest":
return version
self.r(body={
"error_code":
42202,
"message":
"The specified version is not a valid version id. "
"Allowed values are between [1, 2^31-1] and the string \"latest\""
},
content_type=content_type,
status=422)
@staticmethod
def r(body, content_type, status=200):
raise HTTPResponse(
body=body,
status=status,
content_type=content_type,
headers={},
)
def get_offset_from_queue(self, sent_offset):
start_time = time.monotonic()
while True:
self.log.info("Starting to wait for offset: %r from ksr queue",
sent_offset)
offset = self.ksr.queue.get()
if offset == sent_offset:
self.log.info(
"We've consumed back produced offset: %r message back, everything is in sync, took: %.4f",
offset,
time.monotonic() - start_time)
break
self.log.error(
"Put the offset: %r back to queue, someone else is waiting for this?",
offset)
self.ksr.queue.put(offset)
def send_kafka_message(self, key, value):
if isinstance(key, str):
key = key.encode("utf8")
if isinstance(value, str):
value = value.encode("utf8")
future = self.producer.send(self.config["topic_name"],
key=key,
value=value)
self.producer.flush(timeout=self.kafka_timeout)
msg = future.get(self.kafka_timeout)
self.log.debug("Sent kafka msg key: %r, value: %r, offset: %r", key,
value, msg.offset)
self.get_offset_from_queue(msg.offset)
return future
def send_schema_message(self, subject, parsed_schema_json, schema_id,
version, deleted):
key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format(
subject, version)
value = {
"subject": subject,
"version": version,
"id": schema_id,
"schema": json_encode(parsed_schema_json, compact=True),
"deleted": deleted
}
return self.send_kafka_message(key, json_encode(value, compact=True))
def send_config_message(self, compatibility_level, subject=None):
if subject is not None:
key = '{{"subject":"{}","magic":0,"keytype":"CONFIG"}}'.format(
subject)
else:
key = '{"subject":null,"magic":0,"keytype":"CONFIG"}'
value = '{{"compatibilityLevel":"{}"}}'.format(compatibility_level)
return self.send_kafka_message(key, value)
def send_delete_subject_message(self, subject, version):
key = '{{"subject":"{}","magic":0,"keytype":"DELETE_SUBJECT"}}'.format(
subject)
value = '{{"subject":"{}","version":{}}}'.format(subject, version)
return self.send_kafka_message(key, value)
async def root_get(self):
self.r({}, "application/json")
async def compatibility_check(self, content_type, *, subject, version,
request):
"""Check for schema compatibility"""
body = request.json
self.log.info(
"Got request to check subject: %r, version_id: %r compatibility",
subject, version)
old = await self.subject_version_get(content_type=content_type,
subject=subject,
version=version,
return_dict=True)
self.log.info("Existing schema: %r, new_schema: %r", old["schema"],
body["schema"])
try:
new = avro.schema.Parse(body["schema"])
except avro.schema.SchemaParseException:
self.log.warning("Invalid schema: %r", body["schema"])
self.r(body={
"error_code": 44201,
"message": "Invalid Avro schema"
},
content_type=content_type,
status=422)
try:
old_schema = avro.schema.Parse(old["schema"])
except avro.schema.SchemaParseException:
self.log.warning("Invalid existing schema: %r", old["schema"])
self.r(body={
"error_code": 44201,
"message": "Invalid Avro schema"
},
content_type=content_type,
status=422)
compat = Compatibility(source=old_schema,
target=new,
compatibility=old.get(
"compatibility",
self.ksr.config["compatibility"]))
try:
compat.check()
except IncompatibleSchema as ex:
self.log.warning(
"Invalid schema %s found by compatibility check: old: %s new: %s",
ex, old_schema, new)
self.r({"is_compatible": False}, content_type)
self.r({"is_compatible": True}, content_type)
async def schemas_get(self, content_type, *, schema_id):
try:
schema_id_int = int(schema_id)
except ValueError:
self.r({
"error_code": 404,
"message": "HTTP 404 Not Found"
},
content_type,
status=404)
schema = self.ksr.schemas.get(schema_id_int)
if not schema:
self.log.warning("Schema: %r that was requested, not found",
int(schema_id))
self.r(body={
"error_code": 40403,
"message": "Schema not found"
},
content_type=content_type,
status=404)
self.r({"schema": schema}, content_type)
async def config_get(self, content_type):
self.r({"compatibilityLevel": self.ksr.config["compatibility"]},
content_type)
async def config_set(self, content_type, *, request):
if "compatibility" in request.json and request.json[
"compatibility"] in COMPATIBILITY_MODES:
compatibility_level = request.json["compatibility"]
self.send_config_message(compatibility_level=compatibility_level,
subject=None)
else:
self.r(body={
"error_code":
42203,
"message":
"Invalid compatibility level. Valid values are none, backward, forward and full"
},
content_type=content_type,
status=422)
self.r({"compatibility": self.ksr.config["compatibility"]},
content_type)
async def config_subject_get(self, content_type, *, subject):
# Config for a subject can exist without schemas so no need to check for their existence
subject_data = self.ksr.subjects.get(subject)
if "compatibility" in subject_data:
self.r({"compatibilityLevel": subject_data["compatibility"]},
content_type)
self.r({
"error_code": 40401,
"message": "Subject not found."
},
content_type,
status=404)
async def config_subject_set(self, content_type, *, request, subject):
if "compatibility" in request.json and request.json[
"compatibility"] in COMPATIBILITY_MODES:
self.send_config_message(
compatibility_level=request.json["compatibility"],
subject=subject)
else:
self.r(body={
"error_code": 42203,
"message": "Invalid compatibility level"
},
content_type=content_type,
status=422)
self.r({"compatibility": request.json["compatibility"]}, content_type)
async def subjects_list(self, content_type):
subjects_list = [
key for key, val in self.ksr.subjects.items()
if self.ksr.get_schemas(key)
]
self.r(subjects_list, content_type, status=200)
async def subject_delete(self, content_type, *, subject):
self._subject_get(subject, content_type)
version_list = list(self.ksr.get_schemas(subject))
if version_list:
latest_schema_id = version_list[-1]
else:
latest_schema_id = 0
self.send_delete_subject_message(subject, latest_schema_id)
self.r(version_list, content_type, status=200)
async def subject_version_get(self,
content_type,
*,
subject,
version,
return_dict=False):
self._validate_version(content_type, version)
subject_data = self._subject_get(subject, content_type)
max_version = max(subject_data["schemas"])
if version == "latest":
schema = subject_data["schemas"][max(subject_data["schemas"])]
version = max(subject_data["schemas"])
elif int(version) <= max_version:
schema = subject_data["schemas"].get(int(version))
else:
self.r({
"error_code": 40402,
"message": "Version not found."
},
content_type,
status=404)
schema_string = schema["schema"]
schema_id = schema["id"]
ret = {
"subject": subject,
"version": int(version),
"id": schema_id,
"schema": schema_string,
}
if return_dict:
# Return also compatibility information to compatibility check
if subject_data.get("compatibility"):
ret["compatibility"] = subject_data.get("compatibility")
return ret
self.r(ret, content_type)
async def subject_version_delete(self, content_type, *, subject, version):
version = int(version)
subject_data = self._subject_get(subject, content_type)
schema = subject_data["schemas"].get(version, None)
if not schema:
self.r({
"error_code": 40402,
"message": "Version not found."
},
content_type,
status=404)
schema_id = schema["id"]
self.send_schema_message(subject,
schema,
schema_id,
version,
deleted=True)
self.r(str(version), content_type, status=200)
async def subject_version_schema_get(self, content_type, *, subject,
version):
self._validate_version(content_type, version)
subject_data = self._subject_get(subject, content_type)
max_version = max(subject_data["schemas"])
if version == "latest":
schema_data = subject_data["schemas"][max_version]
elif int(version) <= max_version:
schema_data = subject_data["schemas"].get(int(version))
else:
self.r({
"error_code": 40402,
"message": "Version not found."
},
content_type,
status=404)
self.r(schema_data["schema"], content_type)
async def subject_versions_list(self, content_type, *, subject):
subject_data = self._subject_get(subject, content_type)
self.r(list(subject_data["schemas"]), content_type, status=200)
async def get_master(self):
async with self.master_lock:
while True:
master, master_url = self.mc.get_master_info()
if master is None:
self.log.info("No master set: %r, url: %r", master,
master_url)
elif self.ksr.ready is False:
self.log.info("Schema reader isn't ready yet: %r",
self.ksr.ready)
else:
return master, master_url
await asyncio.sleep(1.0)
def _validate_schema_request_body(self, content_type, body):
if not isinstance(body, dict):
self.r({
"error_code": 500,
"message": "Internal Server Error"
},
content_type,
status=500)
for field in body:
if field != "schema":
self.r(body={
"error_code": 422,
"message": f"Unrecognized field: {field}",
},
content_type=content_type,
status=422)
async def subjects_schema_post(self, content_type, *, subject, request):
body = request.json
self._validate_schema_request_body(content_type, body)
subject_data = self._subject_get(subject, content_type)
if "schema" not in body:
self.r({
"error_code": 500,
"message": "Internal Server Error"
},
content_type,
status=500)
try:
new_schema = avro.schema.Parse(body["schema"])
except avro.schema.SchemaParseException:
self.r(body={
"error_code":
500,
"message":
f"Error while looking up schema under subject {subject}"
},
content_type=content_type,
status=500)
new_schema_encoded = json_encode(new_schema.to_json(), compact=True)
for schema in subject_data["schemas"].values():
if schema["schema"] == new_schema_encoded:
ret = {
"subject": subject,
"version": schema["version"],
"id": schema["id"],
"schema": schema["schema"],
}
self.r(ret, content_type)
self.r({
"error_code": 40403,
"message": "Schema not found"
},
content_type,
status=404)
async def subject_post(self, content_type, *, subject, request):
body = request.json
self.log.debug("POST with subject: %r, request: %r", subject, body)
self._validate_schema_request_body(content_type, body)
if "schema" not in body:
self.r({
"error_code": 500,
"message": "Internal Server Error"
},
content_type,
status=500)
are_we_master, master_url = await self.get_master()
if are_we_master:
self.write_new_schema_local(subject, body, content_type)
elif are_we_master is None:
self.r(
{
"error_code": 50003,
"message":
"Error while forwarding the request to the master."
},
content_type,
status=500)
else:
await self.write_new_schema_remote(subject, body, master_url,
content_type)
def write_new_schema_local(self, subject, body, content_type):
"""Since we're the master we get to write the new schema"""
self.log.info("Writing new schema locally since we're the master")
try:
new_schema = avro.schema.Parse(body["schema"])
except avro.schema.SchemaParseException:
self.log.warning("Invalid schema: %r", body["schema"])
self.r(body={
"error_code": 44201,
"message": "Invalid Avro schema"
},
content_type=content_type,
status=422)
if subject not in self.ksr.subjects or not self.ksr.subjects.get(
subject)["schemas"]:
schema_id = self.ksr.get_schema_id(new_schema)
version = 1
self.log.info(
"Registering new subject: %r with version: %r to schema %r, schema_id: %r",
subject, version, new_schema.to_json(), schema_id)
else:
# First check if any of the existing schemas for the subject match
subject_data = self.ksr.subjects[subject]
schemas = self.ksr.get_schemas(subject)
if not schemas: # Previous ones have been deleted by the user.
version = max(self.ksr.subjects[subject]["schemas"]) + 1
schema_id = self.ksr.get_schema_id(new_schema)
self.log.info(
"Registering subject: %r, id: %r new version: %r with schema %r, schema_id: %r",
subject, schema_id, version, new_schema.to_json(),
schema_id)
self.send_schema_message(subject,
new_schema.to_json(),
schema_id,
version,
deleted=False)
self.r({"id": schema_id}, content_type)
schema_versions = sorted(list(schemas))
# Go through these in version order
for version in schema_versions:
schema = subject_data["schemas"][version]
parsed_version_schema = avro.schema.Parse(schema["schema"])
if parsed_version_schema == new_schema:
self.r({"id": schema["id"]}, content_type)
else:
self.log.debug("schema: %s did not match with: %s",
schema["schema"], new_schema.to_json())
compatibility = subject_data.get("compatibility",
self.ksr.config["compatibility"])
# Run a compatibility check between on file schema(s) and the one being submitted now
# the check is either towards the latest one or against all previous ones in case of
# transitive mode
if compatibility in TRANSITIVE_MODES:
check_against = schema_versions
else:
check_against = [schema_versions[-1]]
for old_version in check_against:
old_schema = avro.schema.Parse(
subject_data["schemas"][old_version]["schema"])
compat = Compatibility(old_schema,
new_schema,
compatibility=compatibility)
try:
compat.check()
except IncompatibleSchema as ex:
self.log.warning("Incompatible schema: %s", ex)
self.r(body={
"error_code":
409,
"message":
"Schema being registered is incompatible with an earlier schema"
},
content_type=content_type,
status=409)
# We didn't find an existing schema and the schema is compatible so go and create one
schema_id = self.ksr.get_schema_id(new_schema)
version = max(self.ksr.subjects[subject]["schemas"]) + 1
self.log.info(
"Registering subject: %r, id: %r new version: %r with schema %r, schema_id: %r",
subject, schema_id, version, new_schema.to_json(), schema_id)
self.send_schema_message(subject,
new_schema.to_json(),
schema_id,
version,
deleted=False)
self.r({"id": schema_id}, content_type)
async def write_new_schema_remote(self, subject, body, master_url,
content_type):
self.log.info(
"Writing new schema to remote url: %r since we're not the master",
master_url)
response = await self.http_request(
url="{}/subjects/{}/versions".format(master_url, subject),
method="POST",
json=body,
timeout=60.0)
self.r(body=response.body,
content_type=content_type,
status=response.status)