def update_compatibility(self, level, subject=None, headers=None): """ PUT /config/(string: subject) Update the compatibility level for a subject. Level must be one of: Args: level (str): ex: 'NONE','FULL','FORWARD', or 'BACKWARD' headers (dict): Extra headers to add on the requests Returns: None """ if level not in utils.VALID_LEVELS: raise ClientError(f"Invalid level specified: {level}") url = "/".join([self.url, "config"]) if subject: url += "/" + subject body = {"compatibility": level} result, code = self.request(url, method="PUT", body=body, headers=headers) if status.HTTP_200_OK <= code < status.HTTP_300_MULTIPLE_CHOICES: return result["compatibility"] else: raise ClientError(f"Unable to update level: {level}. Error code: {code}")
async def update_compatibility(self, level: str, subject: str = None, headers: dict = None) -> bool: """ PUT /config/(string: subject) Update the compatibility level. If subject is None, the compatibility level is global. Args: level (str): one of BACKWARD, BACKWARD_TRANSITIVE, FORWARD, FORWARD_TRANSITIVE, FULL, FULL_TRANSITIVE, NONE subject (str): Option subject headers (dict): Extra headers to add on the requests Returns: bool: True if compatibility was updated Raises: ClientError: if the request was unsuccessful or an invalid """ if level not in utils.VALID_LEVELS: raise ClientError(f"Invalid level specified: {level}") url, method = self.url_manager.url_for("update_compatibility", subject=subject) body = {"compatibility": level} result, code = await self.request(url, method=method, body=body, headers=headers) print(result, code) if status.is_success(code): return True raise ClientError(f"Unable to update level: {level}.", http_code=code, server_traceback=result)
def update_compatibility(self, level, subject=None, headers=None): """ PUT /config/(string: subject) Update the compatibility level. If subject is None, the compatibility level is global. Args: level (str): one of BACKWARD, BACKWARD_TRANSITIVE, FORWARD, FORWARD_TRANSITIVE, FULL, FULL_TRANSITIVE, NONE subject (str): Option subject headers (dict): Extra headers to add on the requests Returns: None """ if level not in utils.VALID_LEVELS: raise ClientError(f"Invalid level specified: {level}") url = "/".join([self.url, "config"]) if subject: url += "/" + subject body = {"compatibility": level} result, code = self.request(url, method="PUT", body=body, headers=headers) if status.is_success(code): return result["compatibility"] raise ClientError(f"Unable to update level: {level}.", http_code=code, server_traceback=result)
async def get_compatibility(self, subject: str = None, headers: dict = None) -> str: """ Get the current compatibility level for a subject. Args: subject (str): subject name headers (dict): Extra headers to add on the requests Returns: str: one of BACKWARD, BACKWARD_TRANSITIVE, FORWARD, FORWARD_TRANSITIVE, FULL, FULL_TRANSITIVE, NONE Raises: ClientError: if the request was unsuccessful or an invalid compatibility level was returned """ url, method = self.url_manager.url_for("get_compatibility", subject=subject) result, code = await self.request(url, method=method, headers=headers) if status.is_success(code): compatibility = result.get("compatibilityLevel") if compatibility not in utils.VALID_LEVELS: if compatibility is None: error_msg_suffix = "No compatibility was returned" else: error_msg_suffix = str(compatibility) raise ClientError( f"Invalid compatibility level received: {error_msg_suffix}", http_code=code, server_traceback=result ) return compatibility raise ClientError( f"Unable to fetch compatibility level. Error code: {code}", http_code=code, server_traceback=result )
async def test_compatibility( self, subject: str, avro_schema: AvroSchema, version: typing.Union[int, str] = "latest", headers: dict = None ) -> bool: """ POST /compatibility/subjects/(string: subject)/versions/(versionId: version) Test the compatibility of a candidate parsed schema for a given subject. By default the latest version is checked against. Args: subject (str): subject name avro_schema (avro.schema.RecordSchema): Avro schema headers (dict): Extra headers to add on the requests Returns: bool: True if schema given compatible, False otherwise """ url, method = self.url_manager.url_for("test_compatibility", subject=subject, version=version) body = {"schema": json.dumps(avro_schema.schema)} result, code = await self.request(url, method=method, body=body, headers=headers) if code == status.HTTP_404_NOT_FOUND: logger.error(f"Subject or version not found: {code}") return False elif code == status.HTTP_422_UNPROCESSABLE_ENTITY: logger.error(f"Invalid subject or schema: {code}") return False elif status.is_success(code): return result.get("is_compatible") raise ClientError("Unable to check the compatibility", http_code=code, server_traceback=result)
async def get_versions(self, subject: str, headers: dict = None) -> list: """ GET subjects/{subject}/versions Get a list of versions registered under the specified subject. Args: subject (str): subject name headers (dict): Extra headers to add on the requests Returns: list (str): version of the schema registered under this subject """ url, method = self.url_manager.url_for("get_versions", subject=subject) result, code = await self.request(url, method=method, headers=headers) if status.is_success(code): return result elif code == status.HTTP_404_NOT_FOUND: logger.error(f"Subject {subject} not found") return [] raise ClientError( f"Unable to get the versions for subject {subject}", http_code=code, server_traceback=result, )
async def get_by_id(self, schema_id: int, headers: dict = None) -> typing.Optional[AvroSchema]: """ GET /schemas/ids/{int: id} Retrieve a parsed avro schema by id or None if not found Args: schema_id (int): Schema Id headers (dict): Extra headers to add on the requests Returns: client.schema.AvroSchema: Avro Record schema """ if schema_id in self.id_to_schema: return self.id_to_schema[schema_id] url, method = self.url_manager.url_for("get_by_id", schema_id=schema_id) result, code = await self.request(url, method=method, headers=headers) if code == status.HTTP_404_NOT_FOUND: logger.error(f"Schema not found: {code}") return None elif status.is_success(code): schema_str = result.get("schema") result = AvroSchema(schema_str) # cache the result self._cache_schema(result, schema_id) return result raise ClientError(f"Received bad schema (id {schema_id})", http_code=code, server_traceback=result)
def delete_subject(self, subject: str, headers: dict = None) -> list: """ DELETE /subjects/(string: subject) Deletes the specified subject and its associated compatibility level if registered. It is recommended to use this API only when a topic needs to be recycled or in development environments. Args: subject (str): subject name headers (dict): Extra headers to add on the requests Returns: list (int): version of the schema deleted under this subject """ url, method = self.url_manager.url_for("delete_subject", subject=subject) result, code = self.request(url, method=method, headers=headers) if status.is_success(code): return result elif code == status.HTTP_404_NOT_FOUND: return [] raise ClientError("Unable to delete subject", http_code=code, server_traceback=result)
def get_versions( self, subject: str, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET) -> list: """ GET subjects/{subject}/versions Get a list of versions registered under the specified subject. Args: subject (str): subject name headers (dict): Extra headers to add on the requests timeout (httpx._client.TimeoutTypes): The timeout configuration to use when sending requests. Default UNSET Returns: list (str): version of the schema registered under this subject """ url, method = self.url_manager.url_for("get_versions", subject=subject) result, code = self.request(url, method=method, headers=headers, timeout=timeout) if status.is_success(code): return result elif code == status.HTTP_404_NOT_FOUND: logger.error(f"Subject {subject} not found") return [] raise ClientError(f"Unable to get the versions for subject {subject}", http_code=code, server_traceback=result)
def get_subjects( self, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET) -> list: """ GET /subjects/(string: subject) Get list of all registered subjects in your Schema Registry. Args: subject (str): subject name headers (dict): Extra headers to add on the requests timeout (httpx._client.TimeoutTypes): The timeout configuration to use when sending requests. Default UNSET Returns: list [str]: list of registered subjects. """ url, method = self.url_manager.url_for("get_subjects") result, code = self.request(url, method=method, headers=headers, timeout=timeout) if status.is_success(code): return result raise ClientError("Unable to get subjects", http_code=code, server_traceback=result)
def request( self, url: str, method: str = "GET", body: dict = None, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> tuple: if method not in utils.VALID_METHODS: raise ClientError( f"Method {method} is invalid; valid methods include {utils.VALID_METHODS}" ) _headers = self.prepare_headers(body=body, headers=headers) with self.session as session: response = session.request(method, url, headers=_headers, json=body, timeout=timeout) try: return response.json(), response.status_code except ValueError: return response.content, response.status_code
async def delete_version( self, subject: str, version: typing.Union[int, str] = "latest", headers: dict = None ) -> typing.Optional[int]: """ DELETE /subjects/(string: subject)/versions/(versionId: version) Deletes a specific version of the schema registered under this subject. This only deletes the version and the schema ID remains intact making it still possible to decode data using the schema ID. This API is recommended to be used only in development environments or under extreme circumstances where-in, its required to delete a previously registered schema for compatibility purposes or re-register previously registered schema. Args: subject (str): subject name version (str): Version of the schema to be deleted. Valid values for versionId are between [1,2^31-1] or the string "latest". "latest" deletes the last registered schema under the specified subject. headers (dict): Extra headers to add on the requests Returns: int: version of the schema deleted None: If the subject or version does not exist. """ url, method = self.url_manager.url_for("delete_version", subject=subject, version=version) result, code = await self.request(url, method=method, headers=headers) if status.is_success(code): return result elif status.is_client_error(code): return None raise ClientError("Unable to delete the version", http_code=code, server_traceback=result)
def get_compatibility( self, subject: str = None, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET) -> str: """ Get the current compatibility level for a subject. Args: subject (str): subject name headers (dict): Extra headers to add on the requests timeout (httpx._client.TimeoutTypes): The timeout configuration to use when sending requests. Default UNSET Returns: str: one of BACKWARD, BACKWARD_TRANSITIVE, FORWARD, FORWARD_TRANSITIVE, FULL, FULL_TRANSITIVE, NONE Raises: ClientError: if the request was unsuccessful or an invalid compatibility level was returned """ url, method = self.url_manager.url_for("get_compatibility", subject=subject) result, code = self.request(url, method=method, headers=headers, timeout=timeout) if status.is_success(code): compatibility = result.get("compatibilityLevel") if compatibility not in utils.VALID_LEVELS: if compatibility is None: error_msg_suffix = "No compatibility was returned" else: error_msg_suffix = str(compatibility) raise ClientError( f"Invalid compatibility level received: {error_msg_suffix}", http_code=code, server_traceback=result) return compatibility raise ClientError( f"Unable to fetch compatibility level. Error code: {code}", http_code=code, server_traceback=result)
def register( self, subject: str, avro_schema: AvroSchema, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> int: """ POST /subjects/(string: subject)/versions Register a schema with the registry under the given subject and receive a schema id. avro_schema must be a parsed schema from the python avro library Multiple instances of the same schema will result in cache misses. Args: subject (str): subject name avro_schema (avro.schema.RecordSchema): Avro schema to be registered headers (dict): Extra headers to add on the requests timeout (httpx._client.TimeoutTypes): The timeout configuration to use when sending requests. Default UNSET Returns: int: schema_id """ schemas_to_id = self.subject_to_schema_ids[subject] schema_id = schemas_to_id.get(avro_schema) if schema_id is not None: return schema_id url, method = self.url_manager.url_for("register", subject=subject) body = {"schema": json.dumps(avro_schema.schema)} result, code = self.request(url, method=method, body=body, headers=headers, timeout=timeout) msg = None if code in (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN): msg = "Unauthorized access" elif code == status.HTTP_409_CONFLICT: msg = "Incompatible Avro schema" elif code == status.HTTP_422_UNPROCESSABLE_ENTITY: msg = "Invalid Avro schema" elif not status.is_success(code): msg = "Unable to register schema" if msg is not None: raise ClientError(message=msg, http_code=code, server_traceback=result) schema_id = result["id"] self._cache_schema(avro_schema, schema_id, subject) return schema_id
def check_version( self, subject: str, avro_schema: AvroSchema, headers: dict = None) -> typing.Optional[utils.SchemaVersion]: """ POST /subjects/(string: subject) Check if a schema has already been registered under the specified subject. If so, this returns the schema string along with its globally unique identifier, its version under this subject and the subject name. Args: subject (str): subject name avro_schema (avro.schema.RecordSchema): Avro schema headers (dict): Extra headers to add on the requests Returns: dict: subject (string) -- Name of the subject that this schema is registered under id (int) -- Globally unique identifier of the schema version (int) -- Version of the returned schema schema (dict) -- The Avro schema None: If schema not found. """ schemas_to_version = self.subject_to_schema_versions[subject] version = schemas_to_version.get(avro_schema) schemas_to_id = self.subject_to_schema_ids[subject] schema_id = schemas_to_id.get(avro_schema) if all((version, schema_id)): return utils.SchemaVersion(subject, schema_id, version, avro_schema) url, method = self.url_manager.url_for("check_version", subject=subject) body = {"schema": json.dumps(avro_schema.schema)} result, code = self.request(url, method=method, body=body, headers=headers) if code == status.HTTP_404_NOT_FOUND: logger.error(f"Not found: {code}") return None elif status.is_success(code): schema_id = result.get("id") version = result.get("version") self._cache_schema(avro_schema, schema_id, subject, version) return utils.SchemaVersion(subject, schema_id, version, result.get("schema")) raise ClientError("Unable to get version of a schema", http_code=code, server_traceback=result)
def update_compatibility( self, level: str, subject: str = None, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> bool: """ PUT /config/(string: subject) Update the compatibility level. If subject is None, the compatibility level is global. Args: level (str): one of BACKWARD, BACKWARD_TRANSITIVE, FORWARD, FORWARD_TRANSITIVE, FULL, FULL_TRANSITIVE, NONE subject (str): Option subject headers (dict): Extra headers to add on the requests timeout (httpx._client.TimeoutTypes): The timeout configuration to use when sending requests. Default UNSET Returns: bool: True if compatibility was updated Raises: ClientError: if the request was unsuccessful or an invalid """ if level not in utils.VALID_LEVELS: raise ClientError(f"Invalid level specified: {level}") url, method = self.url_manager.url_for("update_compatibility", subject=subject) body = {"compatibility": level} result, code = self.request(url, method=method, body=body, headers=headers, timeout=timeout) if status.is_success(code): return True raise ClientError(f"Unable to update level: {level}.", http_code=code, server_traceback=result)
def check_version( self, subject: str, avro_schema: AvroSchema, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> typing.Optional[utils.SchemaVersion]: """ POST /subjects/(string: subject) Check if a schema has already been registered under the specified subject. If so, this returns the schema string along with its globally unique identifier, its version under this subject and the subject name. Args: subject (str): subject name avro_schema (avro.schema.RecordSchema): Avro schema headers (dict): Extra headers to add on the requests timeout (httpx._client.TimeoutTypes): The timeout configuration to use when sending requests. Default UNSET Returns: SchemaVersion (nametupled): (subject, schema_id, schema, version) None: If schema not found. """ schemas_to_version = self.subject_to_schema_versions[subject] version = schemas_to_version.get(avro_schema) schemas_to_id = self.subject_to_schema_ids[subject] schema_id = schemas_to_id.get(avro_schema) if all((version, schema_id)): return utils.SchemaVersion(subject, schema_id, version, avro_schema) url, method = self.url_manager.url_for("check_version", subject=subject) body = {"schema": json.dumps(avro_schema.schema)} result, code = self.request(url, method=method, body=body, headers=headers, timeout=timeout) if code == status.HTTP_404_NOT_FOUND: logger.error(f"Not found: {code}") return None elif status.is_success(code): schema_id = result.get("id") version = result.get("version") self._cache_schema(avro_schema, schema_id, subject, version) return utils.SchemaVersion(subject, schema_id, version, result.get("schema")) raise ClientError("Unable to get version of a schema", http_code=code, server_traceback=result)
def get_compatibility(self, subject, headers=None): """ Get the current compatibility level for a subject. Args: subject (str): subject name headers (dict): Extra headers to add on the requests Returns: str: one of 'NONE','FULL','FORWARD', or 'BACKWARD' Raises: ClientError: if the request was unsuccessful or an invalid compatibility level was returned """ url = "/".join([self.url, "config"]) if subject: url = "/".join([url, subject]) result, code = self.request(url, headers=headers) is_successful_request = ( status.HTTP_200_OK <= code < status.HTTP_300_MULTIPLE_CHOICES ) if not is_successful_request: raise ClientError( f"Unable to fetch compatibility level. Error code: {code}" ) compatibility = result.get("compatibilityLevel") if compatibility not in utils.VALID_LEVELS: if compatibility is None: error_msg_suffix = "No compatibility was returned" else: error_msg_suffix = str(compatibility) raise ClientError( f"Invalid compatibility level received: {error_msg_suffix}", code, server_traceback=result, ) return compatibility
def get_compatibility(self, subject, headers=None): """ Get the current compatibility level for a subject. Args: subject (str): subject name headers (dict): Extra headers to add on the requests Returns: str: one of BACKWARD, BACKWARD_TRANSITIVE, FORWARD, FORWARD_TRANSITIVE, FULL, FULL_TRANSITIVE, NONE Raises: ClientError: if the request was unsuccessful or an invalid compatibility level was returned """ url = "/".join([self.url, "config"]) if subject: url = "/".join([url, subject]) result, code = self.request(url, headers=headers) if not status.is_success(code): raise ClientError( f"Unable to fetch compatibility level. Error code: {code}", http_code=code, server_traceback=result, ) compatibility = result.get("compatibilityLevel") if compatibility not in utils.VALID_LEVELS: if compatibility is None: error_msg_suffix = "No compatibility was returned" else: error_msg_suffix = str(compatibility) raise ClientError( f"Invalid compatibility level received: {error_msg_suffix}", http_code=code, server_traceback=result, ) return compatibility
async def request(self, url: str, method: str = "GET", body: dict = None, headers: dict = None) -> tuple: if method not in utils.VALID_METHODS: raise ClientError(f"Method {method} is invalid; valid methods include {utils.VALID_METHODS}") _headers = self.prepare_headers(body=body, headers=headers) response = await self.session.request(method, url, headers=_headers, json=body) await self.session.aclose() try: return response.json(), response.status_code except ValueError: return response.content, response.status_code
def request(self, url, method="GET", body=None, headers=None): if method not in utils.VALID_METHODS: raise ClientError( f"Method {method} is invalid; valid methods include {utils.VALID_METHODS}" ) _headers = self.prepare_headers(body=body, headers=headers) response = super().request(method, url, headers=_headers, json=body) try: return response.json(), response.status_code except ValueError: return response.content, response.status_code
def register(self, subject, avro_schema, headers=None): """ POST /subjects/(string: subject)/versions Register a schema with the registry under the given subject and receive a schema id. avro_schema must be a parsed schema from the python avro library Multiple instances of the same schema will result in cache misses. Args: subject (str): subject name avro_schema (avro.schema.RecordSchema): Avro schema to be registered headers (dict): Extra headers to add on the requests Returns: int: schema_id """ schemas_to_id = self.subject_to_schema_ids[subject] schema_id = schemas_to_id.get(avro_schema.name) if schema_id is not None: return schema_id url = "/".join([self.url, "subjects", subject, "versions"]) body = {"schema": json.dumps(avro_schema.schema)} result, code = self.request(url, method="POST", body=body, headers=headers) msg = None if code in (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN): msg = "Unauthorized access" elif code == status.HTTP_409_CONFLICT: msg = "Incompatible Avro schema" elif code == status.HTTP_422_UNPROCESSABLE_ENTITY: msg = "Invalid Avro schema" elif not status.is_success(code): msg = "Unable to register schema" if msg is not None: raise ClientError(message=msg, http_code=code, server_traceback=result) schema_id = result["id"] self._cache_schema(avro_schema, schema_id, subject) return schema_id
async def get_subjects(self, headers: dict = None) -> list: """ GET /subjects/(string: subject) Get list of all registered subjects in your Schema Registry. Args: subject (str): subject name headers (dict): Extra headers to add on the requests Returns: list [str]: list of registered subjects. """ url, method = self.url_manager.url_for("get_subjects") result, code = await self.request(url, method=method, headers=headers) if status.is_success(code): return result raise ClientError("Unable to get subject", http_code=code, server_traceback=result)
def delete_subject(self, subject, headers=None): """ DELETE /subjects/(string: subject) Deletes the specified subject and its associated compatibility level if registered. It is recommended to use this API only when a topic needs to be recycled or in development environments. Args: subject (str): subject name headers (dict): Extra headers to add on the requests Returns: int: version of the schema deleted under this subject """ url = "/".join([self.url, "subjects", subject]) result, code = self.request(url, method="DELETE", headers=headers) if not (status.HTTP_200_OK <= code < status.HTTP_300_MULTIPLE_CHOICES): raise ClientError("Unable to delete subject", code, server_traceback=result) return result
def get_by_id( self, schema_id: int, headers: dict = None, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET ) -> typing.Optional[AvroSchema]: """ GET /schemas/ids/{int: id} Retrieve a parsed avro schema by id or None if not found Args: schema_id (int): Schema Id headers (dict): Extra headers to add on the requests timeout (httpx._client.TimeoutTypes): The timeout configuration to use when sending requests. Default UNSET Returns: client.schema.AvroSchema: Avro Record schema """ if schema_id in self.id_to_schema: return self.id_to_schema[schema_id] url, method = self.url_manager.url_for("get_by_id", schema_id=schema_id) result, code = self.request(url, method=method, headers=headers, timeout=timeout) if code == status.HTTP_404_NOT_FOUND: logger.error(f"Schema not found: {code}") return None elif status.is_success(code): schema_str = result.get("schema") result = AvroSchema(schema_str) self._cache_schema(result, schema_id) return result raise ClientError(f"Received bad schema (id {schema_id})", http_code=code, server_traceback=result)
def get_by_id(self, schema_id, headers=None): """ GET /schemas/ids/{int: id} Retrieve a parsed avro schema by id or None if not found Args: schema_id (int): Schema Id headers (dict): Extra headers to add on the requests Returns: avro.schema.RecordSchema: Avro Record schema """ if schema_id in self.id_to_schema: return self.id_to_schema[schema_id] # fetch from the registry url = "/".join([self.url, "schemas", "ids", str(schema_id)]) result, code = self.request(url, headers=headers) if code == status.HTTP_404_NOT_FOUND: log.error(f"Schema not found: {code}") elif not status.is_success(code): log.error(f"Unable to get schema for the specific ID: {code}") else: # need to parse the schema schema_str = result.get("schema") try: result = AvroSchema(schema_str) # cache the result self._cache_schema(result, schema_id) return result except ClientError: # bad schema - should not happen raise ClientError( f"Received bad schema (id {schema_id})", http_code=code, server_traceback=result, )