async def test_override_headers(client, deployment_schema, response_klass, async_mock): extra_headers = {"custom-serialization": utils.HEADER_AVRO_JSON} client = SchemaRegistryClient(url=os.getenv("SCHEMA_REGISTRY_URL"), extra_headers=extra_headers) assert client.prepare_headers().get( "custom-serialization") == utils.HEADER_AVRO_JSON subject = "test" override_header = {"custom-serialization": utils.HEADER_AVRO} mock = async_mock(httpx.AsyncClient, "request", returned_value=response_klass(200, content={"id": 1})) with mock: await client.register(subject, deployment_schema, headers=override_header) prepare_headers = client.prepare_headers(body="1") prepare_headers["custom-serialization"] = utils.HEADER_AVRO mock.assert_called_with(headers=prepare_headers)
def client(): url = os.getenv("SCHEMA_REGISTRY_URL", "http://schema-registry-server:8081") client = SchemaRegistryClient(url) yield client subjects = { "test-basic-schema", "test-deployment", "test-country", "test-basic-schema-backup", "test-advance-schema", "test-user-schema", "subject-does-not-exist", "test-logical-types-schema", "test-schema-version", "test-nested-schema", } # Executing the clean up. Delete all the subjects between tests. for subject in subjects: try: client.delete_subject(subject) except errors.ClientError as exc: logger.info(exc.message)
def test_override_headers(client, deployment_schema, mocker, response_klass): extra_headers = {"custom-serialization": "application/x-avro-json"} client = SchemaRegistryClient("https://127.0.0.1:65534", extra_headers=extra_headers) assert client.session.headers.get( "custom-serialization") == "application/x-avro-json" subject = "test" override_header = {"custom-serialization": "application/avro"} request_patch = mocker.patch.object(httpx.Client, "request", return_value=response_klass( 200, content={"id": 1})) client.register(subject, deployment_schema, headers=override_header) prepare_headers = client.prepare_headers(body="1") prepare_headers["custom-serialization"] = "application/avro" request_patch.assert_called_once_with("POST", mocker.ANY, headers=prepare_headers, json=mocker.ANY, timeout=UNSET)
def client(): client = SchemaRegistryClient("http://my-dummy-schema.com") client.register = MagicMock(name="register") client.get_by_id = MagicMock(name="get_by_id") client.register.return_value = 1 client.get_by_id.return_value = AvroSchema(UserModel._schema) return client
async def test_override_headers(client, deployment_schema, response_klass, async_mock): extra_headers = {"custom-serialization": "application/x-avro-json"} client = SchemaRegistryClient(url=os.getenv("SCHEMA_REGISTRY_URL"), extra_headers=extra_headers) assert (client.prepare_headers().get("custom-serialization") == "application/x-avro-json") subject = "test" override_header = {"custom-serialization": "application/avro"} mock = async_mock( requests.sessions.Session, "request", returned_value=response_klass(200, content={"id": 1}), ) with mock: await client.register(subject, deployment_schema, headers=override_header) prepare_headers = client.prepare_headers(body="1") prepare_headers["custom-serialization"] = "application/avro" mock.assert_called_with(headers=prepare_headers)
def delete_subject(client: SchemaRegistryClient, topic: str): subjects = client.get_subjects() key_subject = f"{topic}-key" value_subject = f"{topic}-value" if key_subject in subjects: client.delete_subject(key_subject) if value_subject in subjects: client.delete_subject(value_subject)
def lambda_handler(event, context): responseData = {} print(event) if 'SCHEMA_REEGISTRY_URI' in os.environ: SchemaRegistryURI = os.environ['SCHEMA_REEGISTRY_URI'] print('SchemaRegistryURI is: {}'.format(SchemaRegistryURI)) else: responseData['cause'] = "Schema registry URI not provided" cfnresponse.send(event, context, cfnresponse.FAILED, responseData) if 'SchemaConfig' in event['ResourceProperties']: json_content = json.loads(event['ResourceProperties']['SchemaConfig']) print("Loaded AVRO schema from file {}".format(json_content)) else: responseData['status'] = "Location of Schema file not provided" responseData['cause'] = event['ResourceProperties'] print('SchemaFile not provided') cfnresponse.send(event, context, cfnresponse.FAILED, responseData) if event['RequestType'] == 'Create': if 'type' in json_content: print("Looks like we got valide AVRO schema") try: client = SchemaRegistryClient(url=SchemaRegistryURI) except Exception as e: responseData['cause'] = "Schema registry client cannot be created {} \ ".format(e.__class__.__name__) print(responseData['cause']) cfnresponse.send(event, context, cfnresponse.FAILED, responseData) print('Schema registry client created') avro_schema = schema.AvroSchema(json_content) print('AVRO schema object created') ###Get subjects, just for debug list_registered_subjects = requests.get(SchemaRegistryURI + "/subjects") print("Native http rest response, list of current subjects {}, \ ".format(list_registered_subjects.json())) try: schema_id = client.register(json_content['name'] + "-value", avro_schema, timeout=10) responseData['cause'] = "Schema {} created successfully, schema id is {} \ ".format(json_content['name'], schema_id) print(responseData['cause']) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) except Exception as e: responseData['cause'] = "Some exception in request to schema register happened, {}\ ".format(format(e.__class__.__name__)) print (responseData['cause']) cfnresponse.send(event, context, cfnresponse.FAILED, responseData) else: responseData['cause'] = 'Provided file not an AVRO schema, \ there are no /"type/" field in request object' print(responseData['cause']) cfnresponse.send(event, context, cfnresponse.FAILED, responseData) else: #if event['RequestType'] == 'Delete' or event['RequestType'] == 'Update': responseData['cause'] = 'CloudFormation Delete and Update method not implemented, \ just return cnfresponse=SUCCSESS without any job/modifications' cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
def build_schema_registry_client(url_schema, config: dict): if "schema.registry.basic.auth.user.info" in config: config_schema = { "url": url_schema, "ssl.ca.location": None, "ssl.certificate.location": None, "ssl.key.location": None, "basic.auth.user.info": config["schema.registry.basic.auth.user.info"], "basic.auth.credentials.source": "USER_INFO" } client_schema = SchemaRegistryClient(url=config_schema) return client_schema return SchemaRegistryClient(url_schema)
def test_init_with_dict(): client = SchemaRegistryClient({ "url": "https://127.0.0.1:65534", "ssl.certificate.location": "/path/to/cert", "ssl.key.location": "/path/to/key", }) assert "https://127.0.0.1:65534" == client.url_manager.url
def test_custom_headers(): extra_headers = {"custom-serialization": "application/x-avro-json"} client = SchemaRegistryClient( url="https://127.0.0.1:65534", extra_headers=extra_headers ) assert extra_headers == client.extra_headers
def test_cert_with_key(): client = SchemaRegistryClient( url="https://127.0.0.1:65534", cert_location="/path/to/cert", key_location="/path/to/key", ) assert ("/path/to/cert", "/path/to/key") == client.session.cert
def test_basic_auth_invalid(): with pytest.raises(ValueError): SchemaRegistryClient( { "url": "https://*****:*****@127.0.0.1:65534", "basic.auth.credentials.source": "VAULT", } )
def test_basic_auth_userinfo(): client = SchemaRegistryClient( { "url": "https://*****:*****@127.0.0.1:65534", "basic.auth.credentials.source": "user_info", "basic.auth.user.info": "user_userinfo:secret_userinfo", } ) assert ("user_userinfo", "secret_userinfo") == client.auth
def test_basic_auth_sasl_inherit(): client = SchemaRegistryClient({ "url": "https://*****:*****@127.0.0.1:65534", "basic.auth.credentials.source": "SASL_INHERIT", "sasl.mechanism": "PLAIN", "sasl.username": "******", "sasl.password": "******", }) assert ("user_sasl", "secret_sasl") == client.session.auth
def test_invalid_conf(): with pytest.raises(ValueError): SchemaRegistryClient({ "url": "https://*****:*****@127.0.0.1:65534", "basic.auth.credentials.source": "SASL_INHERIT", "sasl.username": "******", "sasl.password": "******", "invalid.conf": 1, "invalid.conf2": 2, })
def test_cert_with_key(certificates): client = SchemaRegistryClient( url="https://127.0.0.1:65534", cert_location=certificates["certificate"], key_location=certificates["key"], key_password=certificates["password"], ) assert client.conf[ utils.SSL_CERTIFICATE_LOCATION] == certificates["certificate"] assert client.conf[utils.SSL_KEY_LOCATION] == certificates["key"] assert client.conf[utils.SSL_KEY_PASSWORD] == certificates["password"]
def test_init_with_dict(certificates): client = SchemaRegistryClient({ "url": "https://127.0.0.1:65534", "ssl.certificate.location": certificates["certificate"], "ssl.key.location": certificates["key"], "ssl.key.password": certificates["password"], }) assert "https://127.0.0.1:65534/" == client.url_manager.url
def test_custom_httpx_config(): """ Test the SchemaRegistryClient creation with custom httpx config """ timeout = httpx.Timeout(10.0, connect=60.0) pool_limits = httpx.Limits(max_keepalive=5, max_connections=10) client = SchemaRegistryClient( url="https://127.0.0.1:65534", timeout=timeout, pool_limits=pool_limits, ) assert client.timeout == timeout assert client.pool_limits == pool_limits
def avro_to_row(schema_registry): client = SchemaRegistryClient(schema_registry) serializer = MessageSerializer(client) def convert(msg): try: dat = serializer.decode_message(msg) except Exception as e: logging.warning(f'Serialize Error! ({e}) - Payload: {msg}') return [] logging.info(f'Payload: {dat}') return [dat] return convert
def test_override_headers(client, deployment_schema, mocker): extra_headers = {"custom-serialization": "application/x-avro-json"} client = SchemaRegistryClient( "https://127.0.0.1:65534", extra_headers=extra_headers ) assert ( client.prepare_headers().get("custom-serialization") == "application/x-avro-json" ) class Response: def __init__(self, status_code, content=None): self.status_code = status_code if content is None: content = {} self.content = content def json(self): return self.content subject = "test" override_header = {"custom-serialization": "application/avro"} request_patch = mocker.patch.object( requests.sessions.Session, "request", return_value=Response(200, content={"id": 1}), ) client.register(subject, deployment_schema, headers=override_header) prepare_headers = client.prepare_headers(body="1") prepare_headers["custom-serialization"] = "application/avro" request_patch.assert_called_once_with( "POST", mocker.ANY, headers=prepare_headers, json=mocker.ANY )
def test_basic_auth_url(): client = SchemaRegistryClient( {"url": "https://*****:*****@127.0.0.1:65534"}) assert ("user_url", "secret_url") == client.session.auth
def test_invalid_url(): with pytest.raises(AssertionError): SchemaRegistryClient({"url": "example.com:65534"})
def test_invalid_type_url_dict(): with pytest.raises(AttributeError): SchemaRegistryClient({"url": 1})
def test_invalid_type_url(): with pytest.raises(AttributeError): SchemaRegistryClient(url=1)
def test_cert_path(): client = SchemaRegistryClient(url="https://127.0.0.1:65534", ca_location="/path/to/ca") assert "/path/to/ca" == client.session.verify
def test_cert_path(): client = SchemaRegistryClient(url="https://127.0.0.1:65534", ca_location=True) assert client.conf[utils.SSL_CA_LOCATION]
def test_invalid_cert(): with pytest.raises(FileNotFoundError): SchemaRegistryClient(url="https://127.0.0.1:65534", cert_location="/path/to/cert")
from schema_registry.client import SchemaRegistryClient, schema from schema_registry.serializers import FaustSerializer SCHEMA_REGISTRY_URL = "http://localhost:8081" # from faust_project.users.models import AdvanceUserModel # Initialize Schema Registry Client client = SchemaRegistryClient(url=SCHEMA_REGISTRY_URL) avro_medicare_key_schema = schema.AvroSchema({ "type": "record", "name": "providerKey", "namespace": "org.science.medicare", "fields": [ { "name": "npi", "type": "int" } ], "connect.name": "org.science.medicare.providerKey" }) avro_medicare_value_schema = schema.AvroSchema( """ {"type":"record","name":"provider","namespace":"org.science.medicare","fields":[{"name":"npi","type":"int"}, {"name":"nppes_provider_last_org_name","type":["null","string"],"default":"null"}, {"name":"nppes_provider_first_name","type":["null","string"],"default":"null"},{"name":"nppes_provider_mi", "type":["null","string"],"default":"null"},{"name":"nppes_credentials","type":["null","string"], "default":"null"},{"name":"nppes_provider_gender","type":["null","string"],"default":"null"}, {"name":"nppes_entity_code","type":["null","string"],"default":"null"},{"name":"nppes_provider_street1",
from schema_registry.client import SchemaRegistryClient, schema ## Needs testing client = SchemaRegistryClient(url="http://127.0.0.1:8081") deployment_schema = { "type": "record", "namespace": "com.kubertenes", "name": "AvroDeployment", "fields": [ { "name": "image", "type": "string" }, { "name": "replicas", "type": "int" }, { "name": "port", "type": "int" }, ], } avro_schema = schema.AvroSchema(deployment_schema) schema_id = client.register("test-deployment", avro_schema)
def test_empty_url(): with pytest.raises(AssertionError): SchemaRegistryClient({"url": ""})