async def test_override_headers(client, deployment_schema, response_klass,
async_mock):
extra_headers = {"custom-serialization": utils.HEADER_AVRO_JSON}
client = SchemaRegistryClient(url=os.getenv("SCHEMA_REGISTRY_URL"),
extra_headers=extra_headers)
assert client.prepare_headers().get(
"custom-serialization") == utils.HEADER_AVRO_JSON
subject = "test"
override_header = {"custom-serialization": utils.HEADER_AVRO}
mock = async_mock(httpx.AsyncClient,
"request",
returned_value=response_klass(200, content={"id": 1}))
with mock:
await client.register(subject,
deployment_schema,
headers=override_header)
prepare_headers = client.prepare_headers(body="1")
prepare_headers["custom-serialization"] = utils.HEADER_AVRO
mock.assert_called_with(headers=prepare_headers)
def client():
url = os.getenv("SCHEMA_REGISTRY_URL",
"http://schema-registry-server:8081")
client = SchemaRegistryClient(url)
yield client
subjects = {
"test-basic-schema",
"test-deployment",
"test-country",
"test-basic-schema-backup",
"test-advance-schema",
"test-user-schema",
"subject-does-not-exist",
"test-logical-types-schema",
"test-schema-version",
"test-nested-schema",
}
# Executing the clean up. Delete all the subjects between tests.
for subject in subjects:
try:
client.delete_subject(subject)
except errors.ClientError as exc:
logger.info(exc.message)
def test_override_headers(client, deployment_schema, mocker, response_klass):
extra_headers = {"custom-serialization": "application/x-avro-json"}
client = SchemaRegistryClient("https://127.0.0.1:65534",
extra_headers=extra_headers)
assert client.session.headers.get(
"custom-serialization") == "application/x-avro-json"
subject = "test"
override_header = {"custom-serialization": "application/avro"}
request_patch = mocker.patch.object(httpx.Client,
"request",
return_value=response_klass(
200, content={"id": 1}))
client.register(subject, deployment_schema, headers=override_header)
prepare_headers = client.prepare_headers(body="1")
prepare_headers["custom-serialization"] = "application/avro"
request_patch.assert_called_once_with("POST",
mocker.ANY,
headers=prepare_headers,
json=mocker.ANY,
timeout=UNSET)
def client():
client = SchemaRegistryClient("http://my-dummy-schema.com")
client.register = MagicMock(name="register")
client.get_by_id = MagicMock(name="get_by_id")
client.register.return_value = 1
client.get_by_id.return_value = AvroSchema(UserModel._schema)
return client
async def test_override_headers(client, deployment_schema, response_klass,
async_mock):
extra_headers = {"custom-serialization": "application/x-avro-json"}
client = SchemaRegistryClient(url=os.getenv("SCHEMA_REGISTRY_URL"),
extra_headers=extra_headers)
assert (client.prepare_headers().get("custom-serialization") ==
"application/x-avro-json")
subject = "test"
override_header = {"custom-serialization": "application/avro"}
mock = async_mock(
requests.sessions.Session,
"request",
returned_value=response_klass(200, content={"id": 1}),
)
with mock:
await client.register(subject,
deployment_schema,
headers=override_header)
prepare_headers = client.prepare_headers(body="1")
prepare_headers["custom-serialization"] = "application/avro"
mock.assert_called_with(headers=prepare_headers)
def delete_subject(client: SchemaRegistryClient, topic: str):
subjects = client.get_subjects()
key_subject = f"{topic}-key"
value_subject = f"{topic}-value"
if key_subject in subjects:
client.delete_subject(key_subject)
if value_subject in subjects:
client.delete_subject(value_subject)
def lambda_handler(event, context):
responseData = {}
print(event)
if 'SCHEMA_REEGISTRY_URI' in os.environ:
SchemaRegistryURI = os.environ['SCHEMA_REEGISTRY_URI']
print('SchemaRegistryURI is: {}'.format(SchemaRegistryURI))
else:
responseData['cause'] = "Schema registry URI not provided"
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
if 'SchemaConfig' in event['ResourceProperties']:
json_content = json.loads(event['ResourceProperties']['SchemaConfig'])
print("Loaded AVRO schema from file {}".format(json_content))
else:
responseData['status'] = "Location of Schema file not provided"
responseData['cause'] = event['ResourceProperties']
print('SchemaFile not provided')
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
if event['RequestType'] == 'Create':
if 'type' in json_content:
print("Looks like we got valide AVRO schema")
try:
client = SchemaRegistryClient(url=SchemaRegistryURI)
except Exception as e:
responseData['cause'] = "Schema registry client cannot be created {} \
".format(e.__class__.__name__)
print(responseData['cause'])
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
print('Schema registry client created')
avro_schema = schema.AvroSchema(json_content)
print('AVRO schema object created')
###Get subjects, just for debug
list_registered_subjects = requests.get(SchemaRegistryURI + "/subjects")
print("Native http rest response, list of current subjects {}, \
".format(list_registered_subjects.json()))
try:
schema_id = client.register(json_content['name'] + "-value", avro_schema, timeout=10)
responseData['cause'] = "Schema {} created successfully, schema id is {} \
".format(json_content['name'], schema_id)
print(responseData['cause'])
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
except Exception as e:
responseData['cause'] = "Some exception in request to schema register happened, {}\
".format(format(e.__class__.__name__))
print (responseData['cause'])
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
else:
responseData['cause'] = 'Provided file not an AVRO schema, \
there are no /"type/" field in request object'
print(responseData['cause'])
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
else:
#if event['RequestType'] == 'Delete' or event['RequestType'] == 'Update':
responseData['cause'] = 'CloudFormation Delete and Update method not implemented, \
just return cnfresponse=SUCCSESS without any job/modifications'
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
def build_schema_registry_client(url_schema, config: dict):
if "schema.registry.basic.auth.user.info" in config:
config_schema = {
"url": url_schema,
"ssl.ca.location": None,
"ssl.certificate.location": None,
"ssl.key.location": None,
"basic.auth.user.info": config["schema.registry.basic.auth.user.info"],
"basic.auth.credentials.source": "USER_INFO"
}
client_schema = SchemaRegistryClient(url=config_schema)
return client_schema
return SchemaRegistryClient(url_schema)
def test_init_with_dict():
client = SchemaRegistryClient({
"url": "https://127.0.0.1:65534",
"ssl.certificate.location": "/path/to/cert",
"ssl.key.location": "/path/to/key",
})
assert "https://127.0.0.1:65534" == client.url_manager.url
def test_custom_headers():
extra_headers = {"custom-serialization": "application/x-avro-json"}
client = SchemaRegistryClient(
url="https://127.0.0.1:65534", extra_headers=extra_headers
)
assert extra_headers == client.extra_headers
def test_cert_with_key():
client = SchemaRegistryClient(
url="https://127.0.0.1:65534",
cert_location="/path/to/cert",
key_location="/path/to/key",
)
assert ("/path/to/cert", "/path/to/key") == client.session.cert
def test_basic_auth_invalid():
with pytest.raises(ValueError):
SchemaRegistryClient(
{
"url": "https://*****:*****@127.0.0.1:65534",
"basic.auth.credentials.source": "VAULT",
}
)
def test_basic_auth_userinfo():
client = SchemaRegistryClient(
{
"url": "https://*****:*****@127.0.0.1:65534",
"basic.auth.credentials.source": "user_info",
"basic.auth.user.info": "user_userinfo:secret_userinfo",
}
)
assert ("user_userinfo", "secret_userinfo") == client.auth
def test_basic_auth_sasl_inherit():
client = SchemaRegistryClient({
"url": "https://*****:*****@127.0.0.1:65534",
"basic.auth.credentials.source": "SASL_INHERIT",
"sasl.mechanism": "PLAIN",
"sasl.username": "******",
"sasl.password": "******",
})
assert ("user_sasl", "secret_sasl") == client.session.auth
def test_invalid_conf():
with pytest.raises(ValueError):
SchemaRegistryClient({
"url": "https://*****:*****@127.0.0.1:65534",
"basic.auth.credentials.source": "SASL_INHERIT",
"sasl.username": "******",
"sasl.password": "******",
"invalid.conf": 1,
"invalid.conf2": 2,
})
def test_cert_with_key(certificates):
client = SchemaRegistryClient(
url="https://127.0.0.1:65534",
cert_location=certificates["certificate"],
key_location=certificates["key"],
key_password=certificates["password"],
)
assert client.conf[
utils.SSL_CERTIFICATE_LOCATION] == certificates["certificate"]
assert client.conf[utils.SSL_KEY_LOCATION] == certificates["key"]
assert client.conf[utils.SSL_KEY_PASSWORD] == certificates["password"]
def test_init_with_dict(certificates):
client = SchemaRegistryClient({
"url":
"https://127.0.0.1:65534",
"ssl.certificate.location":
certificates["certificate"],
"ssl.key.location":
certificates["key"],
"ssl.key.password":
certificates["password"],
})
assert "https://127.0.0.1:65534/" == client.url_manager.url
def test_custom_httpx_config():
"""
Test the SchemaRegistryClient creation with custom httpx config
"""
timeout = httpx.Timeout(10.0, connect=60.0)
pool_limits = httpx.Limits(max_keepalive=5, max_connections=10)
client = SchemaRegistryClient(
url="https://127.0.0.1:65534",
timeout=timeout,
pool_limits=pool_limits,
)
assert client.timeout == timeout
assert client.pool_limits == pool_limits
def avro_to_row(schema_registry):
client = SchemaRegistryClient(schema_registry)
serializer = MessageSerializer(client)
def convert(msg):
try:
dat = serializer.decode_message(msg)
except Exception as e:
logging.warning(f'Serialize Error! ({e}) - Payload: {msg}')
return []
logging.info(f'Payload: {dat}')
return [dat]
return convert
def test_override_headers(client, deployment_schema, mocker):
extra_headers = {"custom-serialization": "application/x-avro-json"}
client = SchemaRegistryClient(
"https://127.0.0.1:65534", extra_headers=extra_headers
)
assert (
client.prepare_headers().get("custom-serialization")
== "application/x-avro-json"
)
class Response:
def __init__(self, status_code, content=None):
self.status_code = status_code
if content is None:
content = {}
self.content = content
def json(self):
return self.content
subject = "test"
override_header = {"custom-serialization": "application/avro"}
request_patch = mocker.patch.object(
requests.sessions.Session,
"request",
return_value=Response(200, content={"id": 1}),
)
client.register(subject, deployment_schema, headers=override_header)
prepare_headers = client.prepare_headers(body="1")
prepare_headers["custom-serialization"] = "application/avro"
request_patch.assert_called_once_with(
"POST", mocker.ANY, headers=prepare_headers, json=mocker.ANY
)
def test_basic_auth_url():
client = SchemaRegistryClient(
{"url": "https://*****:*****@127.0.0.1:65534"})
assert ("user_url", "secret_url") == client.session.auth
def test_invalid_url():
with pytest.raises(AssertionError):
SchemaRegistryClient({"url": "example.com:65534"})
def test_invalid_type_url_dict():
with pytest.raises(AttributeError):
SchemaRegistryClient({"url": 1})
def test_invalid_type_url():
with pytest.raises(AttributeError):
SchemaRegistryClient(url=1)
def test_cert_path():
client = SchemaRegistryClient(url="https://127.0.0.1:65534",
ca_location="/path/to/ca")
assert "/path/to/ca" == client.session.verify
def test_cert_path():
client = SchemaRegistryClient(url="https://127.0.0.1:65534",
ca_location=True)
assert client.conf[utils.SSL_CA_LOCATION]
def test_invalid_cert():
with pytest.raises(FileNotFoundError):
SchemaRegistryClient(url="https://127.0.0.1:65534",
cert_location="/path/to/cert")
from schema_registry.client import SchemaRegistryClient, schema
from schema_registry.serializers import FaustSerializer
SCHEMA_REGISTRY_URL = "http://localhost:8081"
# from faust_project.users.models import AdvanceUserModel
# Initialize Schema Registry Client
client = SchemaRegistryClient(url=SCHEMA_REGISTRY_URL)
avro_medicare_key_schema = schema.AvroSchema({
"type": "record",
"name": "providerKey",
"namespace": "org.science.medicare",
"fields": [
{
"name": "npi",
"type": "int"
}
],
"connect.name": "org.science.medicare.providerKey"
})
avro_medicare_value_schema = schema.AvroSchema(
"""
{"type":"record","name":"provider","namespace":"org.science.medicare","fields":[{"name":"npi","type":"int"},
{"name":"nppes_provider_last_org_name","type":["null","string"],"default":"null"},
{"name":"nppes_provider_first_name","type":["null","string"],"default":"null"},{"name":"nppes_provider_mi",
"type":["null","string"],"default":"null"},{"name":"nppes_credentials","type":["null","string"],
"default":"null"},{"name":"nppes_provider_gender","type":["null","string"],"default":"null"},
{"name":"nppes_entity_code","type":["null","string"],"default":"null"},{"name":"nppes_provider_street1",
from schema_registry.client import SchemaRegistryClient, schema
## Needs testing
client = SchemaRegistryClient(url="http://127.0.0.1:8081")
deployment_schema = {
"type":
"record",
"namespace":
"com.kubertenes",
"name":
"AvroDeployment",
"fields": [
{
"name": "image",
"type": "string"
},
{
"name": "replicas",
"type": "int"
},
{
"name": "port",
"type": "int"
},
],
}
avro_schema = schema.AvroSchema(deployment_schema)
schema_id = client.register("test-deployment", avro_schema)
def test_empty_url():
with pytest.raises(AssertionError):
SchemaRegistryClient({"url": ""})
