def test_api_register_schema_incompatible(kafka_cluster, load_file):
"""
Attempts to register an incompatible Schema verifies the error.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema1 = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
schema2 = Schema(load_file('adv_schema.avsc'), schema_type='AVRO')
subject = _subject_name('test_register_incompatible')
sr.register_schema(subject, schema1)
with pytest.raises(SchemaRegistryError, match="Schema being registered is"
" incompatible with an"
" earlier schema") as e:
# The default Schema Registry compatibility type is BACKWARD.
# this allows 1) fields to be deleted, 2) optional fields to
# be added. schema2 adds non-optional fields to schema1, so
# registering schema2 after schema1 should fail.
sr.register_schema(subject, schema2)
assert e.value.http_status_code == 409 # conflict
assert e.value.error_code == 409
def register_schema():
client = get_schema_registry_client()
key_schema = Schema(open('schema/KeySchema.avsc', "r").read(), schema_type="AVRO")
key_schema_id = client.register_schema(
subject_name="quote-feedback-key",
schema=key_schema
)
print("Schema with subject {} registered with id: {}".format("quote-feedback-key", key_schema_id))
value_schema = Schema(open('schema/ValueSchema.avsc', "r").read(), schema_type="AVRO")
value_schema_id = client.register_schema(
subject_name="quote-feedback-value",
schema=value_schema
)
print("Schema with subject {} registered with id: {}".format("quote-feedback-value", value_schema_id))
def test_api_get_subjects(kafka_cluster, load_file):
"""
Populates KafkaClusterFixture SR instance with a fixed number of subjects
then verifies the response includes them all.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
avscs = ['basic_schema.avsc', 'primitive_string.avsc',
'primitive_bool.avsc', 'primitive_float.avsc']
subjects = []
for avsc in avscs:
schema = Schema(load_file(avsc), schema_type='AVRO')
subject = _subject_name(avsc)
subjects.append(subject)
sr.register_schema(subject, schema)
registered = sr.get_subjects()
assert all([s in registered for s in subjects])
def test_api_get_subject_versions(kafka_cluster, load_file):
"""
Registers a Schema with a subject, lists the versions associated with that
subject and ensures the versions and their schemas match what was
registered.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor.
"""
sr = kafka_cluster.schema_registry()
subject = _subject_name("list-version-test")
sr.set_compatibility(level="NONE")
avscs = ['basic_schema.avsc', 'primitive_string.avsc',
'primitive_bool.avsc', 'primitive_float.avsc']
schemas = []
for avsc in avscs:
schema = Schema(load_file(avsc), schema_type='AVRO')
schemas.append(schema)
sr.register_schema(subject, schema)
versions = sr.get_versions(subject)
assert len(versions) == len(avscs)
for schema in schemas:
registered_schema = sr.lookup_schema(subject, schema)
assert registered_schema.subject == subject
assert registered_schema.version in versions
def test_register_schema():
name = "test1-key"
client = MockSRClient({})
schema_id = client.register_schema(name, Schema(json.dumps(SCHEMA1), "AVRO"))
assert schema_id == 1
assert name in client.subjects
subject = client.subjects[name]
assert "versions" in subject
assert len(subject["versions"]) == 1
assert subject["versions"][1] == schema_id
# register new version
schema = Schema(json.dumps(SCHEMA2), "AVRO")
schema_id = client.register_schema(name, schema)
subject = client.subjects[name]
assert schema_id == 2
assert len(subject["versions"]) == 2
assert subject["versions"][2] == schema_id
assert client.schemas[schema_id] == schema
def test_api_register_logical_schema(kafka_cluster, load_file):
sr = kafka_cluster.schema_registry()
schema = Schema(load_file('logical_date.avsc'), schema_type='AVRO')
subject = _subject_name('test_logical_registration')
schema_id = sr.register_schema(subject, schema)
registered_schema = sr.lookup_schema(subject, schema)
assert registered_schema.schema_id == schema_id
assert registered_schema.subject == subject
def get_latest_version(self, subject_name):
if subject_name == "temperature-key":
test_schema_str = get_test_avro_schema_topic_keys()
else:
test_schema_str = get_test_avro_schema_topic_values()
schema = Schema(test_schema_str, schema_type="AVRO")
registered_schema = RegisteredSchema(schema_id=1,
schema=schema,
subject=subject_name,
version=1)
return registered_schema
def test_api_get_subject_version_no_version(kafka_cluster, load_file):
sr = kafka_cluster.schema_registry()
# ensures subject exists and has a single version
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name('test-get_subject')
sr.register_schema(subject, schema)
with pytest.raises(SchemaRegistryError, match="Version .*not found") as e:
sr.get_version(subject, version=3)
assert e.value.http_status_code == 404
assert e.value.error_code == 40402
def test_api_register_schema_incompatible(kafka_cluster, load_file):
"""
Attempts to register an incompatible Schema verifies the error.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema1 = Schema(load_file('adv_schema.avsc'), schema_type='AVRO')
schema2 = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name('test_register_incompatible')
sr.register_schema(subject, schema1)
with pytest.raises(SchemaRegistryError, match="Schema being registered is"
" incompatible with an"
" earlier schema") as e:
sr.register_schema(subject, schema2)
assert e.value.http_status_code == 409
assert e.value.error_code == 409
def test_lookup_schema():
name = "test1-key"
schema2 = Schema(json.dumps(SCHEMA2), "AVRO")
client = MockSRClient({})
# Lookup a non existent schema
with pytest.raises(SchemaRegistryError) as e:
client.lookup_schema("DOES_NOT_EXIST", schema2)
assert e.value.error_code == 40403
# add a schema and try again
schema_id = client.register_schema(name, schema2)
assert schema_id == 1
found = client.lookup_schema(name, schema2)
assert isinstance(found, RegisteredSchema)
def test_api_get_register_schema_invalid(kafka_cluster, load_file):
"""
Attempts to obtain registration information with an invalid schema
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
subject = _subject_name("registration_invalid_schema")
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
# register valid schema so we don't hit subject not found exception
sr.register_schema(subject, schema)
schema2 = Schema(load_file('invalid_schema.avsc'), schema_type='AVRO')
with pytest.raises(SchemaRegistryError, match="Invalid schema") as e:
sr.lookup_schema(subject, schema2)
# Not as documented but the caused by is correct.
assert e.value.http_status_code == 500
assert e.value.error_code == 500
def test_api_get_subject_version_invalid(kafka_cluster, load_file):
sr = kafka_cluster.schema_registry()
# ensures subject exists and has a single version
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name('test-get_subject')
sr.register_schema(subject, schema)
with pytest.raises(SchemaRegistryError,
match="The specified version .*is not"
" a valid version id.*") as e:
sr.get_version(subject, version='a')
assert e.value.http_status_code == 422
assert e.value.error_code == 42202
def __init__(self,
schema_str,
schema_registry_client,
to_dict=None,
conf=None):
self._registry = schema_registry_client
self._schema_id = None
# Avoid calling registry if schema is known to be registered
self._known_subjects = set()
if to_dict is not None and not callable(to_dict):
raise ValueError("to_dict must be callable with the signature"
" to_dict(object, SerializationContext)->dict")
self._to_dict = to_dict
# handle configuration
conf_copy = self._default_conf.copy()
if conf is not None:
conf_copy.update(conf)
self._auto_register = conf_copy.pop('auto.register.schemas')
if not isinstance(self._auto_register, bool):
raise ValueError("auto.register.schemas must be a boolean value")
self._use_latest_version = conf_copy.pop('use.latest.version')
if not isinstance(self._use_latest_version, bool):
raise ValueError("use.latest.version must be a boolean value")
if self._use_latest_version and self._auto_register:
raise ValueError(
"cannot enable both use.latest.version and auto.register.schemas"
)
self._subject_name_func = conf_copy.pop('subject.name.strategy')
if not callable(self._subject_name_func):
raise ValueError("subject.name.strategy must be callable")
if len(conf_copy) > 0:
raise ValueError("Unrecognized properties: {}".format(", ".join(
conf_copy.keys())))
schema_dict = json.loads(schema_str)
schema_name = schema_dict.get('title', None)
if schema_name is None:
raise ValueError("Missing required JSON schema annotation title")
self._schema_name = schema_name
self._parsed_schema = schema_dict
self._schema = Schema(schema_str, schema_type="JSON")
def test_api_register_schema_invalid(kafka_cluster, load_file):
"""
Attempts to register an invalid schema, validates the error.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_file('invalid_schema.avsc'), schema_type='AVRO')
subject = _subject_name('test_invalid_schema')
with pytest.raises(SchemaRegistryError) as e:
sr.register_schema(subject, schema)
assert e.value.http_status_code == 422
assert e.value.error_code == 42201
def register(file, subject, references=[]):
schema_bytes = pkgutil.get_data("jlab_jaws", file)
json_dict = json.loads(schema_bytes)
json_str = json.dumps(json_dict)
unregistered_schema = Schema(json_str, 'AVRO', references)
id = client.register_schema(subject, unregistered_schema)
print('Successfully registered {} with id: {}'.format(subject, id))
registered_schema = client.get_latest_version(subject)
return registered_schema
def test_api_delete_subject_version(kafka_cluster, load_file):
"""
Registers a Schema under a specific subject then deletes it.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = str(uuid1())
sr.register_schema(subject, schema)
sr.delete_version(subject, 1)
assert subject not in sr.get_subjects()
def test_api_get_registration_subject_not_found(kafka_cluster, load_file):
"""
Attempts to obtain information about a schema's subject registration for
an unknown subject.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name("registration_subject_not_found")
with pytest.raises(SchemaRegistryError, match="Subject .*not found.*") as e:
sr.lookup_schema(subject, schema)
assert e.value.http_status_code == 404
assert e.value.error_code == 40401
def test_api_delete_subject(kafka_cluster, load_avsc):
"""
Registers a Schema under a specific subject then deletes it.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_avsc (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_avsc('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name("test-delete")
sr.register_schema(subject, schema)
assert subject in sr.get_subjects()
sr.delete_subject(subject)
assert subject not in sr.get_subjects()
def test_api_post_subject_registration(kafka_cluster, load_file):
"""
Registers a schema, fetches that schema by it's subject version id.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name('test_registration')
schema_id = sr.register_schema(subject, schema)
registered_schema = sr.lookup_schema(subject, schema)
assert registered_schema.schema_id == schema_id
assert registered_schema.subject == subject
def test_api_get_schema(kafka_cluster, load_file):
"""
Registers a schema then retrieves it using the schema id returned by the
call to register the Schema.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name('get_schema')
schema_id = sr.register_schema(subject, schema)
registration = sr.lookup_schema(subject, schema)
assert registration.schema_id == schema_id
assert registration.subject == subject
assert schema.schema_str, registration.schema.schema_str
def _schema_loads(schema_str):
"""
Instantiates a Schema instance from a declaration string
Args:
schema_str (str): Avro Schema declaration.
.. _Schema declaration:
https://avro.apache.org/docs/current/spec.html#schemas
Returns:
Schema: Schema instance
"""
schema_str = schema_str.strip(
) # canonical form primitive declarations are not supported
if schema_str[0] != "{":
if schema_str[0] != '"':
schema_str = '{"type":"' + schema_str + '"}'
else:
schema_str = '{"type":' + schema_str + "}"
return Schema(schema_str, schema_type="AVRO")
def test_api_subject_config_update(kafka_cluster, load_file):
"""
Updates a subjects compatibility policy then ensures the same policy
is returned when queried.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO')
subject = str(uuid1())
sr.register_schema(subject, schema)
sr.set_compatibility(subject_name=subject,
level="FULL_TRANSITIVE")
assert sr.get_compatibility(subject_name=subject) == "FULL_TRANSITIVE"
def test_api_register_schema(kafka_cluster, load_file):
"""
Registers a schema, verifies the registration
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_file (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
avsc = 'basic_schema.avsc'
subject = _subject_name(avsc)
schema = Schema(load_file(avsc), schema_type='AVRO')
schema_id = sr.register_schema(subject, schema)
registered_schema = sr.lookup_schema(subject, schema)
assert registered_schema.schema_id == schema_id
assert registered_schema.subject == subject
assert schema.schema_str, registered_schema.schema.schema_str
def test_api_get_subject_version(kafka_cluster, load_avsc):
"""
Registers a schema, fetches that schema by it's subject version id.
Args:
kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
load_avsc (callable(str)): Schema fixture constructor
"""
sr = kafka_cluster.schema_registry()
schema = Schema(load_avsc('basic_schema.avsc'), schema_type='AVRO')
subject = _subject_name('test-get_subject')
sr.register_schema(subject, schema)
registered_schema = sr.lookup_schema(subject, schema)
registered_schema2 = sr.get_version(subject, registered_schema.version)
assert registered_schema2.schema_id == registered_schema.schema_id
assert registered_schema2.schema.schema_str == registered_schema.schema.schema_str
assert registered_schema2.version == registered_schema.version
def main():
kafka_url = configs.KAFKA_URL
admin_client = AdminClient({"bootstrap.servers": kafka_url})
topics_path = configs.KAFKA_TOPIC_INIT_TOPICS_PATH
with open(topics_path) as f:
topics_config = json.load(f)
topic_list = []
schema_list = []
for tc in topics_config:
topic_name = tc["name"]
topic_num_partitions = tc["num_partitions"]
topic_list.append(NewTopic(topic_name, topic_num_partitions, 1))
if "schema" in tc:
schema_list.append(tc)
# Create topics
try:
admin_client.create_topics(topic_list)
except kafka.errors.TopicAlreadyExistsError:
# topics already exists
print("Topics already made")
# Register Schemas
schema_registry_client = SchemaRegistryClient(
{"url": "http://" + configs.KAFKA_SCHEMA_REGISTRY_URL})
for ts in schema_list:
name = ts["schema"]["schema"]["title"]
shcema_type = ts["schema"]["schemaType"]
schema_raw = ts["schema"]["schema"]
schema = Schema(json.dumps(schema_raw), "JSON")
print("Schema Registered")
print(schema_registry_client.register_schema(name, schema))
def test_set_compatibility():
client = MockSRClient({})
# Starting default
assert client.compatibility == "BACKWARD"
# Set global default
client.set_compatibility(subject_name=None, level="FORWARD")
assert client.compatibility == "FORWARD"
assert client.get_compatibility() == {"compatibilityLevel": "FORWARD"}
client.set_compatibility(subject_name=None, level="BACKWARD")
assert client.get_compatibility() == {"compatibilityLevel": "BACKWARD"}
# set compat for a topic
name = "test-key"
with pytest.raises(SchemaRegistryError) as e:
assert client.get_compatibility(name) == {"compatibilityLevel": "BACKWARD"}
assert e.value.error_message == "Subject not found"
assert e.value.error_code == 40401
schema_id = client.register_schema(name, Schema(json.dumps(SCHEMA1), "AVRO"))
assert schema_id == 1
client.set_compatibility(subject_name=name, level="NONE")
assert client.get_compatibility(name) == {"compatibilityLevel": "NONE"}
assert client.get_compatibility() == {"compatibilityLevel": "BACKWARD"}
def create_schemas():
shema_registry_client.register_schema(
'BaseFact',
Schema(schema_str=json.dumps(models.BaseFact.schema()),
schema_type='JSON'),
)
shema_registry_client.register_schema(
'agent-collect-create',
Schema(schema_str=json.dumps(models.Collect.schema()),
schema_type='JSON'),
)
shema_registry_client.register_schema(
'agent-broadcast-ping',
Schema(schema_str=json.dumps(models.PingRequest.schema()),
schema_type='JSON'),
)
shema_registry_client.register_schema(
'agent-connect',
Schema(schema_str=json.dumps(models.AgentInfo.schema()),
schema_type='JSON'),
)
shema_registry_client.register_schema(
'agent-disconnect',
Schema(schema_str=json.dumps(models.AgentInfo.schema()),
schema_type='JSON'),
)
shema_registry_client.register_schema(
'engine-connect',
Schema(schema_str=json.dumps(models.EngineInfo.schema()),
schema_type='JSON'),
)
# print(json.dumps(bf))
# print(models.Collect.schema())
sys.exit(0)
