def test_api_register_schema_incompatible(kafka_cluster, load_file): """ Attempts to register an incompatible Schema verifies the error. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema1 = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') schema2 = Schema(load_file('adv_schema.avsc'), schema_type='AVRO') subject = _subject_name('test_register_incompatible') sr.register_schema(subject, schema1) with pytest.raises(SchemaRegistryError, match="Schema being registered is" " incompatible with an" " earlier schema") as e: # The default Schema Registry compatibility type is BACKWARD. # this allows 1) fields to be deleted, 2) optional fields to # be added. schema2 adds non-optional fields to schema1, so # registering schema2 after schema1 should fail. sr.register_schema(subject, schema2) assert e.value.http_status_code == 409 # conflict assert e.value.error_code == 409
def register_schema(): client = get_schema_registry_client() key_schema = Schema(open('schema/KeySchema.avsc', "r").read(), schema_type="AVRO") key_schema_id = client.register_schema( subject_name="quote-feedback-key", schema=key_schema ) print("Schema with subject {} registered with id: {}".format("quote-feedback-key", key_schema_id)) value_schema = Schema(open('schema/ValueSchema.avsc', "r").read(), schema_type="AVRO") value_schema_id = client.register_schema( subject_name="quote-feedback-value", schema=value_schema ) print("Schema with subject {} registered with id: {}".format("quote-feedback-value", value_schema_id))
def test_api_get_subjects(kafka_cluster, load_file): """ Populates KafkaClusterFixture SR instance with a fixed number of subjects then verifies the response includes them all. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() avscs = ['basic_schema.avsc', 'primitive_string.avsc', 'primitive_bool.avsc', 'primitive_float.avsc'] subjects = [] for avsc in avscs: schema = Schema(load_file(avsc), schema_type='AVRO') subject = _subject_name(avsc) subjects.append(subject) sr.register_schema(subject, schema) registered = sr.get_subjects() assert all([s in registered for s in subjects])
def test_api_get_subject_versions(kafka_cluster, load_file): """ Registers a Schema with a subject, lists the versions associated with that subject and ensures the versions and their schemas match what was registered. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor. """ sr = kafka_cluster.schema_registry() subject = _subject_name("list-version-test") sr.set_compatibility(level="NONE") avscs = ['basic_schema.avsc', 'primitive_string.avsc', 'primitive_bool.avsc', 'primitive_float.avsc'] schemas = [] for avsc in avscs: schema = Schema(load_file(avsc), schema_type='AVRO') schemas.append(schema) sr.register_schema(subject, schema) versions = sr.get_versions(subject) assert len(versions) == len(avscs) for schema in schemas: registered_schema = sr.lookup_schema(subject, schema) assert registered_schema.subject == subject assert registered_schema.version in versions
def test_register_schema(): name = "test1-key" client = MockSRClient({}) schema_id = client.register_schema(name, Schema(json.dumps(SCHEMA1), "AVRO")) assert schema_id == 1 assert name in client.subjects subject = client.subjects[name] assert "versions" in subject assert len(subject["versions"]) == 1 assert subject["versions"][1] == schema_id # register new version schema = Schema(json.dumps(SCHEMA2), "AVRO") schema_id = client.register_schema(name, schema) subject = client.subjects[name] assert schema_id == 2 assert len(subject["versions"]) == 2 assert subject["versions"][2] == schema_id assert client.schemas[schema_id] == schema
def test_api_register_logical_schema(kafka_cluster, load_file): sr = kafka_cluster.schema_registry() schema = Schema(load_file('logical_date.avsc'), schema_type='AVRO') subject = _subject_name('test_logical_registration') schema_id = sr.register_schema(subject, schema) registered_schema = sr.lookup_schema(subject, schema) assert registered_schema.schema_id == schema_id assert registered_schema.subject == subject
def get_latest_version(self, subject_name): if subject_name == "temperature-key": test_schema_str = get_test_avro_schema_topic_keys() else: test_schema_str = get_test_avro_schema_topic_values() schema = Schema(test_schema_str, schema_type="AVRO") registered_schema = RegisteredSchema(schema_id=1, schema=schema, subject=subject_name, version=1) return registered_schema
def test_api_get_subject_version_no_version(kafka_cluster, load_file): sr = kafka_cluster.schema_registry() # ensures subject exists and has a single version schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name('test-get_subject') sr.register_schema(subject, schema) with pytest.raises(SchemaRegistryError, match="Version .*not found") as e: sr.get_version(subject, version=3) assert e.value.http_status_code == 404 assert e.value.error_code == 40402
def test_api_register_schema_incompatible(kafka_cluster, load_file): """ Attempts to register an incompatible Schema verifies the error. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema1 = Schema(load_file('adv_schema.avsc'), schema_type='AVRO') schema2 = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name('test_register_incompatible') sr.register_schema(subject, schema1) with pytest.raises(SchemaRegistryError, match="Schema being registered is" " incompatible with an" " earlier schema") as e: sr.register_schema(subject, schema2) assert e.value.http_status_code == 409 assert e.value.error_code == 409
def test_lookup_schema(): name = "test1-key" schema2 = Schema(json.dumps(SCHEMA2), "AVRO") client = MockSRClient({}) # Lookup a non existent schema with pytest.raises(SchemaRegistryError) as e: client.lookup_schema("DOES_NOT_EXIST", schema2) assert e.value.error_code == 40403 # add a schema and try again schema_id = client.register_schema(name, schema2) assert schema_id == 1 found = client.lookup_schema(name, schema2) assert isinstance(found, RegisteredSchema)
def test_api_get_register_schema_invalid(kafka_cluster, load_file): """ Attempts to obtain registration information with an invalid schema Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() subject = _subject_name("registration_invalid_schema") schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') # register valid schema so we don't hit subject not found exception sr.register_schema(subject, schema) schema2 = Schema(load_file('invalid_schema.avsc'), schema_type='AVRO') with pytest.raises(SchemaRegistryError, match="Invalid schema") as e: sr.lookup_schema(subject, schema2) # Not as documented but the caused by is correct. assert e.value.http_status_code == 500 assert e.value.error_code == 500
def test_api_get_subject_version_invalid(kafka_cluster, load_file): sr = kafka_cluster.schema_registry() # ensures subject exists and has a single version schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name('test-get_subject') sr.register_schema(subject, schema) with pytest.raises(SchemaRegistryError, match="The specified version .*is not" " a valid version id.*") as e: sr.get_version(subject, version='a') assert e.value.http_status_code == 422 assert e.value.error_code == 42202
def __init__(self, schema_str, schema_registry_client, to_dict=None, conf=None): self._registry = schema_registry_client self._schema_id = None # Avoid calling registry if schema is known to be registered self._known_subjects = set() if to_dict is not None and not callable(to_dict): raise ValueError("to_dict must be callable with the signature" " to_dict(object, SerializationContext)->dict") self._to_dict = to_dict # handle configuration conf_copy = self._default_conf.copy() if conf is not None: conf_copy.update(conf) self._auto_register = conf_copy.pop('auto.register.schemas') if not isinstance(self._auto_register, bool): raise ValueError("auto.register.schemas must be a boolean value") self._use_latest_version = conf_copy.pop('use.latest.version') if not isinstance(self._use_latest_version, bool): raise ValueError("use.latest.version must be a boolean value") if self._use_latest_version and self._auto_register: raise ValueError( "cannot enable both use.latest.version and auto.register.schemas" ) self._subject_name_func = conf_copy.pop('subject.name.strategy') if not callable(self._subject_name_func): raise ValueError("subject.name.strategy must be callable") if len(conf_copy) > 0: raise ValueError("Unrecognized properties: {}".format(", ".join( conf_copy.keys()))) schema_dict = json.loads(schema_str) schema_name = schema_dict.get('title', None) if schema_name is None: raise ValueError("Missing required JSON schema annotation title") self._schema_name = schema_name self._parsed_schema = schema_dict self._schema = Schema(schema_str, schema_type="JSON")
def test_api_register_schema_invalid(kafka_cluster, load_file): """ Attempts to register an invalid schema, validates the error. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_file('invalid_schema.avsc'), schema_type='AVRO') subject = _subject_name('test_invalid_schema') with pytest.raises(SchemaRegistryError) as e: sr.register_schema(subject, schema) assert e.value.http_status_code == 422 assert e.value.error_code == 42201
def register(file, subject, references=[]): schema_bytes = pkgutil.get_data("jlab_jaws", file) json_dict = json.loads(schema_bytes) json_str = json.dumps(json_dict) unregistered_schema = Schema(json_str, 'AVRO', references) id = client.register_schema(subject, unregistered_schema) print('Successfully registered {} with id: {}'.format(subject, id)) registered_schema = client.get_latest_version(subject) return registered_schema
def test_api_delete_subject_version(kafka_cluster, load_file): """ Registers a Schema under a specific subject then deletes it. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = str(uuid1()) sr.register_schema(subject, schema) sr.delete_version(subject, 1) assert subject not in sr.get_subjects()
def test_api_get_registration_subject_not_found(kafka_cluster, load_file): """ Attempts to obtain information about a schema's subject registration for an unknown subject. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name("registration_subject_not_found") with pytest.raises(SchemaRegistryError, match="Subject .*not found.*") as e: sr.lookup_schema(subject, schema) assert e.value.http_status_code == 404 assert e.value.error_code == 40401
def test_api_delete_subject(kafka_cluster, load_avsc): """ Registers a Schema under a specific subject then deletes it. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_avsc (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_avsc('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name("test-delete") sr.register_schema(subject, schema) assert subject in sr.get_subjects() sr.delete_subject(subject) assert subject not in sr.get_subjects()
def test_api_post_subject_registration(kafka_cluster, load_file): """ Registers a schema, fetches that schema by it's subject version id. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name('test_registration') schema_id = sr.register_schema(subject, schema) registered_schema = sr.lookup_schema(subject, schema) assert registered_schema.schema_id == schema_id assert registered_schema.subject == subject
def test_api_get_schema(kafka_cluster, load_file): """ Registers a schema then retrieves it using the schema id returned by the call to register the Schema. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name('get_schema') schema_id = sr.register_schema(subject, schema) registration = sr.lookup_schema(subject, schema) assert registration.schema_id == schema_id assert registration.subject == subject assert schema.schema_str, registration.schema.schema_str
def _schema_loads(schema_str): """ Instantiates a Schema instance from a declaration string Args: schema_str (str): Avro Schema declaration. .. _Schema declaration: https://avro.apache.org/docs/current/spec.html#schemas Returns: Schema: Schema instance """ schema_str = schema_str.strip( ) # canonical form primitive declarations are not supported if schema_str[0] != "{": if schema_str[0] != '"': schema_str = '{"type":"' + schema_str + '"}' else: schema_str = '{"type":' + schema_str + "}" return Schema(schema_str, schema_type="AVRO")
def test_api_subject_config_update(kafka_cluster, load_file): """ Updates a subjects compatibility policy then ensures the same policy is returned when queried. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_file('basic_schema.avsc'), schema_type='AVRO') subject = str(uuid1()) sr.register_schema(subject, schema) sr.set_compatibility(subject_name=subject, level="FULL_TRANSITIVE") assert sr.get_compatibility(subject_name=subject) == "FULL_TRANSITIVE"
def test_api_register_schema(kafka_cluster, load_file): """ Registers a schema, verifies the registration Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_file (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() avsc = 'basic_schema.avsc' subject = _subject_name(avsc) schema = Schema(load_file(avsc), schema_type='AVRO') schema_id = sr.register_schema(subject, schema) registered_schema = sr.lookup_schema(subject, schema) assert registered_schema.schema_id == schema_id assert registered_schema.subject == subject assert schema.schema_str, registered_schema.schema.schema_str
def test_api_get_subject_version(kafka_cluster, load_avsc): """ Registers a schema, fetches that schema by it's subject version id. Args: kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture load_avsc (callable(str)): Schema fixture constructor """ sr = kafka_cluster.schema_registry() schema = Schema(load_avsc('basic_schema.avsc'), schema_type='AVRO') subject = _subject_name('test-get_subject') sr.register_schema(subject, schema) registered_schema = sr.lookup_schema(subject, schema) registered_schema2 = sr.get_version(subject, registered_schema.version) assert registered_schema2.schema_id == registered_schema.schema_id assert registered_schema2.schema.schema_str == registered_schema.schema.schema_str assert registered_schema2.version == registered_schema.version
def main(): kafka_url = configs.KAFKA_URL admin_client = AdminClient({"bootstrap.servers": kafka_url}) topics_path = configs.KAFKA_TOPIC_INIT_TOPICS_PATH with open(topics_path) as f: topics_config = json.load(f) topic_list = [] schema_list = [] for tc in topics_config: topic_name = tc["name"] topic_num_partitions = tc["num_partitions"] topic_list.append(NewTopic(topic_name, topic_num_partitions, 1)) if "schema" in tc: schema_list.append(tc) # Create topics try: admin_client.create_topics(topic_list) except kafka.errors.TopicAlreadyExistsError: # topics already exists print("Topics already made") # Register Schemas schema_registry_client = SchemaRegistryClient( {"url": "http://" + configs.KAFKA_SCHEMA_REGISTRY_URL}) for ts in schema_list: name = ts["schema"]["schema"]["title"] shcema_type = ts["schema"]["schemaType"] schema_raw = ts["schema"]["schema"] schema = Schema(json.dumps(schema_raw), "JSON") print("Schema Registered") print(schema_registry_client.register_schema(name, schema))
def test_set_compatibility(): client = MockSRClient({}) # Starting default assert client.compatibility == "BACKWARD" # Set global default client.set_compatibility(subject_name=None, level="FORWARD") assert client.compatibility == "FORWARD" assert client.get_compatibility() == {"compatibilityLevel": "FORWARD"} client.set_compatibility(subject_name=None, level="BACKWARD") assert client.get_compatibility() == {"compatibilityLevel": "BACKWARD"} # set compat for a topic name = "test-key" with pytest.raises(SchemaRegistryError) as e: assert client.get_compatibility(name) == {"compatibilityLevel": "BACKWARD"} assert e.value.error_message == "Subject not found" assert e.value.error_code == 40401 schema_id = client.register_schema(name, Schema(json.dumps(SCHEMA1), "AVRO")) assert schema_id == 1 client.set_compatibility(subject_name=name, level="NONE") assert client.get_compatibility(name) == {"compatibilityLevel": "NONE"} assert client.get_compatibility() == {"compatibilityLevel": "BACKWARD"}
def create_schemas(): shema_registry_client.register_schema( 'BaseFact', Schema(schema_str=json.dumps(models.BaseFact.schema()), schema_type='JSON'), ) shema_registry_client.register_schema( 'agent-collect-create', Schema(schema_str=json.dumps(models.Collect.schema()), schema_type='JSON'), ) shema_registry_client.register_schema( 'agent-broadcast-ping', Schema(schema_str=json.dumps(models.PingRequest.schema()), schema_type='JSON'), ) shema_registry_client.register_schema( 'agent-connect', Schema(schema_str=json.dumps(models.AgentInfo.schema()), schema_type='JSON'), ) shema_registry_client.register_schema( 'agent-disconnect', Schema(schema_str=json.dumps(models.AgentInfo.schema()), schema_type='JSON'), ) shema_registry_client.register_schema( 'engine-connect', Schema(schema_str=json.dumps(models.EngineInfo.schema()), schema_type='JSON'), ) # print(json.dumps(bf)) # print(models.Collect.schema()) sys.exit(0)