def sync(dry_run, config): """ Synchronize Kafka config to YAML files """ configuration = Config(filename=config) topic_admin, schema_admin, mds_admin = get_admin_clients(configuration) # Reconcile topics parser = InputParser(configuration.get_input_patterns()) topic_admin.reconcile_topics(parser.get_topics(), dry_run=dry_run) topics_context = topic_admin.get_dry_run_plan() # Reconcile schemas schemas = parser.get_schemas() schema_admin.reconcile_schemas(schemas, dry_run=dry_run) schema_context = schema_admin.get_dry_run_plan() # TODO no reconcile yet for Clients... # mds_admin.do_consumer_for("SKATA", "arcanum") mds_admin.reconcile_roles(parser.get_clients(), dry_run=dry_run) if dry_run: client_context = mds_admin.get_dry_run_plan() report = Report( client_context=client_context, schema_context=schema_context, topics_context=topics_context, ) print(report.render())
def test_populate_schema_cache(): client = SchemaAdmin(MockSRClient({})) client._populate_subject_cache() assert len(client.subject_cache) == 0 parser = InputParser(SAMPLE_PATH) schemas = parser.get_schemas() client.reconcile_schemas(schemas) client._populate_subject_cache() assert len(client.subject_cache) == 3
def test_get_subjects_to_update(): client = SchemaAdmin(MockSRClient({})) parser = InputParser(SAMPLE_PATH) schemas = parser.get_schemas() to_update = client.get_subjects_to_update(schemas) assert len(to_update) == 3 client.reconcile_schemas(schemas) to_update = client.get_subjects_to_update(schemas) assert len(to_update) == 0
def test_clients(): """ test client """ parser = InputParser(SAMPLE_PATH) client_list = parser.get_clients() assert isinstance(client_list, list) clients = {x.principal: x for x in client_list} assert "User:poutanaola" in clients assert "Group:malakes" in clients assert "User:produser" in clients
def test_compatibility(): parser = InputParser(SAMPLE_PATH) schemas = parser.get_schemas() assert isinstance(schemas, list) s1 = schemas[0] assert isinstance(s1, Schema) assert s1.subject_name == "SKATA.VROMIA.POLY-key" assert s1.compatibility == "backward" s2 = schemas[1] assert isinstance(s1, Schema) assert s2.subject_name == "SKATA.VROMIA.POLY-value" assert s2.compatibility == "none"
def test_inputparser_get_schemas(): parser = InputParser(SAMPLE_PATH) schemas = parser.get_schemas_as_dict() for schema in schemas.values(): assert isinstance(schema, Schema) assert schemas[ "SKATA.VROMIA.POLY-key"].subject_name == "SKATA.VROMIA.POLY-key" assert schemas[ "SKATA.VROMIA.POLY-value"].subject_name == "SKATA.VROMIA.POLY-value" # Topic 1 has not compatibility set assert schemas["SKATA.VROMIA.POLY-value"].compatibility == "none" assert schemas["SKATA.VROMIA.POLY-key"].compatibility == "backward" assert schemas["SKATA.VROMIA.LIGO-key"].compatibility is None
def test_resolve_patterns(): patterns = [ "tests/data/patterns/*.yaml", ] parser = InputParser(patterns) assert isinstance(parser, InputParser) patterns2 = patterns + ["tests/data/patterns/faildir/*.yaml"] with pytest.raises(DuplicateResourceException) as e: parser = InputParser(patterns2) assert "already declared" in str(e.value) pattern3 = ["tests/data/patterns/sample.yaml"] parser = InputParser(pattern3) assert isinstance(parser, InputParser) assert "tests/data/patterns/sample.yaml" in parser.filenames assert len(parser.filenames) == 1
def test_inputparser_get_topics(): parser = InputParser(SAMPLE_PATH) topics = parser.get_topics() assert isinstance(topics, list) for topic in topics: assert isinstance(topic, Topic) # make sure we get the same data as in sample assert topics[0].name == "SKATA.VROMIA.POLY" assert topics[0].partitions == 6 assert topics[0].replication_factor == 1 assert topics[0].configs is not None assert topics[0].configs["cleanup.policy"] == "delete" assert topics[0].configs["min.insync.replicas"] == 1 assert topics[0].configs["retention.ms"] == 10000000 assert topics[0].schema["value"]["fromFile"] == "schema.json" assert topics[0].schema["value"]["compatibility"] == "NONE" assert topics[0].schema["key"]["fromFile"] == "schema-key.json" assert topics[0].schema["key"]["compatibility"] == "BACKWARD"
def test_inputparser_load(): parser = InputParser(SAMPLE_PATH) assert isinstance(parser, InputParser)
def test_register_schema(): client = SchemaAdmin(MockSRClient({})) parser = InputParser(SAMPLE_PATH) schemas = parser.get_schemas() schema1 = schemas[0] client.update_or_create_schema(schema1)