def sync(dry_run, config):
"""
Synchronize Kafka config to YAML files
"""
configuration = Config(filename=config)
topic_admin, schema_admin, mds_admin = get_admin_clients(configuration)
# Reconcile topics
parser = InputParser(configuration.get_input_patterns())
topic_admin.reconcile_topics(parser.get_topics(), dry_run=dry_run)
topics_context = topic_admin.get_dry_run_plan()
# Reconcile schemas
schemas = parser.get_schemas()
schema_admin.reconcile_schemas(schemas, dry_run=dry_run)
schema_context = schema_admin.get_dry_run_plan()
# TODO no reconcile yet for Clients...
# mds_admin.do_consumer_for("SKATA", "arcanum")
mds_admin.reconcile_roles(parser.get_clients(), dry_run=dry_run)
if dry_run:
client_context = mds_admin.get_dry_run_plan()
report = Report(
client_context=client_context,
schema_context=schema_context,
topics_context=topics_context,
)
print(report.render())
def test_populate_schema_cache():
client = SchemaAdmin(MockSRClient({}))
client._populate_subject_cache()
assert len(client.subject_cache) == 0
parser = InputParser(SAMPLE_PATH)
schemas = parser.get_schemas()
client.reconcile_schemas(schemas)
client._populate_subject_cache()
assert len(client.subject_cache) == 3
def test_get_subjects_to_update():
client = SchemaAdmin(MockSRClient({}))
parser = InputParser(SAMPLE_PATH)
schemas = parser.get_schemas()
to_update = client.get_subjects_to_update(schemas)
assert len(to_update) == 3
client.reconcile_schemas(schemas)
to_update = client.get_subjects_to_update(schemas)
assert len(to_update) == 0
def test_clients():
"""
test client
"""
parser = InputParser(SAMPLE_PATH)
client_list = parser.get_clients()
assert isinstance(client_list, list)
clients = {x.principal: x for x in client_list}
assert "User:poutanaola" in clients
assert "Group:malakes" in clients
assert "User:produser" in clients
def test_compatibility():
parser = InputParser(SAMPLE_PATH)
schemas = parser.get_schemas()
assert isinstance(schemas, list)
s1 = schemas[0]
assert isinstance(s1, Schema)
assert s1.subject_name == "SKATA.VROMIA.POLY-key"
assert s1.compatibility == "backward"
s2 = schemas[1]
assert isinstance(s1, Schema)
assert s2.subject_name == "SKATA.VROMIA.POLY-value"
assert s2.compatibility == "none"
def test_inputparser_get_schemas():
parser = InputParser(SAMPLE_PATH)
schemas = parser.get_schemas_as_dict()
for schema in schemas.values():
assert isinstance(schema, Schema)
assert schemas[
"SKATA.VROMIA.POLY-key"].subject_name == "SKATA.VROMIA.POLY-key"
assert schemas[
"SKATA.VROMIA.POLY-value"].subject_name == "SKATA.VROMIA.POLY-value"
# Topic 1 has not compatibility set
assert schemas["SKATA.VROMIA.POLY-value"].compatibility == "none"
assert schemas["SKATA.VROMIA.POLY-key"].compatibility == "backward"
assert schemas["SKATA.VROMIA.LIGO-key"].compatibility is None
def test_resolve_patterns():
patterns = [
"tests/data/patterns/*.yaml",
]
parser = InputParser(patterns)
assert isinstance(parser, InputParser)
patterns2 = patterns + ["tests/data/patterns/faildir/*.yaml"]
with pytest.raises(DuplicateResourceException) as e:
parser = InputParser(patterns2)
assert "already declared" in str(e.value)
pattern3 = ["tests/data/patterns/sample.yaml"]
parser = InputParser(pattern3)
assert isinstance(parser, InputParser)
assert "tests/data/patterns/sample.yaml" in parser.filenames
assert len(parser.filenames) == 1
def test_inputparser_get_topics():
parser = InputParser(SAMPLE_PATH)
topics = parser.get_topics()
assert isinstance(topics, list)
for topic in topics:
assert isinstance(topic, Topic)
# make sure we get the same data as in sample
assert topics[0].name == "SKATA.VROMIA.POLY"
assert topics[0].partitions == 6
assert topics[0].replication_factor == 1
assert topics[0].configs is not None
assert topics[0].configs["cleanup.policy"] == "delete"
assert topics[0].configs["min.insync.replicas"] == 1
assert topics[0].configs["retention.ms"] == 10000000
assert topics[0].schema["value"]["fromFile"] == "schema.json"
assert topics[0].schema["value"]["compatibility"] == "NONE"
assert topics[0].schema["key"]["fromFile"] == "schema-key.json"
assert topics[0].schema["key"]["compatibility"] == "BACKWARD"
def test_inputparser_load():
parser = InputParser(SAMPLE_PATH)
assert isinstance(parser, InputParser)
def test_register_schema():
client = SchemaAdmin(MockSRClient({}))
parser = InputParser(SAMPLE_PATH)
schemas = parser.get_schemas()
schema1 = schemas[0]
client.update_or_create_schema(schema1)