def test_master_selection(kafka_server, strategy):
config_aa = set_config_defaults({})
config_aa["advertised_hostname"] = "127.0.0.1"
config_aa["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"])
config_aa["client_id"] = "aa"
config_aa["port"] = 1234
config_aa["master_election_strategy"] = strategy
mc_aa = init_admin(config_aa)
config_bb = set_config_defaults({})
config_bb["advertised_hostname"] = "127.0.0.1"
config_bb["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"])
config_bb["client_id"] = "bb"
config_bb["port"] = 5678
config_bb["master_election_strategy"] = strategy
mc_bb = init_admin(config_bb)
while True:
if not (mc_aa.sc or mc_bb.sc):
time.sleep(1.0)
continue
if not (mc_aa.sc.master or mc_bb.sc.master or mc_aa.sc.master_url or mc_bb.sc.master_url):
time.sleep(1.0)
continue
assert mc_bb.sc.election_strategy == strategy
assert mc_aa.sc.election_strategy == strategy
aa_master = strategy == "lowest"
bb_master = strategy == "highest"
assert mc_aa.sc.master is aa_master
assert mc_bb.sc.master is bb_master
master_config = config_aa if strategy == "lowest" else config_bb
master_url = "http://{}:{}".format(master_config["host"], master_config["port"])
assert mc_aa.sc.master_url == master_url
assert mc_bb.sc.master_url == master_url
break
mc_aa.close()
mc_bb.close()
def test_master_selection(kafka_server):
config_aa = set_config_defaults({})
config_aa["advertised_hostname"] = "127.0.0.1"
config_aa["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"])
config_aa["client_id"] = "aa"
config_aa["port"] = 1234
mc_aa = init_admin(config_aa)
config_bb = set_config_defaults({})
config_bb["advertised_hostname"] = "127.0.0.1"
config_bb["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"])
config_bb["client_id"] = "bb"
config_bb["port"] = 5678
mc_bb = init_admin(config_bb)
while True:
if not (mc_aa.sc or mc_bb.sc):
time.sleep(1.0)
continue
if not (mc_aa.sc.master or mc_bb.sc.master or mc_aa.sc.master_url or mc_bb.sc.master_url):
time.sleep(1.0)
continue
assert mc_aa.sc.master is True
assert mc_bb.sc.master is False
master_url = "http://{}:{}".format(config_aa["host"], config_aa["port"])
assert mc_aa.sc.master_url == master_url
assert mc_bb.sc.master_url == master_url
break
mc_aa.close()
mc_bb.close()
async def fixture_registry_async(
tmp_path: Path, kafka_server: Optional[KafkaConfig]
) -> AsyncIterator[KarapaceSchemaRegistry]:
if not kafka_server:
assert REGISTRY_URI in os.environ or REST_URI in os.environ
instance, _ = mock_factory("registry")()
yield instance
else:
config_path = tmp_path / "karapace_config.json"
kafka_port = kafka_server.kafka_port
config = set_config_defaults({
"log_level":
"WARNING",
"bootstrap_uri":
f"127.0.0.1:{kafka_port}",
"topic_name":
new_random_name(),
"group_id":
new_random_name("schema_registry")
})
write_config(config_path, config)
registry = KarapaceSchemaRegistry(config_file_path=str(config_path),
config=set_config_defaults(config))
await registry.get_master()
try:
yield registry
finally:
registry.close()
async def fixture_registry_async(
request,
loop, # pylint: disable=unused-argument
tmp_path: Path,
kafka_servers: KafkaServers,
) -> AsyncIterator[Optional[KarapaceSchemaRegistry]]:
# Do not start a registry when the user provided an external service. Doing
# so would cause this node to join the existing group and participate in
# the election process. Without proper configuration for the listeners that
# won't work and will cause test failures.
rest_url = request.config.getoption("registry_url")
if rest_url:
yield None
return
config_path = tmp_path / "karapace_config.json"
config = set_config_defaults({
"bootstrap_uri":
kafka_servers.bootstrap_servers,
# Using the default settings instead of random values, otherwise it
# would not be possible to run the tests with external services.
# Because of this every test must be written in such a way that it can
# be executed twice with the same servers.
# "topic_name": new_random_name("topic"),
# "group_id": new_random_name("schema_registry")
})
write_config(config_path, config)
registry = KarapaceSchemaRegistry(config=config)
await registry.get_master()
try:
yield registry
finally:
await registry.close()
async def fixture_rest_async(
request,
loop, # pylint: disable=unused-argument
tmp_path: Path,
kafka_servers: KafkaServers,
registry_async_client: Client,
) -> AsyncIterator[Optional[KafkaRest]]:
# Do not start a REST api when the user provided an external service. Doing
# so would cause this node to join the existing group and participate in
# the election process. Without proper configuration for the listeners that
# won't work and will cause test failures.
rest_url = request.config.getoption("rest_url")
if rest_url:
yield None
return
config_path = tmp_path / "karapace_config.json"
config = set_config_defaults({
"bootstrap_uri": kafka_servers.bootstrap_servers,
"admin_metadata_max_age": 0
})
write_config(config_path, config)
rest = KafkaRest(config=config)
assert rest.serializer.registry_client
assert rest.consumer_manager.deserializer.registry_client
rest.serializer.registry_client.client = registry_async_client
rest.consumer_manager.deserializer.registry_client.client = registry_async_client
try:
yield rest
finally:
await rest.close()
async def fixture_rest_async(
session_tmpdir: TempDirCreator, kafka_server: Optional[KafkaConfig], registry_async_client: Client
) -> AsyncIterator[KafkaRest]:
if not kafka_server:
assert REST_URI in os.environ
instance, _ = mock_factory("rest")()
yield instance
else:
config_path = os.path.join(session_tmpdir(), "karapace_config.json")
kafka_port = kafka_server.kafka_port
config = set_config_defaults({
"log_level": "WARNING",
"bootstrap_uri": f"127.0.0.1:{kafka_port}",
"admin_metadata_max_age": 0
})
write_config(config_path, config)
rest = KafkaRest(config_file_path=config_path, config=config)
assert rest.serializer.registry_client
assert rest.consumer_manager.deserializer.registry_client
rest.serializer.registry_client.client = registry_async_client
rest.consumer_manager.deserializer.registry_client.client = registry_async_client
try:
yield rest
finally:
rest.close()
await rest.close_producers()
def test_master_selection(kafka_servers: KafkaServers, strategy: str) -> None:
# Use random port to allow for parallel runs.
port1 = get_random_port(port_range=TESTS_PORT_RANGE, blacklist=[])
port2 = get_random_port(port_range=TESTS_PORT_RANGE, blacklist=[port1])
port_aa, port_bb = sorted((port1, port2))
client_id_aa = new_random_name("master_selection_aa_")
client_id_bb = new_random_name("master_selection_bb_")
group_id = new_random_name("group_id")
config_aa = set_config_defaults({
"advertised_hostname": "127.0.0.1",
"bootstrap_uri": kafka_servers.bootstrap_servers,
"client_id": client_id_aa,
"group_id": group_id,
"port": port_aa,
"master_election_strategy": strategy,
})
config_bb = set_config_defaults({
"advertised_hostname": "127.0.0.1",
"bootstrap_uri": kafka_servers.bootstrap_servers,
"client_id": client_id_bb,
"group_id": group_id,
"port": port_bb,
"master_election_strategy": strategy,
})
with closing(init_admin(config_aa)) as mc_aa, closing(init_admin(config_bb)) as mc_bb:
if strategy == "lowest":
master = mc_aa
slave = mc_bb
else:
master = mc_bb
slave = mc_aa
# Wait for the election to happen
while not is_master(master):
time.sleep(0.3)
while not has_master(slave):
time.sleep(0.3)
# Make sure the end configuration is as expected
master_url = f'http://{master.config["host"]}:{master.config["port"]}'
assert master.sc.election_strategy == strategy
assert slave.sc.election_strategy == strategy
assert master.sc.master_url == master_url
assert slave.sc.master_url == master_url
def read_config(config_path):
if os.path.exists(config_path):
try:
config = json.loads(open(config_path, "r").read())
config = set_config_defaults(config)
return config
except Exception as ex:
raise InvalidConfiguration(ex)
else:
raise InvalidConfiguration()
async def test_backup_get(registry_async_client, kafka_servers: KafkaServers, tmp_path: Path):
_ = await insert_data(registry_async_client)
# Get the backup
backup_location = tmp_path / "schemas.log"
config = set_config_defaults({"bootstrap_uri": kafka_servers.bootstrap_servers})
sb = SchemaBackup(config, str(backup_location))
sb.request_backup()
# The backup file has been created
assert os.path.exists(backup_location)
def test_master_selection(kafka_server: Optional[KafkaConfig],
strategy: str) -> None:
assert kafka_server, f"test_master_selection can not be used if the env variable `{REGISTRY_URI}` or `{REST_URI}` is set"
config_aa = set_config_defaults({})
config_aa["advertised_hostname"] = "127.0.0.1"
config_aa["bootstrap_uri"] = f"127.0.0.1:{kafka_server.kafka_port}"
config_aa["client_id"] = "aa"
config_aa["port"] = 1234
config_aa["master_election_strategy"] = strategy
mc_aa = init_admin(config_aa)
config_bb = set_config_defaults({})
config_bb["advertised_hostname"] = "127.0.0.1"
config_bb["bootstrap_uri"] = f"127.0.0.1:{kafka_server.kafka_port}"
config_bb["client_id"] = "bb"
config_bb["port"] = 5678
config_bb["master_election_strategy"] = strategy
mc_bb = init_admin(config_bb)
if strategy == "lowest":
master = mc_aa
slave = mc_bb
else:
master = mc_bb
slave = mc_aa
# Wait for the election to happen
while not is_master(master):
time.sleep(0.3)
while not has_master(slave):
time.sleep(0.3)
# Make sure the end configuration is as expected
master_url = f'http://{master.config["host"]}:{master.config["port"]}'
assert master.sc.election_strategy == strategy
assert slave.sc.election_strategy == strategy
assert master.sc.master_url == master_url
assert slave.sc.master_url == master_url
mc_aa.close()
mc_bb.close()
def read_config(self):
if os.path.exists(self.config_path):
try:
config = json.loads(open(self.config_path, "r").read())
self.config = set_config_defaults(config)
try:
logging.getLogger().setLevel(config["log_level"])
except ValueError:
self.log.excption("Problem with log_level: %r",
config["log_level"])
except Exception as ex:
raise InvalidConfiguration(ex)
else:
raise InvalidConfiguration()
def test_master_selection(kafka_server: Optional[KafkaConfig], strategy: str) -> None:
assert kafka_server, f"test_master_selection can not be used if the env variable `{REGISTRY_URI}` or `{REST_URI}` is set"
config_aa = set_config_defaults({})
config_aa["advertised_hostname"] = "127.0.0.1"
config_aa["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server.kafka_port)
config_aa["client_id"] = "aa"
config_aa["port"] = 1234
config_aa["master_election_strategy"] = strategy
mc_aa = init_admin(config_aa)
config_bb = set_config_defaults({})
config_bb["advertised_hostname"] = "127.0.0.1"
config_bb["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server.kafka_port)
config_bb["client_id"] = "bb"
config_bb["port"] = 5678
config_bb["master_election_strategy"] = strategy
mc_bb = init_admin(config_bb)
while True:
if not (mc_aa.sc or mc_bb.sc):
time.sleep(1.0)
continue
if not (mc_aa.sc.master or mc_bb.sc.master or mc_aa.sc.master_url or mc_bb.sc.master_url):
time.sleep(1.0)
continue
assert mc_bb.sc.election_strategy == strategy
assert mc_aa.sc.election_strategy == strategy
aa_master = strategy == "lowest"
bb_master = strategy == "highest"
assert mc_aa.sc.master is aa_master
assert mc_bb.sc.master is bb_master
master_config = config_aa if strategy == "lowest" else config_bb
master_url = "http://{}:{}".format(master_config["host"], master_config["port"])
assert mc_aa.sc.master_url == master_url
assert mc_bb.sc.master_url == master_url
break
mc_aa.close()
mc_bb.close()
def test_no_eligible_master(kafka_servers: KafkaServers) -> None:
client_id = new_random_name("master_selection_")
group_id = new_random_name("group_id")
config_aa = set_config_defaults({
"advertised_hostname": "127.0.0.1",
"bootstrap_uri": kafka_servers.bootstrap_servers,
"client_id": client_id,
"group_id": group_id,
"port": get_random_port(port_range=TESTS_PORT_RANGE, blacklist=[]),
"master_eligibility": False,
})
with closing(init_admin(config_aa)) as mc:
# Wait for the election to happen, ie. flag is not None
while not mc.sc or mc.sc.are_we_master is None:
time.sleep(0.3)
# Make sure the end configuration is as expected
assert mc.sc.are_we_master is False
assert mc.sc.master_url is None
async def test_backup_restore(
registry_async_client: Client,
kafka_servers: KafkaServers,
tmp_path: Path,
) -> None:
subject = new_random_name("subject")
restore_location = tmp_path / "restore.log"
with restore_location.open("w") as fp:
jsonlib.dump(
[[
{
"subject": subject,
"version": 1,
"magic": 1,
"keytype": "SCHEMA",
},
{
"deleted": False,
"id": 1,
"schema": "\"string\"",
"subject": subject,
"version": 1,
},
]],
fp=fp,
)
config = set_config_defaults({"bootstrap_uri": kafka_servers.bootstrap_servers})
sb = SchemaBackup(config, str(restore_location))
sb.restore_backup()
# The restored karapace should have the previously created subject
all_subjects = []
expiration = Expiration.from_timeout(timeout=10)
while subject not in all_subjects:
expiration.raise_if_expired(msg=f"{subject} not in {all_subjects}")
res = await registry_async_client.get("subjects")
assert res.status_code == 200
all_subjects = res.json()
# Test a few exotic scenarios
subject = new_random_name("subject")
res = await registry_async_client.put(f"config/{subject}", json={"compatibility": "NONE"})
assert res.status == 200
assert res.json()["compatibility"] == "NONE"
# Restore a compatibility config remove message
with open(restore_location, "w") as fp:
fp.write(
"""
[
[
{{
"subject": "{subject_value}",
"magic": 0,
"keytype": "CONFIG"
}},
null
]
]
""".format(subject_value=subject)
)
res = await registry_async_client.get(f"config/{subject}")
assert res.status == 200
sb.restore_backup()
time.sleep(1.0)
res = await registry_async_client.get(f"config/{subject}")
assert res.status == 404
# Restore a complete schema delete message
subject = new_random_name("subject")
res = await registry_async_client.put(f"config/{subject}", json={"compatibility": "NONE"})
res = await registry_async_client.post(f"subjects/{subject}/versions", json={"schema": '{"type": "int"}'})
res = await registry_async_client.post(f"subjects/{subject}/versions", json={"schema": '{"type": "float"}'})
res = await registry_async_client.get(f"subjects/{subject}/versions")
assert res.status == 200
assert res.json() == [1, 2]
with open(restore_location, "w") as fp:
fp.write(
"""
[
[
{{
"subject": "{subject_value}",
"magic": 1,
"keytype": "SCHEMA",
"version": 2
}},
null
]
]
""".format(subject_value=subject)
)
sb.restore_backup()
time.sleep(1.0)
res = await registry_async_client.get(f"subjects/{subject}/versions")
assert res.status == 200
assert res.json() == [1]
# Schema delete for a nonexistent subject version is ignored
subject = new_random_name("subject")
res = await registry_async_client.post(f"subjects/{subject}/versions", json={"schema": '{"type": "string"}'})
with open(restore_location, "w") as fp:
fp.write(
"""
[
[
{{
"subject": "{subject_value}",
"magic": 1,
"keytype": "SCHEMA",
"version": 2
}},
null
]
]
""".format(subject_value=subject)
)
sb.restore_backup()
time.sleep(1.0)
res = await registry_async_client.get(f"subjects/{subject}/versions")
assert res.status == 200
assert res.json() == [1]