def test_master_selection(kafka_server, strategy): config_aa = set_config_defaults({}) config_aa["advertised_hostname"] = "127.0.0.1" config_aa["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"]) config_aa["client_id"] = "aa" config_aa["port"] = 1234 config_aa["master_election_strategy"] = strategy mc_aa = init_admin(config_aa) config_bb = set_config_defaults({}) config_bb["advertised_hostname"] = "127.0.0.1" config_bb["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"]) config_bb["client_id"] = "bb" config_bb["port"] = 5678 config_bb["master_election_strategy"] = strategy mc_bb = init_admin(config_bb) while True: if not (mc_aa.sc or mc_bb.sc): time.sleep(1.0) continue if not (mc_aa.sc.master or mc_bb.sc.master or mc_aa.sc.master_url or mc_bb.sc.master_url): time.sleep(1.0) continue assert mc_bb.sc.election_strategy == strategy assert mc_aa.sc.election_strategy == strategy aa_master = strategy == "lowest" bb_master = strategy == "highest" assert mc_aa.sc.master is aa_master assert mc_bb.sc.master is bb_master master_config = config_aa if strategy == "lowest" else config_bb master_url = "http://{}:{}".format(master_config["host"], master_config["port"]) assert mc_aa.sc.master_url == master_url assert mc_bb.sc.master_url == master_url break mc_aa.close() mc_bb.close()
def test_master_selection(kafka_server): config_aa = set_config_defaults({}) config_aa["advertised_hostname"] = "127.0.0.1" config_aa["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"]) config_aa["client_id"] = "aa" config_aa["port"] = 1234 mc_aa = init_admin(config_aa) config_bb = set_config_defaults({}) config_bb["advertised_hostname"] = "127.0.0.1" config_bb["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server["kafka_port"]) config_bb["client_id"] = "bb" config_bb["port"] = 5678 mc_bb = init_admin(config_bb) while True: if not (mc_aa.sc or mc_bb.sc): time.sleep(1.0) continue if not (mc_aa.sc.master or mc_bb.sc.master or mc_aa.sc.master_url or mc_bb.sc.master_url): time.sleep(1.0) continue assert mc_aa.sc.master is True assert mc_bb.sc.master is False master_url = "http://{}:{}".format(config_aa["host"], config_aa["port"]) assert mc_aa.sc.master_url == master_url assert mc_bb.sc.master_url == master_url break mc_aa.close() mc_bb.close()
async def fixture_registry_async( tmp_path: Path, kafka_server: Optional[KafkaConfig] ) -> AsyncIterator[KarapaceSchemaRegistry]: if not kafka_server: assert REGISTRY_URI in os.environ or REST_URI in os.environ instance, _ = mock_factory("registry")() yield instance else: config_path = tmp_path / "karapace_config.json" kafka_port = kafka_server.kafka_port config = set_config_defaults({ "log_level": "WARNING", "bootstrap_uri": f"127.0.0.1:{kafka_port}", "topic_name": new_random_name(), "group_id": new_random_name("schema_registry") }) write_config(config_path, config) registry = KarapaceSchemaRegistry(config_file_path=str(config_path), config=set_config_defaults(config)) await registry.get_master() try: yield registry finally: registry.close()
async def fixture_registry_async( request, loop, # pylint: disable=unused-argument tmp_path: Path, kafka_servers: KafkaServers, ) -> AsyncIterator[Optional[KarapaceSchemaRegistry]]: # Do not start a registry when the user provided an external service. Doing # so would cause this node to join the existing group and participate in # the election process. Without proper configuration for the listeners that # won't work and will cause test failures. rest_url = request.config.getoption("registry_url") if rest_url: yield None return config_path = tmp_path / "karapace_config.json" config = set_config_defaults({ "bootstrap_uri": kafka_servers.bootstrap_servers, # Using the default settings instead of random values, otherwise it # would not be possible to run the tests with external services. # Because of this every test must be written in such a way that it can # be executed twice with the same servers. # "topic_name": new_random_name("topic"), # "group_id": new_random_name("schema_registry") }) write_config(config_path, config) registry = KarapaceSchemaRegistry(config=config) await registry.get_master() try: yield registry finally: await registry.close()
async def fixture_rest_async( request, loop, # pylint: disable=unused-argument tmp_path: Path, kafka_servers: KafkaServers, registry_async_client: Client, ) -> AsyncIterator[Optional[KafkaRest]]: # Do not start a REST api when the user provided an external service. Doing # so would cause this node to join the existing group and participate in # the election process. Without proper configuration for the listeners that # won't work and will cause test failures. rest_url = request.config.getoption("rest_url") if rest_url: yield None return config_path = tmp_path / "karapace_config.json" config = set_config_defaults({ "bootstrap_uri": kafka_servers.bootstrap_servers, "admin_metadata_max_age": 0 }) write_config(config_path, config) rest = KafkaRest(config=config) assert rest.serializer.registry_client assert rest.consumer_manager.deserializer.registry_client rest.serializer.registry_client.client = registry_async_client rest.consumer_manager.deserializer.registry_client.client = registry_async_client try: yield rest finally: await rest.close()
async def fixture_rest_async( session_tmpdir: TempDirCreator, kafka_server: Optional[KafkaConfig], registry_async_client: Client ) -> AsyncIterator[KafkaRest]: if not kafka_server: assert REST_URI in os.environ instance, _ = mock_factory("rest")() yield instance else: config_path = os.path.join(session_tmpdir(), "karapace_config.json") kafka_port = kafka_server.kafka_port config = set_config_defaults({ "log_level": "WARNING", "bootstrap_uri": f"127.0.0.1:{kafka_port}", "admin_metadata_max_age": 0 }) write_config(config_path, config) rest = KafkaRest(config_file_path=config_path, config=config) assert rest.serializer.registry_client assert rest.consumer_manager.deserializer.registry_client rest.serializer.registry_client.client = registry_async_client rest.consumer_manager.deserializer.registry_client.client = registry_async_client try: yield rest finally: rest.close() await rest.close_producers()
def test_master_selection(kafka_servers: KafkaServers, strategy: str) -> None: # Use random port to allow for parallel runs. port1 = get_random_port(port_range=TESTS_PORT_RANGE, blacklist=[]) port2 = get_random_port(port_range=TESTS_PORT_RANGE, blacklist=[port1]) port_aa, port_bb = sorted((port1, port2)) client_id_aa = new_random_name("master_selection_aa_") client_id_bb = new_random_name("master_selection_bb_") group_id = new_random_name("group_id") config_aa = set_config_defaults({ "advertised_hostname": "127.0.0.1", "bootstrap_uri": kafka_servers.bootstrap_servers, "client_id": client_id_aa, "group_id": group_id, "port": port_aa, "master_election_strategy": strategy, }) config_bb = set_config_defaults({ "advertised_hostname": "127.0.0.1", "bootstrap_uri": kafka_servers.bootstrap_servers, "client_id": client_id_bb, "group_id": group_id, "port": port_bb, "master_election_strategy": strategy, }) with closing(init_admin(config_aa)) as mc_aa, closing(init_admin(config_bb)) as mc_bb: if strategy == "lowest": master = mc_aa slave = mc_bb else: master = mc_bb slave = mc_aa # Wait for the election to happen while not is_master(master): time.sleep(0.3) while not has_master(slave): time.sleep(0.3) # Make sure the end configuration is as expected master_url = f'http://{master.config["host"]}:{master.config["port"]}' assert master.sc.election_strategy == strategy assert slave.sc.election_strategy == strategy assert master.sc.master_url == master_url assert slave.sc.master_url == master_url
def read_config(config_path): if os.path.exists(config_path): try: config = json.loads(open(config_path, "r").read()) config = set_config_defaults(config) return config except Exception as ex: raise InvalidConfiguration(ex) else: raise InvalidConfiguration()
async def test_backup_get(registry_async_client, kafka_servers: KafkaServers, tmp_path: Path): _ = await insert_data(registry_async_client) # Get the backup backup_location = tmp_path / "schemas.log" config = set_config_defaults({"bootstrap_uri": kafka_servers.bootstrap_servers}) sb = SchemaBackup(config, str(backup_location)) sb.request_backup() # The backup file has been created assert os.path.exists(backup_location)
def test_master_selection(kafka_server: Optional[KafkaConfig], strategy: str) -> None: assert kafka_server, f"test_master_selection can not be used if the env variable `{REGISTRY_URI}` or `{REST_URI}` is set" config_aa = set_config_defaults({}) config_aa["advertised_hostname"] = "127.0.0.1" config_aa["bootstrap_uri"] = f"127.0.0.1:{kafka_server.kafka_port}" config_aa["client_id"] = "aa" config_aa["port"] = 1234 config_aa["master_election_strategy"] = strategy mc_aa = init_admin(config_aa) config_bb = set_config_defaults({}) config_bb["advertised_hostname"] = "127.0.0.1" config_bb["bootstrap_uri"] = f"127.0.0.1:{kafka_server.kafka_port}" config_bb["client_id"] = "bb" config_bb["port"] = 5678 config_bb["master_election_strategy"] = strategy mc_bb = init_admin(config_bb) if strategy == "lowest": master = mc_aa slave = mc_bb else: master = mc_bb slave = mc_aa # Wait for the election to happen while not is_master(master): time.sleep(0.3) while not has_master(slave): time.sleep(0.3) # Make sure the end configuration is as expected master_url = f'http://{master.config["host"]}:{master.config["port"]}' assert master.sc.election_strategy == strategy assert slave.sc.election_strategy == strategy assert master.sc.master_url == master_url assert slave.sc.master_url == master_url mc_aa.close() mc_bb.close()
def read_config(self): if os.path.exists(self.config_path): try: config = json.loads(open(self.config_path, "r").read()) self.config = set_config_defaults(config) try: logging.getLogger().setLevel(config["log_level"]) except ValueError: self.log.excption("Problem with log_level: %r", config["log_level"]) except Exception as ex: raise InvalidConfiguration(ex) else: raise InvalidConfiguration()
def test_master_selection(kafka_server: Optional[KafkaConfig], strategy: str) -> None: assert kafka_server, f"test_master_selection can not be used if the env variable `{REGISTRY_URI}` or `{REST_URI}` is set" config_aa = set_config_defaults({}) config_aa["advertised_hostname"] = "127.0.0.1" config_aa["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server.kafka_port) config_aa["client_id"] = "aa" config_aa["port"] = 1234 config_aa["master_election_strategy"] = strategy mc_aa = init_admin(config_aa) config_bb = set_config_defaults({}) config_bb["advertised_hostname"] = "127.0.0.1" config_bb["bootstrap_uri"] = "127.0.0.1:{}".format(kafka_server.kafka_port) config_bb["client_id"] = "bb" config_bb["port"] = 5678 config_bb["master_election_strategy"] = strategy mc_bb = init_admin(config_bb) while True: if not (mc_aa.sc or mc_bb.sc): time.sleep(1.0) continue if not (mc_aa.sc.master or mc_bb.sc.master or mc_aa.sc.master_url or mc_bb.sc.master_url): time.sleep(1.0) continue assert mc_bb.sc.election_strategy == strategy assert mc_aa.sc.election_strategy == strategy aa_master = strategy == "lowest" bb_master = strategy == "highest" assert mc_aa.sc.master is aa_master assert mc_bb.sc.master is bb_master master_config = config_aa if strategy == "lowest" else config_bb master_url = "http://{}:{}".format(master_config["host"], master_config["port"]) assert mc_aa.sc.master_url == master_url assert mc_bb.sc.master_url == master_url break mc_aa.close() mc_bb.close()
def test_no_eligible_master(kafka_servers: KafkaServers) -> None: client_id = new_random_name("master_selection_") group_id = new_random_name("group_id") config_aa = set_config_defaults({ "advertised_hostname": "127.0.0.1", "bootstrap_uri": kafka_servers.bootstrap_servers, "client_id": client_id, "group_id": group_id, "port": get_random_port(port_range=TESTS_PORT_RANGE, blacklist=[]), "master_eligibility": False, }) with closing(init_admin(config_aa)) as mc: # Wait for the election to happen, ie. flag is not None while not mc.sc or mc.sc.are_we_master is None: time.sleep(0.3) # Make sure the end configuration is as expected assert mc.sc.are_we_master is False assert mc.sc.master_url is None
async def test_backup_restore( registry_async_client: Client, kafka_servers: KafkaServers, tmp_path: Path, ) -> None: subject = new_random_name("subject") restore_location = tmp_path / "restore.log" with restore_location.open("w") as fp: jsonlib.dump( [[ { "subject": subject, "version": 1, "magic": 1, "keytype": "SCHEMA", }, { "deleted": False, "id": 1, "schema": "\"string\"", "subject": subject, "version": 1, }, ]], fp=fp, ) config = set_config_defaults({"bootstrap_uri": kafka_servers.bootstrap_servers}) sb = SchemaBackup(config, str(restore_location)) sb.restore_backup() # The restored karapace should have the previously created subject all_subjects = [] expiration = Expiration.from_timeout(timeout=10) while subject not in all_subjects: expiration.raise_if_expired(msg=f"{subject} not in {all_subjects}") res = await registry_async_client.get("subjects") assert res.status_code == 200 all_subjects = res.json() # Test a few exotic scenarios subject = new_random_name("subject") res = await registry_async_client.put(f"config/{subject}", json={"compatibility": "NONE"}) assert res.status == 200 assert res.json()["compatibility"] == "NONE" # Restore a compatibility config remove message with open(restore_location, "w") as fp: fp.write( """ [ [ {{ "subject": "{subject_value}", "magic": 0, "keytype": "CONFIG" }}, null ] ] """.format(subject_value=subject) ) res = await registry_async_client.get(f"config/{subject}") assert res.status == 200 sb.restore_backup() time.sleep(1.0) res = await registry_async_client.get(f"config/{subject}") assert res.status == 404 # Restore a complete schema delete message subject = new_random_name("subject") res = await registry_async_client.put(f"config/{subject}", json={"compatibility": "NONE"}) res = await registry_async_client.post(f"subjects/{subject}/versions", json={"schema": '{"type": "int"}'}) res = await registry_async_client.post(f"subjects/{subject}/versions", json={"schema": '{"type": "float"}'}) res = await registry_async_client.get(f"subjects/{subject}/versions") assert res.status == 200 assert res.json() == [1, 2] with open(restore_location, "w") as fp: fp.write( """ [ [ {{ "subject": "{subject_value}", "magic": 1, "keytype": "SCHEMA", "version": 2 }}, null ] ] """.format(subject_value=subject) ) sb.restore_backup() time.sleep(1.0) res = await registry_async_client.get(f"subjects/{subject}/versions") assert res.status == 200 assert res.json() == [1] # Schema delete for a nonexistent subject version is ignored subject = new_random_name("subject") res = await registry_async_client.post(f"subjects/{subject}/versions", json={"schema": '{"type": "string"}'}) with open(restore_location, "w") as fp: fp.write( """ [ [ {{ "subject": "{subject_value}", "magic": 1, "keytype": "SCHEMA", "version": 2 }}, null ] ] """.format(subject_value=subject) ) sb.restore_backup() time.sleep(1.0) res = await registry_async_client.get(f"subjects/{subject}/versions") assert res.status == 200 assert res.json() == [1]