def test_kafka_connect_config(self):
self._check_folder_exist(KAFKA_CONNECT_DIR)
# check config has been generated
connect.setup_configs(self.complete_producer_conf)
assert os.path.isfile(KAFKA_CONNECT_CFG_PATH)
actual_config = {}
with open(KAFKA_CONNECT_CFG_PATH, "r") as f:
for line in f.readlines():
tmp_line = line.strip().split("=")
actual_config[tmp_line[0]] = tmp_line[1]
assert actual_config["bootstrap.servers"] == os.environ.get(
"MX_MyFirstModule.Kafka_broker_url")
# verify postgres whitelists
debezium_config = json.loads(
debezium_generator.generate_config(self.complete_producer_conf))
assert (debezium_config["config"]["table.whitelist"] ==
".*MyFirstModule.company.*,.*MyFirstModule.project.*")
assert (
debezium_config["config"]["column.whitelist"] ==
".*MyFirstModule.company.*,MyFirstModule.company.id,.*MyFirstModule.project.*,MyFirstModule.project.id"
)
def run(complete_conf):
setup_configs(complete_conf)
java_path = os.path.join(os.getcwd(), LOCAL, "bin")
os.environ["PATH"] += os.pathsep + java_path
os.environ["JMX_PORT"] = KAFKA_CONNECT_JMX_PORT
os.environ["KAFKA_LOG4J_OPTS"] = (
"-Dlog4j.configuration=file:"
+ os.path.join(os.getcwd(), LOG4J_CFG_PATH)
)
kafka_connect_heap_opts = os.environ.get(
"DATABROKER_KAFKA_CONNECT_HEAP_OPTS", "-Xms512M -Xmx2G"
)
os.environ["KAFKA_HEAP_OPTS"] = kafka_connect_heap_opts
env = dict(os.environ)
kafka_connect_process = DataBrokerProcess(
PROCESS_NAME, (KAFKA_CONNECT_START_PATH, KAFKA_CONNECT_CFG_PATH), env,
)
# Wait for kafka connect to initialize and then issue a request for debezium connector
time.sleep(INITIAL_WAIT)
debezium_config = json.loads(
debezium_generator.generate_config(complete_conf)
)
def backoff_hdlr(details):
logging.warn(
"Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format(
details["tries"], MAX_RETRIES
)
)
def giveup_hdlr(details):
logging.error("Databroker: Kafka Connect wait retries exhaused")
raise Exception("Databroker: Kafka Connect failed to start")
@backoff.on_predicate(
backoff.constant,
interval=BACKOFF_TIME,
max_tries=MAX_RETRIES,
on_backoff=backoff_hdlr,
on_giveup=giveup_hdlr,
)
@backoff.on_exception(
backoff.constant,
Exception,
interval=BACKOFF_TIME,
max_tries=MAX_RETRIES,
on_backoff=backoff_hdlr,
on_giveup=giveup_hdlr,
)
def start_debezium_connector():
return requests.put(
"{}/{}/{}".format(CONNECT_URL, debezium_config["name"], "config"),
json=debezium_config["config"],
)
start_debezium_connector()
return kafka_connect_process
def run(complete_conf):
setup_configs(complete_conf)
java_path = os.path.join(os.getcwd(), LOCAL, "bin")
os.environ["PATH"] += os.pathsep + java_path
os.environ["JMX_PORT"] = KAFKA_CONNECT_JMX_PORT
env = dict(os.environ)
kafka_connect_process = DataBrokerProcess(
PROCESS_NAME, (KAFKA_CONNECT_START_PATH, KAFKA_CONNECT_CFG_PATH), env,
)
# Wait for kafka connect to initialize and then issue a request for debezium connector
time.sleep(INITIAL_WAIT)
debezium_config = json.loads(
debezium_generator.generate_config(complete_conf)
)
retry_count = 1
success = False
while retry_count < MAX_RETRIES:
try:
response = requests.put(
"{}/{}/{}".format(
CONNECT_URL, debezium_config["name"], "config"
),
json=debezium_config["config"],
)
if response.status_code in (200, 201):
success = True
break
else:
logging.warn(
"Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format(
retry_count, MAX_RETRIES
)
)
retry_count += 1
time.sleep(BACKOFF_TIME)
except Exception as ex:
logging.warn(
"Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format(
retry_count, MAX_RETRIES
)
)
logging.debug("Exception message: {}".format(str(ex)))
retry_count += 1
time.sleep(BACKOFF_TIME)
if success:
logging.debug(
"Databroker: connect and debezium successfully initialized"
)
else:
logging.error("Databroker: Kafka Connect wait retries exhaused")
raise Exception("Databroker: Kafka Connect failed to start")
return kafka_connect_process