def run(complete_conf):
setup_configs(complete_conf)
java_path = os.path.join(os.getcwd(), LOCAL, "bin")
os.environ["PATH"] += os.pathsep + java_path
os.environ["JMX_PORT"] = KAFKA_CONNECT_JMX_PORT
os.environ["KAFKA_LOG4J_OPTS"] = (
"-Dlog4j.configuration=file:"
+ os.path.join(os.getcwd(), LOG4J_CFG_PATH)
)
kafka_connect_heap_opts = os.environ.get(
"DATABROKER_KAFKA_CONNECT_HEAP_OPTS", "-Xms512M -Xmx2G"
)
os.environ["KAFKA_HEAP_OPTS"] = kafka_connect_heap_opts
env = dict(os.environ)
kafka_connect_process = DataBrokerProcess(
PROCESS_NAME, (KAFKA_CONNECT_START_PATH, KAFKA_CONNECT_CFG_PATH), env,
)
# Wait for kafka connect to initialize and then issue a request for debezium connector
time.sleep(INITIAL_WAIT)
debezium_config = json.loads(
debezium_generator.generate_config(complete_conf)
)
def backoff_hdlr(details):
logging.warn(
"Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format(
details["tries"], MAX_RETRIES
)
)
def giveup_hdlr(details):
logging.error("Databroker: Kafka Connect wait retries exhaused")
raise Exception("Databroker: Kafka Connect failed to start")
@backoff.on_predicate(
backoff.constant,
interval=BACKOFF_TIME,
max_tries=MAX_RETRIES,
on_backoff=backoff_hdlr,
on_giveup=giveup_hdlr,
)
@backoff.on_exception(
backoff.constant,
Exception,
interval=BACKOFF_TIME,
max_tries=MAX_RETRIES,
on_backoff=backoff_hdlr,
on_giveup=giveup_hdlr,
)
def start_debezium_connector():
return requests.put(
"{}/{}/{}".format(CONNECT_URL, debezium_config["name"], "config"),
json=debezium_config["config"],
)
start_debezium_connector()
return kafka_connect_process
def run(complete_conf):
setup_configs(complete_conf)
java_path = os.path.join(os.getcwd(), LOCAL, "bin")
os.environ["PATH"] += os.pathsep + java_path
os.environ["JMX_PORT"] = KAFKA_CONNECT_JMX_PORT
env = dict(os.environ)
kafka_connect_process = DataBrokerProcess(
PROCESS_NAME, (KAFKA_CONNECT_START_PATH, KAFKA_CONNECT_CFG_PATH), env,
)
# Wait for kafka connect to initialize and then issue a request for debezium connector
time.sleep(INITIAL_WAIT)
debezium_config = json.loads(
debezium_generator.generate_config(complete_conf)
)
retry_count = 1
success = False
while retry_count < MAX_RETRIES:
try:
response = requests.put(
"{}/{}/{}".format(
CONNECT_URL, debezium_config["name"], "config"
),
json=debezium_config["config"],
)
if response.status_code in (200, 201):
success = True
break
else:
logging.warn(
"Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format(
retry_count, MAX_RETRIES
)
)
retry_count += 1
time.sleep(BACKOFF_TIME)
except Exception as ex:
logging.warn(
"Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format(
retry_count, MAX_RETRIES
)
)
logging.debug("Exception message: {}".format(str(ex)))
retry_count += 1
time.sleep(BACKOFF_TIME)
if success:
logging.debug(
"Databroker: connect and debezium successfully initialized"
)
else:
logging.error("Databroker: Kafka Connect wait retries exhaused")
raise Exception("Databroker: Kafka Connect failed to start")
return kafka_connect_process
def run(complete_conf):
setup_configs(complete_conf)
java_path = os.path.join(os.getcwd(), LOCAL, "bin")
os.environ["PATH"] += os.pathsep + java_path
os.environ["JMX_PORT"] = KAFKA_STREAMS_JMX_PORT
env = dict(os.environ)
kafka_streams_process = DataBrokerProcess(
PROCESS_NAME,
(
"java",
"-Dconfig.file=" + AZKARRA_CONF_PATH,
"-Dcom.sun.management.jmxremote",
"-Dcom.sun.management.jmxremote.authenticate=false",
"-Dcom.sun.management.jmxremote.ssl=false",
"-Dcom.sun.management.jmxremote.port=" + KAFKA_STREAMS_JMX_PORT,
"-jar",
PDR_STREAMS_JAR,
),
env,
)
return kafka_streams_process
def run(complete_conf):
version = get_pdr_stream_version()
setup_configs(complete_conf, version)
java_path = os.path.join(os.getcwd(), LOCAL, "bin")
os.environ["PATH"] += os.pathsep + java_path
os.environ["JMX_PORT"] = KAFKA_STREAMS_JMX_PORT
os.environ["LOG_LEVEL"] = LOG_LEVEL
env = dict(os.environ)
kafka_streams_process = DataBrokerProcess(
PROCESS_NAME,
(
"java",
"-Dconfig.file=" + _get_azkarra_conf_path(version),
"-Dcom.sun.management.jmxremote",
"-Dcom.sun.management.jmxremote.authenticate=false",
"-Dcom.sun.management.jmxremote.ssl=false",
"-Dcom.sun.management.jmxremote.port=" + KAFKA_STREAMS_JMX_PORT,
"-jar",
_get_pdr_streams_jar(version),
),
env,
)
return kafka_streams_process