def run(complete_conf): setup_configs(complete_conf) java_path = os.path.join(os.getcwd(), LOCAL, "bin") os.environ["PATH"] += os.pathsep + java_path os.environ["JMX_PORT"] = KAFKA_CONNECT_JMX_PORT os.environ["KAFKA_LOG4J_OPTS"] = ( "-Dlog4j.configuration=file:" + os.path.join(os.getcwd(), LOG4J_CFG_PATH) ) kafka_connect_heap_opts = os.environ.get( "DATABROKER_KAFKA_CONNECT_HEAP_OPTS", "-Xms512M -Xmx2G" ) os.environ["KAFKA_HEAP_OPTS"] = kafka_connect_heap_opts env = dict(os.environ) kafka_connect_process = DataBrokerProcess( PROCESS_NAME, (KAFKA_CONNECT_START_PATH, KAFKA_CONNECT_CFG_PATH), env, ) # Wait for kafka connect to initialize and then issue a request for debezium connector time.sleep(INITIAL_WAIT) debezium_config = json.loads( debezium_generator.generate_config(complete_conf) ) def backoff_hdlr(details): logging.warn( "Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format( details["tries"], MAX_RETRIES ) ) def giveup_hdlr(details): logging.error("Databroker: Kafka Connect wait retries exhaused") raise Exception("Databroker: Kafka Connect failed to start") @backoff.on_predicate( backoff.constant, interval=BACKOFF_TIME, max_tries=MAX_RETRIES, on_backoff=backoff_hdlr, on_giveup=giveup_hdlr, ) @backoff.on_exception( backoff.constant, Exception, interval=BACKOFF_TIME, max_tries=MAX_RETRIES, on_backoff=backoff_hdlr, on_giveup=giveup_hdlr, ) def start_debezium_connector(): return requests.put( "{}/{}/{}".format(CONNECT_URL, debezium_config["name"], "config"), json=debezium_config["config"], ) start_debezium_connector() return kafka_connect_process
def run(complete_conf): setup_configs(complete_conf) java_path = os.path.join(os.getcwd(), LOCAL, "bin") os.environ["PATH"] += os.pathsep + java_path os.environ["JMX_PORT"] = KAFKA_CONNECT_JMX_PORT env = dict(os.environ) kafka_connect_process = DataBrokerProcess( PROCESS_NAME, (KAFKA_CONNECT_START_PATH, KAFKA_CONNECT_CFG_PATH), env, ) # Wait for kafka connect to initialize and then issue a request for debezium connector time.sleep(INITIAL_WAIT) debezium_config = json.loads( debezium_generator.generate_config(complete_conf) ) retry_count = 1 success = False while retry_count < MAX_RETRIES: try: response = requests.put( "{}/{}/{}".format( CONNECT_URL, debezium_config["name"], "config" ), json=debezium_config["config"], ) if response.status_code in (200, 201): success = True break else: logging.warn( "Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format( retry_count, MAX_RETRIES ) ) retry_count += 1 time.sleep(BACKOFF_TIME) except Exception as ex: logging.warn( "Databroker: Failed to receive successful response from connect. Retrying...({}/{})".format( retry_count, MAX_RETRIES ) ) logging.debug("Exception message: {}".format(str(ex))) retry_count += 1 time.sleep(BACKOFF_TIME) if success: logging.debug( "Databroker: connect and debezium successfully initialized" ) else: logging.error("Databroker: Kafka Connect wait retries exhaused") raise Exception("Databroker: Kafka Connect failed to start") return kafka_connect_process
def run(complete_conf): setup_configs(complete_conf) java_path = os.path.join(os.getcwd(), LOCAL, "bin") os.environ["PATH"] += os.pathsep + java_path os.environ["JMX_PORT"] = KAFKA_STREAMS_JMX_PORT env = dict(os.environ) kafka_streams_process = DataBrokerProcess( PROCESS_NAME, ( "java", "-Dconfig.file=" + AZKARRA_CONF_PATH, "-Dcom.sun.management.jmxremote", "-Dcom.sun.management.jmxremote.authenticate=false", "-Dcom.sun.management.jmxremote.ssl=false", "-Dcom.sun.management.jmxremote.port=" + KAFKA_STREAMS_JMX_PORT, "-jar", PDR_STREAMS_JAR, ), env, ) return kafka_streams_process
def run(complete_conf): version = get_pdr_stream_version() setup_configs(complete_conf, version) java_path = os.path.join(os.getcwd(), LOCAL, "bin") os.environ["PATH"] += os.pathsep + java_path os.environ["JMX_PORT"] = KAFKA_STREAMS_JMX_PORT os.environ["LOG_LEVEL"] = LOG_LEVEL env = dict(os.environ) kafka_streams_process = DataBrokerProcess( PROCESS_NAME, ( "java", "-Dconfig.file=" + _get_azkarra_conf_path(version), "-Dcom.sun.management.jmxremote", "-Dcom.sun.management.jmxremote.authenticate=false", "-Dcom.sun.management.jmxremote.ssl=false", "-Dcom.sun.management.jmxremote.port=" + KAFKA_STREAMS_JMX_PORT, "-jar", _get_pdr_streams_jar(version), ), env, ) return kafka_streams_process