def logger_init(args):
logging.basicConfig(level=logging.DEBUG,
format='%(module)15s %(asctime)s %(message)s',
datefmt='%H:%M:%S')
if args.log_to_file:
log_filename = os.path.join(
args.log_dir,
args.log_prefix + datetime.datetime.now().strftime("%m%d%H%M%S"))
logging.getLogget().addHandler(logging.FileHandler(log_filename))
import json
import logging
import requests
logger = logging.getLogget(__name__)
KAFKA_CONNECT_URL = "http://localhost:8083/connectors"
CONNECTOR_NAME = "stations"
def configure_connector():
logging.debug("Creating or updating kafka connect connector...")
resp = requests.get(f"{KAFKA_CONNECT_URL}/{CONNECTOR_NAME}")
if resp.status_code == 200:
logging.debug("connector already created skipping recreation")
return
resp = requests.post(
KAFKA_CONNECT_URL,
headers={"Content-Type": "application/json"},
data=json.dumps({
"name": CONNECTOR_NAME,
"config": {
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
"connection.url": "jdbc:postgresql://postgres:5432/cta",
"connection.user": "******",
"connection.password": "******",
"topic.prefix": "org.chicago.cta.",
"poll.interval.ms": "500000",
"table.whitelist": "stations",