- Python 3.9
- librdkafka
brew install librdkafka # Macos
apt-get install librdkafka-dev # Ubuntu
python setup.py install
start_producer --config config_file_path.json
start_consumer --config config_file_path.json
docker build -t kafka_http_monitor .
docker run -v local_config_path.json:container_config_path.json -it kafka_http_monitor start_consumer --config container_config_path.json
docker run -v local_config_path.json:container_config_path.json -it kafka_http_monitor start_producer --config container_config_path.json
jsonschema for configuration file
{
"db_uri": "postgres://user:pass@dbhost:dbport/defaultdb?sslmode=require",
"transport": "kafka",
"bootstrap.servers": "localhost",
"kafka_topic": "httpwatcher",
"auto.offset.reset": "earliest",
"group.id": "my-group",
"enable.auto.commit": "true",
"auto.commit.interval.ms": "500",
"ssl.ca.location": "/Users/mghosal/Downloads/ca.pem",
"ssl.certificate.location": "/Users/mghosal/Downloads/service.cert",
"ssl.key.location": "/Users/mghosal/Downloads/service.key",
"security.protocol": "SSL"
}{
"monitoring_uri": "https://www.python.org",
"regex_pattern": "python",
"interval": 5,
"transport": "kafka",
"metric_collector_type": "http",
"bootstrap.servers": "localhost:23908",
"kafka_topic": "httpwatcher",
"ssl.ca.location": "/Users/mghosal/Downloads/ca.pem",
"ssl.certificate.location": "/Users/mghosal/Downloads/service.cert",
"ssl.key.location": "/Users/mghosal/Downloads/service.key",
"security.protocol": "SSL"
}pip install poetry==1.1.4
cd kafka-http-monitor
poetry install
poetry run pytest