def create_app():
broker = None
if config.get('broker'):
broker = f'kafka://{config["kafka.bootstrap.servers"]}',
broker_credentials = None
if 'auth' in config:
broker_credentials = faust.SASLCredentials(
username=config['auth']['username'],
password=config['auth']['password'],
ssl_context=ssl.create_default_context()
)
# create your application
global app
app = faust.App(
config['app']['id'],
autodiscover=True,
origin='pyconsumer',
broker=broker,
broker_credentials=broker_credentials,
consumer_auto_offset_reset=config['app'].get('auto_offset_reset', None),
stream_wait_empty=config['app'].get('enable_auto_commit', None),
topic_disable_leader=True
)
create_consumer(app)
return app
def build_faust_config(brokers: str, config: dict):
ssl_context = ssl.create_default_context()
ssl_context.load_verify_locations(cafile=certifi.where())
faust_config = {}
faust_config.update(broker=brokers)
faust_config.update(broker_session_timeout=20)
faust_config.update(topic_replication_factor=config["replication.factor"])
if "SASL_AUTH" in config:
faust_config.update(broker_credentials=faust.SASLCredentials(
username=config["SASL_AUTH"]["username"],
password=config["SASL_AUTH"]["password"],
ssl_context=ssl.create_default_context(),
mechanism="PLAIN"
))
return faust_config
def create_kafka_app(broker, user, pwd, auto_offset_reset, enable_auto_commit):
credentials = None
if user is not None and pwd is not None:
credentials = faust.SASLCredentials(
username=user,
password=pwd,
ssl_context=ssl.create_default_context()
)
# create your application
app = faust.App(
'sofia',
autodiscover=False,
broker=broker,
broker_credentials=credentials,
consumer_auto_offset_reset=auto_offset_reset,
stream_wait_empty=enable_auto_commit,
topic_disable_leader=True)
return app
def create_app():
broker = None
if config.get('kafka.bootstrap.servers'):
broker = f'kafka://{config["kafka.bootstrap.servers"]}',
broker_credentials = None
if 'auth' in config:
broker_credentials = faust.SASLCredentials(
username=config['auth']['username'],
password=config['auth']['password'],
ssl_context=ssl.create_default_context())
# create your application
global app
app = faust.App(config['app']['id'],
autodiscover=True,
origin='pyproducer',
broker=broker,
broker_credentials=broker_credentials)
create_producer(app)
return app
from dotenv import load_dotenv
import os
load_dotenv(dotenv_path="../.env")
ssl_context = ssl.create_default_context()
ssl_context.load_verify_locations(cafile=certifi.where())
kafka_broker = os.getenv("kafka_broker")
kafka_user = os.getenv("kafka_user")
kafka_password = os.getenv("kafka_password")
app = faust.App(id='page_views',
broker=kafka_broker,
broker_credentials=faust.SASLCredentials(
username=kafka_user,
password=kafka_password,
ssl_context=ssl_context),
store='rocksdb://',
version=1,
topic_replication_factor=3)
class PageView(faust.Record):
id: str
user: str
page_view_topic = app.topic('page_views', value_type=PageView)
page_views = app.Table('page_views', default=int)
from db import ConnPool
from models import StatusEvent
if settings.USE_SASL:
ssl_context = ssl.create_default_context(
purpose=ssl.Purpose.SERVER_AUTH, cafile=settings.CA_FILE
)
app = faust.App(
"checkwebsite",
topic_disable_leader=True,
broker=settings.KAFKA_BROKER_URL,
broker_credentials=faust.SASLCredentials(
ssl_context=ssl_context,
username=settings.KAFKA_USER,
password=settings.KAFKA_PASSWORD,
),
)
else:
app = faust.App("checkwebsite", broker=settings.KAFKA_BROKER_URL)
events_topic = app.topic("events", value_type=StatusEvent)
log = logging.getLogger(__name__)
@app.agent(events_topic)
async def process_event(events: typing.List[StatusEvent]):
"""
Process events in 'events' topic.