def make_consumers(application):
INTERRUPT_EVENT = Event()
logging.info("Conectando no Kafka: " + os.environ.get("KAFKA_BROKERS") +
" / " + os.environ.get("KAFKA_SECURITY_PROTOCOL"))
bus = FlaskKafka(
INTERRUPT_EVENT,
security_protocol=os.environ.get("KAFKA_SECURITY_PROTOCOL"),
bootstrap_servers=os.environ.get("KAFKA_BROKERS"),
group_id="financeiro",
value_deserializer=lambda v: json.loads(v),
)
@bus.handle("consulta-atualizada")
def handler_consulta_atualizada(msg):
logging.info("mensagem recebida " + msg)
return bus
else: # prediction if tax
prediction = model_tax_type.predict(model_input)
input_data['TaxType'], input_data['PStatus'] = from_categorical(prediction, 0.9, 0, input_data['PStatus'])
prediction = model_pmnt_type_tax.predict(model_input)
input_data['PmntType'], input_data['PStatus'] = from_percent(prediction, 0.9, input_data['PStatus'])
del input_data['TextData']
del input_data['Tax']
return input_data
def json_deserializer(data):
return json.loads(data.decode('utf-8'))
bus = FlaskKafka(INTERRUPT_EVENT,
bootstrap_servers=",".join(["localhost:9092"]),
value_deserializer = json_deserializer,
group_id="consumer-grp-id"
)
@bus.handle('TO_ML')
def test_topic_handler(msg):
print("consumed data from TO_ML topic")
producer.send_message(predict_data(msg.value))
if __name__ == '__main__':
bus.run()
app.run(debug=True, port=5004)
from threading import Event
from flask import Flask, make_response, request, g
from flask_restx import Resource, Api, reqparse
from flask_httpauth import HTTPTokenAuth
from flask_kafka import FlaskKafka
app = Flask(__name__)
api = Api(app)
auth = HTTPTokenAuth(scheme='Token')
RECOMMENDATIONS_TOPIC = os.environ.get('RECOMMENDATIONS_TOPIC')
INTERRUPT_EVENT = Event()
kafka = FlaskKafka(
INTERRUPT_EVENT,
bootstrap_servers=os.environ.get('KAFKA_BROKER_URI'),
)
def connect_db():
conn = psycopg2.connect(dbname=os.environ.get('PG_DB'),
user=os.environ.get('PG_USER'),
password=os.environ.get('PG_PASS'),
host=os.environ.get('PG_HOST'))
return conn
# Auth check token
@auth.verify_token
def verify_token(token):
if not (token):
import logging
import signal
from threading import Event
from flask_kafka import FlaskKafka
import settings
from services.persuasion_services import PersuasionServices
logger = logging.getLogger("persuasion_engine")
INTERRUPT_EVENT = Event()
consumer = FlaskKafka(INTERRUPT_EVENT,
bootstrap_servers=settings.KAFKA_SERVER["host"],
group_id=settings.KAFKA_SERVER["group"]["persuasion"],
enable_auto_commit=False,
request_timeout_ms=30000)
def listen_kill_server():
signal.signal(signal.SIGTERM, consumer.interrupted_process)
signal.signal(signal.SIGINT, consumer.interrupted_process)
signal.signal(signal.SIGQUIT, consumer.interrupted_process)
signal.signal(signal.SIGHUP, consumer.interrupted_process)
@consumer.handle(settings.KAFKA_SERVER["topic"]["persuasion"])
def kafka_consumer_listener(con):
"""
Watson kafka consumer to create persuasions on real time
from flask import Flask
from threading import Event
import signal
from flask_kafka import FlaskKafka
app = Flask(__name__)
INTERRUPT_EVENT = Event()
kafka_params = {
"bootstrap.servers": ",".join(["localhost:9092"]),
"group.id": "consumer-grp-id"
}
bus = FlaskKafka(INTERRUPT_EVENT, **kafka_params)
# Register termination listener
def listen_kill_server():
signal.signal(signal.SIGTERM, bus.interrupted_process)
signal.signal(signal.SIGINT, bus.interrupted_process)
signal.signal(signal.SIGQUIT, bus.interrupted_process)
signal.signal(signal.SIGHUP, bus.interrupted_process)
# Handle message received from a Kafka topic
@bus.handle('test-topic')
def test_topic_handler(msg):
print("consumed {} from test-topic".format(msg))