def run(config: Config):
log.info(f'Running consumer with config {config}')
kafka_consumer = create_kafka_consumer(
config.kafka_bootstrap,
config.kafka_topic,
config.kafka_ca_path,
config.kafka_key_path,
config.kafka_cert_path
)
db = create_db_client(config.postgres_uri)
consumer = Consumer(config, kafka_consumer, db)
consumer.run()
async def get_messages(): # DEPRECATED!!!
while True:
try:
consumer = Consumer()
break
except Exception as e:
print(e)
ioloop = asyncio.get_event_loop()
tasks = [
ioloop.create_task(consumer.fetch_messages()),
ioloop.create_task(consumer.commit_per_second(2))
]
ioloop.run_until_complete(asyncio.wait(tasks))
ioloop.close()
return text("start listening")
def test_consumer_handle_event():
config = Config(kafka_bootstrap='localhost:9002',
kafka_topic='checks',
kafka_ca_path='./',
kafka_key_path='./',
kafka_cert_path='./',
postgres_uri='postgres://*****:*****@localhost:port/db')
check = Check(response_time=100, status_code=201, url='http://google.com')
event = Mock()
event.value = check.serialize()
kafka_consumer = [event]
db = Mock()
cursor = Mock()
db.cursor = Mock(return_value=cursor)
consumer = Consumer(config, kafka_consumer, db) # type: ignore
consumer.run()
cursor.execute.assert_called()
def main(args):
mtcnn = MTCNN('./models/mtcnn/', False)
print('loading gallery....')
if (os.path.exists('gallery.pickle')):
with open('gallery.pickle', 'rb') as handle:
gallery = pickle.load(handle)
else:
gallery = prepare_gallery(args.gallery_path)
print('gallery loaded')
print('characters in gallery->', gallery.keys())
consumer = Consumer(args.video_path)
ret, frame = consumer.get_frame()
task_q = multiprocessing.Queue()
result_q = multiprocessing.Queue()
e = multiprocessing.Event()
facenet = multiprocessing.Process(name='facenet',
target=facenet_run,
args=(task_q, result_q, e))
display = multiprocessing.Process(name='display',
target=display_run,
args=(gallery, result_q, e))
facenet.start()
display.start()
try:
while (ret):
mtcnn_run(frame, task_q, mtcnn)
ret, frame = consumer.get_frame()
e.set()
task_q.close()
consumer.release()
facenet.join()
display.join()
except Exception:
e.set()
task_q.close()
result_q.close()
consumer.release()
facenet.join()
display.join()
from producer import APP as producer_app
from consumer import APP as consumer_app
import time
from sys import argv
import asyncio
from consumer.consumer import Consumer
from producer.producer import Producer
if __name__ == '__main__':
if 'producer' in argv:
producer_app.run(host='0.0.0.0', port=5000)
if 'consumer' in argv:
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(Consumer.listen_and_commit(loop=loop, item=10, second=10))
except Exception as e:
print("some issue2")
finally:
loop.close()
#!/usr/bin/env python3
from consumer.consumer import Consumer
from os import environ
RABBIT_MQ_HOST = environ.get("RABBITMQ_HOST") or "localhost"
print(f"Use {RABBIT_MQ_HOST} as RabbitMQ host")
if __name__ == "__main__":
try:
consumer = Consumer(host=RABBIT_MQ_HOST)
consumer.start_consuming()
print(f"Connected to RabbitMQ")
except:
print("RabbitMQ currently unavailable, {action}".format(
action="exit" if RABBIT_MQ_HOST ==
"localhost" else "trying to reconnect..."))
def test_no_brokers_available_error():
with pytest.raises(NoBrokersAvailable):
c = Consumer('12.345.678.90', '9092')
from os import environ
import prometheus_client
from apscheduler.schedulers.background import BlockingScheduler
from consumer.consumer import Consumer
from mail_manager import MailManager
if __name__ == '__main__':
scheduler = BlockingScheduler()
prometheus_client.start_http_server(5000)
SENDINBLUE_URL = environ.get('SENDINBLUE_URL', 'https://api.sendinblue.com/v3')
SENDINBLUE_API_KEY = environ.get('SENDINBLUE_API_KEY', '')
SENDINBLUE_MAILS_PER_HOUR = int(environ.get('SENDINBLUE_MAILS_PER_HOUR', 12))
QUEUE_SERVICE_URL = environ.get('QUEUE_SERVICE_URL', 'amqp://*****:*****@rabbitmq:5672/%2f')
mail_manager = MailManager(SENDINBLUE_API_KEY, SENDINBLUE_URL)
consumer = Consumer(mail_manager, SENDINBLUE_MAILS_PER_HOUR, QUEUE_SERVICE_URL)
scheduler.add_job(consumer.consume_mails_per_hour, 'cron',
year='*', month='*', day='*', week='*', day_of_week='*', hour='*', minute=0, second=0,
id='mails-every-hour')
scheduler.add_job(consumer.consume_send_last_mails, 'cron',
year='*', month='*', day='*', week='*', day_of_week='*', hour=23, minute=5, second=0,
id='last-mails')
print('Application started!')
scheduler.start()