def main(profile: str):
"""
Celery worker main entry point
Args:
profile: profile used to run the app
"""
load_config(profile, CONFIGS_PATH, config, 'NLP_SERVICE')
initialize_summary_service()
load()
publisher = container.get('exchange_publisher')
if not publisher.test_connection():
LOGGER.error('Error connecting to the queue provider. Exiting...')
sys.exit(1)
add_logstash_handler(LOG_CONFIG, config.logstash.host, config.logstash.port)
CELERY_APP.configure(task_queue_name='nlp-worker',
broker_config=config.rabbit,
worker_concurrency=config.celery.concurrency,
result_backend_url=build_redis_url(**config.redis))
apm_client = Client(config={
'SERVICE_NAME': config.elastic_apm.service_name,
'SECRET_TOKEN': config.elastic_apm.secret_token,
'SERVER_URL': config.elastic_apm.url
})
register_instrumentation(apm_client)
register_exception_tracking(apm_client)
CELERY_APP.run()
def setup_event_bus():
"""
Setup the event bus with the provided application data
"""
global bus
redis_url = build_redis_url(**config.redis)
if redis_health_check(redis_url):
LOGGER.info('Starting event bus on %s', redis_url)
bus = lightbus.create(
config=dict(
service_name='search-engine',
process_name='search-engine-process',
bus=dict(
schema=dict(
transport=dict(
redis=dict(
url=redis_url
)
)
)
),
apis=dict(
default=dict(
event_transport=dict(
redis=dict(
url=redis_url
)
),
rpc_transport=dict(
redis=dict(
url=redis_url
)
),
result_transport=dict(
redis=dict(
url=redis_url
)
)
)
)
))
bus.client.register_api(UserEvents())
UserCreatedListener(name='handle_user_creation',
event_api='user',
event_name='user_created',
storage_config=config.storage).add_to_bus(bus)
UserDeletedListener(name='handle_user_deletion',
event_api='user',
event_name='user_deleted',
storage_config=config.storage).add_to_bus(bus)
p = Process(target=event_bus_runner, args=(bus,))
p.start()
else:
LOGGER.error(f'Redis service not available. Exiting...')
sys.exit(1)
def setup_event_bus():
"""
Setup the events bus for the provided app
"""
global bus
redis_url = build_redis_url(**config.redis)
if redis_health_check(redis_url):
LOGGER.info('Starting event bus on %s', redis_url)
bus = lightbus.create(
config=dict(
bus=dict(
schema=dict(
transport=dict(
redis=dict(
url=redis_url
)
)
)
),
apis=dict(
default=dict(
event_transport=dict(
redis=dict(
url=redis_url
)
),
rpc_transport=dict(
redis=dict(
url=redis_url
)
),
result_transport=dict(
redis=dict(
url=redis_url
)
)
)
)
))
bus.client.register_api(UserEvents())
else:
LOGGER.error(f'Redis service not available. Exiting...')
sys.exit(1)
def test_build_url_with_password(self):
"""
Test building a redis URL with password correctly builds it
"""
redis_url = build_redis_url('test', 123, 5, password='******')
self.assertEqual(redis_url, 'redis://:testpass@test:123/5')