def init_app(self, app, **defaults):
self.app = app
if not self.client:
self.client = make_client(self.client_cls, app, **defaults)
# 0 is a valid log level (NOTSET), so we need to check explicitly for it
if self.logging or self.logging is 0:
if self.logging is not True:
kwargs = {"level": self.logging}
else:
kwargs = {}
setup_logging(LoggingHandler(self.client, **kwargs))
signals.got_request_exception.connect(self.handle_exception,
sender=app,
weak=False)
try:
from elasticapm.contrib.celery import register_exception_tracking
register_exception_tracking(self.client)
except ImportError:
pass
# Instrument to get spans
if self.client.config.instrument:
elasticapm.instrumentation.control.instrument()
signals.request_started.connect(self.request_started, sender=app)
signals.request_finished.connect(self.request_finished, sender=app)
try:
from elasticapm.contrib.celery import register_instrumentation
register_instrumentation(self.client)
except ImportError:
pass
else:
logger.debug(
"Skipping instrumentation. INSTRUMENT is set to False.")
@app.context_processor
def rum_tracing():
"""
Adds APM related IDs to the context used for correlating the backend transaction with the RUM transaction
"""
transaction = get_transaction()
if transaction and transaction.trace_parent:
return {
"apm": {
"trace_id":
transaction.trace_parent.trace_id,
"span_id":
lambda: transaction.ensure_parent_id(),
"is_sampled":
transaction.is_sampled,
"is_sampled_js":
"true" if transaction.is_sampled else "false",
}
}
return {}
def register_handlers(client):
from django.core.signals import got_request_exception, request_started, request_finished
from elasticapm.contrib.django.handlers import exception_handler
# Connect to Django's internal signal handlers
got_request_exception.disconnect(dispatch_uid=ERROR_DISPATCH_UID)
got_request_exception.connect(partial(exception_handler, client),
dispatch_uid=ERROR_DISPATCH_UID,
weak=False)
request_started.disconnect(dispatch_uid=REQUEST_START_DISPATCH_UID)
request_started.connect(partial(_request_started_handler, client),
dispatch_uid=REQUEST_START_DISPATCH_UID,
weak=False)
request_finished.disconnect(dispatch_uid=REQUEST_FINISH_DISPATCH_UID)
request_finished.connect(
lambda sender, **kwargs: client.end_transaction()
if _should_start_transaction(client) else None,
dispatch_uid=REQUEST_FINISH_DISPATCH_UID,
weak=False,
)
# If we can import celery, register ourselves as exception handler
try:
import celery # noqa F401
from elasticapm.contrib.celery import register_exception_tracking
try:
register_exception_tracking(client)
except Exception as e:
client.logger.exception(
"Failed installing django-celery hook: %s" % e)
except ImportError:
client.logger.debug("Not instrumenting Celery, couldn't import")
def init_app(self, app, **defaults):
self.app = app
if not self.client:
self.client = make_client(self.client_cls, app, **defaults)
if self.logging:
setup_logging(LoggingHandler(self.client))
signals.got_request_exception.connect(self.handle_exception,
sender=app,
weak=False)
try:
from elasticapm.contrib.celery import register_exception_tracking
register_exception_tracking(self.client)
except ImportError:
pass
# Instrument to get spans
if self.client.config.instrument:
elasticapm.instrumentation.control.instrument()
signals.request_started.connect(self.request_started, sender=app)
signals.request_finished.connect(self.request_finished, sender=app)
try:
from elasticapm.contrib.celery import register_instrumentation
register_instrumentation(self.client)
except ImportError:
pass
else:
logger.debug(
"Skipping instrumentation. INSTRUMENT is set to False.")
def main(profile: str):
"""
Celery app main entry point
Args:
profile: profile used to run the app
"""
load_config(profile, CONFIGS_PATH, config, 'NEWS_DISCOVERY')
load()
publisher = container.get('exchange_publisher')
if not publisher.test_connection():
LOGGER.error('Error connecting to the queue provider. Exiting...')
sys.exit(1)
add_logstash_handler(LOG_CONFIG, config.logstash.host,
config.logstash.port)
CELERY_APP.configure(task_queue_name='news-discovery',
broker_config=config.rabbit,
worker_concurrency=config.celery.concurrency)
apm_client = Client(
config={
'SERVICE_NAME': 'news-discovery-app',
'SECRET_TOKEN': config.elastic_apm.secret_token,
'SERVER_URL': config.elastic_apm.url
})
register_instrumentation(apm_client)
register_exception_tracking(apm_client)
CELERY_APP.run()
def main(profile: str):
"""
Celery worker main entry point
Args:
profile: profile used to run the app
"""
load_config(profile, CONFIGS_PATH, config, 'NLP_SERVICE')
initialize_summary_service()
load()
publisher = container.get('exchange_publisher')
if not publisher.test_connection():
LOGGER.error('Error connecting to the queue provider. Exiting...')
sys.exit(1)
add_logstash_handler(LOG_CONFIG, config.logstash.host, config.logstash.port)
CELERY_APP.configure(task_queue_name='nlp-worker',
broker_config=config.rabbit,
worker_concurrency=config.celery.concurrency,
result_backend_url=build_redis_url(**config.redis))
apm_client = Client(config={
'SERVICE_NAME': config.elastic_apm.service_name,
'SECRET_TOKEN': config.elastic_apm.secret_token,
'SERVER_URL': config.elastic_apm.url
})
register_instrumentation(apm_client)
register_exception_tracking(apm_client)
CELERY_APP.run()
def test_failing_celery_task(django_elasticapm_client):
register_exception_tracking(django_elasticapm_client)
t = failing_task.delay()
assert t.state == "FAILURE"
assert len(django_elasticapm_client.events[ERROR]) == 1
assert len(django_elasticapm_client.events[TRANSACTION]) == 1
error = django_elasticapm_client.events[ERROR][0]
assert error["culprit"] == "tests.contrib.django.testapp.tasks.failing_task"
assert error["exception"]["message"] == "ValueError: foo"
assert error["exception"]["handled"] is False
transaction = django_elasticapm_client.events[TRANSACTION][0]
assert transaction["name"] == "tests.contrib.django.testapp.tasks.failing_task"
assert transaction["type"] == "celery"
assert transaction["result"] == "FAILURE"
def register_handlers(client):
from django.core.signals import got_request_exception
from elasticapm.contrib.django.handlers import exception_handler
# Connect to Django's internal signal handler
got_request_exception.connect(exception_handler)
# If we can import celery, register ourselves as exception handler
try:
import celery # noqa F401
from elasticapm.contrib.celery import register_exception_tracking
try:
register_exception_tracking(client)
except Exception as e:
client.logger.exception(
'Failed installing django-celery hook: %s' % e)
except ImportError:
client.logger.debug("Not instrumenting Celery, couldn't import")
def test_failing_celery_task(django_elasticapm_client):
register_exception_tracking(django_elasticapm_client)
with mock.patch("elasticapm.traces.TransactionsStore.should_collect"
) as should_collect_mock:
should_collect_mock.return_value = True
t = failing_task.delay()
assert t.state == "FAILURE"
assert len(django_elasticapm_client.events) == 2
error = django_elasticapm_client.events[0]["errors"][0]
assert error[
"culprit"] == "tests.contrib.django.testapp.tasks.failing_task"
assert error["exception"]["message"] == "ValueError: foo"
assert error["exception"]["handled"] is False
transaction = django_elasticapm_client.events[1]["transactions"][0]
assert transaction[
"name"] == "tests.contrib.django.testapp.tasks.failing_task"
assert transaction["type"] == "celery"
assert transaction["result"] == "FAILURE"
def test_failing_celery_task(django_elasticapm_client):
register_exception_tracking(django_elasticapm_client)
with mock.patch('elasticapm.traces.TransactionsStore.should_collect'
) as should_collect_mock:
should_collect_mock.return_value = True
t = failing_task.delay()
assert t.state == 'FAILURE'
assert len(django_elasticapm_client.events) == 2
error = django_elasticapm_client.events[0]['errors'][0]
assert error[
'culprit'] == 'tests.contrib.django.testapp.tasks.failing_task'
assert error['exception']['message'] == 'ValueError: foo'
assert error['handled'] is False
transaction = django_elasticapm_client.events[1]['transactions'][0]
assert transaction[
'name'] == 'tests.contrib.django.testapp.tasks.failing_task'
assert transaction['type'] == 'celery'
assert transaction['result'] == 'FAILURE'
def _setup_exception_manager(self):
"""
Setup global exception handler where all unhandled exception can be caught and tracked to APM server
:return:
"""
# noinspection PyUnusedLocal
async def _handler(request: Request, exception: BaseException):
if not self._client:
return
self._client.capture_exception(
exc_info=sys.exc_info(),
context={
"request": await get_request_info(config=self._client.config, request=request),
},
handled=True,
)
self._setup_transaction_name(request=request)
set_transaction_result(result="HTTP 5xx", override=False)
set_transaction_outcome(outcome=constants.OUTCOME.FAILURE, override=False)
elastic_context(data={"status_code": 500}, key="response")
self._client.end_transaction()
if not isinstance(self._app.error_handler, ElasticAPMPatchedErrorHandler):
patched_client = ElasticAPMPatchedErrorHandler(current_handler=self._app.error_handler)
patched_client.setup_apm_handler(apm_handler=_handler)
self._app.error_handler = patched_client
else:
self._app.error_handler.setup_apm_handler(apm_handler=_handler)
try:
from elasticapm.contrib.celery import register_exception_tracking
register_exception_tracking(client=self._client)
except ImportError:
self._logger.debug(
"Failed to setup Exception tracking. "
"Please install requirements for elasticapm.contrib.celery if exception tracking is required"
)
pass
def main(profile: str):
"""
Celery beat main entry point
Args:
profile: profile used to run the beat
"""
load_config(profile, CONFIGS_PATH, config, 'NEWS_DISCOVERY')
add_logstash_handler(LOG_CONFIG, config.logstash.host, config.logstash.port)
CELERY_BEAT.configure(task_queue_name='news-discovery',
broker_config=config.rabbit)
apm_client = Client(config={
'SERVICE_NAME': 'news-discovery-beat',
'SECRET_TOKEN': config.elastic_apm.secret_token,
'SERVER_URL': config.elastic_apm.url
})
register_instrumentation(apm_client)
register_exception_tracking(apm_client)
CELERY_BEAT.run(beat=True)
def init_app(self, app):
self.app = app
if not self.client:
self.client = make_client(
self.client_cls,
app,
self.app_name,
self.secret_token,
)
if self.logging:
setup_logging(LoggingHandler(self.client))
signals.got_request_exception.connect(self.handle_exception,
sender=app,
weak=False)
try:
from elasticapm.contrib.celery import register_exception_tracking
register_exception_tracking(self.client)
except ImportError:
pass
# Instrument to get traces
skip_env_var = 'SKIP_INSTRUMENT'
if skip_env_var in os.environ:
logger.debug("Skipping instrumentation. %s is set.", skip_env_var)
else:
elasticapm.instrumentation.control.instrument()
signals.request_started.connect(self.request_started, sender=app)
signals.request_finished.connect(self.request_finished, sender=app)
try:
from elasticapm.contrib.celery import register_instrumentation
register_instrumentation(self.client)
except ImportError:
pass
from celery import Celery
# <SFTRACE-CONFIG> add the below agent specific configuration
from elasticapm import Client, instrument
from elasticapm.contrib.celery import register_exception_tracking, register_instrumentation
import os
instrument()
# <SFTRACE-CONFIG> Replace <service_name> with appropriate value. The service_name is used to identify and filter the traces related to an application and should be named appropriately to distinctly identify it. Service name must only contain characters from the ASCII alphabet, numbers, dashes, underscores and spaces.
apm_client = Client(server_url=os.getenv('SFTRACE_SERVER_URL', None),
global_labels=os.getenv('SFTRACE_GLOBAL_LABELS', None),
service_name='<service_name>',
verify_server_cert=os.getenv('SFTRACE_VERFIY_SERVER_CERT',
None))
register_exception_tracking(apm_client)
register_instrumentation(apm_client)
# sfagent config finish
app = Celery('tasks', broker='amqp://guest@localhost:5672')
@app.task
def add(x, y):
return x + y
