def start_span(
self,
span=None, # type: Optional[Span]
**kwargs # type: Any
):
# type: (...) -> Span
"""
Create a new span whose parent span is the currently active
span, if any. The return value is the span object that can
be used as a context manager to start and stop timing.
Note that you will not see any span that is not contained
within a transaction. Create a transaction with
``start_span(transaction="my transaction")`` if an
integration doesn't already do this for you.
"""
client, scope = self._stack[-1]
kwargs.setdefault("hub", self)
if span is None:
if scope.span is not None:
span = scope.span.new_span(**kwargs)
else:
span = Span(**kwargs)
if span.sampled is None and span.transaction is not None:
sample_rate = client and client.options["traces_sample_rate"] or 0
span.sampled = random.random() < sample_rate
if span.sampled:
span.init_finished_spans()
return span
async def inner():
# type: () -> Any
hub = Hub.current
if hub.get_integration(AioHttpIntegration) is None:
return await old_handle(self, request, *args, **kwargs)
weak_request = weakref.ref(request)
with Hub(Hub.current) as hub:
with hub.configure_scope() as scope:
scope.clear_breadcrumbs()
scope.add_event_processor(
_make_request_processor(weak_request))
span = Span.continue_from_headers(request.headers)
span.op = "http.server"
# If this transaction name makes it to the UI, AIOHTTP's
# URL resolver did not find a route or died trying.
span.transaction = "generic AIOHTTP request"
with hub.start_span(span):
try:
response = await old_handle(self, request)
except HTTPException:
raise
except Exception:
reraise(*_capture_exception(hub))
return response
def _inner(*args, **kwargs):
# type: (*Any, **Any) -> Any
hub = Hub.current
if hub.get_integration(CeleryIntegration) is None:
return f(*args, **kwargs)
with hub.push_scope() as scope:
scope._name = "celery"
scope.clear_breadcrumbs()
scope.add_event_processor(_make_event_processor(task, *args, **kwargs))
span = Span.continue_from_headers(args[3].get("headers") or {})
span.op = "celery.task"
span.transaction = "unknown celery task"
# Could possibly use a better hook than this one
span.set_status("ok")
with capture_internal_exceptions():
# Celery task objects are not a thing to be trusted. Even
# something such as attribute access can fail.
span.transaction = task.name
with hub.start_span(span):
return f(*args, **kwargs)
def __call__(self, environ, start_response):
# type: (Dict[str, str], Callable[..., Any]) -> _ScopedResponse
if _wsgi_middleware_applied.get(False):
return self.app(environ, start_response)
_wsgi_middleware_applied.set(True)
try:
hub = Hub(Hub.current)
with hub:
with capture_internal_exceptions():
with hub.configure_scope() as scope:
scope.clear_breadcrumbs()
scope._name = "wsgi"
scope.add_event_processor(
_make_wsgi_event_processor(environ))
span = Span.continue_from_environ(environ)
span.op = "http.server"
span.transaction = "generic WSGI request"
with hub.start_span(span) as span:
try:
rv = self.app(
environ,
functools.partial(_sentry_start_response,
start_response, span),
)
except BaseException:
reraise(*_capture_exception(hub))
finally:
_wsgi_middleware_applied.set(False)
return _ScopedResponse(hub, rv)
def process_event(cache_key, start_time=None, event_id=None, data_has_changed=None, **kwargs):
"""
Handles event processing (for those events that need it)
This excludes symbolication via symbolicator service (see symbolicate_event).
:param string cache_key: the cache key for the event data
:param int start_time: the timestamp when the event was ingested
:param string event_id: the event identifier
:param boolean data_has_changed: set to True if the event data was changed in previous tasks
"""
with sentry_sdk.start_span(
Span(
op="tasks.store.process_event",
transaction="TaskProcessEvent",
sampled=sample_process_event_apm(),
)
):
return _do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_event,
data_has_changed=data_has_changed,
)
async def _run_app(self, scope, callback):
if _asgi_middleware_applied.get(False):
return await callback()
_asgi_middleware_applied.set(True)
try:
hub = Hub(Hub.current)
with hub:
with hub.configure_scope() as sentry_scope:
sentry_scope.clear_breadcrumbs()
sentry_scope._name = "asgi"
processor = functools.partial(self.event_processor,
asgi_scope=scope)
sentry_scope.add_event_processor(processor)
span = Span.continue_from_headers(dict(scope["headers"]))
span.op = "http.server"
span.transaction = "generic ASGI request"
with hub.start_span(span) as span:
try:
return await callback()
except Exception as exc:
_capture_exception(hub, exc)
raise exc from None
finally:
_asgi_middleware_applied.set(False)
def sentry_patched_perform_job(self, job, *args, **kwargs):
# type: (Any, Job, *Queue, **Any) -> bool
hub = Hub.current
integration = hub.get_integration(RqIntegration)
if integration is None:
return old_perform_job(self, job, *args, **kwargs)
client = hub.client
assert client is not None
with hub.push_scope() as scope:
scope.clear_breadcrumbs()
scope.add_event_processor(
_make_event_processor(weakref.ref(job)))
span = Span.continue_from_headers(
job.meta.get("_sentry_trace_headers") or {})
with capture_internal_exceptions():
span.transaction = job.func_name
with hub.span(span):
rv = old_perform_job(self, job, *args, **kwargs)
if self.is_horse:
# We're inside of a forked process and RQ is
# about to call `os._exit`. Make sure that our
# events get sent out.
client.flush()
return rv
def run(self):
logger.debug("Starting snuba query subscriber")
self.offsets.clear()
conf = {
"bootstrap.servers": self.bootstrap_servers,
"group.id": self.group_id,
"session.timeout.ms": 6000,
"auto.offset.reset": self.initial_offset_reset,
"enable.auto.commit": "false",
"enable.auto.offset.store": "false",
"enable.partition.eof": "false",
"default.topic.config": {
"auto.offset.reset": self.initial_offset_reset
},
}
def on_revoke(consumer, partitions):
self.commit_offsets()
self.consumer = Consumer(conf)
self.consumer.subscribe([self.topic], on_revoke=on_revoke)
try:
i = 0
while True:
message = self.consumer.poll(0.1)
if message is None:
continue
error = message.error()
if error is not None:
raise KafkaException(error)
i = i + 1
with sentry_sdk.start_span(
Span(
op="handle_message",
transaction=
"query_subscription_consumer_process_message",
sampled=True,
)), metrics.timer(
"snuba_query_subscriber.handle_message"):
self.handle_message(message)
# Track latest completed message here, for use in `shutdown` handler.
self.offsets[message.partition()] = message.offset() + 1
if i % self.commit_batch_size == 0:
logger.debug("Committing offsets")
self.commit_offsets()
except KeyboardInterrupt:
pass
self.shutdown()
def start_span(
self,
span=None, # type: Optional[Span]
**kwargs # type: Any
):
# type: (...) -> Span
client, scope = self._stack[-1]
if span is None:
if scope.span is not None:
span = scope.span.new_span(**kwargs)
else:
span = Span(**kwargs)
if span.sampled is None and span.transaction is not None:
sample_rate = client and client.options["traces_sample_rate"] or 0
span.sampled = random.random() < sample_rate
return span
def symbolicate_event_from_reprocessing(cache_key, start_time=None, event_id=None, **kwargs):
with sentry_sdk.start_span(
Span(
op="tasks.store.symbolicate_event_from_reprocessing",
transaction="TaskSymbolicateEvent",
sampled=sample_symbolicate_event_apm(),
)
):
return _do_symbolicate_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
symbolicate_task=symbolicate_event_from_reprocessing,
)
def test_continue_from_headers(sentry_init, capture_events, sampled):
sentry_init(traces_sample_rate=1.0, traceparent_v2=True)
events = capture_events()
with Hub.current.start_span(transaction="hi"):
with Hub.current.start_span() as old_span:
old_span.sampled = sampled
headers = dict(Hub.current.iter_trace_propagation_headers())
header = headers["sentry-trace"]
if sampled is True:
assert header.endswith("-1")
if sampled is False:
assert header.endswith("-0")
if sampled is None:
assert header.endswith("-")
span = Span.continue_from_headers(headers)
span.transaction = "WRONG"
assert span is not None
assert span.sampled == sampled
assert span.trace_id == old_span.trace_id
assert span.same_process_as_parent is False
assert span.parent_span_id == old_span.span_id
assert span.span_id != old_span.span_id
with Hub.current.start_span(span):
with Hub.current.configure_scope() as scope:
scope.transaction = "ho"
capture_message("hello")
if sampled is False:
trace1, message = events
assert trace1["transaction"] == "hi"
else:
trace1, message, trace2 = events
assert trace1["transaction"] == "hi"
assert trace2["transaction"] == "ho"
assert (
trace1["contexts"]["trace"]["trace_id"]
== trace2["contexts"]["trace"]["trace_id"]
== span.trace_id
== message["contexts"]["trace"]["trace_id"]
)
assert message["message"] == "hello"
def test_start_span_to_start_transaction(sentry_init, capture_events):
# XXX: this only exists for backwards compatibility with code before
# Transaction / start_transaction were introduced.
sentry_init(traces_sample_rate=1.0)
events = capture_events()
with start_span(transaction="/1/"):
pass
with start_span(Span(transaction="/2/")):
pass
assert len(events) == 2
assert events[0]["transaction"] == "/1/"
assert events[1]["transaction"] == "/2/"
def process_event_from_reprocessing(
cache_key, start_time=None, event_id=None, data_has_changed=None, **kwargs
):
with sentry_sdk.start_span(
Span(
op="tasks.store.process_event_from_reprocessing",
transaction="TaskProcessEvent",
sampled=sample_process_event_apm(),
)
):
return _do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_event_from_reprocessing,
data_has_changed=data_has_changed,
)
def start_span(
self,
span=None, # type: Optional[Span]
**kwargs # type: Any
):
# type: (...) -> Span
"""
Create and start timing a new span whose parent is the currently active
span or transaction, if any. The return value is a span instance,
typically used as a context manager to start and stop timing in a `with`
block.
Only spans contained in a transaction are sent to Sentry. Most
integrations start a transaction at the appropriate time, for example
for every incoming HTTP request. Use `start_transaction` to start a new
transaction when one is not already in progress.
"""
# TODO: consider removing this in a future release.
# This is for backwards compatibility with releases before
# start_transaction existed, to allow for a smoother transition.
if isinstance(span, Transaction) or "transaction" in kwargs:
deprecation_msg = (
"Deprecated: use start_transaction to start transactions and "
"Transaction.start_child to start spans."
)
if isinstance(span, Transaction):
logger.warning(deprecation_msg)
return self.start_transaction(span)
if "transaction" in kwargs:
logger.warning(deprecation_msg)
name = kwargs.pop("transaction")
return self.start_transaction(name=name, **kwargs)
if span is not None:
return span
kwargs.setdefault("hub", self)
span = self.scope.span
if span is not None:
return span.start_child(**kwargs)
return Span(**kwargs)
def symbolicate_event(cache_key, start_time=None, event_id=None, **kwargs):
"""
Handles event symbolication using the external service: symbolicator.
:param string cache_key: the cache key for the event data
:param int start_time: the timestamp when the event was ingested
:param string event_id: the event identifier
"""
with sentry_sdk.start_span(
Span(
op="tasks.store.symbolicate_event",
transaction="TaskSymbolicateEvent",
sampled=sample_symbolicate_event_apm(),
)):
return _do_symbolicate_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
symbolicate_task=symbolicate_event,
)
def _inner(*args, **kwargs):
hub = Hub.current
if hub.get_integration(CeleryIntegration) is None:
return f(*args, **kwargs)
with hub.push_scope() as scope:
scope._name = "celery"
scope.clear_breadcrumbs()
scope.add_event_processor(_make_event_processor(task, *args, **kwargs))
span = Span.continue_from_headers(args[3].get("headers") or {})
span.transaction = "unknown celery task"
with capture_internal_exceptions():
# Celery task objects are not a thing to be trusted. Even
# something such as attribute access can fail.
span.transaction = task.name
with hub.span(span):
return f(*args, **kwargs)
def inner():
# type: () -> Any
hub = Hub.current
if hub.get_integration(AioHttpIntegration) is None:
old_handle_response = yield from old_handle(
self, request, *args, **kwargs)
return old_handle_response
weak_request = weakref.ref(request)
with Hub(Hub.current) as hub:
with hub.configure_scope() as scope:
scope.clear_breadcrumbs()
scope.add_event_processor(
_make_request_processor(weak_request))
span = Span.continue_from_headers(request.headers)
span.op = "http.server"
# If this transaction name makes it to the UI, AIOHTTP's
# URL resolver did not find a route or died trying.
span.transaction = "generic AIOHTTP request"
with hub.start_span(span):
try:
response = yield from old_handle(self, request)
except HTTPException as e:
span.set_http_status(e.status_code)
raise
except asyncio.CancelledError:
span.set_status("cancelled")
raise
except Exception:
# This will probably map to a 500 but seems like we
# have no way to tell. Do not set span status.
reraise(*_capture_exception(hub))
span.set_http_status(response.status)
return response
def __call__(self, environ, start_response):
# type: (Dict[str, str], Callable) -> _ScopedResponse
hub = Hub(Hub.current)
with hub:
with capture_internal_exceptions():
with hub.configure_scope() as scope:
scope.clear_breadcrumbs()
scope._name = "wsgi"
scope.add_event_processor(
_make_wsgi_event_processor(environ))
span = Span.continue_from_environ(environ)
span.op = "http.server"
span.transaction = "generic WSGI request"
with hub.span(span):
try:
rv = self.app(environ, start_response)
except BaseException:
reraise(*_capture_exception(hub))
return _ScopedResponse(hub, rv)
def _run_app(self, scope, callback):
# type: (Any, Any) -> Any
if _asgi_middleware_applied.get(False):
response = yield from callback()
return response
_asgi_middleware_applied.set(True)
try:
hub = Hub(Hub.current)
with hub:
with hub.configure_scope() as sentry_scope:
sentry_scope.clear_breadcrumbs()
sentry_scope._name = "asgi"
processor = functools.partial(self.event_processor,
asgi_scope=scope)
sentry_scope.add_event_processor(processor)
if scope["type"] in ("http", "websocket"):
span = Span.continue_from_headers(dict(scope["headers"]))
span.op = "{}.server".format(scope["type"])
else:
span = Span()
span.op = "asgi.server"
span.set_tag("asgi.type", scope["type"])
span.transaction = "generic ASGI request"
with hub.start_span(span) as span:
# XXX: Would be cool to have correct span status, but we
# would have to wrap send(). That is a bit hard to do with
# the current abstraction over ASGI 2/3.
try:
response = yield from callback()
return response
except Exception as exc:
_capture_exception(hub, exc)
raise exc from None
finally:
_asgi_middleware_applied.set(False)
def test_to_tracestate(sentry_init):
sentry_init(
dsn=
"https://[email protected]/12312012",
environment="dogpark",
release="off.leash.park",
)
# it correctly uses the value from the transaction itself or the span's
# containing transaction
transaction_no_third_party = Transaction(
trace_id="12312012123120121231201212312012",
sentry_tracestate="sentry=doGsaREgReaT",
)
non_orphan_span = Span()
non_orphan_span._containing_transaction = transaction_no_third_party
assert transaction_no_third_party.to_tracestate() == "sentry=doGsaREgReaT"
assert non_orphan_span.to_tracestate() == "sentry=doGsaREgReaT"
# it combines sentry and third-party values correctly
transaction_with_third_party = Transaction(
trace_id="12312012123120121231201212312012",
sentry_tracestate="sentry=doGsaREgReaT",
third_party_tracestate="maisey=silly",
)
assert (transaction_with_third_party.to_tracestate() ==
"sentry=doGsaREgReaT,maisey=silly")
# it computes a tracestate from scratch for orphan transactions
orphan_span = Span(trace_id="12312012123120121231201212312012", )
assert orphan_span._containing_transaction is None
assert orphan_span.to_tracestate() == "sentry=" + compute_tracestate_value(
{
"trace_id": "12312012123120121231201212312012",
"environment": "dogpark",
"release": "off.leash.park",
"public_key": "dogsarebadatkeepingsecrets",
})
def handle_message(self, message):
"""
Parses the value from Kafka, and if valid passes the payload to the callback defined by the
subscription. If the subscription has been removed, or no longer has a valid callback then
just log metrics/errors and continue.
:param message:
:return:
"""
with sentry_sdk.push_scope() as scope:
try:
contents = self.parse_message_value(message.value())
except InvalidMessageError:
# If the message is in an invalid format, just log the error
# and continue
logger.exception(
"Subscription update could not be parsed",
extra={
"offset": message.offset(),
"partition": message.partition(),
"value": message.value(),
},
)
return
scope.set_tag("query_subscription_id", contents["subscription_id"])
try:
subscription = QuerySubscription.objects.get_from_cache(
subscription_id=contents["subscription_id"])
except QuerySubscription.DoesNotExist:
metrics.incr(
"snuba_query_subscriber.subscription_doesnt_exist")
logger.error(
"Received subscription update, but subscription does not exist",
extra={
"offset": message.offset(),
"partition": message.partition(),
"value": message.value(),
},
)
return
if subscription.type not in subscriber_registry:
metrics.incr(
"snuba_query_subscriber.subscription_type_not_registered")
logger.error(
"Received subscription update, but no subscription handler registered",
extra={
"offset": message.offset(),
"partition": message.partition(),
"value": message.value(),
},
)
return
logger.info(
"query-subscription-consumer.handle_message",
extra={
"timestamp": contents["timestamp"],
"query_subscription_id": contents["subscription_id"],
"contents": contents,
"offset": message.offset(),
"partition": message.partition(),
"value": message.value(),
},
)
callback = subscriber_registry[subscription.type]
with sentry_sdk.start_span(
Span(
op="process_message",
transaction=
"query_subscription_consumer_process_message",
sampled=True,
)) as span, metrics.timer(
"snuba_query_subscriber.callback.duration",
instance=subscription.type):
span.set_data("payload", contents)
callback(contents, subscription)
def post(self, request):
with Hub.current.start_span(
Span(op="http.server",
transaction="RelayProjectConfigsEndpoint",
sampled=_sample_apm())):
return self._post(request)
def post_process_group(event, is_new, is_regression, is_new_group_environment, **kwargs):
"""
Fires post processing hooks for a group.
"""
set_current_project(event.project_id)
from sentry.utils import snuba
with snuba.options_override({"consistent": True}):
if check_event_already_post_processed(event):
logger.info(
"post_process.skipped",
extra={
"project_id": event.project_id,
"event_id": event.event_id,
"reason": "duplicate",
},
)
return
# NOTE: we must pass through the full Event object, and not an
# event_id since the Event object may not actually have been stored
# in the database due to sampling.
from sentry.models import Project, Organization, EventDict
from sentry.models.group import get_group_with_redirect
from sentry.rules.processor import RuleProcessor
from sentry.tasks.servicehooks import process_service_hook
# Re-bind node data to avoid renormalization. We only want to
# renormalize when loading old data from the database.
event.data = EventDict(event.data, skip_renormalization=True)
if event.group_id:
# Re-bind Group since we're pickling the whole Event object
# which may contain a stale Project.
event.group, _ = get_group_with_redirect(event.group_id)
event.group_id = event.group.id
# Re-bind Project and Org since we're pickling the whole Event object
# which may contain stale parent models.
event.project = Project.objects.get_from_cache(id=event.project_id)
event.project._organization_cache = Organization.objects.get_from_cache(
id=event.project.organization_id
)
_capture_stats(event, is_new)
if event.group_id:
# we process snoozes before rules as it might create a regression
# but not if it's new because you can't immediately snooze a new group
has_reappeared = False if is_new else process_snoozes(event.group)
handle_owner_assignment(event.project, event.group, event)
rp = RuleProcessor(
event, is_new, is_regression, is_new_group_environment, has_reappeared
)
has_alert = False
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
has_alert = True
with sentry_sdk.start_span(
Span(op="post_process_group", transaction="rule_processor_apply", sampled=True)
):
safe_execute(callback, event, futures)
if features.has("projects:servicehooks", project=event.project):
allowed_events = set(["event.created"])
if has_alert:
allowed_events.add("event.alert")
if allowed_events:
for servicehook_id, events in _get_service_hooks(project_id=event.project_id):
if any(e in allowed_events for e in events):
process_service_hook.delay(servicehook_id=servicehook_id, event=event)
from sentry.tasks.sentry_apps import process_resource_change_bound
if event.get_event_type() == "error" and _should_send_error_created_hooks(
event.project
):
process_resource_change_bound.delay(
action="created", sender="Error", instance_id=event.event_id, instance=event
)
if is_new:
process_resource_change_bound.delay(
action="created", sender="Group", instance_id=event.group_id
)
from sentry.plugins.base import plugins
for plugin in plugins.for_project(event.project):
plugin_post_process_group(
plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression
)
event_processed.send_robust(
sender=post_process_group,
project=event.project,
event=event,
primary_hash=kwargs.get("primary_hash"),
)
def wrapped(request, *args, **kwargs):
with Hub.current.start_span(
Span(op="http.server", transaction=endpoint,
sampled=True)):
return func(request, *args, **kwargs)
