def start_span( self, span=None, # type: Optional[Span] **kwargs # type: Any ): # type: (...) -> Span """ Create a new span whose parent span is the currently active span, if any. The return value is the span object that can be used as a context manager to start and stop timing. Note that you will not see any span that is not contained within a transaction. Create a transaction with ``start_span(transaction="my transaction")`` if an integration doesn't already do this for you. """ client, scope = self._stack[-1] kwargs.setdefault("hub", self) if span is None: if scope.span is not None: span = scope.span.new_span(**kwargs) else: span = Span(**kwargs) if span.sampled is None and span.transaction is not None: sample_rate = client and client.options["traces_sample_rate"] or 0 span.sampled = random.random() < sample_rate if span.sampled: span.init_finished_spans() return span
async def inner(): # type: () -> Any hub = Hub.current if hub.get_integration(AioHttpIntegration) is None: return await old_handle(self, request, *args, **kwargs) weak_request = weakref.ref(request) with Hub(Hub.current) as hub: with hub.configure_scope() as scope: scope.clear_breadcrumbs() scope.add_event_processor( _make_request_processor(weak_request)) span = Span.continue_from_headers(request.headers) span.op = "http.server" # If this transaction name makes it to the UI, AIOHTTP's # URL resolver did not find a route or died trying. span.transaction = "generic AIOHTTP request" with hub.start_span(span): try: response = await old_handle(self, request) except HTTPException: raise except Exception: reraise(*_capture_exception(hub)) return response
def _inner(*args, **kwargs): # type: (*Any, **Any) -> Any hub = Hub.current if hub.get_integration(CeleryIntegration) is None: return f(*args, **kwargs) with hub.push_scope() as scope: scope._name = "celery" scope.clear_breadcrumbs() scope.add_event_processor(_make_event_processor(task, *args, **kwargs)) span = Span.continue_from_headers(args[3].get("headers") or {}) span.op = "celery.task" span.transaction = "unknown celery task" # Could possibly use a better hook than this one span.set_status("ok") with capture_internal_exceptions(): # Celery task objects are not a thing to be trusted. Even # something such as attribute access can fail. span.transaction = task.name with hub.start_span(span): return f(*args, **kwargs)
def __call__(self, environ, start_response): # type: (Dict[str, str], Callable[..., Any]) -> _ScopedResponse if _wsgi_middleware_applied.get(False): return self.app(environ, start_response) _wsgi_middleware_applied.set(True) try: hub = Hub(Hub.current) with hub: with capture_internal_exceptions(): with hub.configure_scope() as scope: scope.clear_breadcrumbs() scope._name = "wsgi" scope.add_event_processor( _make_wsgi_event_processor(environ)) span = Span.continue_from_environ(environ) span.op = "http.server" span.transaction = "generic WSGI request" with hub.start_span(span) as span: try: rv = self.app( environ, functools.partial(_sentry_start_response, start_response, span), ) except BaseException: reraise(*_capture_exception(hub)) finally: _wsgi_middleware_applied.set(False) return _ScopedResponse(hub, rv)
def process_event(cache_key, start_time=None, event_id=None, data_has_changed=None, **kwargs): """ Handles event processing (for those events that need it) This excludes symbolication via symbolicator service (see symbolicate_event). :param string cache_key: the cache key for the event data :param int start_time: the timestamp when the event was ingested :param string event_id: the event identifier :param boolean data_has_changed: set to True if the event data was changed in previous tasks """ with sentry_sdk.start_span( Span( op="tasks.store.process_event", transaction="TaskProcessEvent", sampled=sample_process_event_apm(), ) ): return _do_process_event( cache_key=cache_key, start_time=start_time, event_id=event_id, process_task=process_event, data_has_changed=data_has_changed, )
async def _run_app(self, scope, callback): if _asgi_middleware_applied.get(False): return await callback() _asgi_middleware_applied.set(True) try: hub = Hub(Hub.current) with hub: with hub.configure_scope() as sentry_scope: sentry_scope.clear_breadcrumbs() sentry_scope._name = "asgi" processor = functools.partial(self.event_processor, asgi_scope=scope) sentry_scope.add_event_processor(processor) span = Span.continue_from_headers(dict(scope["headers"])) span.op = "http.server" span.transaction = "generic ASGI request" with hub.start_span(span) as span: try: return await callback() except Exception as exc: _capture_exception(hub, exc) raise exc from None finally: _asgi_middleware_applied.set(False)
def sentry_patched_perform_job(self, job, *args, **kwargs): # type: (Any, Job, *Queue, **Any) -> bool hub = Hub.current integration = hub.get_integration(RqIntegration) if integration is None: return old_perform_job(self, job, *args, **kwargs) client = hub.client assert client is not None with hub.push_scope() as scope: scope.clear_breadcrumbs() scope.add_event_processor( _make_event_processor(weakref.ref(job))) span = Span.continue_from_headers( job.meta.get("_sentry_trace_headers") or {}) with capture_internal_exceptions(): span.transaction = job.func_name with hub.span(span): rv = old_perform_job(self, job, *args, **kwargs) if self.is_horse: # We're inside of a forked process and RQ is # about to call `os._exit`. Make sure that our # events get sent out. client.flush() return rv
def run(self): logger.debug("Starting snuba query subscriber") self.offsets.clear() conf = { "bootstrap.servers": self.bootstrap_servers, "group.id": self.group_id, "session.timeout.ms": 6000, "auto.offset.reset": self.initial_offset_reset, "enable.auto.commit": "false", "enable.auto.offset.store": "false", "enable.partition.eof": "false", "default.topic.config": { "auto.offset.reset": self.initial_offset_reset }, } def on_revoke(consumer, partitions): self.commit_offsets() self.consumer = Consumer(conf) self.consumer.subscribe([self.topic], on_revoke=on_revoke) try: i = 0 while True: message = self.consumer.poll(0.1) if message is None: continue error = message.error() if error is not None: raise KafkaException(error) i = i + 1 with sentry_sdk.start_span( Span( op="handle_message", transaction= "query_subscription_consumer_process_message", sampled=True, )), metrics.timer( "snuba_query_subscriber.handle_message"): self.handle_message(message) # Track latest completed message here, for use in `shutdown` handler. self.offsets[message.partition()] = message.offset() + 1 if i % self.commit_batch_size == 0: logger.debug("Committing offsets") self.commit_offsets() except KeyboardInterrupt: pass self.shutdown()
def start_span( self, span=None, # type: Optional[Span] **kwargs # type: Any ): # type: (...) -> Span client, scope = self._stack[-1] if span is None: if scope.span is not None: span = scope.span.new_span(**kwargs) else: span = Span(**kwargs) if span.sampled is None and span.transaction is not None: sample_rate = client and client.options["traces_sample_rate"] or 0 span.sampled = random.random() < sample_rate return span
def symbolicate_event_from_reprocessing(cache_key, start_time=None, event_id=None, **kwargs): with sentry_sdk.start_span( Span( op="tasks.store.symbolicate_event_from_reprocessing", transaction="TaskSymbolicateEvent", sampled=sample_symbolicate_event_apm(), ) ): return _do_symbolicate_event( cache_key=cache_key, start_time=start_time, event_id=event_id, symbolicate_task=symbolicate_event_from_reprocessing, )
def test_continue_from_headers(sentry_init, capture_events, sampled): sentry_init(traces_sample_rate=1.0, traceparent_v2=True) events = capture_events() with Hub.current.start_span(transaction="hi"): with Hub.current.start_span() as old_span: old_span.sampled = sampled headers = dict(Hub.current.iter_trace_propagation_headers()) header = headers["sentry-trace"] if sampled is True: assert header.endswith("-1") if sampled is False: assert header.endswith("-0") if sampled is None: assert header.endswith("-") span = Span.continue_from_headers(headers) span.transaction = "WRONG" assert span is not None assert span.sampled == sampled assert span.trace_id == old_span.trace_id assert span.same_process_as_parent is False assert span.parent_span_id == old_span.span_id assert span.span_id != old_span.span_id with Hub.current.start_span(span): with Hub.current.configure_scope() as scope: scope.transaction = "ho" capture_message("hello") if sampled is False: trace1, message = events assert trace1["transaction"] == "hi" else: trace1, message, trace2 = events assert trace1["transaction"] == "hi" assert trace2["transaction"] == "ho" assert ( trace1["contexts"]["trace"]["trace_id"] == trace2["contexts"]["trace"]["trace_id"] == span.trace_id == message["contexts"]["trace"]["trace_id"] ) assert message["message"] == "hello"
def test_start_span_to_start_transaction(sentry_init, capture_events): # XXX: this only exists for backwards compatibility with code before # Transaction / start_transaction were introduced. sentry_init(traces_sample_rate=1.0) events = capture_events() with start_span(transaction="/1/"): pass with start_span(Span(transaction="/2/")): pass assert len(events) == 2 assert events[0]["transaction"] == "/1/" assert events[1]["transaction"] == "/2/"
def process_event_from_reprocessing( cache_key, start_time=None, event_id=None, data_has_changed=None, **kwargs ): with sentry_sdk.start_span( Span( op="tasks.store.process_event_from_reprocessing", transaction="TaskProcessEvent", sampled=sample_process_event_apm(), ) ): return _do_process_event( cache_key=cache_key, start_time=start_time, event_id=event_id, process_task=process_event_from_reprocessing, data_has_changed=data_has_changed, )
def start_span( self, span=None, # type: Optional[Span] **kwargs # type: Any ): # type: (...) -> Span """ Create and start timing a new span whose parent is the currently active span or transaction, if any. The return value is a span instance, typically used as a context manager to start and stop timing in a `with` block. Only spans contained in a transaction are sent to Sentry. Most integrations start a transaction at the appropriate time, for example for every incoming HTTP request. Use `start_transaction` to start a new transaction when one is not already in progress. """ # TODO: consider removing this in a future release. # This is for backwards compatibility with releases before # start_transaction existed, to allow for a smoother transition. if isinstance(span, Transaction) or "transaction" in kwargs: deprecation_msg = ( "Deprecated: use start_transaction to start transactions and " "Transaction.start_child to start spans." ) if isinstance(span, Transaction): logger.warning(deprecation_msg) return self.start_transaction(span) if "transaction" in kwargs: logger.warning(deprecation_msg) name = kwargs.pop("transaction") return self.start_transaction(name=name, **kwargs) if span is not None: return span kwargs.setdefault("hub", self) span = self.scope.span if span is not None: return span.start_child(**kwargs) return Span(**kwargs)
def symbolicate_event(cache_key, start_time=None, event_id=None, **kwargs): """ Handles event symbolication using the external service: symbolicator. :param string cache_key: the cache key for the event data :param int start_time: the timestamp when the event was ingested :param string event_id: the event identifier """ with sentry_sdk.start_span( Span( op="tasks.store.symbolicate_event", transaction="TaskSymbolicateEvent", sampled=sample_symbolicate_event_apm(), )): return _do_symbolicate_event( cache_key=cache_key, start_time=start_time, event_id=event_id, symbolicate_task=symbolicate_event, )
def _inner(*args, **kwargs): hub = Hub.current if hub.get_integration(CeleryIntegration) is None: return f(*args, **kwargs) with hub.push_scope() as scope: scope._name = "celery" scope.clear_breadcrumbs() scope.add_event_processor(_make_event_processor(task, *args, **kwargs)) span = Span.continue_from_headers(args[3].get("headers") or {}) span.transaction = "unknown celery task" with capture_internal_exceptions(): # Celery task objects are not a thing to be trusted. Even # something such as attribute access can fail. span.transaction = task.name with hub.span(span): return f(*args, **kwargs)
def inner(): # type: () -> Any hub = Hub.current if hub.get_integration(AioHttpIntegration) is None: old_handle_response = yield from old_handle( self, request, *args, **kwargs) return old_handle_response weak_request = weakref.ref(request) with Hub(Hub.current) as hub: with hub.configure_scope() as scope: scope.clear_breadcrumbs() scope.add_event_processor( _make_request_processor(weak_request)) span = Span.continue_from_headers(request.headers) span.op = "http.server" # If this transaction name makes it to the UI, AIOHTTP's # URL resolver did not find a route or died trying. span.transaction = "generic AIOHTTP request" with hub.start_span(span): try: response = yield from old_handle(self, request) except HTTPException as e: span.set_http_status(e.status_code) raise except asyncio.CancelledError: span.set_status("cancelled") raise except Exception: # This will probably map to a 500 but seems like we # have no way to tell. Do not set span status. reraise(*_capture_exception(hub)) span.set_http_status(response.status) return response
def __call__(self, environ, start_response): # type: (Dict[str, str], Callable) -> _ScopedResponse hub = Hub(Hub.current) with hub: with capture_internal_exceptions(): with hub.configure_scope() as scope: scope.clear_breadcrumbs() scope._name = "wsgi" scope.add_event_processor( _make_wsgi_event_processor(environ)) span = Span.continue_from_environ(environ) span.op = "http.server" span.transaction = "generic WSGI request" with hub.span(span): try: rv = self.app(environ, start_response) except BaseException: reraise(*_capture_exception(hub)) return _ScopedResponse(hub, rv)
def _run_app(self, scope, callback): # type: (Any, Any) -> Any if _asgi_middleware_applied.get(False): response = yield from callback() return response _asgi_middleware_applied.set(True) try: hub = Hub(Hub.current) with hub: with hub.configure_scope() as sentry_scope: sentry_scope.clear_breadcrumbs() sentry_scope._name = "asgi" processor = functools.partial(self.event_processor, asgi_scope=scope) sentry_scope.add_event_processor(processor) if scope["type"] in ("http", "websocket"): span = Span.continue_from_headers(dict(scope["headers"])) span.op = "{}.server".format(scope["type"]) else: span = Span() span.op = "asgi.server" span.set_tag("asgi.type", scope["type"]) span.transaction = "generic ASGI request" with hub.start_span(span) as span: # XXX: Would be cool to have correct span status, but we # would have to wrap send(). That is a bit hard to do with # the current abstraction over ASGI 2/3. try: response = yield from callback() return response except Exception as exc: _capture_exception(hub, exc) raise exc from None finally: _asgi_middleware_applied.set(False)
def test_to_tracestate(sentry_init): sentry_init( dsn= "https://[email protected]/12312012", environment="dogpark", release="off.leash.park", ) # it correctly uses the value from the transaction itself or the span's # containing transaction transaction_no_third_party = Transaction( trace_id="12312012123120121231201212312012", sentry_tracestate="sentry=doGsaREgReaT", ) non_orphan_span = Span() non_orphan_span._containing_transaction = transaction_no_third_party assert transaction_no_third_party.to_tracestate() == "sentry=doGsaREgReaT" assert non_orphan_span.to_tracestate() == "sentry=doGsaREgReaT" # it combines sentry and third-party values correctly transaction_with_third_party = Transaction( trace_id="12312012123120121231201212312012", sentry_tracestate="sentry=doGsaREgReaT", third_party_tracestate="maisey=silly", ) assert (transaction_with_third_party.to_tracestate() == "sentry=doGsaREgReaT,maisey=silly") # it computes a tracestate from scratch for orphan transactions orphan_span = Span(trace_id="12312012123120121231201212312012", ) assert orphan_span._containing_transaction is None assert orphan_span.to_tracestate() == "sentry=" + compute_tracestate_value( { "trace_id": "12312012123120121231201212312012", "environment": "dogpark", "release": "off.leash.park", "public_key": "dogsarebadatkeepingsecrets", })
def handle_message(self, message): """ Parses the value from Kafka, and if valid passes the payload to the callback defined by the subscription. If the subscription has been removed, or no longer has a valid callback then just log metrics/errors and continue. :param message: :return: """ with sentry_sdk.push_scope() as scope: try: contents = self.parse_message_value(message.value()) except InvalidMessageError: # If the message is in an invalid format, just log the error # and continue logger.exception( "Subscription update could not be parsed", extra={ "offset": message.offset(), "partition": message.partition(), "value": message.value(), }, ) return scope.set_tag("query_subscription_id", contents["subscription_id"]) try: subscription = QuerySubscription.objects.get_from_cache( subscription_id=contents["subscription_id"]) except QuerySubscription.DoesNotExist: metrics.incr( "snuba_query_subscriber.subscription_doesnt_exist") logger.error( "Received subscription update, but subscription does not exist", extra={ "offset": message.offset(), "partition": message.partition(), "value": message.value(), }, ) return if subscription.type not in subscriber_registry: metrics.incr( "snuba_query_subscriber.subscription_type_not_registered") logger.error( "Received subscription update, but no subscription handler registered", extra={ "offset": message.offset(), "partition": message.partition(), "value": message.value(), }, ) return logger.info( "query-subscription-consumer.handle_message", extra={ "timestamp": contents["timestamp"], "query_subscription_id": contents["subscription_id"], "contents": contents, "offset": message.offset(), "partition": message.partition(), "value": message.value(), }, ) callback = subscriber_registry[subscription.type] with sentry_sdk.start_span( Span( op="process_message", transaction= "query_subscription_consumer_process_message", sampled=True, )) as span, metrics.timer( "snuba_query_subscriber.callback.duration", instance=subscription.type): span.set_data("payload", contents) callback(contents, subscription)
def post(self, request): with Hub.current.start_span( Span(op="http.server", transaction="RelayProjectConfigsEndpoint", sampled=_sample_apm())): return self._post(request)
def post_process_group(event, is_new, is_regression, is_new_group_environment, **kwargs): """ Fires post processing hooks for a group. """ set_current_project(event.project_id) from sentry.utils import snuba with snuba.options_override({"consistent": True}): if check_event_already_post_processed(event): logger.info( "post_process.skipped", extra={ "project_id": event.project_id, "event_id": event.event_id, "reason": "duplicate", }, ) return # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Project, Organization, EventDict from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor from sentry.tasks.servicehooks import process_service_hook # Re-bind node data to avoid renormalization. We only want to # renormalize when loading old data from the database. event.data = EventDict(event.data, skip_renormalization=True) if event.group_id: # Re-bind Group since we're pickling the whole Event object # which may contain a stale Project. event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id # Re-bind Project and Org since we're pickling the whole Event object # which may contain stale parent models. event.project = Project.objects.get_from_cache(id=event.project_id) event.project._organization_cache = Organization.objects.get_from_cache( id=event.project.organization_id ) _capture_stats(event, is_new) if event.group_id: # we process snoozes before rules as it might create a regression # but not if it's new because you can't immediately snooze a new group has_reappeared = False if is_new else process_snoozes(event.group) handle_owner_assignment(event.project, event.group, event) rp = RuleProcessor( event, is_new, is_regression, is_new_group_environment, has_reappeared ) has_alert = False # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): has_alert = True with sentry_sdk.start_span( Span(op="post_process_group", transaction="rule_processor_apply", sampled=True) ): safe_execute(callback, event, futures) if features.has("projects:servicehooks", project=event.project): allowed_events = set(["event.created"]) if has_alert: allowed_events.add("event.alert") if allowed_events: for servicehook_id, events in _get_service_hooks(project_id=event.project_id): if any(e in allowed_events for e in events): process_service_hook.delay(servicehook_id=servicehook_id, event=event) from sentry.tasks.sentry_apps import process_resource_change_bound if event.get_event_type() == "error" and _should_send_error_created_hooks( event.project ): process_resource_change_bound.delay( action="created", sender="Error", instance_id=event.event_id, instance=event ) if is_new: process_resource_change_bound.delay( action="created", sender="Group", instance_id=event.group_id ) from sentry.plugins.base import plugins for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression ) event_processed.send_robust( sender=post_process_group, project=event.project, event=event, primary_hash=kwargs.get("primary_hash"), )
def wrapped(request, *args, **kwargs): with Hub.current.start_span( Span(op="http.server", transaction=endpoint, sampled=True)): return func(request, *args, **kwargs)