def _start_response(status, response_headers, *args, **kwargs):
otel_wsgi.add_response_attributes(span, status, response_headers)
response = start_response(status, response_headers, *args,
**kwargs)
activation.__exit__(None, None, None)
context.detach(token)
return response
def use_span(self,
span: trace_api.Span,
end_on_exit: bool = False) -> Iterator[trace_api.Span]:
try:
token = context_api.attach(context_api.set_value(SPAN_KEY, span))
try:
yield span
finally:
context_api.detach(token)
except Exception as error: # pylint: disable=broad-except
if (isinstance(span, Span) and span.status is None
and span._set_status_on_exception # pylint:disable=protected-access # noqa
):
span.set_status(
Status(
canonical_code=StatusCanonicalCode.UNKNOWN,
description="{}: {}".format(
type(error).__name__, error),
))
raise
finally:
if end_on_exit:
span.end()
def _finish_span(tracer, handler, error=None):
status_code = handler.get_status()
reason = getattr(handler, "_reason")
finish_args = (None, None, None)
ctx = getattr(handler, _HANDLER_CONTEXT_KEY, None)
if error:
if isinstance(error, tornado.web.HTTPError):
status_code = error.status_code
if not ctx and status_code == 404:
ctx = _start_span(tracer, handler, _time_ns())
if status_code != 404:
finish_args = (
type(error),
error,
getattr(error, "__traceback__", None),
)
status_code = 500
reason = None
if not ctx:
return
if ctx.span.is_recording():
ctx.span.set_attribute("http.status_code", status_code)
ctx.span.set_status(
Status(
status_code=http_status_to_status_code(status_code),
description=reason,
))
ctx.activation.__exit__(*finish_args) # pylint: disable=E1101
context.detach(ctx.token)
delattr(handler, _HANDLER_CONTEXT_KEY)
def use_span(
self,
span: trace_api.Span,
end_on_exit: bool = False,
record_exception: bool = True,
) -> Iterator[trace_api.Span]:
try:
token = context_api.attach(context_api.set_value(SPAN_KEY, span))
try:
yield span
finally:
context_api.detach(token)
except Exception as error: # pylint: disable=broad-except
# pylint:disable=protected-access
if isinstance(span, Span):
if record_exception:
span.record_exception(error)
if span.status is None and span._set_status_on_exception:
span.set_status(
Status(
canonical_code=getattr(
error,
EXCEPTION_STATUS_FIELD,
StatusCanonicalCode.UNKNOWN,
),
description="{}: {}".format(
type(error).__name__, error),
))
raise
finally:
if end_on_exit:
span.end()
def use_span(
self,
span: trace_api.Span,
end_on_exit: bool = False,
) -> Iterator[trace_api.Span]:
try:
token = context_api.attach(context_api.set_value(SPAN_KEY, span))
try:
yield span
finally:
context_api.detach(token)
except Exception as exc: # pylint: disable=broad-except
# Record the exception as an event
if isinstance(span, Span) and span.is_recording():
# pylint:disable=protected-access
if span._record_exception:
span.record_exception(exc)
# Records status if use_span is used
# i.e. with tracer.start_as_current_span() as span:
if (span.status.status_code is StatusCode.UNSET
and span._set_status_on_exception):
span.set_status(
Status(
status_code=StatusCode.ERROR,
description="{}: {}".format(
type(exc).__name__, exc),
))
raise
finally:
if end_on_exit:
span.end()
def decorated_callback(
channel: Channel,
method: Basic.Deliver,
properties: BasicProperties,
body: bytes,
) -> Any:
if not properties:
properties = BasicProperties(headers={})
if properties.headers is None:
properties.headers = {}
ctx = propagate.extract(properties.headers, getter=_pika_getter)
if not ctx:
ctx = context.get_current()
token = context.attach(ctx)
span = _get_span(
tracer,
channel,
properties,
destination=method.exchange
if method.exchange else method.routing_key,
span_kind=SpanKind.CONSUMER,
task_name=task_name,
operation=MessagingOperationValues.RECEIVE,
)
try:
with trace.use_span(span, end_on_exit=True):
try:
consume_hook(span, body, properties)
except Exception as hook_exception: # pylint: disable=W0703
_LOG.exception(hook_exception)
retval = callback(channel, method, properties, body)
finally:
context.detach(token)
return retval
def _suppress_further_instrumentation():
token = context.attach(
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
try:
yield
finally:
context.detach(token)
def close(self):
"""Closes the `ScopeShim`. If the `ScopeShim` was created from a
context manager, calling this method sets the active span in the
OpenTelemetry tracer back to the span which was active before this
`ScopeShim` was created. In addition, if the span represented by this
`ScopeShim` was activated with the *finish_on_close* argument set to
`True`, calling this method will end the span.
Warning:
In the current state of the implementation it is possible to create
a `ScopeShim` directly from a `SpanShim`, that is - without using
:meth:`from_context_manager()`. For that reason we need to be able
to end the span represented by the `ScopeShim` in this case, too.
Please note that closing a `ScopeShim` created this way (for
example as returned by :meth:`ScopeManagerShim.active`) **always
ends the associated span**, regardless of the value passed in
*finish_on_close* when activating the span.
"""
detach(self._token)
if self._span_cm is not None:
# We don't have error information to pass to `__exit__()` so we
# pass `None` in all arguments. If the OpenTelemetry tracer
# implementation requires this information, the `__exit__()` method
# on `opentracing.Scope` should be overridden and modified to pass
# the relevant values to this `close()` method.
self._span_cm.__exit__(None, None, None)
else:
self._span.unwrap().end()
def export(self) -> None:
"""Exports at most max_export_batch_size spans."""
idx = 0
notify_flush = False
# currently only a single thread acts as consumer, so queue.pop() will
# not raise an exception
while idx < self.max_export_batch_size and self.queue:
span = self.queue.pop()
if span is self._FLUSH_TOKEN_SPAN:
notify_flush = True
else:
self.spans_list[idx] = span
idx += 1
token = attach(set_value("suppress_instrumentation", True))
try:
# Ignore type b/c the Optional[None]+slicing is too "clever"
# for mypy
self.span_exporter.export(self.spans_list[:idx]) # type: ignore
# pylint: disable=broad-except
except Exception:
logger.exception("Exception while exporting Span batch.")
detach(token)
if notify_flush:
with self.flush_condition:
self.flush_condition.notify()
# clean up list
for index in range(idx):
self.spans_list[index] = None
def _instrumented_open_call(_, request, call_wrapped,
get_or_create_headers): # pylint: disable=too-many-locals
if context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY) or context.get_value(
_SUPPRESS_HTTP_INSTRUMENTATION_KEY):
return call_wrapped()
method = request.get_method().upper()
url = request.full_url
span_name = f"HTTP {method}".strip()
url = remove_url_credentials(url)
labels = {
SpanAttributes.HTTP_METHOD: method,
SpanAttributes.HTTP_URL: url,
}
with tracer.start_as_current_span(span_name,
kind=SpanKind.CLIENT,
attributes=labels) as span:
exception = None
if callable(request_hook):
request_hook(span, request)
headers = get_or_create_headers()
inject(headers)
token = context.attach(
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "file", None)
finally:
context.detach(token)
if result is not None:
code_ = result.getcode()
labels[SpanAttributes.HTTP_STATUS_CODE] = str(code_)
if span.is_recording():
span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, code_)
span.set_status(Status(http_status_to_status_code(code_)))
ver_ = str(getattr(result, "version", ""))
if ver_:
labels[
SpanAttributes.HTTP_FLAVOR] = f"{ver_[:1]}.{ver_[:-1]}"
if callable(response_hook):
response_hook(span, request, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def __call__(self, environ, start_response):
"""The WSGI application
Args:
environ: A WSGI environment.
start_response: The WSGI start_response callable.
"""
token = context.attach(
propagators.extract(get_header_from_environ, environ))
span_name = get_default_span_name(environ)
span = self.tracer.start_span(
span_name,
kind=trace.SpanKind.SERVER,
attributes=collect_request_attributes(environ),
)
try:
with self.tracer.use_span(span):
start_response = self._create_start_response(
span, start_response)
iterable = self.wsgi(environ, start_response)
return _end_span_after_iterating(iterable, span, self.tracer,
token)
except: # noqa
# TODO Set span status (cf. https://github.com/open-telemetry/opentelemetry-python/issues/292)
span.end()
context.detach(token)
raise
async def suppressed_request(server: aiohttp.test_utils.TestServer):
async with aiohttp.test_utils.TestClient(server) as client:
token = context.attach(
context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)
)
await client.get(TestAioHttpClientInstrumentor.URL)
context.detach(token)
def __call__(self, environ, start_response):
"""The WSGI application
Args:
environ: A WSGI environment.
start_response: The WSGI start_response callable.
"""
token = context.attach(propagators.extract(carrier_getter, environ))
span_name = self.name_callback(environ)
span = self.tracer.start_span(
span_name,
kind=trace.SpanKind.SERVER,
attributes=collect_request_attributes(environ),
)
try:
with self.tracer.use_span(span):
start_response = self._create_start_response(
span, start_response)
iterable = self.wsgi(environ, start_response)
return _end_span_after_iterating(iterable, span, self.tracer,
token)
except Exception as ex:
if span.is_recording():
span.set_status(Status(StatusCode.ERROR, str(ex)))
span.end()
context.detach(token)
raise
def process_response(self, request, response):
if self._excluded_urls.url_disabled(request.build_absolute_uri("?")):
return response
if (self._environ_activation_key in request.META.keys()
and self._environ_span_key in request.META.keys()):
add_response_attributes(
request.META[self._environ_span_key],
"{} {}".format(response.status_code, response.reason_phrase),
response,
)
request.META.pop(self._environ_span_key)
exception = request.META.pop(self._environ_exception_key, None)
if exception:
request.META[self._environ_activation_key].__exit__(
type(exception),
exception,
getattr(exception, "__traceback__", None),
)
else:
request.META[self._environ_activation_key].__exit__(
None, None, None)
request.META.pop(self._environ_activation_key)
if self._environ_token in request.META.keys():
detach(request.environ.get(self._environ_token))
request.META.pop(self._environ_token)
return response
def _instrumented_requests_call(method: str, url: str, call_wrapped,
get_or_create_headers):
if context.get_value("suppress_instrumentation") or context.get_value(
_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY):
return call_wrapped()
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-client
method = method.upper()
span_name = ""
if name_callback is not None:
span_name = name_callback(method, url)
if not span_name or not isinstance(span_name, str):
span_name = get_default_span_name(method)
labels = {}
labels["http.method"] = method
labels["http.url"] = url
with get_tracer(__name__, __version__,
tracer_provider).start_as_current_span(
span_name, kind=SpanKind.CLIENT) as span:
exception = None
if span.is_recording():
span.set_attribute("http.method", method)
span.set_attribute("http.url", url)
headers = get_or_create_headers()
inject(type(headers).__setitem__, headers)
token = context.attach(
context.set_value(_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY,
True))
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
finally:
context.detach(token)
if isinstance(result, Response):
if span.is_recording():
span.set_attribute("http.status_code", result.status_code)
span.set_attribute("http.status_text", result.reason)
span.set_status(
Status(http_status_to_status_code(result.status_code)))
labels["http.status_code"] = str(result.status_code)
if result.raw and result.raw.version:
labels["http.flavor"] = (str(result.raw.version)[:1] +
"." +
str(result.raw.version)[:-1])
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def use_context(parent_context: "Context") -> Generator[None, None, None]:
"""Uses the Ray trace context for the span."""
new_context = parent_context if parent_context is not None else Context()
token = context.attach(new_context)
try:
yield
finally:
context.detach(token)
def test_set_current(self):
context.attach(context.set_value("a", "yyy"))
token = context.attach(context.set_value("a", "zzz"))
self.assertEqual("zzz", context.get_value("a"))
context.detach(token)
self.assertEqual("yyy", context.get_value("a"))
def _start_response(status, response_headers, *args, **kwargs):
response = start_response(
status, response_headers, *args, **kwargs
)
activation.__exit__(None, None, None)
if token is not None:
context.detach(token)
return response
def on_end(self, span: Span) -> None:
token = attach(set_value("suppress_instrumentation", True))
try:
self.span_exporter.export((span,))
# pylint: disable=broad-except
except Exception:
logger.exception("Exception while exporting Span.")
detach(token)
async def __call__(self, scope, receive, send):
"""The ASGI application
Args:
scope: A ASGI environment.
receive: An awaitable callable yielding dictionaries
send: An awaitable callable taking a single dictionary as argument.
"""
if scope["type"] not in ("http", "websocket"):
return await self.app(scope, receive, send)
_, _, url = get_host_port_url_tuple(scope)
if self.excluded_urls and self.excluded_urls.url_disabled(url):
return await self.app(scope, receive, send)
token = context.attach(propagators.extract(carrier_getter, scope))
span_name, additional_attributes = self.span_details_callback(scope)
try:
with self.tracer.start_as_current_span(
span_name + " asgi",
kind=trace.SpanKind.SERVER,
) as span:
if span.is_recording():
attributes = collect_request_attributes(scope)
attributes.update(additional_attributes)
for key, value in attributes.items():
span.set_attribute(key, value)
@wraps(receive)
async def wrapped_receive():
with self.tracer.start_as_current_span(
span_name + " asgi." + scope["type"] +
".receive") as receive_span:
message = await receive()
if receive_span.is_recording():
if message["type"] == "websocket.receive":
set_status_code(receive_span, 200)
receive_span.set_attribute("type", message["type"])
return message
@wraps(send)
async def wrapped_send(message):
with self.tracer.start_as_current_span(
span_name + " asgi." + scope["type"] +
".send") as send_span:
if send_span.is_recording():
if message["type"] == "http.response.start":
status_code = message["status"]
set_status_code(send_span, status_code)
elif message["type"] == "websocket.send":
set_status_code(send_span, 200)
send_span.set_attribute("type", message["type"])
await send(message)
await self.app(scope, wrapped_receive, wrapped_send)
finally:
context.detach(token)
def tick(self):
# Collect all of the meter's metrics to be exported
self.meter.collect()
token = attach(set_value("suppress_instrumentation", True))
# Export the given metrics in the batcher
self.exporter.export(self.meter.batcher.checkpoint_set())
detach(token)
# Perform post-exporting logic based on batcher configuration
self.meter.batcher.finished_collection()
def test_detach_out_of_order(self):
t1 = context.attach(context.set_value("c", 1))
self.assertEqual(context.get_current(), {"c": 1})
t2 = context.attach(context.set_value("c", 2))
self.assertEqual(context.get_current(), {"c": 2})
context.detach(t1)
self.assertEqual(context.get_current(), {})
context.detach(t2)
self.assertEqual(context.get_current(), {"c": 1})
def _safe_end_processing_span(receipt_handle: str) -> None:
started_span: Span = Boto3SQSInstrumentor.received_messages_spans.pop(
receipt_handle, None)
if started_span:
if (Boto3SQSInstrumentor.current_span_related_to_token ==
started_span):
context.detach(Boto3SQSInstrumentor.current_context_token)
Boto3SQSInstrumentor.current_context_token = None
started_span.end()
def test_stream_stream_with_suppress_key(self):
token = context.attach(
context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
try:
bidirectional_streaming_method(self._stub)
spans = self.memory_exporter.get_finished_spans()
finally:
context.detach(token)
self.assertEqual(len(spans), 0)
def tick(self):
# Collect all of the meter's metrics to be exported
self.accumulator.collect()
# Export the collected metrics
token = attach(set_value("suppress_instrumentation", True))
self.exporter.export(self.accumulator.processor.checkpoint_set())
detach(token)
# Perform post-exporting logic
self.accumulator.processor.finished_collection()
def _receive_metrics(self, metrics: Iterable[Metric]) -> None:
if metrics is None:
return
token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
try:
self._exporter.export(metrics)
except Exception as e: # pylint: disable=broad-except,invalid-name
_logger.exception("Exception while exporting metrics %s", str(e))
detach(token)
def test_suppress_http_instrumentation_xray_client(self):
xray_client = self._make_client("xray")
token = attach(set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
try:
xray_client.put_trace_segments(TraceSegmentDocuments=["str1"])
xray_client.put_trace_segments(TraceSegmentDocuments=["str2"])
finally:
detach(token)
self.assertEqual(2, len(self.get_finished_spans()))
def test_suppress_instrumentation(self):
token = context.attach(
context.set_value("suppress_instrumentation", True))
try:
run_with_test_server(self.get_default_request(), self.URL,
self.default_handler)
finally:
context.detach(token)
self.assert_spans(0)
def _instrumented_requests_call(
method: str, url: str, call_wrapped, get_or_create_headers
):
if context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY
) or context.get_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY):
return call_wrapped()
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-client
method = method.upper()
span_name = ""
if name_callback is not None:
span_name = name_callback(method, url)
if not span_name or not isinstance(span_name, str):
span_name = get_default_span_name(method)
url = remove_url_credentials(url)
with tracer.start_as_current_span(
span_name, kind=SpanKind.CLIENT
) as span:
exception = None
if span.is_recording():
span.set_attribute(SpanAttributes.HTTP_METHOD, method)
span.set_attribute(SpanAttributes.HTTP_URL, url)
headers = get_or_create_headers()
inject(headers)
token = context.attach(
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True)
)
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
finally:
context.detach(token)
if isinstance(result, Response):
if span.is_recording():
span.set_attribute(
SpanAttributes.HTTP_STATUS_CODE, result.status_code
)
span.set_status(
Status(http_status_to_status_code(result.status_code))
)
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def _instrumented_requests_call(
method: str, url: str, call_wrapped, get_or_create_headers
):
if context.get_value("suppress_instrumentation") or context.get_value(
_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY
):
return call_wrapped()
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/http.md#http-client
method = method.upper()
span_name = "HTTP {}".format(method)
exception = None
with get_tracer(
__name__, __version__, tracer_provider
).start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
span.set_attribute("component", "http")
span.set_attribute("http.method", method.upper())
span.set_attribute("http.url", url)
headers = get_or_create_headers()
propagators.inject(type(headers).__setitem__, headers)
token = context.attach(
context.set_value(_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY, True)
)
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
finally:
context.detach(token)
if exception is not None:
span.set_status(
Status(_exception_to_canonical_code(exception))
)
span.record_exception(exception)
if result is not None:
span.set_attribute("http.status_code", result.status_code)
span.set_attribute("http.status_text", result.reason)
span.set_status(
Status(http_status_to_canonical_code(result.status_code))
)
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
