async def _opentracing_middleware(
request: Request,
handler: Callable[[Request], Awaitable[StreamResponse]],
) -> StreamResponse:
tracer = get_tracer(__name__)
attach(extract(_headers_getter, request))
with tracer.start_as_current_span(
request.path,
kind=SpanKind.SERVER,
attributes={
"component": "http",
"http.method": request.method,
"http.scheme": request.scheme,
"http.host": request.host,
},
) as span:
try:
response = await handler(request)
except HTTPException as e:
code = _utils.status_to_canonical_code(e.status)
span.set_status(Status(code))
span.set_attribute("http.status_text", e.reason)
raise
else:
code = _utils.status_to_canonical_code(response.status)
span.set_status(Status(code))
span.set_attribute("http.status_text", response.reason)
return response
def test_set_current(self):
context.attach(context.set_value("a", "yyy"))
token = context.attach(context.set_value("a", "zzz"))
self.assertEqual("zzz", context.get_value("a"))
context.detach(token)
self.assertEqual("yyy", context.get_value("a"))
def test_detach_out_of_order(self):
t1 = context.attach(context.set_value("c", 1))
self.assertEqual(context.get_current(), {"c": 1})
t2 = context.attach(context.set_value("c", 2))
self.assertEqual(context.get_current(), {"c": 2})
context.detach(t1)
self.assertEqual(context.get_current(), {})
context.detach(t2)
self.assertEqual(context.get_current(), {"c": 1})
def test_attach(self):
context.attach(context.set_value("a", "yyy"))
token = context.attach(context.set_value("a", "zzz"))
self.assertEqual("zzz", context.get_value("a"))
context.detach(token)
self.assertEqual("yyy", context.get_value("a"))
with self.assertLogs(level=ERROR):
context.detach("some garbage")
async def _recv_request(event: RecvRequest) -> None:
tracer = get_tracer(__name__)
attach(extract(_metadata_getter, event.metadata))
span_ctx = tracer.start_as_current_span(
event.method_name,
kind=SpanKind.SERVER,
attributes={
"component": "grpc",
"grpc.method": event.method_name
},
)
span_ctx.__enter__()
_server_span_ctx.set(span_ctx)
def process_view(self, request, view_func, view_args, view_kwargs): # pylint: disable=unused-argument
# request.META is a dictionary containing all available HTTP headers
# Read more about request.META here:
# https://docs.djangoproject.com/en/3.0/ref/request-response/#django.http.HttpRequest.META
# environ = {
# key.lower().replace('_', '-').replace("http-", "", 1): value
# for key, value in request.META.items()
# }
environ = request.META
token = attach(extract(get_header_from_environ, environ))
tracer = get_tracer(__name__, __version__)
attributes = collect_request_attributes(environ)
span = tracer.start_span(
view_func.__name__,
kind=SpanKind.SERVER,
attributes=attributes,
start_time=environ.get(
"opentelemetry-instrumentor-django.starttime_key"),
)
activation = tracer.use_span(span, end_on_exit=True)
activation.__enter__()
request.META[self._environ_activation_key] = activation
request.META[self._environ_span_key] = span
request.META[self._environ_token] = token
def __call__(self, environ, start_response):
"""The WSGI application
Args:
environ: A WSGI environment.
start_response: The WSGI start_response callable.
"""
token = context.attach(
propagators.extract(get_header_from_environ, environ)
)
span_name = self.name_callback(environ)
span = self.tracer.start_span(
span_name,
kind=trace.SpanKind.SERVER,
attributes=collect_request_attributes(environ),
)
try:
with self.tracer.use_span(span):
start_response = self._create_start_response(
span, start_response
)
iterable = self.wsgi(environ, start_response)
return _end_span_after_iterating(
iterable, span, self.tracer, token
)
except: # noqa
# TODO Set span status (cf. https://github.com/open-telemetry/opentelemetry-python/issues/292)
span.end()
context.detach(token)
raise
def _suppress_further_instrumentation():
token = context.attach(
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
try:
yield
finally:
context.detach(token)
def _start_span(tracer, handler, start_time) -> _TraceContext:
token = context.attach(extract(handler.request.headers))
span = tracer.start_span(
_get_operation_name(handler, handler.request),
kind=trace.SpanKind.SERVER,
start_time=start_time,
)
if span.is_recording():
attributes = _get_attributes_from_request(handler.request)
for key, value in attributes.items():
span.set_attribute(key, value)
activation = trace.use_span(span, end_on_exit=True)
activation.__enter__() # pylint: disable=E1101
ctx = _TraceContext(activation, span, token)
setattr(handler, _HANDLER_CONTEXT_KEY, ctx)
# finish handler is called after the response is sent back to
# the client so it is too late to inject trace response headers
# there.
propagator = get_global_response_propagator()
if propagator:
propagator.inject(handler, setter=response_propagation_setter)
return ctx
async def suppressed_request(server: aiohttp.test_utils.TestServer):
async with aiohttp.test_utils.TestClient(server) as client:
token = context.attach(
context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)
)
await client.get(TestAioHttpClientInstrumentor.URL)
context.detach(token)
def __call__(self, environ, start_response):
"""The WSGI application
Args:
environ: A WSGI environment.
start_response: The WSGI start_response callable.
"""
token = context.attach(extract(environ, getter=wsgi_getter))
span_name = self.name_callback(environ)
span = self.tracer.start_span(
span_name,
kind=trace.SpanKind.SERVER,
attributes=collect_request_attributes(environ),
)
try:
with trace.use_span(span):
start_response = self._create_start_response(
span, start_response)
iterable = self.wsgi(environ, start_response)
return _end_span_after_iterating(iterable, span, self.tracer,
token)
except Exception as ex:
if span.is_recording():
span.set_status(Status(StatusCode.ERROR, str(ex)))
span.end()
context.detach(token)
raise
def export(self, spans) -> SpanExportResult:
logger.info("--- XraySpanExporter emitter ---")
# TODO: merge segments to one request, else batch processor does not make sense
for span in spans:
segment = self._translate_to_segment(span)
if segment == "":
continue
entity = json.dumps(segment)
# emit segment to daemon or xray ...
# self._emitter.send_entity(entity)
logger.info(entity)
# suppress boto3 instumentation
token = attach(set_value("suppress_instrumentation", True))
response = self._xray_client.put_trace_segments(
TraceSegmentDocuments=[
entity,
]
)
detach(token)
logger.info(response)
return SpanExportResult.SUCCESS
def _start_internal_or_server_span(
tracer, span_name, start_time, context_carrier, context_getter
):
"""Returns internal or server span along with the token which can be used by caller to reset context
Args:
tracer : tracer in use by given instrumentation library
name (string): name of the span
start_time : start time of the span
context_carrier : object which contains values that are
used to construct a Context. This object
must be paired with an appropriate getter
which understands how to extract a value from it.
context_getter : an object which contains a get function that can retrieve zero
or more values from the carrier and a keys function that can get all the keys
from carrier.
"""
token = ctx = span_kind = None
if trace.get_current_span() is trace.INVALID_SPAN:
ctx = extract(context_carrier, getter=context_getter)
token = context.attach(ctx)
span_kind = trace.SpanKind.SERVER
else:
ctx = context.get_current()
span_kind = trace.SpanKind.INTERNAL
span = tracer.start_span(
name=span_name,
context=ctx,
kind=span_kind,
start_time=start_time,
)
return span, token
async def on_request_start(
unused_session: aiohttp.ClientSession,
trace_config_ctx: types.SimpleNamespace,
params: aiohttp.TraceRequestStartParams,
):
if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY):
trace_config_ctx.span = None
return
http_method = params.method.upper()
request_span_name = f"HTTP {http_method}"
request_url = (remove_url_credentials(
trace_config_ctx.url_filter(params.url)) if callable(
trace_config_ctx.url_filter) else remove_url_credentials(
str(params.url)))
span_attributes = {
SpanAttributes.HTTP_METHOD: http_method,
SpanAttributes.HTTP_URL: request_url,
}
trace_config_ctx.span = trace_config_ctx.tracer.start_span(
request_span_name,
kind=SpanKind.CLIENT,
attributes=span_attributes)
if callable(request_hook):
request_hook(trace_config_ctx.span, params)
trace_config_ctx.token = context_api.attach(
trace.set_span_in_context(trace_config_ctx.span))
inject(params.headers)
def _before_flask_request():
environ = flask.request.environ
span_name = (flask.request.endpoint
or otel_wsgi.get_default_span_name(environ))
token = context.attach(
propagators.extract(otel_wsgi.get_header_from_environ,
environ))
tracer = trace.get_tracer(__name__, __version__)
attributes = otel_wsgi.collect_request_attributes(environ)
if flask.request.url_rule:
# For 404 that result from no route found, etc, we
# don't have a url_rule.
attributes["http.route"] = flask.request.url_rule.rule
span = tracer.start_span(
span_name,
kind=trace.SpanKind.SERVER,
attributes=attributes,
start_time=environ.get(_ENVIRON_STARTTIME_KEY),
)
activation = tracer.use_span(span, end_on_exit=True)
activation.__enter__()
environ[_ENVIRON_ACTIVATION_KEY] = activation
environ[_ENVIRON_SPAN_KEY] = span
environ[_ENVIRON_TOKEN] = token
async def suppressed_request(server: aiohttp.test_utils.TestServer):
async with aiohttp.test_utils.TestClient(server) as client:
token = context.attach(
context.set_value("suppress_instrumentation", True)
)
await client.get(TestAioHttpClientInstrumentor.URL)
context.detach(token)
async def on_request_start(
unused_session: aiohttp.ClientSession,
trace_config_ctx: types.SimpleNamespace,
params: aiohttp.TraceRequestStartParams,
):
http_method = params.method.upper()
if trace_config_ctx.span_name is None:
request_span_name = http_method
elif callable(trace_config_ctx.span_name):
request_span_name = str(trace_config_ctx.span_name(params))
else:
request_span_name = str(trace_config_ctx.span_name)
trace_config_ctx.span = trace_config_ctx.tracer.start_span(
request_span_name,
kind=SpanKind.CLIENT,
attributes={
"component": "http",
"http.method": http_method,
"http.url": trace_config_ctx.url_filter(params.url)
if callable(trace_config_ctx.url_filter)
else str(params.url),
},
)
trace_config_ctx.token = context_api.attach(
trace.propagation.set_span_in_context(trace_config_ctx.span)
)
propagators.inject(
tracer, type(params.headers).__setitem__, params.headers
)
def use_span(
self,
span: trace_api.Span,
end_on_exit: bool = False,
record_exception: bool = True,
) -> Iterator[trace_api.Span]:
try:
token = context_api.attach(context_api.set_value(SPAN_KEY, span))
try:
yield span
finally:
context_api.detach(token)
except Exception as error: # pylint: disable=broad-except
# pylint:disable=protected-access
if isinstance(span, Span):
if record_exception:
span.record_exception(error)
if span.status is None and span._set_status_on_exception:
span.set_status(
Status(
canonical_code=getattr(
error,
EXCEPTION_STATUS_FIELD,
StatusCanonicalCode.UNKNOWN,
),
description="{}: {}".format(
type(error).__name__, error),
))
raise
finally:
if end_on_exit:
span.end()
def export(self) -> None:
"""Exports at most max_export_batch_size spans."""
idx = 0
notify_flush = False
# currently only a single thread acts as consumer, so queue.pop() will
# not raise an exception
while idx < self.max_export_batch_size and self.queue:
span = self.queue.pop()
if span is self._FLUSH_TOKEN_SPAN:
notify_flush = True
else:
self.spans_list[idx] = span
idx += 1
token = attach(set_value("suppress_instrumentation", True))
try:
# Ignore type b/c the Optional[None]+slicing is too "clever"
# for mypy
self.span_exporter.export(self.spans_list[:idx]) # type: ignore
# pylint: disable=broad-except
except Exception:
logger.exception("Exception while exporting Span batch.")
detach(token)
if notify_flush:
with self.flush_condition:
self.flush_condition.notify()
# clean up list
for index in range(idx):
self.spans_list[index] = None
def __init__(self,
manager: "ScopeManagerShim",
span: SpanShim,
span_cm=None):
super().__init__(manager, span)
self._span_cm = span_cm
self._token = attach(set_value("scope_shim", self))
def _instrumented_open_call(_, request, call_wrapped,
get_or_create_headers): # pylint: disable=too-many-locals
if context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY) or context.get_value(
_SUPPRESS_HTTP_INSTRUMENTATION_KEY):
return call_wrapped()
method = request.get_method().upper()
url = request.full_url
span_name = f"HTTP {method}".strip()
url = remove_url_credentials(url)
labels = {
SpanAttributes.HTTP_METHOD: method,
SpanAttributes.HTTP_URL: url,
}
with tracer.start_as_current_span(span_name,
kind=SpanKind.CLIENT,
attributes=labels) as span:
exception = None
if callable(request_hook):
request_hook(span, request)
headers = get_or_create_headers()
inject(headers)
token = context.attach(
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "file", None)
finally:
context.detach(token)
if result is not None:
code_ = result.getcode()
labels[SpanAttributes.HTTP_STATUS_CODE] = str(code_)
if span.is_recording():
span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, code_)
span.set_status(Status(http_status_to_status_code(code_)))
ver_ = str(getattr(result, "version", ""))
if ver_:
labels[
SpanAttributes.HTTP_FLAVOR] = f"{ver_[:1]}.{ver_[:-1]}"
if callable(response_hook):
response_hook(span, request, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def use_span(self,
span: trace_api.Span,
end_on_exit: bool = False) -> Iterator[trace_api.Span]:
try:
token = context_api.attach(context_api.set_value(SPAN_KEY, span))
try:
yield span
finally:
context_api.detach(token)
except Exception as error: # pylint: disable=broad-except
if (isinstance(span, Span) and span.status is None
and span._set_status_on_exception # pylint:disable=protected-access # noqa
):
span.set_status(
Status(
canonical_code=StatusCanonicalCode.UNKNOWN,
description="{}: {}".format(
type(error).__name__, error),
))
raise
finally:
if end_on_exit:
span.end()
def __call__(self, environ, start_response):
"""The WSGI application
Args:
environ: A WSGI environment.
start_response: The WSGI start_response callable.
"""
token = context.attach(
propagators.extract(get_header_from_environ, environ))
span_name = self.name_callback(environ)
span = self.tracer.start_span(
span_name,
kind=trace.SpanKind.SERVER,
attributes=collect_request_attributes(environ),
)
try:
with self.tracer.use_span(span):
start_response = self._create_start_response(
span, start_response)
iterable = self.wsgi(environ, start_response)
return _end_span_after_iterating(iterable, span, self.tracer,
token)
except Exception as ex:
span.set_status(Status(StatusCanonicalCode.INTERNAL, str(ex)))
span.end()
context.detach(token)
raise
def _before_request():
if _excluded_urls.url_disabled(flask.request.url):
return
flask_request_environ = flask.request.environ
span_name = name_callback()
token = context.attach(
extract(flask_request_environ, getter=otel_wsgi.wsgi_getter))
tracer = trace.get_tracer(__name__, __version__)
span = tracer.start_span(
span_name,
kind=trace.SpanKind.SERVER,
start_time=flask_request_environ.get(_ENVIRON_STARTTIME_KEY),
)
if span.is_recording():
attributes = otel_wsgi.collect_request_attributes(
flask_request_environ)
if flask.request.url_rule:
# For 404 that result from no route found, etc, we
# don't have a url_rule.
attributes["http.route"] = flask.request.url_rule.rule
for key, value in attributes.items():
span.set_attribute(key, value)
activation = trace.use_span(span, end_on_exit=True)
activation.__enter__() # pylint: disable=E1101
flask_request_environ[_ENVIRON_ACTIVATION_KEY] = activation
flask_request_environ[_ENVIRON_SPAN_KEY] = span
flask_request_environ[_ENVIRON_TOKEN] = token
def process_request(self, request):
# request.META is a dictionary containing all available HTTP headers
# Read more about request.META here:
# https://docs.djangoproject.com/en/3.0/ref/request-response/#django.http.HttpRequest.META
if self._excluded_urls.url_disabled(request.build_absolute_uri("?")):
return
environ = request.META
token = attach(extract(get_header_from_environ, environ))
tracer = get_tracer(__name__, __version__)
attributes = collect_request_attributes(environ)
for attr in self._traced_request_attrs:
value = getattr(request, attr, None)
if value is not None:
attributes[attr] = str(value)
span = tracer.start_span(
self._get_span_name(request),
kind=SpanKind.SERVER,
attributes=attributes,
start_time=environ.get(
"opentelemetry-instrumentor-django.starttime_key"),
)
activation = tracer.use_span(span, end_on_exit=True)
activation.__enter__()
request.META[self._environ_activation_key] = activation
request.META[self._environ_span_key] = span
request.META[self._environ_token] = token
def use_span(
self,
span: trace_api.Span,
end_on_exit: bool = False,
) -> Iterator[trace_api.Span]:
try:
token = context_api.attach(context_api.set_value(SPAN_KEY, span))
try:
yield span
finally:
context_api.detach(token)
except Exception as exc: # pylint: disable=broad-except
# Record the exception as an event
if isinstance(span, Span) and span.is_recording():
# pylint:disable=protected-access
if span._record_exception:
span.record_exception(exc)
# Records status if use_span is used
# i.e. with tracer.start_as_current_span() as span:
if (span.status.status_code is StatusCode.UNSET
and span._set_status_on_exception):
span.set_status(
Status(
status_code=StatusCode.ERROR,
description="{}: {}".format(
type(exc).__name__, exc),
))
raise
finally:
if end_on_exit:
span.end()
def decorated_callback(
channel: Channel,
method: Basic.Deliver,
properties: BasicProperties,
body: bytes,
) -> Any:
if not properties:
properties = BasicProperties(headers={})
if properties.headers is None:
properties.headers = {}
ctx = propagate.extract(properties.headers, getter=_pika_getter)
if not ctx:
ctx = context.get_current()
token = context.attach(ctx)
span = _get_span(
tracer,
channel,
properties,
destination=method.exchange
if method.exchange else method.routing_key,
span_kind=SpanKind.CONSUMER,
task_name=task_name,
operation=MessagingOperationValues.RECEIVE,
)
try:
with trace.use_span(span, end_on_exit=True):
try:
consume_hook(span, body, properties)
except Exception as hook_exception: # pylint: disable=W0703
_LOG.exception(hook_exception)
retval = callback(channel, method, properties, body)
finally:
context.detach(token)
return retval
def _instrumented_requests_call(method: str, url: str, call_wrapped,
get_or_create_headers):
if context.get_value("suppress_instrumentation") or context.get_value(
_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY):
return call_wrapped()
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-client
method = method.upper()
span_name = ""
if name_callback is not None:
span_name = name_callback(method, url)
if not span_name or not isinstance(span_name, str):
span_name = get_default_span_name(method)
labels = {}
labels["http.method"] = method
labels["http.url"] = url
with get_tracer(__name__, __version__,
tracer_provider).start_as_current_span(
span_name, kind=SpanKind.CLIENT) as span:
exception = None
if span.is_recording():
span.set_attribute("http.method", method)
span.set_attribute("http.url", url)
headers = get_or_create_headers()
inject(type(headers).__setitem__, headers)
token = context.attach(
context.set_value(_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY,
True))
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
finally:
context.detach(token)
if isinstance(result, Response):
if span.is_recording():
span.set_attribute("http.status_code", result.status_code)
span.set_attribute("http.status_text", result.reason)
span.set_status(
Status(http_status_to_status_code(result.status_code)))
labels["http.status_code"] = str(result.status_code)
if result.raw and result.raw.version:
labels["http.flavor"] = (str(result.raw.version)[:1] +
"." +
str(result.raw.version)[:-1])
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def setUp(self):
self.token = context.attach(context.Context())
self.tracer_provider = trace.TracerProvider()
self.tracer = self.tracer_provider.get_tracer(__name__)
self.memory_exporter = InMemorySpanExporter()
span_processor = export.SimpleExportSpanProcessor(self.memory_exporter)
self.tracer_provider.add_span_processor(span_processor)
self.loop = asyncio.get_event_loop()
def use_context(parent_context: "Context") -> Generator[None, None, None]:
"""Uses the Ray trace context for the span."""
new_context = parent_context if parent_context is not None else Context()
token = context.attach(new_context)
try:
yield
finally:
context.detach(token)
