def _instrumented_open_call(_, request, call_wrapped,
get_or_create_headers): # pylint: disable=too-many-locals
if context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY) or context.get_value(
_SUPPRESS_HTTP_INSTRUMENTATION_KEY):
return call_wrapped()
method = request.get_method().upper()
url = request.full_url
span_name = f"HTTP {method}".strip()
url = remove_url_credentials(url)
labels = {
SpanAttributes.HTTP_METHOD: method,
SpanAttributes.HTTP_URL: url,
}
with tracer.start_as_current_span(span_name,
kind=SpanKind.CLIENT,
attributes=labels) as span:
exception = None
if callable(request_hook):
request_hook(span, request)
headers = get_or_create_headers()
inject(headers)
token = context.attach(
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "file", None)
finally:
context.detach(token)
if result is not None:
code_ = result.getcode()
labels[SpanAttributes.HTTP_STATUS_CODE] = str(code_)
if span.is_recording():
span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, code_)
span.set_status(Status(http_status_to_status_code(code_)))
ver_ = str(getattr(result, "version", ""))
if ver_:
labels[
SpanAttributes.HTTP_FLAVOR] = f"{ver_[:1]}.{ver_[:-1]}"
if callable(response_hook):
response_hook(span, request, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def _instrumented_requests_call(method: str, url: str, call_wrapped,
get_or_create_headers):
if context.get_value("suppress_instrumentation") or context.get_value(
_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY):
return call_wrapped()
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-client
method = method.upper()
span_name = ""
if name_callback is not None:
span_name = name_callback(method, url)
if not span_name or not isinstance(span_name, str):
span_name = get_default_span_name(method)
labels = {}
labels["http.method"] = method
labels["http.url"] = url
with get_tracer(__name__, __version__,
tracer_provider).start_as_current_span(
span_name, kind=SpanKind.CLIENT) as span:
exception = None
if span.is_recording():
span.set_attribute("http.method", method)
span.set_attribute("http.url", url)
headers = get_or_create_headers()
inject(type(headers).__setitem__, headers)
token = context.attach(
context.set_value(_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY,
True))
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
finally:
context.detach(token)
if isinstance(result, Response):
if span.is_recording():
span.set_attribute("http.status_code", result.status_code)
span.set_attribute("http.status_text", result.reason)
span.set_status(
Status(http_status_to_status_code(result.status_code)))
labels["http.status_code"] = str(result.status_code)
if result.raw and result.raw.version:
labels["http.flavor"] = (str(result.raw.version)[:1] +
"." +
str(result.raw.version)[:-1])
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def test_context_key(self):
key1 = context.create_key("say")
key2 = context.create_key("say")
self.assertNotEqual(key1, key2)
first = context.set_value(key1, "foo")
second = context.set_value(key2, "bar")
self.assertEqual(context.get_value(key1, context=first), "foo")
self.assertEqual(context.get_value(key2, context=second), "bar")
def test_set_value(self):
first = context.set_value("a", "yyy")
second = context.set_value("a", "zzz")
third = context.set_value("a", "---", first)
self.assertEqual("yyy", context.get_value("a", context=first))
self.assertEqual("zzz", context.get_value("a", context=second))
self.assertEqual("---", context.get_value("a", context=third))
self.assertEqual(None, context.get_value("a"))
def test_set_current(self):
context.attach(context.set_value("a", "yyy"))
token = context.attach(context.set_value("a", "zzz"))
self.assertEqual("zzz", context.get_value("a"))
context.detach(token)
self.assertEqual("yyy", context.get_value("a"))
def _instrumented_requests_call(
method: str, url: str, call_wrapped, get_or_create_headers
):
if context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY
) or context.get_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY):
return call_wrapped()
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-client
method = method.upper()
span_name = ""
if name_callback is not None:
span_name = name_callback(method, url)
if not span_name or not isinstance(span_name, str):
span_name = get_default_span_name(method)
url = remove_url_credentials(url)
with tracer.start_as_current_span(
span_name, kind=SpanKind.CLIENT
) as span:
exception = None
if span.is_recording():
span.set_attribute(SpanAttributes.HTTP_METHOD, method)
span.set_attribute(SpanAttributes.HTTP_URL, url)
headers = get_or_create_headers()
inject(headers)
token = context.attach(
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True)
)
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
finally:
context.detach(token)
if isinstance(result, Response):
if span.is_recording():
span.set_attribute(
SpanAttributes.HTTP_STATUS_CODE, result.status_code
)
span.set_status(
Status(http_status_to_status_code(result.status_code))
)
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def _instrumented_requests_call(
method: str, url: str, call_wrapped, get_or_create_headers
):
if context.get_value("suppress_instrumentation") or context.get_value(
_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY
):
return call_wrapped()
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/http.md#http-client
method = method.upper()
span_name = "HTTP {}".format(method)
exception = None
with get_tracer(
__name__, __version__, tracer_provider
).start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
span.set_attribute("component", "http")
span.set_attribute("http.method", method.upper())
span.set_attribute("http.url", url)
headers = get_or_create_headers()
propagators.inject(type(headers).__setitem__, headers)
token = context.attach(
context.set_value(_SUPPRESS_REQUESTS_INSTRUMENTATION_KEY, True)
)
try:
result = call_wrapped() # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
finally:
context.detach(token)
if exception is not None:
span.set_status(
Status(_exception_to_canonical_code(exception))
)
span.record_exception(exception)
if result is not None:
span.set_attribute("http.status_code", result.status_code)
span.set_attribute("http.status_text", result.reason)
span.set_status(
Status(http_status_to_canonical_code(result.status_code))
)
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def test_attach(self):
context.attach(context.set_value("a", "yyy"))
token = context.attach(context.set_value("a", "zzz"))
self.assertEqual("zzz", context.get_value("a"))
context.detach(token)
self.assertEqual("yyy", context.get_value("a"))
with self.assertLogs(level=ERROR):
context.detach("some garbage")
def test_push_controller_suppress_instrumentation(self):
meter = mock.Mock()
exporter = mock.Mock()
exporter.export = lambda x: self.assertIsNotNone(
get_value("suppress_instrumentation"))
with mock.patch(
"opentelemetry.context._RUNTIME_CONTEXT") as context_patch:
controller = PushController(meter, exporter, 30.0)
controller.tick()
self.assertEqual(context_patch.attach.called, True)
self.assertEqual(context_patch.detach.called, True)
self.assertEqual(get_value("suppress_instrumentation"), None)
async def on_request_start(
unused_session: aiohttp.ClientSession,
trace_config_ctx: types.SimpleNamespace,
params: aiohttp.TraceRequestStartParams,
):
if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY):
trace_config_ctx.span = None
return
http_method = params.method.upper()
request_span_name = f"HTTP {http_method}"
request_url = (remove_url_credentials(
trace_config_ctx.url_filter(params.url)) if callable(
trace_config_ctx.url_filter) else remove_url_credentials(
str(params.url)))
span_attributes = {
SpanAttributes.HTTP_METHOD: http_method,
SpanAttributes.HTTP_URL: request_url,
}
trace_config_ctx.span = trace_config_ctx.tracer.start_span(
request_span_name,
kind=SpanKind.CLIENT,
attributes=span_attributes)
if callable(request_hook):
request_hook(trace_config_ctx.span, params)
trace_config_ctx.token = context_api.attach(
trace.set_span_in_context(trace_config_ctx.span))
inject(params.headers)
def started(self, event: monitoring.CommandStartedEvent):
""" Method to handle a pymongo CommandStartedEvent """
if not self.is_enabled or context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY):
return
command = event.command.get(event.command_name, "")
name = event.command_name
statement = event.command_name
if command:
name += "." + str(command)
statement += " " + str(command)
try:
span = self._tracer.start_span(name, kind=SpanKind.CLIENT)
if span.is_recording():
span.set_attribute(SpanAttributes.DB_SYSTEM,
DbSystemValues.MONGODB.value)
span.set_attribute(SpanAttributes.DB_NAME, event.database_name)
span.set_attribute(SpanAttributes.DB_STATEMENT, statement)
if event.connection_id is not None:
span.set_attribute(SpanAttributes.NET_PEER_NAME,
event.connection_id[0])
span.set_attribute(SpanAttributes.NET_PEER_PORT,
event.connection_id[1])
# Add Span to dictionary
self._span_dict[_get_span_dict_key(event)] = span
except Exception as ex: # noqa pylint: disable=broad-except
if span is not None and span.is_recording():
span.set_status(Status(StatusCode.ERROR, str(ex)))
span.end()
self._pop_span(event)
def _span_end(
self,
span_name: str,
status: Optional["status.StatusCode"] = None,
) -> bool:
# Check that the current context matches the lifecycle
if context.get_value(self.lifecycle_key) != span_name:
logger.warning(f"Requested span not active: {span_name}")
return False
span = trace.get_current_span()
# Set span status
if status is not None:
span.set_status(status)
# End current span
span.end()
# Pop context stack
context_detach_token: Optional[str] = None
try:
context_detach_token = self.context_detach_tokens[-1]
except IndexError:
pass
if context_detach_token:
context.detach(context_detach_token)
self.context_detach_tokens.pop()
else:
logger.warning(f"Context detach token missing: {span_name}")
return True
def active(self) -> "ScopeShim":
"""Returns a :class:`ScopeShim` object representing the
currently-active span in the OpenTelemetry tracer.
Returns:
A :class:`ScopeShim` representing the active span in the
OpenTelemetry tracer, or `None` if no span is currently active.
Warning:
Calling :meth:`ScopeShim.close` on the :class:`ScopeShim` returned
by this property **always ends the corresponding span**, regardless
of the *finish_on_close* value used when activating the span. This
is a limitation of the current implementation of the OpenTracing
shim and is likely to be handled in future versions.
"""
span = get_current_span()
if span.get_context() == INVALID_SPAN_CONTEXT:
return None
try:
return get_value("scope_shim")
except KeyError:
span_context = SpanContextShim(span.get_context())
wrapped_span = SpanShim(self._tracer, span_context, span)
return ScopeShim(self, span=wrapped_span)
def dependency_patch(*args, **kwargs) -> None:
start_time = time.time()
try:
result = ORIGINAL_REQUEST(*args, **kwargs)
except Exception as exc: # pylint: disable=broad-except
exception = exc
result = getattr(exc, "response", None)
end_time = time.time()
# Only collect request metric if sent from non-exporter thread
if context.get_value("suppress_instrumentation") is None:
# We don't want multiple threads updating this at once
with _dependency_lock:
try:
# Update duration
duration = dependency_map.get("duration", 0)
dependency_map["duration"] = duration + (end_time - start_time)
# Update count
count = dependency_map.get("count", 0)
dependency_map["count"] = count + 1
# Update failed count
if (result is not None and result.status_code < 200 and
result.status_code >= 300) or exception is not None:
failed_count = dependency_map.get("failed_count", 0)
dependency_map["failed_count"] = failed_count + 1
except Exception: # pylint: disable=broad-except
logger.warning("Error handling failed dependency metrics.")
return result
def instrumented_request(self, method, url, *args, **kwargs):
if context.get_value("suppress_instrumentation"):
return wrapped(self, method, url, *args, **kwargs)
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/data-semantic-conventions.md#http-client
try:
parsed_url = urlparse(url)
except ValueError as exc: # Invalid URL
path = "<Unparsable URL: {}>".format(exc)
else:
if parsed_url is None:
path = "<URL parses to None>"
path = parsed_url.path
with tracer.start_as_current_span(path, kind=SpanKind.CLIENT) as span:
span.set_attribute("component", "http")
span.set_attribute("http.method", method.upper())
span.set_attribute("http.url", url)
# TODO: Propagate the trace context via headers once we have a way
# to access propagators.
headers = kwargs.setdefault("headers", {})
propagators.inject(tracer, type(headers).__setitem__, headers)
result = wrapped(self, method, url, *args, **kwargs) # *** PROCEED
span.set_attribute("http.status_code", result.status_code)
span.set_attribute("http.status_text", result.reason)
return result
def succeeded(self, event: monitoring.CommandSucceededEvent):
""" Method to handle a pymongo CommandSucceededEvent """
if not self.is_enabled or context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY):
return
span = self._pop_span(event)
if span is None:
return
span.end()
def _patched_api_call(self, original_func, instance, args, kwargs):
if context_api.get_value("suppress_instrumentation"):
return original_func(*args, **kwargs)
endpoint_name = deep_getattr(instance, "_endpoint._endpoint_prefix")
with self._tracer.start_as_current_span(
"{}.command".format(endpoint_name), kind=SpanKind.CONSUMER,
) as span:
operation = None
if args and span.is_recording():
operation = args[0]
span.resource = Resource(
attributes={
"endpoint": endpoint_name,
"operation": operation.lower(),
}
)
else:
span.resource = Resource(
attributes={"endpoint": endpoint_name}
)
add_span_arg_tags(
span,
endpoint_name,
args,
("action", "params", "path", "verb"),
{"params", "path", "verb"},
)
if span.is_recording():
region_name = deep_getattr(instance, "meta.region_name")
meta = {
"aws.agent": "botocore",
"aws.operation": operation,
"aws.region": region_name,
}
for key, value in meta.items():
span.set_attribute(key, value)
result = original_func(*args, **kwargs)
if span.is_recording():
span.set_attribute(
"http.status_code",
result["ResponseMetadata"]["HTTPStatusCode"],
)
span.set_attribute(
"retry_attempts",
result["ResponseMetadata"]["RetryAttempts"],
)
return result
def dependency_patch(*args, **kwargs) -> None:
result = ORIGINAL_REQUEST(*args, **kwargs)
# Only collect request metric if sent from non-exporter thread
if context.get_value("suppress_instrumentation") is None:
# We don't want multiple threads updating this at once
with _dependency_lock:
count = dependency_map.get("count", 0)
dependency_map["count"] = count + 1
return result
def _patched_api_call(self, original_func, instance, args, kwargs):
if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY):
return original_func(*args, **kwargs)
call_context = _determine_call_context(instance, args)
if call_context is None:
return original_func(*args, **kwargs)
extension = _find_extension(call_context)
if not extension.should_trace_service_call():
return original_func(*args, **kwargs)
attributes = {
SpanAttributes.RPC_SYSTEM: "aws-api",
SpanAttributes.RPC_SERVICE: call_context.service_id,
SpanAttributes.RPC_METHOD: call_context.operation,
# TODO: update when semantic conventions exist
"aws.region": call_context.region,
}
_safe_invoke(extension.extract_attributes, attributes)
with self._tracer.start_as_current_span(
call_context.span_name,
kind=call_context.span_kind,
attributes=attributes,
) as span:
# inject trace context into payload headers for lambda Invoke
if BotocoreInstrumentor._is_lambda_invoke(call_context):
BotocoreInstrumentor._patch_lambda_invoke(call_context.params)
_safe_invoke(extension.before_service_call, span)
self._call_request_hook(span, call_context)
token = context_api.attach(
context_api.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY,
True))
result = None
try:
result = original_func(*args, **kwargs)
except ClientError as error:
result = getattr(error, "response", None)
_apply_response_attributes(span, result)
_safe_invoke(extension.on_error, span, error)
raise
else:
_apply_response_attributes(span, result)
_safe_invoke(extension.on_success, span, result)
finally:
context_api.detach(token)
_safe_invoke(extension.after_service_call)
self._call_response_hook(span, call_context, result)
return result
async def handle_async_request(
self,
method: bytes,
url: URL,
headers: typing.Optional[Headers] = None,
stream: typing.Optional[httpx.AsyncByteStream] = None,
extensions: typing.Optional[dict] = None,
) -> typing.Tuple[int, "Headers", httpx.AsyncByteStream, dict]:
"""Add request info to span."""
if context.get_value("suppress_instrumentation"):
return await self._transport.handle_async_request(
method,
url,
headers=headers,
stream=stream,
extensions=extensions,
)
span_attributes = _prepare_attributes(method, url)
_headers = _prepare_headers(headers)
span_name = _get_default_span_name(
span_attributes[SpanAttributes.HTTP_METHOD]
)
request = RequestInfo(method, url, headers, stream, extensions)
with self._tracer.start_as_current_span(
span_name, kind=SpanKind.CLIENT, attributes=span_attributes
) as span:
if self._request_hook is not None:
await self._request_hook(span, request)
inject(_headers)
(
status_code,
headers,
stream,
extensions,
) = await self._transport.handle_async_request(
method,
url,
headers=_headers.raw,
stream=stream,
extensions=extensions,
)
_apply_status_code(span, status_code)
if self._response_hook is not None:
await self._response_hook(
span,
request,
ResponseInfo(status_code, headers, stream, extensions),
)
return status_code, headers, stream, extensions
def _get_span(
tracer: Tracer,
channel: Channel,
properties: BasicProperties,
task_name: str,
destination: str,
span_kind: SpanKind,
operation: Optional[MessagingOperationValues] = None,
) -> Optional[Span]:
if context.get_value("suppress_instrumentation") or context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY):
return None
task_name = properties.type if properties.type else task_name
span = tracer.start_span(
name=_generate_span_name(destination, operation),
kind=span_kind,
)
if span.is_recording():
_enrich_span(span, channel, properties, task_name, operation)
return span
def intercept_stream(self, request_or_iterator, metadata, client_info,
invoker):
if context.get_value(_SUPPRESS_INSTRUMENTATION_KEY):
return invoker(request_or_iterator, metadata)
if client_info.is_server_stream:
return self._intercept_server_stream(request_or_iterator, metadata,
client_info, invoker)
return self._intercept(request_or_iterator, metadata, client_info,
invoker)
def failed(self, event: monitoring.CommandFailedEvent):
""" Method to handle a pymongo CommandFailedEvent """
if not self.is_enabled or context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY):
return
span = self._pop_span(event)
if span is None:
return
if span.is_recording():
span.set_status(Status(StatusCode.ERROR, event.failure))
span.end()
def instrumented_request(self, method, url, *args, **kwargs):
if context.get_value("suppress_instrumentation"):
return wrapped(self, method, url, *args, **kwargs)
# See
# https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/http.md#http-client
try:
parsed_url = urlparse(url)
span_name = parsed_url.path
except ValueError as exc: # Invalid URL
span_name = "<Unparsable URL: {}>".format(exc)
exception = None
with get_tracer(
__name__, __version__, tracer_provider
).start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
span.set_attribute("component", "http")
span.set_attribute("http.method", method.upper())
span.set_attribute("http.url", url)
headers = kwargs.get("headers", {}) or {}
propagators.inject(type(headers).__setitem__, headers)
kwargs["headers"] = headers
try:
result = wrapped(
self, method, url, *args, **kwargs
) # *** PROCEED
except Exception as exc: # pylint: disable=W0703
exception = exc
result = getattr(exc, "response", None)
if exception is not None:
span.set_status(
Status(_exception_to_canonical_code(exception))
)
if result is not None:
span.set_attribute("http.status_code", result.status_code)
span.set_attribute("http.status_text", result.reason)
span.set_status(
Status(http_status_to_canonical_code(result.status_code))
)
if span_callback is not None:
span_callback(span, result)
if exception is not None:
raise exception.with_traceback(exception.__traceback__)
return result
def get_current_span(context: Optional[Context] = None) -> Span:
"""Retrieve the current span.
Args:
context: A Context object. If one is not passed, the
default current context is used instead.
Returns:
The Span set in the context if it exists. INVALID_SPAN otherwise.
"""
span = get_value(_SPAN_KEY, context=context)
if span is None or not isinstance(span, Span):
return INVALID_SPAN
return span
def get_correlations(
context: typing.Optional[Context] = None, ) -> typing.Mapping[str, object]:
"""Returns the name/value pairs in the CorrelationContext
Args:
context: The Context to use. If not set, uses current Context
Returns:
Name/value pairs in the CorrelationContext
"""
correlations = get_value(_CORRELATION_CONTEXT_KEY, context=context)
if isinstance(correlations, dict):
return MappingProxyType(correlations.copy())
return MappingProxyType({})
def get_all(
context: typing.Optional[Context] = None, ) -> typing.Mapping[str, object]:
"""Returns the name/value pairs in the Baggage
Args:
context: The Context to use. If not set, uses current Context
Returns:
The name/value pairs in the Baggage
"""
baggage = get_value(_BAGGAGE_KEY, context=context)
if isinstance(baggage, dict):
return MappingProxyType(baggage)
return MappingProxyType({})
async def instrumented_async_send(wrapped, instance, args, kwargs):
if context.get_value("suppress_instrumentation"):
return await wrapped(*args, **kwargs)
transport = instance._transport or httpx.AsyncHTTPTransport()
telemetry_transport = AsyncOpenTelemetryTransport(
transport,
tracer_provider=tracer_provider,
request_hook=request_hook,
response_hook=response_hook,
)
instance._transport = telemetry_transport
return await wrapped(*args, **kwargs)
def succeeded(self, event: monitoring.CommandSucceededEvent):
"""Method to handle a pymongo CommandSucceededEvent"""
if not self.is_enabled or context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY):
return
span = self._pop_span(event)
if span is None:
return
if span.is_recording():
try:
self.success_hook(span, event)
except Exception as hook_exception: # noqa pylint: disable=broad-except
_LOG.exception(hook_exception)
span.end()
def failed(self, event: monitoring.CommandFailedEvent):
"""Method to handle a pymongo CommandFailedEvent"""
if not self.is_enabled or context.get_value(
_SUPPRESS_INSTRUMENTATION_KEY):
return
span = self._pop_span(event)
if span is None:
return
if span.is_recording():
span.set_status(Status(StatusCode.ERROR, event.failure))
try:
self.failed_hook(span, event)
except Exception as hook_exception: # noqa pylint: disable=broad-except
_LOG.exception(hook_exception)
span.end()