def fetch_async(tracer, func, _, args, kwargs):
start_time = _time_ns()
# Return immediately if no args were provided (error)
# or original_request is set (meaning we are in a redirect step).
if len(args) == 0 or hasattr(args[0], "original_request"):
return func(*args, **kwargs)
# Force the creation of a HTTPRequest object if needed,
# so we can inject the context into the headers.
args, kwargs = _normalize_request(args, kwargs)
request = args[0]
span = tracer.start_span(
request.method,
kind=trace.SpanKind.CLIENT,
start_time=start_time,
)
if span.is_recording():
attributes = {
"http.url": request.url,
"http.method": request.method,
}
for key, value in attributes.items():
span.set_attribute(key, value)
with trace.use_span(span):
inject(request.headers)
future = func(*args, **kwargs)
future.add_done_callback(
functools.partial(_finish_tracing_callback, span=span))
return future
def _finish_span(tracer, handler, error=None):
status_code = handler.get_status()
reason = getattr(handler, "_reason")
finish_args = (None, None, None)
ctx = getattr(handler, _HANDLER_CONTEXT_KEY, None)
if error:
if isinstance(error, tornado.web.HTTPError):
status_code = error.status_code
if not ctx and status_code == 404:
ctx = _start_span(tracer, handler, _time_ns())
if status_code != 404:
finish_args = (
type(error),
error,
getattr(error, "__traceback__", None),
)
status_code = 500
reason = None
if not ctx:
return
if ctx.span.is_recording():
ctx.span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, status_code)
ctx.span.set_status(
Status(
status_code=http_status_to_status_code(status_code),
description=reason,
))
ctx.activation.__exit__(*finish_args) # pylint: disable=E1101
context.detach(ctx.token)
delattr(handler, _HANDLER_CONTEXT_KEY)
def collect(self) -> Optional[Sum]:
"""
Atomically return a point for the current value of the metric and
reset the aggregation value.
"""
now = _time_ns()
if self._instrument_temporality is AggregationTemporality.DELTA:
with self._lock:
value = self._value
start_time_unix_nano = self._start_time_unix_nano
self._value = 0
self._start_time_unix_nano = now + 1
return Sum(
aggregation_temporality=AggregationTemporality.DELTA,
is_monotonic=self._instrument_is_monotonic,
start_time_unix_nano=start_time_unix_nano,
time_unix_nano=now,
value=value,
)
if self._value is None:
return None
return Sum(
aggregation_temporality=AggregationTemporality.CUMULATIVE,
is_monotonic=self._instrument_is_monotonic,
start_time_unix_nano=self._start_time_unix_nano,
time_unix_nano=now,
value=self._value,
)
def _prepare(tracer, func, handler, args, kwargs):
start_time = _time_ns()
request = handler.request
if _excluded_urls.url_disabled(request.uri):
return func(*args, **kwargs)
_start_span(tracer, handler, start_time)
return func(*args, **kwargs)
def _wrapped_app(wrapped_app_environ, start_response):
# We want to measure the time for route matching, etc.
# In theory, we could start the span here and use
# update_name later but that API is "highly discouraged" so
# we better avoid it.
wrapped_app_environ[_ENVIRON_STARTTIME_KEY] = _time_ns()
def _start_response(status, response_headers, *args, **kwargs):
if not _excluded_urls.url_disabled(flask.request.url):
span = flask.request.environ.get(_ENVIRON_SPAN_KEY)
propagator = get_global_response_propagator()
if propagator:
propagator.inject(
response_headers,
setter=otel_wsgi.default_response_propagation_setter,
)
if span:
otel_wsgi.add_response_attributes(span, status,
response_headers)
else:
_logger.warning(
"Flask environ's OpenTelemetry span "
"missing at _start_response(%s)",
status,
)
return start_response(status, response_headers, *args, **kwargs)
return wsgi_app(wrapped_app_environ, _start_response)
def __init__(self, uuid, name, status, result):
self.uuid = uuid
self.name = name
self.status = status
self.result = result
if sys.version_info >= (3, 7):
self.finish = time.time_ns()
else:
self.finish = _time_ns()
def _prepare(tracer, request_hook, func, handler, args, kwargs):
start_time = _time_ns()
request = handler.request
if _excluded_urls.url_disabled(request.uri):
return func(*args, **kwargs)
ctx = _start_span(tracer, handler, start_time)
if request_hook:
request_hook(ctx.span, handler)
return func(*args, **kwargs)
def worker(self):
timeout = self._schedule_delay_millis / 1e3
flush_request = None # type: Optional[_FlushRequest]
while not self._shutdown:
with self._condition:
if self._shutdown:
# shutdown may have been called, avoid further processing
break
flush_request = self._get_and_unset_flush_request()
if (
len(self._queue) < self._max_export_batch_size
and self._flush_request is None
):
self._condition.wait(timeout)
flush_request = self._get_and_unset_flush_request()
if not self._queue:
timeout = self._schedule_delay_millis / 1e3
self._notify_flush_request_finished(flush_request)
flush_request = None
continue
if self._shutdown:
break
start_ns = _time_ns()
self._export(flush_request)
end_ns = _time_ns()
# subtract the duration of this export call to the next timeout
timeout = self._schedule_delay_millis / 1e3 - (
(end_ns - start_ns) / 1e9
)
self._notify_flush_request_finished(flush_request)
flush_request = None
# there might have been a new flush request while export was running
# and before the done flag switched to true
with self._condition:
shutdown_flush_request = self._get_and_unset_flush_request()
# flush the remaining logs
self._drain_queue()
self._notify_flush_request_finished(flush_request)
self._notify_flush_request_finished(shutdown_flush_request)
def worker(self):
timeout = self.schedule_delay_millis / 1e3
flush_request = None # type: typing.Optional[_FlushRequest]
while not self.done:
with self.condition:
if self.done:
# done flag may have changed, avoid waiting
break
flush_request = self._get_and_unset_flush_request()
if (len(self.queue) < self.max_export_batch_size
and flush_request is None):
self.condition.wait(timeout)
flush_request = self._get_and_unset_flush_request()
if not self.queue:
# spurious notification, let's wait again, reset timeout
timeout = self.schedule_delay_millis / 1e3
self._notify_flush_request_finished(flush_request)
flush_request = None
continue
if self.done:
# missing spans will be sent when calling flush
break
# subtract the duration of this export call to the next timeout
start = _time_ns()
self._export(flush_request)
end = _time_ns()
duration = (end - start) / 1e9
timeout = self.schedule_delay_millis / 1e3 - duration
self._notify_flush_request_finished(flush_request)
flush_request = None
# there might have been a new flush request while export was running
# and before the done flag switched to true
with self.condition:
shutdown_flush_request = self._get_and_unset_flush_request()
# be sure that all spans are sent
self._drain_queue()
self._notify_flush_request_finished(flush_request)
self._notify_flush_request_finished(shutdown_flush_request)
def trace_tween(request):
# pylint: disable=E1101
if _excluded_urls.url_disabled(request.url):
request.environ[_ENVIRON_ENABLED_KEY] = False
# short-circuit when we don't want to trace anything
return handler(request)
request.environ[_ENVIRON_ENABLED_KEY] = True
request.environ[_ENVIRON_STARTTIME_KEY] = _time_ns()
try:
response = handler(request)
response_or_exception = response
except HTTPException as exc:
# If the exception is a pyramid HTTPException,
# that's still valuable information that isn't necessarily
# a 500. For instance, HTTPFound is a 302.
# As described in docs, Pyramid exceptions are all valid
# response types
response_or_exception = exc
raise
finally:
span = request.environ.get(_ENVIRON_SPAN_KEY)
enabled = request.environ.get(_ENVIRON_ENABLED_KEY)
if not span and enabled:
_logger.warning(
"Pyramid environ's OpenTelemetry span missing."
"If the OpenTelemetry tween was added manually, make sure"
"PyramidInstrumentor().instrument_config(config) is called"
)
elif enabled:
otel_wsgi.add_response_attributes(
span,
response_or_exception.status,
response_or_exception.headers,
)
propagator = get_global_response_propagator()
if propagator:
propagator.inject(response.headers)
activation = request.environ.get(_ENVIRON_ACTIVATION_KEY)
if isinstance(response_or_exception, HTTPException):
activation.__exit__(
type(response_or_exception),
response_or_exception,
getattr(response_or_exception, "__traceback__", None),
)
else:
activation.__exit__(None, None, None)
context.detach(request.environ.get(_ENVIRON_TOKEN))
return response
def collect(self) -> Optional[Gauge]:
"""
Atomically return a point for the current value of the metric.
"""
if self._value is None:
return None
return Gauge(
time_unix_nano=_time_ns(),
value=self._value,
)
def end(self, end_time: Optional[int] = None) -> None:
with self._lock:
if self._start_time is None:
raise RuntimeError("Calling end() on a not started span.")
if self._end_time is not None:
logger.warning("Calling end() on an ended span.")
return
self._end_time = end_time if end_time is not None else _time_ns()
self._span_processor.on_end(self._readable_span())
def _wrapped_app(wrapped_app_environ, start_response):
# We want to measure the time for route matching, etc.
# In theory, we could start the span here and use
# update_name later but that API is "highly discouraged" so
# we better avoid it.
wrapped_app_environ[_ENVIRON_STARTTIME_KEY] = _time_ns()
start = default_timer()
attributes = otel_wsgi.collect_request_attributes(wrapped_app_environ)
active_requests_count_attrs = (
otel_wsgi._parse_active_request_count_attrs(attributes))
duration_attrs = otel_wsgi._parse_duration_attrs(attributes)
active_requests_counter.add(1, active_requests_count_attrs)
def _start_response(status, response_headers, *args, **kwargs):
if flask.request and (
excluded_urls is None
or not excluded_urls.url_disabled(flask.request.url)):
span = flask.request.environ.get(_ENVIRON_SPAN_KEY)
propagator = get_global_response_propagator()
if propagator:
propagator.inject(
response_headers,
setter=otel_wsgi.default_response_propagation_setter,
)
if span:
otel_wsgi.add_response_attributes(span, status,
response_headers)
status_code = otel_wsgi._parse_status_code(status)
if status_code is not None:
duration_attrs[
SpanAttributes.HTTP_STATUS_CODE] = status_code
if (span.is_recording()
and span.kind == trace.SpanKind.SERVER):
custom_attributes = otel_wsgi.collect_custom_response_headers_attributes(
response_headers)
if len(custom_attributes) > 0:
span.set_attributes(custom_attributes)
else:
_logger.warning(
"Flask environ's OpenTelemetry span "
"missing at _start_response(%s)",
status,
)
if response_hook is not None:
response_hook(span, status, response_headers)
return start_response(status, response_headers, *args, **kwargs)
result = wsgi_app(wrapped_app_environ, _start_response)
duration = max(round((default_timer() - start) * 1000), 0)
duration_histogram.record(duration, duration_attrs)
active_requests_counter.add(-1, active_requests_count_attrs)
return result
def __init__(self, uuid, name, path, play, action, args):
self.uuid = uuid
self.name = name
self.path = path
self.play = play
self.host_data = OrderedDict()
if sys.version_info >= (3, 7):
self.start = time.time_ns()
else:
self.start = _time_ns()
self.action = action
self.args = args
def start(
self,
start_time: Optional[int] = None,
parent_context: Optional[context_api.Context] = None,
) -> None:
with self._lock:
if self._start_time is not None:
logger.warning("Calling start() on a started span.")
return
self._start_time = (start_time
if start_time is not None else _time_ns())
self._span_processor.on_start(self, parent_context=parent_context)
def force_flush(self, timeout_millis: int = 30000) -> bool:
"""Force flush the log processors one by one
Args:
timeout_millis: The maximum amount of time to wait for logs to be
exported. If the first n log processors exceeded the timeout
then remaining log processors will not be flushed.
Returns:
True if all the log processors flushes the logs within timeout,
False otherwise.
"""
deadline_ns = _time_ns() + timeout_millis * 1000000
for lp in self._log_processors:
current_ts = _time_ns()
if current_ts >= deadline_ns:
return False
if not lp.force_flush((deadline_ns - current_ts) // 1000000):
return False
return True
def __call__(self, env, start_response):
# pylint: disable=E1101
if self._otel_excluded_urls.url_disabled(env.get("PATH_INFO", "/")):
return super().__call__(env, start_response)
start_time = _time_ns()
span, token = _start_internal_or_server_span(
tracer=self._otel_tracer,
span_name=otel_wsgi.get_default_span_name(env),
start_time=start_time,
context_carrier=env,
context_getter=otel_wsgi.wsgi_getter,
)
if span.is_recording():
attributes = otel_wsgi.collect_request_attributes(env)
for key, value in attributes.items():
span.set_attribute(key, value)
if span.is_recording() and span.kind == trace.SpanKind.SERVER:
custom_attributes = (
otel_wsgi.collect_custom_request_headers_attributes(env)
)
if len(custom_attributes) > 0:
span.set_attributes(custom_attributes)
activation = trace.use_span(span, end_on_exit=True)
activation.__enter__()
env[_ENVIRON_SPAN_KEY] = span
env[_ENVIRON_ACTIVATION_KEY] = activation
def _start_response(status, response_headers, *args, **kwargs):
response = start_response(
status, response_headers, *args, **kwargs
)
activation.__exit__(None, None, None)
if token is not None:
context.detach(token)
return response
try:
return super().__call__(env, _start_response)
except Exception as exc:
activation.__exit__(
type(exc),
exc,
getattr(exc, "__traceback__", None),
)
if token is not None:
context.detach(token)
raise
def worker(self):
timeout = self.schedule_delay_millis / 1e3
while not self.done:
if not self._flushing:
with self.condition:
self.condition.wait(timeout)
if not self.check_traces_queue:
# spurious notification, let's wait again, reset timeout
timeout = self.schedule_delay_millis / 1e3
continue
if self.done:
# missing spans will be sent when calling flush
break
# substract the duration of this export call to the next timeout
start = _time_ns()
self.export()
end = _time_ns()
duration = (end - start) / 1e9
timeout = self.schedule_delay_millis / 1e3 - duration
# be sure that all spans are sent
self._drain_queue()
def add_event(
self,
name: str,
attributes: types.Attributes = None,
timestamp: Optional[int] = None,
) -> None:
_filter_attribute_values(attributes)
attributes = _create_immutable_attributes(attributes)
self._add_event(
Event(
name=name,
attributes=attributes,
timestamp=_time_ns() if timestamp is None else timestamp,
))
def force_flush(self, timeout_millis: int = 30000) -> bool:
"""Sequentially calls force_flush on all underlying
:class:`SpanProcessor`
Args:
timeout_millis: The maximum amount of time over all span processors
to wait for spans to be exported. In case the first n span
processors exceeded the timeout followup span processors will be
skipped.
Returns:
True if all span processors flushed their spans within the
given timeout, False otherwise.
"""
deadline_ns = _time_ns() + timeout_millis * 1000000
for sp in self._span_processors:
current_time_ns = _time_ns()
if current_time_ns >= deadline_ns:
return False
if not sp.force_flush((deadline_ns - current_time_ns) // 1000000):
return False
return True
def __init__(
self,
instrument_is_monotonic: bool,
instrument_temporality: AggregationTemporality,
):
super().__init__()
self._start_time_unix_nano = _time_ns()
self._instrument_temporality = instrument_temporality
self._instrument_is_monotonic = instrument_is_monotonic
if self._instrument_temporality is AggregationTemporality.DELTA:
self._value = 0
else:
self._value = None
def _finish_span(tracer, handler, error=None):
status_code = handler.get_status()
reason = getattr(handler, "_reason")
finish_args = (None, None, None)
ctx = getattr(handler, _HANDLER_CONTEXT_KEY, None)
if error:
if isinstance(error, tornado.web.HTTPError):
status_code = error.status_code
if not ctx and status_code == 404:
ctx = _start_span(tracer, handler, _time_ns())
else:
status_code = 500
reason = None
if status_code >= 500:
finish_args = (
type(error),
error,
getattr(error, "__traceback__", None),
)
if not ctx:
return
if ctx.span.is_recording():
ctx.span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, status_code)
otel_status_code = http_status_to_status_code(status_code,
server_span=True)
otel_status_description = None
if otel_status_code is StatusCode.ERROR:
otel_status_description = reason
ctx.span.set_status(
Status(
status_code=otel_status_code,
description=otel_status_description,
))
if ctx.span.is_recording() and ctx.span.kind == trace.SpanKind.SERVER:
custom_attributes = _collect_custom_response_headers_attributes(
handler._headers)
if len(custom_attributes) > 0:
ctx.span.set_attributes(custom_attributes)
ctx.activation.__exit__(*finish_args) # pylint: disable=E1101
if ctx.token:
context.detach(ctx.token)
delattr(handler, _HANDLER_CONTEXT_KEY)
def test_events(self):
self.assertEqual(trace_api.get_current_span(), trace_api.INVALID_SPAN)
with self.tracer.start_as_current_span("root") as root:
# only event name
root.add_event("event0")
# event name and attributes
root.add_event("event1", {
"name": "pluto",
"some_bools": [True, False]
})
# event name, attributes and timestamp
now = _time_ns()
root.add_event("event2", {"name": ["birthday"]}, now)
mutable_list = ["original_contents"]
root.add_event("event3", {"name": mutable_list})
self.assertEqual(len(root.events), 4)
self.assertEqual(root.events[0].name, "event0")
self.assertEqual(root.events[0].attributes, {})
self.assertEqual(root.events[1].name, "event1")
self.assertEqual(
root.events[1].attributes,
{
"name": "pluto",
"some_bools": (True, False)
},
)
self.assertEqual(root.events[2].name, "event2")
self.assertEqual(root.events[2].attributes,
{"name": ("birthday", )})
self.assertEqual(root.events[2].timestamp, now)
self.assertEqual(root.events[3].name, "event3")
self.assertEqual(root.events[3].attributes,
{"name": ("original_contents", )})
mutable_list = ["new_contents"]
self.assertEqual(root.events[3].attributes,
{"name": ("original_contents", )})
def __call__(self, env, start_response):
# pylint: disable=E1101
if _excluded_urls.url_disabled(env.get("PATH_INFO", "/")):
return super().__call__(env, start_response)
start_time = _time_ns()
token = context.attach(extract(env, getter=otel_wsgi.wsgi_getter))
span = self._tracer.start_span(
otel_wsgi.get_default_span_name(env),
kind=trace.SpanKind.SERVER,
start_time=start_time,
)
if span.is_recording():
attributes = otel_wsgi.collect_request_attributes(env)
for key, value in attributes.items():
span.set_attribute(key, value)
activation = trace.use_span(span, end_on_exit=True)
activation.__enter__()
env[_ENVIRON_SPAN_KEY] = span
env[_ENVIRON_ACTIVATION_KEY] = activation
def _start_response(status, response_headers, *args, **kwargs):
otel_wsgi.add_response_attributes(span, status, response_headers)
response = start_response(status, response_headers, *args,
**kwargs)
activation.__exit__(None, None, None)
context.detach(token)
return response
try:
return super().__call__(env, _start_response)
except Exception as exc:
activation.__exit__(
type(exc),
exc,
getattr(exc, "__traceback__", None),
)
context.detach(token)
raise
def collect(self) -> Histogram:
"""
Atomically return a point for the current value of the metric.
"""
now = _time_ns()
with self._lock:
value = self._bucket_counts
start_time_unix_nano = self._start_time_unix_nano
self._bucket_counts = self._get_empty_bucket_counts()
self._start_time_unix_nano = now + 1
return Histogram(
start_time_unix_nano=start_time_unix_nano,
time_unix_nano=now,
bucket_counts=tuple(value),
explicit_bounds=self._boundaries,
aggregation_temporality=AggregationTemporality.DELTA,
sum=self._sum,
)
def __init__(
self,
boundaries: Sequence[float] = (
0.0,
5.0,
10.0,
25.0,
50.0,
75.0,
100.0,
250.0,
500.0,
1000.0,
),
record_min_max: bool = True,
):
super().__init__()
self._boundaries = tuple(boundaries)
self._bucket_counts = self._get_empty_bucket_counts()
self._min = inf
self._max = -inf
self._sum = 0
self._record_min_max = record_min_max
self._start_time_unix_nano = _time_ns()
def test_time_seconds_from_ns(self):
time_nanoseconds = _time_ns()
result = time_seconds_from_ns(time_nanoseconds)
self.assertEqual(result, time_nanoseconds / 1e9)
return True
def shutdown(self):
self._shutdown = True
metrics_list = [
Metric(
name="sum_name",
attributes={},
description="",
instrumentation_info=None,
resource=Resource.create(),
unit="",
point=Sum(
start_time_unix_nano=_time_ns(),
time_unix_nano=_time_ns(),
value=2,
aggregation_temporality=1,
is_monotonic=True,
),
),
Metric(
name="gauge_name",
attributes={},
description="",
instrumentation_info=None,
resource=Resource.create(),
unit="",
point=Gauge(
time_unix_nano=_time_ns(),
def __init__(self, name: str, timestamp: Optional[int] = None) -> None:
self._name = name
if timestamp is None:
self._timestamp = _time_ns()
else:
self._timestamp = timestamp
def trace_tween(request):
# pylint: disable=E1101
if _excluded_urls.url_disabled(request.url):
request.environ[_ENVIRON_ENABLED_KEY] = False
# short-circuit when we don't want to trace anything
return handler(request)
request.environ[_ENVIRON_ENABLED_KEY] = True
request.environ[_ENVIRON_STARTTIME_KEY] = _time_ns()
response = None
status = None
try:
response = handler(request)
except HTTPException as exc:
# If the exception is a pyramid HTTPException,
# that's still valuable information that isn't necessarily
# a 500. For instance, HTTPFound is a 302.
# As described in docs, Pyramid exceptions are all valid
# response types
response = exc
raise
except BaseException:
# In the case that a non-HTTPException is bubbled up we
# should infer a internal server error and raise
status = "500 InternalServerError"
raise
finally:
span = request.environ.get(_ENVIRON_SPAN_KEY)
enabled = request.environ.get(_ENVIRON_ENABLED_KEY)
if not span and enabled:
_logger.warning(
"Pyramid environ's OpenTelemetry span missing."
"If the OpenTelemetry tween was added manually, make sure"
"PyramidInstrumentor().instrument_config(config) is called"
)
elif enabled:
status = getattr(response, "status", status)
if status is not None:
otel_wsgi.add_response_attributes(
span,
status,
getattr(response, "headerlist", None),
)
if span.is_recording() and span.kind == trace.SpanKind.SERVER:
custom_attributes = (
otel_wsgi.collect_custom_response_headers_attributes(
getattr(response, "headerlist", None)))
if len(custom_attributes) > 0:
span.set_attributes(custom_attributes)
propagator = get_global_response_propagator()
if propagator and hasattr(response, "headers"):
propagator.inject(response.headers)
activation = request.environ.get(_ENVIRON_ACTIVATION_KEY)
# Only considering HTTPServerError
# to make sure 200, 300 and 400 exceptions are not reported as error
if isinstance(response, HTTPServerError):
activation.__exit__(
type(response),
response,
getattr(response, "__traceback__", None),
)
else:
activation.__exit__(None, None, None)
env_token = request.environ.get(_ENVIRON_TOKEN, None)
if env_token is not None:
context.detach(env_token)
return response