def __call__(self, *args, **kwargs):
data = to_json(request.json)
result = self.action(data)
if isinstance(result, requests.models.Response):
self.response.status_code = result.status_code
self.response.set_data(result.content)
try:
res_json = json.loads(result.content)
if result.status_code == 402:
trace.get_current_span().add_event(
"exception", {
"exception.code": int(result.status_code),
"exception.message": res_json['error']
}, time_ns())
except:
pass
else:
self.response.status_code = result.code
self.response.set_data(
str({
'msg': result.msg,
'error': result.error
}))
if result.code == 402:
trace.get_current_span().add_event(
"exception", {
"exception.code": int(result.code),
"exception.message": str(result.error)
}, time_ns())
return self.response
def worker(self):
timeout = self.schedule_delay_millis / 1e3
while not self.done:
if (
len(self.queue) < self.max_export_batch_size
and not self._flushing
):
with self.condition:
self.condition.wait(timeout)
if not self.queue:
# spurious notification, let's wait again
continue
if self.done:
# missing spans will be sent when calling flush
break
# substract the duration of this export call to the next timeout
start = time_ns()
self.export()
end = time_ns()
duration = (end - start) / 1e9
timeout = self.schedule_delay_millis / 1e3 - duration
# be sure that all spans are sent
self._drain_queue()
def test_events(self):
self.assertIsNone(self.tracer.get_current_span())
with self.tracer.start_as_current_span("root") as root:
# only event name
root.add_event("event0")
# event name and attributes
now = time_ns()
root.add_event("event1", {"name": "pluto"})
# event name, attributes and timestamp
now = time_ns()
root.add_event("event2", {"name": "birthday"}, now)
# lazy event
root.add_lazy_event(
trace_api.Event("event3", {"name": "hello"}, now))
self.assertEqual(len(root.events), 4)
self.assertEqual(root.events[0].name, "event0")
self.assertEqual(root.events[0].attributes, {})
self.assertEqual(root.events[1].name, "event1")
self.assertEqual(root.events[1].attributes, {"name": "pluto"})
self.assertEqual(root.events[2].name, "event2")
self.assertEqual(root.events[2].attributes, {"name": "birthday"})
self.assertEqual(root.events[2].timestamp, now)
self.assertEqual(root.events[3].name, "event3")
self.assertEqual(root.events[3].attributes, {"name": "hello"})
self.assertEqual(root.events[3].timestamp, now)
def test_events(self):
self.assertEqual(trace_api.get_current_span(), trace_api.INVALID_SPAN)
with self.tracer.start_as_current_span("root") as root:
# only event name
root.add_event("event0")
# event name and attributes
now = time_ns()
root.add_event("event1", {
"name": "pluto",
"some_bools": [True, False]
})
# event name, attributes and timestamp
now = time_ns()
root.add_event("event2", {"name": ["birthday"]}, now)
mutable_list = ["original_contents"]
root.add_event("event3", {"name": mutable_list})
def event_formatter():
return {"name": "hello"}
# lazy event
root.add_lazy_event("event4", event_formatter, now)
self.assertEqual(len(root.events), 5)
self.assertEqual(root.events[0].name, "event0")
self.assertEqual(root.events[0].attributes, {})
self.assertEqual(root.events[1].name, "event1")
self.assertEqual(
root.events[1].attributes,
{
"name": "pluto",
"some_bools": (True, False)
},
)
self.assertEqual(root.events[2].name, "event2")
self.assertEqual(root.events[2].attributes,
{"name": ("birthday", )})
self.assertEqual(root.events[2].timestamp, now)
self.assertEqual(root.events[3].name, "event3")
self.assertEqual(root.events[3].attributes,
{"name": ("original_contents", )})
mutable_list = ["new_contents"]
self.assertEqual(root.events[3].attributes,
{"name": ("original_contents", )})
self.assertEqual(root.events[4].name, "event4")
self.assertEqual(root.events[4].attributes, {"name": "hello"})
self.assertEqual(root.events[4].timestamp, now)
def record(self, value: metrics_api.ValueT) -> None:
"""See `opentelemetry.metrics.MeasureHandle.record`."""
if self._validate_update(value):
if self.monotonic and value < 0:
logger.warning("Monotonic measure cannot accept negatives.")
return
self.last_update_timestamp = time_ns()
def _prepare(tracer, func, handler, args, kwargs):
start_time = time_ns()
request = handler.request
if _excluded_urls.url_disabled(request.uri):
return func(*args, **kwargs)
_start_span(tracer, handler, start_time)
return func(*args, **kwargs)
def _wrapped_app(wrapped_app_environ, start_response):
# We want to measure the time for route matching, etc.
# In theory, we could start the span here and use
# update_name later but that API is "highly discouraged" so
# we better avoid it.
wrapped_app_environ[_ENVIRON_STARTTIME_KEY] = time_ns()
def _start_response(status, response_headers, *args, **kwargs):
if not _excluded_urls.url_disabled(flask.request.url):
span = flask.request.environ.get(_ENVIRON_SPAN_KEY)
if span:
otel_wsgi.add_response_attributes(
span, status, response_headers
)
else:
_logger.warning(
"Flask environ's OpenTelemetry span "
"missing at _start_response(%s)",
status,
)
return start_response(status, response_headers, *args, **kwargs)
return wsgi_app(wrapped_app_environ, _start_response)
def fetch_async(tracer, func, _, args, kwargs):
start_time = time_ns()
# Return immediately if no args were provided (error)
# or original_request is set (meaning we are in a redirect step).
if len(args) == 0 or hasattr(args[0], "original_request"):
return func(*args, **kwargs)
# Force the creation of a HTTPRequest object if needed,
# so we can inject the context into the headers.
args, kwargs = _normalize_request(args, kwargs)
request = args[0]
span = tracer.start_span(
request.method, kind=trace.SpanKind.CLIENT, start_time=start_time,
)
if span.is_recording():
attributes = {
"component": "tornado",
"http.url": request.url,
"http.method": request.method,
}
for key, value in attributes.items():
span.set_attribute(key, value)
with tracer.use_span(span):
propagators.inject(type(request.headers).__setitem__, request.headers)
future = func(*args, **kwargs)
future.add_done_callback(
functools.partial(_finish_tracing_callback, span=span)
)
return future
def test_invalid_event_attributes(self):
self.assertEqual(trace_api.get_current_span(), trace_api.INVALID_SPAN)
now = time_ns()
with self.tracer.start_as_current_span("root") as root:
root.add_event("event0", {"attr1": True, "attr2": ["hi", False]})
root.add_event("event0", {"attr1": dict()})
root.add_event("event0", {"attr1": [[True]]})
root.add_event("event0", {"attr1": [dict()], "attr2": [1, 2]})
self.assertEqual(len(root.events), 4)
self.assertEqual(root.events[0].attributes, {"attr1": True})
self.assertEqual(root.events[1].attributes, {})
self.assertEqual(root.events[2].attributes, {})
self.assertEqual(root.events[3].attributes, {"attr2": (1, 2)})
def event_formatter():
properties = {}
properties["attr1"] = dict()
properties["attr2"] = "hello"
return properties
root.add_lazy_event("event4", event_formatter, now)
self.assertEqual(len(root.events), 5)
self.assertEqual(root.events[4].name, "event4")
self.assertEqual(root.events[4].attributes, {"attr2": "hello"})
self.assertEqual(root.events[4].timestamp, now)
def set(self, value: metrics_api.ValueT) -> None:
"""See `opentelemetry.metrics.GaugeHandle.set`."""
if self._validate_update(value):
if self.monotonic and value < self.data:
logger.warning("Monotonic gauge cannot descend.")
return
self.last_update_timestamp = time_ns()
self.data = value
def add(self, value: metrics_api.ValueT) -> None:
"""See `opentelemetry.metrics.CounterHandle.add`."""
if self._validate_update(value):
if self.monotonic and value < 0:
logger.warning("Monotonic counter cannot descend.")
return
self.last_update_timestamp = time_ns()
self.data += value
def worker(self):
timeout = self.schedule_delay_millis / 1e3
flush_request = None # type: typing.Optional[_FlushRequest]
while not self.done:
with self.condition:
if self.done:
# done flag may have changed, avoid waiting
break
flush_request = self._get_and_unset_flush_request()
if (
len(self.queue) < self.max_export_batch_size
and flush_request is None
):
self.condition.wait(timeout)
flush_request = self._get_and_unset_flush_request()
if not self.queue:
# spurious notification, let's wait again, reset timeout
timeout = self.schedule_delay_millis / 1e3
self._notify_flush_request_finished(flush_request)
flush_request = None
continue
if self.done:
# missing spans will be sent when calling flush
break
# subtract the duration of this export call to the next timeout
start = time_ns()
self._export(flush_request)
end = time_ns()
duration = (end - start) / 1e9
timeout = self.schedule_delay_millis / 1e3 - duration
self._notify_flush_request_finished(flush_request)
flush_request = None
# there might have been a new flush request while export was running
# and before the done flag switched to true
with self.condition:
shutdown_flush_request = self._get_and_unset_flush_request()
# be sure that all spans are sent
self._drain_queue()
self._notify_flush_request_finished(flush_request)
self._notify_flush_request_finished(shutdown_flush_request)
def test_export(self):
trace_id = "6e0c63257de34c92bf9efcd03927272e"
span_id = "95bb5edabd45950f"
# Create span and associated data.
resource_info = Resource(
{
"cloud.account.id": 123,
"host.id": "host",
"cloud.zone": "US",
"cloud.provider": "gcp",
"gcp.resource_type": "gce_instance",
}
)
span = Span(
name="span_name",
context=SpanContext(
trace_id=int(trace_id, 16),
span_id=int(span_id, 16),
is_remote=False,
),
parent=None,
kind=SpanKind.INTERNAL,
resource=resource_info,
attributes={"attr_key": "attr_value"},
)
# pylint: disable=protected-access
span._start_time = int(time_ns() - (60 * 1e9))
span._end_time = time_ns()
span_data = [span]
# Setup the trace exporter.
channel = grpc.insecure_channel(self.address)
transport = trace_service_grpc_transport.TraceServiceGrpcTransport(
channel=channel
)
client = TraceServiceClient(transport=transport)
trace_exporter = CloudTraceSpanExporter(self.project_id, client=client)
# Export the spans and verify the results.
result = trace_exporter.export(span_data)
self.assertEqual(result, SpanExportResult.SUCCESS)
def __init__(
self,
value_type: Type[metrics_api.ValueT],
enabled: bool,
aggregator: Aggregator,
):
self.value_type = value_type
self.enabled = enabled
self.aggregator = aggregator
self.last_update_timestamp = time_ns()
def end(self, end_time: Optional[int] = None) -> None:
with self._lock:
if self.start_time is None:
raise RuntimeError("Calling end() on a not started span.")
if self.end_time is not None:
logger.warning("Calling end() on an ended span.")
return
self._end_time = end_time if end_time is not None else time_ns()
self.span_processor.on_end(self)
def __init__(
self,
value_type: Type[metrics_api.ValueT],
enabled: bool,
monotonic: bool,
):
self.data = value_type()
self.value_type = value_type
self.enabled = enabled
self.monotonic = monotonic
self.last_update_timestamp = time_ns()
def __init__(
self,
value_type: Type[metrics_api.ValueT],
enabled: bool,
aggregator: Aggregator,
):
self.value_type = value_type
self.enabled = enabled
self.aggregator = aggregator
self.last_update_timestamp = time_ns()
self._ref_count = 0
self._ref_count_lock = threading.Lock()
def update(self, value):
with self._lock:
if self.current is self._EMPTY:
self.current = self._TYPE(value, value, value, 1)
else:
self.current = self._TYPE(
min(self.current.min, value),
max(self.current.max, value),
self.current.sum + value,
self.current.count + 1,
)
self.last_update_timestamp = time_ns()
def add_lazy_event(
self,
name: str,
event_formatter: types.AttributesFormatter,
timestamp: Optional[int] = None,
) -> None:
self._add_event(
LazyEvent(
name=name,
event_formatter=event_formatter,
timestamp=time_ns() if timestamp is None else timestamp,
))
def start(self, start_time: Optional[int] = None) -> None:
with self._lock:
if not self.is_recording_events():
return
has_started = self.start_time is not None
if not has_started:
self._start_time = (start_time
if start_time is not None else time_ns())
if has_started:
logger.warning("Calling start() on a started span.")
return
self.span_processor.on_start(self)
def add_event(
self,
name: str,
timestamp: int = None,
attributes: types.Attributes = None,
) -> None:
self.add_lazy_event(
trace_api.Event(
name,
time_ns() if timestamp is None else timestamp,
Span.empty_attributes if attributes is None else attributes,
))
def start(
self,
start_time: Optional[int] = None,
parent_context: Optional[context_api.Context] = None,
) -> None:
with self._lock:
if self.start_time is not None:
logger.warning("Calling start() on a started span.")
return
self._start_time = (start_time
if start_time is not None else time_ns())
self.span_processor.on_start(self, parent_context=parent_context)
def update(self, value):
with self._lock:
if self.current is None:
self.current = [0 for ii in range(len(self._boundaries) + 1)]
# greater than max value
if value >= self._boundaries[len(self._boundaries) - 1]:
self.current[">"] += 1
else:
for bb in self._boundaries:
# find first bucket that value is less than
if value < bb:
self.current[bb] += 1
break
self.last_update_timestamp = time_ns()
def trace_tween(request):
if _excluded_urls.url_disabled(request.url):
request.environ[_ENVIRON_ENABLED_KEY] = False
# short-circuit when we don't want to trace anything
return handler(request)
request.environ[_ENVIRON_ENABLED_KEY] = True
request.environ[_ENVIRON_STARTTIME_KEY] = time_ns()
try:
response = handler(request)
response_or_exception = response
except HTTPException as exc:
# If the exception is a pyramid HTTPException,
# that's still valuable information that isn't necessarily
# a 500. For instance, HTTPFound is a 302.
# As described in docs, Pyramid exceptions are all valid
# response types
response_or_exception = exc
raise
finally:
span = request.environ.get(_ENVIRON_SPAN_KEY)
enabled = request.environ.get(_ENVIRON_ENABLED_KEY)
if not span and enabled:
_logger.warning(
"Pyramid environ's OpenTelemetry span missing."
"If the OpenTelemetry tween was added manually, make sure"
"PyramidInstrumentor().instrument_config(config) is called"
)
elif enabled:
otel_wsgi.add_response_attributes(
span,
response_or_exception.status,
response_or_exception.headers,
)
activation = request.environ.get(_ENVIRON_ACTIVATION_KEY)
if isinstance(response_or_exception, HTTPException):
activation.__exit__(
type(response_or_exception),
response_or_exception,
getattr(response_or_exception, "__traceback__", None),
)
else:
activation.__exit__(None, None, None)
context.detach(request.environ.get(_ENVIRON_TOKEN))
return response
def end(self, end_time: int = None) -> None:
with self._lock:
if not self.is_recording_events():
return
if self.start_time is None:
raise RuntimeError("Calling end() on a not started span.")
has_ended = self.end_time is not None
if not has_ended:
self.end_time = end_time if end_time is not None else time_ns()
if has_ended:
logger.warning("Calling end() on an ended span.")
return
self.span_processor.on_end(self)
def add_event(
self,
name: str,
attributes: types.Attributes = None,
timestamp: Optional[int] = None,
) -> None:
if attributes is None:
attributes = Span._empty_attributes
self._add_event(
Event(
name=name,
attributes=attributes,
timestamp=time_ns() if timestamp is None else timestamp,
))
def add_event(
self,
name: str,
attributes: types.Attributes = None,
timestamp: Optional[int] = None,
) -> None:
_filter_attribute_values(attributes)
attributes = _create_immutable_attributes(attributes)
self._add_event(
Event(
name=name,
attributes=attributes,
timestamp=time_ns() if timestamp is None else timestamp,
))
def add_event(
self,
name: str,
attributes: types.Attributes = None,
timestamp: Optional[int] = None,
) -> None:
_filter_attribute_values(attributes)
if not attributes:
attributes = self._new_attributes()
self._add_event(
Event(
name=name,
attributes=attributes,
timestamp=time_ns() if timestamp is None else timestamp,
))
def force_flush(self, timeout_millis: int = 30000) -> bool:
"""Sequentially calls force_flush on all underlying
:class:`SpanProcessor`
Args:
timeout_millis: The maximum amount of time over all span processors
to wait for spans to be exported. In case the first n span
processors exceeded the timeout followup span processors will be
skipped.
Returns:
True if all span processors flushed their spans within the
given timeout, False otherwise.
"""
deadline_ns = time_ns() + timeout_millis * 1000000
for sp in self._span_processors:
current_time_ns = time_ns()
if current_time_ns >= deadline_ns:
return False
if not sp.force_flush((deadline_ns - current_time_ns) // 1000000):
return False
return True
def end(self, end_time: Optional[int] = None) -> None:
with self._lock:
if not self.is_recording_events():
return
if self.start_time is None:
raise RuntimeError("Calling end() on a not started span.")
has_ended = self.end_time is not None
if not has_ended:
self.end_time = end_time if end_time is not None else time_ns()
if has_ended:
logger.warning("Calling end() on an ended span.")
return
if self.status is None:
self.set_status(Status(canonical_code=StatusCanonicalCode.OK))
self.span_processor.on_end(self)
