def fetch_async(tracer, func, _, args, kwargs): start_time = _time_ns() # Return immediately if no args were provided (error) # or original_request is set (meaning we are in a redirect step). if len(args) == 0 or hasattr(args[0], "original_request"): return func(*args, **kwargs) # Force the creation of a HTTPRequest object if needed, # so we can inject the context into the headers. args, kwargs = _normalize_request(args, kwargs) request = args[0] span = tracer.start_span( request.method, kind=trace.SpanKind.CLIENT, start_time=start_time, ) if span.is_recording(): attributes = { "http.url": request.url, "http.method": request.method, } for key, value in attributes.items(): span.set_attribute(key, value) with trace.use_span(span): inject(request.headers) future = func(*args, **kwargs) future.add_done_callback( functools.partial(_finish_tracing_callback, span=span)) return future
def _finish_span(tracer, handler, error=None): status_code = handler.get_status() reason = getattr(handler, "_reason") finish_args = (None, None, None) ctx = getattr(handler, _HANDLER_CONTEXT_KEY, None) if error: if isinstance(error, tornado.web.HTTPError): status_code = error.status_code if not ctx and status_code == 404: ctx = _start_span(tracer, handler, _time_ns()) if status_code != 404: finish_args = ( type(error), error, getattr(error, "__traceback__", None), ) status_code = 500 reason = None if not ctx: return if ctx.span.is_recording(): ctx.span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, status_code) ctx.span.set_status( Status( status_code=http_status_to_status_code(status_code), description=reason, )) ctx.activation.__exit__(*finish_args) # pylint: disable=E1101 context.detach(ctx.token) delattr(handler, _HANDLER_CONTEXT_KEY)
def collect(self) -> Optional[Sum]: """ Atomically return a point for the current value of the metric and reset the aggregation value. """ now = _time_ns() if self._instrument_temporality is AggregationTemporality.DELTA: with self._lock: value = self._value start_time_unix_nano = self._start_time_unix_nano self._value = 0 self._start_time_unix_nano = now + 1 return Sum( aggregation_temporality=AggregationTemporality.DELTA, is_monotonic=self._instrument_is_monotonic, start_time_unix_nano=start_time_unix_nano, time_unix_nano=now, value=value, ) if self._value is None: return None return Sum( aggregation_temporality=AggregationTemporality.CUMULATIVE, is_monotonic=self._instrument_is_monotonic, start_time_unix_nano=self._start_time_unix_nano, time_unix_nano=now, value=self._value, )
def _prepare(tracer, func, handler, args, kwargs): start_time = _time_ns() request = handler.request if _excluded_urls.url_disabled(request.uri): return func(*args, **kwargs) _start_span(tracer, handler, start_time) return func(*args, **kwargs)
def _wrapped_app(wrapped_app_environ, start_response): # We want to measure the time for route matching, etc. # In theory, we could start the span here and use # update_name later but that API is "highly discouraged" so # we better avoid it. wrapped_app_environ[_ENVIRON_STARTTIME_KEY] = _time_ns() def _start_response(status, response_headers, *args, **kwargs): if not _excluded_urls.url_disabled(flask.request.url): span = flask.request.environ.get(_ENVIRON_SPAN_KEY) propagator = get_global_response_propagator() if propagator: propagator.inject( response_headers, setter=otel_wsgi.default_response_propagation_setter, ) if span: otel_wsgi.add_response_attributes(span, status, response_headers) else: _logger.warning( "Flask environ's OpenTelemetry span " "missing at _start_response(%s)", status, ) return start_response(status, response_headers, *args, **kwargs) return wsgi_app(wrapped_app_environ, _start_response)
def __init__(self, uuid, name, status, result): self.uuid = uuid self.name = name self.status = status self.result = result if sys.version_info >= (3, 7): self.finish = time.time_ns() else: self.finish = _time_ns()
def _prepare(tracer, request_hook, func, handler, args, kwargs): start_time = _time_ns() request = handler.request if _excluded_urls.url_disabled(request.uri): return func(*args, **kwargs) ctx = _start_span(tracer, handler, start_time) if request_hook: request_hook(ctx.span, handler) return func(*args, **kwargs)
def worker(self): timeout = self._schedule_delay_millis / 1e3 flush_request = None # type: Optional[_FlushRequest] while not self._shutdown: with self._condition: if self._shutdown: # shutdown may have been called, avoid further processing break flush_request = self._get_and_unset_flush_request() if ( len(self._queue) < self._max_export_batch_size and self._flush_request is None ): self._condition.wait(timeout) flush_request = self._get_and_unset_flush_request() if not self._queue: timeout = self._schedule_delay_millis / 1e3 self._notify_flush_request_finished(flush_request) flush_request = None continue if self._shutdown: break start_ns = _time_ns() self._export(flush_request) end_ns = _time_ns() # subtract the duration of this export call to the next timeout timeout = self._schedule_delay_millis / 1e3 - ( (end_ns - start_ns) / 1e9 ) self._notify_flush_request_finished(flush_request) flush_request = None # there might have been a new flush request while export was running # and before the done flag switched to true with self._condition: shutdown_flush_request = self._get_and_unset_flush_request() # flush the remaining logs self._drain_queue() self._notify_flush_request_finished(flush_request) self._notify_flush_request_finished(shutdown_flush_request)
def worker(self): timeout = self.schedule_delay_millis / 1e3 flush_request = None # type: typing.Optional[_FlushRequest] while not self.done: with self.condition: if self.done: # done flag may have changed, avoid waiting break flush_request = self._get_and_unset_flush_request() if (len(self.queue) < self.max_export_batch_size and flush_request is None): self.condition.wait(timeout) flush_request = self._get_and_unset_flush_request() if not self.queue: # spurious notification, let's wait again, reset timeout timeout = self.schedule_delay_millis / 1e3 self._notify_flush_request_finished(flush_request) flush_request = None continue if self.done: # missing spans will be sent when calling flush break # subtract the duration of this export call to the next timeout start = _time_ns() self._export(flush_request) end = _time_ns() duration = (end - start) / 1e9 timeout = self.schedule_delay_millis / 1e3 - duration self._notify_flush_request_finished(flush_request) flush_request = None # there might have been a new flush request while export was running # and before the done flag switched to true with self.condition: shutdown_flush_request = self._get_and_unset_flush_request() # be sure that all spans are sent self._drain_queue() self._notify_flush_request_finished(flush_request) self._notify_flush_request_finished(shutdown_flush_request)
def trace_tween(request): # pylint: disable=E1101 if _excluded_urls.url_disabled(request.url): request.environ[_ENVIRON_ENABLED_KEY] = False # short-circuit when we don't want to trace anything return handler(request) request.environ[_ENVIRON_ENABLED_KEY] = True request.environ[_ENVIRON_STARTTIME_KEY] = _time_ns() try: response = handler(request) response_or_exception = response except HTTPException as exc: # If the exception is a pyramid HTTPException, # that's still valuable information that isn't necessarily # a 500. For instance, HTTPFound is a 302. # As described in docs, Pyramid exceptions are all valid # response types response_or_exception = exc raise finally: span = request.environ.get(_ENVIRON_SPAN_KEY) enabled = request.environ.get(_ENVIRON_ENABLED_KEY) if not span and enabled: _logger.warning( "Pyramid environ's OpenTelemetry span missing." "If the OpenTelemetry tween was added manually, make sure" "PyramidInstrumentor().instrument_config(config) is called" ) elif enabled: otel_wsgi.add_response_attributes( span, response_or_exception.status, response_or_exception.headers, ) propagator = get_global_response_propagator() if propagator: propagator.inject(response.headers) activation = request.environ.get(_ENVIRON_ACTIVATION_KEY) if isinstance(response_or_exception, HTTPException): activation.__exit__( type(response_or_exception), response_or_exception, getattr(response_or_exception, "__traceback__", None), ) else: activation.__exit__(None, None, None) context.detach(request.environ.get(_ENVIRON_TOKEN)) return response
def collect(self) -> Optional[Gauge]: """ Atomically return a point for the current value of the metric. """ if self._value is None: return None return Gauge( time_unix_nano=_time_ns(), value=self._value, )
def end(self, end_time: Optional[int] = None) -> None: with self._lock: if self._start_time is None: raise RuntimeError("Calling end() on a not started span.") if self._end_time is not None: logger.warning("Calling end() on an ended span.") return self._end_time = end_time if end_time is not None else _time_ns() self._span_processor.on_end(self._readable_span())
def _wrapped_app(wrapped_app_environ, start_response): # We want to measure the time for route matching, etc. # In theory, we could start the span here and use # update_name later but that API is "highly discouraged" so # we better avoid it. wrapped_app_environ[_ENVIRON_STARTTIME_KEY] = _time_ns() start = default_timer() attributes = otel_wsgi.collect_request_attributes(wrapped_app_environ) active_requests_count_attrs = ( otel_wsgi._parse_active_request_count_attrs(attributes)) duration_attrs = otel_wsgi._parse_duration_attrs(attributes) active_requests_counter.add(1, active_requests_count_attrs) def _start_response(status, response_headers, *args, **kwargs): if flask.request and ( excluded_urls is None or not excluded_urls.url_disabled(flask.request.url)): span = flask.request.environ.get(_ENVIRON_SPAN_KEY) propagator = get_global_response_propagator() if propagator: propagator.inject( response_headers, setter=otel_wsgi.default_response_propagation_setter, ) if span: otel_wsgi.add_response_attributes(span, status, response_headers) status_code = otel_wsgi._parse_status_code(status) if status_code is not None: duration_attrs[ SpanAttributes.HTTP_STATUS_CODE] = status_code if (span.is_recording() and span.kind == trace.SpanKind.SERVER): custom_attributes = otel_wsgi.collect_custom_response_headers_attributes( response_headers) if len(custom_attributes) > 0: span.set_attributes(custom_attributes) else: _logger.warning( "Flask environ's OpenTelemetry span " "missing at _start_response(%s)", status, ) if response_hook is not None: response_hook(span, status, response_headers) return start_response(status, response_headers, *args, **kwargs) result = wsgi_app(wrapped_app_environ, _start_response) duration = max(round((default_timer() - start) * 1000), 0) duration_histogram.record(duration, duration_attrs) active_requests_counter.add(-1, active_requests_count_attrs) return result
def __init__(self, uuid, name, path, play, action, args): self.uuid = uuid self.name = name self.path = path self.play = play self.host_data = OrderedDict() if sys.version_info >= (3, 7): self.start = time.time_ns() else: self.start = _time_ns() self.action = action self.args = args
def start( self, start_time: Optional[int] = None, parent_context: Optional[context_api.Context] = None, ) -> None: with self._lock: if self._start_time is not None: logger.warning("Calling start() on a started span.") return self._start_time = (start_time if start_time is not None else _time_ns()) self._span_processor.on_start(self, parent_context=parent_context)
def force_flush(self, timeout_millis: int = 30000) -> bool: """Force flush the log processors one by one Args: timeout_millis: The maximum amount of time to wait for logs to be exported. If the first n log processors exceeded the timeout then remaining log processors will not be flushed. Returns: True if all the log processors flushes the logs within timeout, False otherwise. """ deadline_ns = _time_ns() + timeout_millis * 1000000 for lp in self._log_processors: current_ts = _time_ns() if current_ts >= deadline_ns: return False if not lp.force_flush((deadline_ns - current_ts) // 1000000): return False return True
def __call__(self, env, start_response): # pylint: disable=E1101 if self._otel_excluded_urls.url_disabled(env.get("PATH_INFO", "/")): return super().__call__(env, start_response) start_time = _time_ns() span, token = _start_internal_or_server_span( tracer=self._otel_tracer, span_name=otel_wsgi.get_default_span_name(env), start_time=start_time, context_carrier=env, context_getter=otel_wsgi.wsgi_getter, ) if span.is_recording(): attributes = otel_wsgi.collect_request_attributes(env) for key, value in attributes.items(): span.set_attribute(key, value) if span.is_recording() and span.kind == trace.SpanKind.SERVER: custom_attributes = ( otel_wsgi.collect_custom_request_headers_attributes(env) ) if len(custom_attributes) > 0: span.set_attributes(custom_attributes) activation = trace.use_span(span, end_on_exit=True) activation.__enter__() env[_ENVIRON_SPAN_KEY] = span env[_ENVIRON_ACTIVATION_KEY] = activation def _start_response(status, response_headers, *args, **kwargs): response = start_response( status, response_headers, *args, **kwargs ) activation.__exit__(None, None, None) if token is not None: context.detach(token) return response try: return super().__call__(env, _start_response) except Exception as exc: activation.__exit__( type(exc), exc, getattr(exc, "__traceback__", None), ) if token is not None: context.detach(token) raise
def worker(self): timeout = self.schedule_delay_millis / 1e3 while not self.done: if not self._flushing: with self.condition: self.condition.wait(timeout) if not self.check_traces_queue: # spurious notification, let's wait again, reset timeout timeout = self.schedule_delay_millis / 1e3 continue if self.done: # missing spans will be sent when calling flush break # substract the duration of this export call to the next timeout start = _time_ns() self.export() end = _time_ns() duration = (end - start) / 1e9 timeout = self.schedule_delay_millis / 1e3 - duration # be sure that all spans are sent self._drain_queue()
def add_event( self, name: str, attributes: types.Attributes = None, timestamp: Optional[int] = None, ) -> None: _filter_attribute_values(attributes) attributes = _create_immutable_attributes(attributes) self._add_event( Event( name=name, attributes=attributes, timestamp=_time_ns() if timestamp is None else timestamp, ))
def force_flush(self, timeout_millis: int = 30000) -> bool: """Sequentially calls force_flush on all underlying :class:`SpanProcessor` Args: timeout_millis: The maximum amount of time over all span processors to wait for spans to be exported. In case the first n span processors exceeded the timeout followup span processors will be skipped. Returns: True if all span processors flushed their spans within the given timeout, False otherwise. """ deadline_ns = _time_ns() + timeout_millis * 1000000 for sp in self._span_processors: current_time_ns = _time_ns() if current_time_ns >= deadline_ns: return False if not sp.force_flush((deadline_ns - current_time_ns) // 1000000): return False return True
def __init__( self, instrument_is_monotonic: bool, instrument_temporality: AggregationTemporality, ): super().__init__() self._start_time_unix_nano = _time_ns() self._instrument_temporality = instrument_temporality self._instrument_is_monotonic = instrument_is_monotonic if self._instrument_temporality is AggregationTemporality.DELTA: self._value = 0 else: self._value = None
def _finish_span(tracer, handler, error=None): status_code = handler.get_status() reason = getattr(handler, "_reason") finish_args = (None, None, None) ctx = getattr(handler, _HANDLER_CONTEXT_KEY, None) if error: if isinstance(error, tornado.web.HTTPError): status_code = error.status_code if not ctx and status_code == 404: ctx = _start_span(tracer, handler, _time_ns()) else: status_code = 500 reason = None if status_code >= 500: finish_args = ( type(error), error, getattr(error, "__traceback__", None), ) if not ctx: return if ctx.span.is_recording(): ctx.span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, status_code) otel_status_code = http_status_to_status_code(status_code, server_span=True) otel_status_description = None if otel_status_code is StatusCode.ERROR: otel_status_description = reason ctx.span.set_status( Status( status_code=otel_status_code, description=otel_status_description, )) if ctx.span.is_recording() and ctx.span.kind == trace.SpanKind.SERVER: custom_attributes = _collect_custom_response_headers_attributes( handler._headers) if len(custom_attributes) > 0: ctx.span.set_attributes(custom_attributes) ctx.activation.__exit__(*finish_args) # pylint: disable=E1101 if ctx.token: context.detach(ctx.token) delattr(handler, _HANDLER_CONTEXT_KEY)
def test_events(self): self.assertEqual(trace_api.get_current_span(), trace_api.INVALID_SPAN) with self.tracer.start_as_current_span("root") as root: # only event name root.add_event("event0") # event name and attributes root.add_event("event1", { "name": "pluto", "some_bools": [True, False] }) # event name, attributes and timestamp now = _time_ns() root.add_event("event2", {"name": ["birthday"]}, now) mutable_list = ["original_contents"] root.add_event("event3", {"name": mutable_list}) self.assertEqual(len(root.events), 4) self.assertEqual(root.events[0].name, "event0") self.assertEqual(root.events[0].attributes, {}) self.assertEqual(root.events[1].name, "event1") self.assertEqual( root.events[1].attributes, { "name": "pluto", "some_bools": (True, False) }, ) self.assertEqual(root.events[2].name, "event2") self.assertEqual(root.events[2].attributes, {"name": ("birthday", )}) self.assertEqual(root.events[2].timestamp, now) self.assertEqual(root.events[3].name, "event3") self.assertEqual(root.events[3].attributes, {"name": ("original_contents", )}) mutable_list = ["new_contents"] self.assertEqual(root.events[3].attributes, {"name": ("original_contents", )})
def __call__(self, env, start_response): # pylint: disable=E1101 if _excluded_urls.url_disabled(env.get("PATH_INFO", "/")): return super().__call__(env, start_response) start_time = _time_ns() token = context.attach(extract(env, getter=otel_wsgi.wsgi_getter)) span = self._tracer.start_span( otel_wsgi.get_default_span_name(env), kind=trace.SpanKind.SERVER, start_time=start_time, ) if span.is_recording(): attributes = otel_wsgi.collect_request_attributes(env) for key, value in attributes.items(): span.set_attribute(key, value) activation = trace.use_span(span, end_on_exit=True) activation.__enter__() env[_ENVIRON_SPAN_KEY] = span env[_ENVIRON_ACTIVATION_KEY] = activation def _start_response(status, response_headers, *args, **kwargs): otel_wsgi.add_response_attributes(span, status, response_headers) response = start_response(status, response_headers, *args, **kwargs) activation.__exit__(None, None, None) context.detach(token) return response try: return super().__call__(env, _start_response) except Exception as exc: activation.__exit__( type(exc), exc, getattr(exc, "__traceback__", None), ) context.detach(token) raise
def collect(self) -> Histogram: """ Atomically return a point for the current value of the metric. """ now = _time_ns() with self._lock: value = self._bucket_counts start_time_unix_nano = self._start_time_unix_nano self._bucket_counts = self._get_empty_bucket_counts() self._start_time_unix_nano = now + 1 return Histogram( start_time_unix_nano=start_time_unix_nano, time_unix_nano=now, bucket_counts=tuple(value), explicit_bounds=self._boundaries, aggregation_temporality=AggregationTemporality.DELTA, sum=self._sum, )
def __init__( self, boundaries: Sequence[float] = ( 0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 1000.0, ), record_min_max: bool = True, ): super().__init__() self._boundaries = tuple(boundaries) self._bucket_counts = self._get_empty_bucket_counts() self._min = inf self._max = -inf self._sum = 0 self._record_min_max = record_min_max self._start_time_unix_nano = _time_ns()
def test_time_seconds_from_ns(self): time_nanoseconds = _time_ns() result = time_seconds_from_ns(time_nanoseconds) self.assertEqual(result, time_nanoseconds / 1e9)
return True def shutdown(self): self._shutdown = True metrics_list = [ Metric( name="sum_name", attributes={}, description="", instrumentation_info=None, resource=Resource.create(), unit="", point=Sum( start_time_unix_nano=_time_ns(), time_unix_nano=_time_ns(), value=2, aggregation_temporality=1, is_monotonic=True, ), ), Metric( name="gauge_name", attributes={}, description="", instrumentation_info=None, resource=Resource.create(), unit="", point=Gauge( time_unix_nano=_time_ns(),
def __init__(self, name: str, timestamp: Optional[int] = None) -> None: self._name = name if timestamp is None: self._timestamp = _time_ns() else: self._timestamp = timestamp
def trace_tween(request): # pylint: disable=E1101 if _excluded_urls.url_disabled(request.url): request.environ[_ENVIRON_ENABLED_KEY] = False # short-circuit when we don't want to trace anything return handler(request) request.environ[_ENVIRON_ENABLED_KEY] = True request.environ[_ENVIRON_STARTTIME_KEY] = _time_ns() response = None status = None try: response = handler(request) except HTTPException as exc: # If the exception is a pyramid HTTPException, # that's still valuable information that isn't necessarily # a 500. For instance, HTTPFound is a 302. # As described in docs, Pyramid exceptions are all valid # response types response = exc raise except BaseException: # In the case that a non-HTTPException is bubbled up we # should infer a internal server error and raise status = "500 InternalServerError" raise finally: span = request.environ.get(_ENVIRON_SPAN_KEY) enabled = request.environ.get(_ENVIRON_ENABLED_KEY) if not span and enabled: _logger.warning( "Pyramid environ's OpenTelemetry span missing." "If the OpenTelemetry tween was added manually, make sure" "PyramidInstrumentor().instrument_config(config) is called" ) elif enabled: status = getattr(response, "status", status) if status is not None: otel_wsgi.add_response_attributes( span, status, getattr(response, "headerlist", None), ) if span.is_recording() and span.kind == trace.SpanKind.SERVER: custom_attributes = ( otel_wsgi.collect_custom_response_headers_attributes( getattr(response, "headerlist", None))) if len(custom_attributes) > 0: span.set_attributes(custom_attributes) propagator = get_global_response_propagator() if propagator and hasattr(response, "headers"): propagator.inject(response.headers) activation = request.environ.get(_ENVIRON_ACTIVATION_KEY) # Only considering HTTPServerError # to make sure 200, 300 and 400 exceptions are not reported as error if isinstance(response, HTTPServerError): activation.__exit__( type(response), response, getattr(response, "__traceback__", None), ) else: activation.__exit__(None, None, None) env_token = request.environ.get(_ENVIRON_TOKEN, None) if env_token is not None: context.detach(env_token) return response