def test_follows_from(tracer):
span = tracer.start_span('test')
span1 = tracer.start_span('test2')
follow_span = tracer.start_span('follow-span', references=[follows_from(span.context),
follows_from(span1.context),
follows_from(None)])
span.finish()
span1.finish()
follow_span.finish()
tracer.reporter.report_span.assert_called()
assert len(follow_span.references) == 2
assert follow_span.context.parent_id == span.context.span_id
for reference in follow_span.references:
assert reference.referenced_context is not None
tracer.reporter = mock.MagicMock()
span = tracer.start_span('test')
follow_span = tracer.start_span(references=follows_from(span.context))
span.finish()
follow_span.finish()
tracer.reporter.report_span.assert_called()
assert isinstance(follow_span.references, list)
tracer.reporter = mock.MagicMock()
span = tracer.start_span('test')
parent_span = tracer.start_span('test-parent')
child_span = tracer.start_span('test-child', child_of=parent_span,
references=follows_from(span.context))
span.finish()
parent_span.finish()
child_span.finish()
tracer.reporter.report_span.assert_called()
assert child_span.context.parent_id == parent_span.context.span_id
assert len(child_span.references) == 1
tracer.close()
def produce(self, topic, value=None, *args, **kwargs):
"""
overridden method
:param topic:
:param value:
:param args:
:param kwargs:
:return:
"""
msg_header_dict = dict() if kwargs.get('headers') is None else dict(
kwargs.get('headers'))
parent_context = self.tracer.extract(Format.TEXT_MAP, msg_header_dict)
user_tags = {}
for span_tag_provider in self.span_tags_providers:
user_tags = merge_two_dicts(
user_tags, span_tag_provider(topic, kwargs.get('key'), value))
span = self.tracer.start_span(
self.span_name_provider(topic, kwargs.get('key'), value),
references=[follows_from(parent_context)],
tags=merge_two_dicts(producer_span_tags_provider(topic),
user_tags))
# Inject created span context into message header for sending to kafka queue
self.tracer.inject(span.context, Format.TEXT_MAP, msg_header_dict)
kwargs['headers'] = list(msg_header_dict.items())
kwargs['on_delivery'] = create_tracing_delivery_callback(
kwargs['on_delivery'], span)
Producer.produce(self, topic, value, *args, **kwargs)
def get_new_span(f,
func_args,
func_kwargs,
operation_name=None,
inspect_stack=True,
ignore_parent_span=False,
span_extractor=None,
use_follows_from=False):
parent_span = None
span_arg_name = DEFAULT_SPAN_ARG_NAME
if not ignore_parent_span:
if callable(span_extractor):
try:
parent_span = span_extractor(*func_args, **func_kwargs)
except Exception:
logger.exception('Failed to extract span from: {}'.format(
span_extractor.__name__))
if not parent_span:
span_arg_name, parent_span = get_parent_span(
inspect_stack=inspect_stack, **func_kwargs)
op_name = f.__name__ if not operation_name else operation_name
references = None
if parent_span:
references = [
follows_from(parent_span.context)
] if use_follows_from else [child_of(parent_span.context)]
return span_arg_name, opentracing.tracer.start_span(operation_name=op_name,
references=references)
def start_active_span_follows_from(operation_name, contexts):
if opentracing is None:
return noop_context_manager()
references = [opentracing.follows_from(context) for context in contexts]
scope = start_active_span(operation_name, references=references)
return scope
def decorator(*args, **kwargs):
parent_span = get_current_span() if not ignore_active_span else None
if parent_span is None and require_active_trace:
return func(*args, **kwargs)
tracer = global_tracer()
if async_result:
reference_span = follows_from(
parent_span.context) if parent_span is not None else None
parent_span = None
else:
reference_span = None
with tracer.start_active_span(
operation_name,
child_of=parent_span,
references=reference_span,
ignore_active_span=ignore_active_span,
tags=tags,
finish_on_close=finish_on_close) as scope:
span = scope.span
if callable(on_start):
on_start(span, *args, **kwargs)
# We explicitly invoke deactivation callback for the StackContext,
# because there are scenarios when it gets retained forever, for
# example when a Periodic Callback is scheduled lazily while in the
# scope of a tracing StackContext.
try:
d = func(*args, **kwargs)
# Twisted routines and inlineCallbacks generate defers
if isinstance(d, defer.Deferred):
def done(results):
span.finish()
return results
def failed(reason):
# Set error in span
span.log(event='exception', payload=reason)
span.set_tag('error', 'true')
span.finish()
return reason
if d.called:
span.finish()
else:
d.addCallbacks(done, failed)
else:
span.finish()
return d
except Exception as e:
span.set_tag(tags.ERROR, 'true')
span.log(event='exception', payload=e)
span.finish()
raise
def extract_tracing_span(carrier, use_follows_from=False):
try:
span_context = opentracing.tracer.extract(opentracing.Format.TEXT_MAP, carrier)
references = [follows_from(span_context)] if use_follows_from else [child_of(span_context)]
return opentracing.tracer.start_span(references=references)
except Exception:
return opentracing.tracer.start_span()
def test_parse_span_references(tracer):
span = tracer.start_span('test')
span2 = tracer.start_span('test2')
follow_span = tracer.start_span(
'test-follow',
references=[follows_from(span.context),
child_of(span2.context)])
span.finish()
span2.finish()
follow_span.finish()
_marshall_span(follow_span)
def extract_tracing_span(carrier, use_follows_from=False):
try:
span_context = opentracing.tracer.extract(opentracing.Format.TEXT_MAP,
carrier)
references = [follows_from(span_context)
] if use_follows_from else [child_of(span_context)]
return opentracing.tracer.start_span(references=references)
except Exception:
return opentracing.tracer.start_span()
def test_start_span_with_parent(self):
tracer = self.tracer()
parent_span = tracer.start_span(operation_name='parent')
assert parent_span is not None
span = tracer.start_span(operation_name='Leela', child_of=parent_span)
span.finish()
span = tracer.start_span(
operation_name='Leela',
references=[opentracing.follows_from(parent_span.context)],
tags={'birthplace': 'sewers'})
span.finish()
parent_span.finish()
def test_start_span_with_parent(self):
tracer = self.tracer()
parent_span = tracer.start_span(operation_name='parent')
assert parent_span is not None
span = tracer.start_span(
operation_name='Leela',
child_of=parent_span)
span.finish()
span = tracer.start_span(
operation_name='Leela',
references=[opentracing.follows_from(parent_span.context)],
tags={'birthplace': 'sewers'})
span.finish()
parent_span.finish()
def _span(self,
record: aiokafka.ConsumerRecord) -> opentracing.SpanContext:
tracer = opentracing.tracer
headers = {x[0]: x[1].decode() for x in record.headers or []}
parent = tracer.extract(opentracing.Format.TEXT_MAP, headers)
context = tracer.start_active_span(
record.topic,
tags={
"message_bus.destination": record.topic,
"message_bus.partition": record.partition,
"message_bus.group_id": self.group_id,
},
references=[opentracing.follows_from(parent)],
)
return context.span
def test_follows_from_references(self):
"""Test span creation using the `references` argument with a follows from relationship."""
with self.shim.start_span("ParentSpan") as parent:
ref = opentracing.follows_from(parent.context)
with self.shim.start_active_span(
"FollowingSpan", references=[ref]
) as child:
self.assertEqual(
child.span.unwrap().links[0].context,
parent.context.unwrap(),
)
self.assertEqual(
child.span.unwrap().parent,
parent.context.unwrap(),
)
def get_new_span(
f, func_args, func_kwargs, operation_name=None, inspect_stack=True, ignore_parent_span=False,
span_extractor=None, use_follows_from=False):
parent_span = None
using_scope_manager = False
span_arg_name = None
if not ignore_parent_span:
if callable(span_extractor):
try:
parent_span = span_extractor(*func_args, **func_kwargs)
except Exception:
logger.exception('Failed to extract span from: {}'.format(span_extractor.__name__))
if not parent_span:
span_arg_name, parent_span = get_span_from_kwargs(**func_kwargs)
if not parent_span:
try:
# We try inspecting ``active_span`` managed by ``tracer.scope_manager``.
parent_span = opentracing.tracer.active_span
using_scope_manager = True if parent_span else False
except AttributeError:
# Old opentracing lib!
pass
# Finally, try to inspect call stack frames.
if not parent_span:
span_arg_name, parent_span = get_parent_span(
inspect_stack=inspect_stack, inspect_kwargs=False, **func_kwargs)
op_name = f.__name__ if not operation_name else operation_name
references = None
if parent_span:
references = [follows_from(parent_span.context)] if use_follows_from else [child_of(parent_span.context)]
span_arg_name = span_arg_name or DEFAULT_SPAN_ARG_NAME
return (
span_arg_name,
using_scope_manager,
opentracing.tracer.start_span(operation_name=op_name, references=references)
)
def start_active_span_follows_from(
operation_name: str,
contexts: Collection,
child_of=None,
start_time: Optional[float] = None,
*,
inherit_force_tracing=False,
tracer=None,
):
"""Starts an active opentracing span, with additional references to previous spans
Args:
operation_name: name of the operation represented by the new span
contexts: the previous spans to inherit from
child_of: optionally override the parent span. If unset, the currently active
span will be the parent. (If there is no currently active span, the first
span in `contexts` will be the parent.)
start_time: optional override for the start time of the created span. Seconds
since the epoch.
inherit_force_tracing: if set, and any of the previous contexts have had tracing
forced, the new span will also have tracing forced.
tracer: override the opentracing tracer. By default the global tracer is used.
"""
if opentracing is None:
return contextlib.nullcontext() # type: ignore[unreachable]
references = [opentracing.follows_from(context) for context in contexts]
scope = start_active_span(
operation_name,
child_of=child_of,
references=references,
start_time=start_time,
tracer=tracer,
)
if inherit_force_tracing and any(
is_context_forced_tracing(ctx) for ctx in contexts):
force_tracing(scope.span)
return scope
def build_and_finish_child_span(self, msg):
msg_header_text_dict = dict((key, binary_value.decode("utf-8"))
for key, binary_value in msg.headers())
parent_context = self.tracer.extract(Format.TEXT_MAP,
msg_header_text_dict)
user_tags = {}
for span_tag_provider in self.span_tags_providers:
user_tags = merge_two_dicts(user_tags, span_tag_provider(msg))
span = self.tracer.start_span(
self.span_name_provider(msg),
references=[follows_from(parent_context)],
tags=merge_two_dicts(consumer_span_tags_provider(msg), user_tags))
span.finish()
# Inject created span context into message header for extraction by client to continue span chain
self.tracer.inject(span.context, Format.TEXT_MAP, msg_header_text_dict)
msg.set_headers(list(msg_header_text_dict.items()))
def start_active_span_follows_from(operation_name: str,
contexts: Collection,
inherit_force_tracing=False):
"""Starts an active opentracing span, with additional references to previous spans
Args:
operation_name: name of the operation represented by the new span
contexts: the previous spans to inherit from
inherit_force_tracing: if set, and any of the previous contexts have had tracing
forced, the new span will also have tracing forced.
"""
if opentracing is None:
return noop_context_manager()
references = [opentracing.follows_from(context) for context in contexts]
scope = start_active_span(operation_name, references=references)
if inherit_force_tracing and any(
is_context_forced_tracing(ctx) for ctx in contexts):
force_tracing(scope.span)
return scope
def _deliver_to(self, consumers):
try:
result = False
for c in consumers:
if c.credit:
msg = self.queue.popleft()
with tracer.start_active_span('dequeue-message', ignore_active_span=True, references=follows_from(msg.qspan.context)):
c.send(msg)
result = True
return result
except IndexError: # no more messages
return False
# for _ in range(100000):
# tr = tracker.SummaryTracker()
# tr = tracker.SummaryTracker()
while True:
with tracer.start_span("TestSpan") as span:
span.set_tag("test_int", 25)
span.set_tag("test_str", "foobar")
span.set_tag("test_double", 1.25)
span.set_tag("test_bool", True)
with tracer.start_span(
"ChildSpan",
references=opentracing.follows_from(span.context),
) as span2:
pass
i += 1
if i % 10000 == 0:
# print(reporter.get_stats())
gc.collect()
# print(len(objgraph.get_leaking_objects()))
# all_objects = muppy.get_objects()
# print("objects", len(all_objects))
# tr.print_diff()
# roots = objgraph.get_leaking_objects()
# print("Roots", len(roots))
def _pub(self,
message,
exchange_name,
routing_key,
parent_span=None,
span_enqueued=None,
dont_close_span=False):
"""
Just send the message. Sends BasicDeliver + header + body.
BECAUSE OF publish THIS CAN GET CALLED BY FOREIGN THREAD.
:param message: Message instance
:param exchange_name: exchange to use
:param routing_key: routing key to use
:type exchange_name: bytes
:param routing_key: bytes
"""
span = None
if parent_span is not None:
import opentracing
span_enqueued.finish()
span = self.cluster.tracer.start_span(
'Sending',
child_of=parent_span,
references=opentracing.follows_from(span_enqueued))
# Break down large bodies
bodies = []
body = memoryview(message.body)
max_body_size = self.connection.frame_max - AMQPBodyFrame.FRAME_SIZE_WITHOUT_PAYLOAD - 16
while len(body) > 0:
bodies.append(body[:max_body_size])
body = body[max_body_size:]
frames_to_send = [
AMQPMethodFrame(
self.channel_id,
BasicPublish(exchange_name, routing_key, False, False)),
AMQPHeaderFrame(self.channel_id, Basic.INDEX, 0, len(message.body),
message.properties)
]
if len(bodies) == 1:
frames_to_send.append(AMQPBodyFrame(self.channel_id, bodies[0]))
if self.content_flow and not self.blocked:
self.connection.send(frames_to_send)
if len(bodies) > 1:
while self.content_flow and not self.blocked and len(
bodies) > 0:
self.connection.send(
[AMQPBodyFrame(self.channel_id, bodies[0])])
del bodies[0]
if not self.content_flow and not self.blocked and len(
bodies) > 0:
for body in bodies:
self.frames_to_send.append(
AMQPBodyFrame(self.channel_id, body))
else:
self.frames_to_send.extend(frames_to_send)
if len(bodies) > 1:
for body in bodies:
self.frames_to_send.append(
AMQPBodyFrame(self.channel_id, body))
if span is not None:
span.finish()
if parent_span is not None and not dont_close_span:
parent_span.finish()
# Create span2, span3 child of span1.
span2 = tracer.start_span(operation_name="span2",
references=child_of(span1.context),
tags=[("span2_key", "span2_val")])
span3 = tracer.start_span(operation_name="span3",
child_of=span1,
tags=[("span3_key", "span3_val")])
time.sleep(2)
span2.finish()
time.sleep(1)
span3.finish()
# Create span4 follows from span3.
span4 = tracer.start_span(operation_name="span4",
references=follows_from(span3.context),
tags=[("span4_key", "span4_val")])
time.sleep(2)
span4.finish()
span5 = tracer.start_span(operation_name="span5")
time.sleep(1)
span5.finish()
time.sleep(1)
# Close the scope
scope.close()
# Close the tracer
tracer.close()
async def _handle_message(self,
record: aiokafka.structs.ConsumerRecord) -> None:
tracer = opentracing.tracer
headers = {x[0]: x[1].decode() for x in record.headers or []}
parent = tracer.extract(opentracing.Format.TEXT_MAP, headers)
context = tracer.start_active_span(
record.topic,
tags={
"message_bus.destination": record.topic,
"message_bus.partition": record.partition,
"message_bus.group_id": self._subscription.group,
},
references=[opentracing.follows_from(parent)],
)
CONSUMER_TOPIC_OFFSET.labels(
group_id=self._subscription.group,
stream_id=record.topic,
partition=record.partition,
).set(record.offset)
# Calculate the time since the message is send until is successfully consumed
lead_time = time.time() - record.timestamp / 1000 # type: ignore
MESSAGE_LEAD_TIME.labels(
stream_id=record.topic,
partition=record.partition,
group_id=self._subscription.group,
).observe(lead_time)
try:
logger.debug(f"Handling msg: {record}")
with CONSUMED_MESSAGE_TIME.labels(
stream_id=record.topic,
partition=record.partition,
group_id=self._subscription.group,
).time():
await self._handler.handle(record, context)
# No error metric
CONSUMED_MESSAGES.labels(
stream_id=record.topic,
error=NOERROR,
partition=record.partition,
group_id=self._subscription.group,
).inc()
self._last_error = False
await self._maybe_commit()
except Exception as err:
self._last_error = True
CONSUMED_MESSAGES.labels(
stream_id=record.topic,
partition=record.partition,
error=err.__class__.__name__,
group_id=self._subscription.group,
).inc()
await self.retry_policy(record=record, exception=err)
# we didn't bubble, so no error here
self._last_error = False
await self._maybe_commit()
finally:
context.close()
await self.emit("message", record=record)
