def start_span(self, context=None, parent_id=None):
if not self._trace:
log('start_span called but no trace is active')
return None
span_id = generate_span_id()
if parent_id:
parent_span_id = parent_id
else:
parent_span_id = self._trace.stack[-1].id if self._trace.stack else None
ev = self._client.new_event(data=self._trace.fields)
if context:
ev.add(data=context)
ev.add(data={
'trace.trace_id': self._trace.id,
'trace.parent_id': parent_span_id,
'trace.span_id': span_id,
})
is_root = len(self._trace.stack) == 0
span = Span(trace_id=self._trace.id, parent_id=parent_span_id,
id=span_id, event=ev, is_root=is_root)
self._trace.stack.append(span)
return span
def __call__(self, name, trace_id=None, parent_id=None):
try:
span = None
if self.get_active_trace_id() and trace_id is None:
span = self.start_span(
context={'name': name}, parent_id=parent_id)
if span:
log('tracer context manager started new span, id = %s',
span.id)
else:
span = self.start_trace(
context={'name': name}, trace_id=trace_id, parent_span_id=parent_id)
if span:
log('tracer context manager started new trace, id = %s',
span.trace_id)
yield span
except Exception as e:
if span:
span.add_context({
"app.exception_type": str(type(e)),
"app.exception_string": stringify_exception(e),
})
raise
finally:
if span:
if span.is_root():
log('tracer context manager ending trace, id = %s',
span.trace_id)
self.finish_trace(span)
else:
log('tracer context manager ending span, id = %s',
span.id)
self.finish_span(span)
else:
log('tracer context manager span for %s was unexpectedly None', name)
def remove_trace_field(self, name):
key = "app.%s" % name
self.remove_context_field(key)
if not self._trace:
log('warning: removing trace field without an active trace')
return
self._trace.fields.pop(key)
def propagate_and_start_trace(self, context, request):
err = None
propagation_context = None
try:
propagation_context = self.parse_http_trace(request)
except Exception:
err = sys.exc_info()[0]
log('error: http_trace_parser_hook returned exception: %s',
sys.exc_info()[0])
if propagation_context:
return self.start_trace(
context=context,
trace_id=propagation_context.trace_id,
parent_span_id=propagation_context.parent_id,
dataset=propagation_context.dataset)
for k, v in propagation_context.trace_fields:
self.add_trace_field(k, v)
else:
# Initialize a new trace from scratch
if err is not None:
context['parser_hook_error'] = repr(err)
return self.start_trace(context,
trace_id=None,
parent_span_id=None)
pass
def marshal_trace_context(self):
if not self._trace:
log('warning: marshal_trace_context called, but no active trace')
return
return marshal_trace_context(self._trace.id, self._trace.stack[-1].id,
self._trace.fields)
def unmarshal_trace_context(trace_context):
# the first value is the trace payload version
# at this time there is only one version, but we should warn
# if another version comes through
version, data = trace_context.split(';', 1)
if version != "1":
log('warning: trace_context version %s is unsupported', version)
return None, None, None
kv_pairs = data.split(',')
trace_id, parent_id, context = None, None, None
# For version 1, we expect three kv pairs. If there's anything else, the
# payload is malformed and we do nothing.
if len(kv_pairs) == 3:
for pair in kv_pairs:
k, v = pair.split('=', 1)
if k == 'trace_id':
trace_id = v
elif k == 'parent_id':
parent_id = v
elif k == 'context':
context = json.loads(base64.b64decode(v.encode()).decode())
return trace_id, parent_id, context
def unmarshal_trace_context(trace_context):
# the first value is the trace payload version
# at this time there is only one version, but we should warn
# if another version comes through
version, data = trace_context.split(';', 1)
if version != "1":
log('warning: trace_context version %s is unsupported', version)
return None, None, None
kv_pairs = data.split(',')
trace_id, parent_id, context = None, None, None
# Some beelines send "dataset" but we do not handle that yet
for pair in kv_pairs:
k, v = pair.split('=', 1)
if k == 'trace_id':
trace_id = v
elif k == 'parent_id':
parent_id = v
elif k == 'context':
context = json.loads(base64.b64decode(v.encode()).decode())
# context should be a dict
if context is None:
context = {}
return trace_id, parent_id, context
def add_trace_field(self, name, value):
# prefix with app to avoid key conflicts
key = "app.%s" % name
# also add to current span
self.add_context_field(key, value)
if not self._trace:
log('warning: adding trace field without an active trace')
return
self._trace.fields[key] = value
def start_trace(self, context=None, trace_id=None, parent_span_id=None, dataset=None):
if trace_id:
if self._trace:
log('warning: start_trace got explicit trace_id but we are already in a trace. '
'starting new trace with id = %s', trace_id)
self._trace = Trace(trace_id, dataset)
else:
self._trace = Trace(generate_trace_id(), dataset)
# start the root span
return self.start_span(context=context, parent_id=parent_span_id)
def finish_span(self, span):
# avoid exception if called with None
if span is None:
return
# send the span's event. Even if the stack is in an unhealthy state,
# it's probably better to send event data than not
if span.event:
if self._trace:
if self._trace.dataset:
span.event.dataset = self._trace.dataset
# add the trace's rollup fields to the root span
if span.is_root():
for k, v in self._trace.rollup_fields.items():
span.event.add_field(k, v)
for k, v in span.rollup_fields.items():
span.event.add_field(k, v)
# propagate trace fields that may have been added in later spans
for k, v in self._trace.fields.items():
# don't overwrite existing values because they may be different
if k not in span.event.fields():
span.event.add_field(k, v)
duration = datetime.datetime.now() - span.event.start_time
duration_ms = duration.total_seconds() * 1000.0
span.event.add_field('duration_ms', duration_ms)
self._run_hooks_and_send(span)
else:
log('warning: span has no event, was it initialized correctly?')
if not self._trace:
log('warning: span finished without an active trace')
return
if span.trace_id != self._trace.id:
log(
'warning: finished span called for span in inactive trace. '
'current trace_id = %s, span trace_id = %s', self._trace.id,
span.trace_id)
return
if not self._trace.stack:
log('warning: finish span called but stack is empty')
return
if self._trace.stack[-1].id != span.id:
log('warning: finished span is not the currently active span')
return
self._trace.stack.pop()
def add_rollup_field(self, name, value):
value = float(value)
span = self.get_active_span()
if span:
span.rollup_fields[name] += value
if not self._trace:
log('warning: adding rollup field without an active trace')
return
self._trace.rollup_fields["rollup.%s" % name] += value
def _run_hooks_and_send(self, span):
''' internal - run any defined hooks on the event and send
kind of hacky: we fetch the hooks from the beeline, but they are only
used here. Pass them to the tracer implementation?
'''
presampled = False
if self.sampler_hook:
log("executing sampler hook on event ev = %s", span.event.fields())
keep, new_rate = self.sampler_hook(span.event.fields())
if not keep:
log("skipping event due to sampler hook sampling ev = %s",
span.event.fields())
return
span.event.sample_rate = new_rate
presampled = True
if self.presend_hook:
log("executing presend hook on event ev = %s", span.event.fields())
self.presend_hook(span.event.fields())
if presampled:
log("enqueuing presampled event ev = %s", span.event.fields())
span.event.send_presampled()
elif _should_sample(span.trace_id, span.event.sample_rate):
# if our sampler hook wasn't used, use deterministic sampling
span.event.send_presampled()
def start_trace(self, context=None, trace_id=None, parent_span_id=None):
if trace_id:
if self._state.trace_id:
log('warning: start_trace got explicit trace_id but we are already in a trace. '
'starting new trace with id = %s', trace_id)
self._state.trace_id = trace_id
else:
self._state.trace_id = str(uuid.uuid4())
# reset our stack and context on new traces
self._state.stack = []
self._state.trace_fields = {}
# start the root span
return self.start_span(context=context, parent_id=parent_span_id)
def add_trace_field(self, name, value):
# prefix with app to avoid key conflicts
# add the app prefix if it's missing
if (type(name) == str and not name.startswith("app.")) or type(name) != str:
key = "app.%s" % name
else:
key = name
# also add to current span
self.add_context_field(key, value)
if not self._trace:
log('warning: adding trace field without an active trace')
return
self._trace.fields[key] = value
