def collect(self):
"""
Collects all metrics attached to this metricset, and returns it as a generator
with one or more elements. More than one element is returned if labels are used.
The format of the return value should be
{
"samples": {"metric.name": {"value": some_float}, ...},
"timestamp": unix epoch in microsecond precision
}
"""
self.before_collect()
timestamp = int(time.time() * 1000000)
samples = defaultdict(dict)
if self._counters:
# iterate over a copy of the dict to avoid threading issues, see #717
for (name, labels), c in compat.iteritems(self._counters.copy()):
if c is not noop_metric:
val = c.val
if val or not c.reset_on_collect:
samples[labels].update({name: {"value": val}})
if c.reset_on_collect:
c.reset()
if self._gauges:
for (name, labels), g in compat.iteritems(self._gauges.copy()):
if g is not noop_metric:
val = g.val
if val or not g.reset_on_collect:
samples[labels].update({name: {"value": val}})
if g.reset_on_collect:
g.reset()
if self._timers:
for (name, labels), t in compat.iteritems(self._timers.copy()):
if t is not noop_metric:
val, count = t.val
if val or not t.reset_on_collect:
samples[labels].update(
{name + ".sum.us": {
"value": int(val * 1000000)
}})
samples[labels].update(
{name + ".count": {
"value": count
}})
if t.reset_on_collect:
t.reset()
if samples:
for labels, sample in compat.iteritems(samples):
result = {"samples": sample, "timestamp": timestamp}
if labels:
result["tags"] = {k: v for k, v in labels}
yield self.before_yield(result)
def update_config(self):
if not self.transport:
logger.warning("No transport set for config updates, skipping")
return
logger.debug("Checking for new config...")
keys = {"service": {"name": self.service_name}}
if self.environment:
keys["service"]["environment"] = self.environment
new_version, new_config, next_run = self.transport.get_config(
self.config_version, keys)
if new_version and new_config:
errors = self.update(new_version, **new_config)
if errors:
logger.error("Error applying new configuration: %s",
repr(errors))
else:
logger.info(
"Applied new configuration: %s",
"; ".join("%s=%s" %
(compat.text_type(k), compat.text_type(v))
for k, v in compat.iteritems(new_config)),
)
elif new_version == self.config_version:
logger.debug("Remote config unchanged")
elif not new_config and self.changed:
logger.debug("Remote config disappeared, resetting to original")
self.reset()
return next_run
def start_span(
self, operation_name=None, child_of=None, references=None, tags=None, start_time=None, ignore_active_span=False
):
if isinstance(child_of, OTSpanContext):
parent_context = child_of
elif isinstance(child_of, OTSpan):
parent_context = child_of.context
elif references and references[0].type == ReferenceType.CHILD_OF:
parent_context = references[0].referenced_context
else:
parent_context = None
transaction = traces.execution_context.get_transaction()
if not transaction:
trace_parent = parent_context.trace_parent if parent_context else None
transaction = self._agent.begin_transaction("custom", trace_parent=trace_parent)
transaction.name = operation_name
span_context = OTSpanContext(trace_parent=transaction.trace_parent)
ot_span = OTSpan(self, span_context, transaction)
else:
# to allow setting an explicit parent span, we check if the parent_context is set
# and if it is a span. In all other cases, the parent is found implicitly through the
# execution context.
parent_span_id = (
parent_context.span.zuqa_ref.id
if parent_context and parent_context.span and not parent_context.span.is_transaction
else None
)
span = transaction._begin_span(operation_name, None, parent_span_id=parent_span_id)
trace_parent = parent_context.trace_parent if parent_context else transaction.trace_parent
span_context = OTSpanContext(trace_parent=trace_parent.copy_from(span_id=span.id))
ot_span = OTSpan(self, span_context, span)
if tags:
for k, v in compat.iteritems(tags):
ot_span.set_tag(k, v)
return ot_span
def __init__(self, app, config, client_cls=Client):
client_config = {
key[11:]: val
for key, val in compat.iteritems(config) if key.startswith("zuqa.")
}
client = client_cls(**client_config)
super(ZUQA, self).__init__(app, client)
def add_context_lines_to_frames(client, event):
# divide frames up into source files before reading from disk. This should help
# with utilizing the disk cache better
#
# TODO: further optimize by only opening each file once and reading all needed source
# TODO: blocks at once.
per_file = defaultdict(list)
_process_stack_frames(
event,
lambda frame: per_file[frame["context_metadata"][0]].append(frame)
if "context_metadata" in frame else None,
)
for filename, frames in compat.iteritems(per_file):
for frame in frames:
# context_metadata key has been set in zuqa.utils.stacks.get_frame_info for
# all frames for which we should gather source code context lines
fname, lineno, context_lines, loader, module_name = frame.pop(
"context_metadata")
pre_context, context_line, post_context = get_lines_from_file(
fname, lineno, context_lines, loader, module_name)
if context_line:
frame["pre_context"] = pre_context
frame["context_line"] = context_line
frame["post_context"] = post_context
return event
def end(self, skip_frames=0, duration=None):
self.duration = duration if duration is not None else (_time_func() - self.start_time)
if self._transaction_metrics:
self._transaction_metrics.timer(
"transaction.duration",
reset_on_collect=True,
**{"transaction.name": self.name, "transaction.type": self.transaction_type}
).update(self.duration)
if self._breakdown:
for (span_type, span_subtype), timer in compat.iteritems(self._span_timers):
labels = {
"span.type": span_type,
"transaction.name": self.name,
"transaction.type": self.transaction_type,
}
if span_subtype:
labels["span.subtype"] = span_subtype
self._breakdown.timer("span.self_time", reset_on_collect=True, **labels).update(*timer.val)
labels = {"transaction.name": self.name, "transaction.type": self.transaction_type}
if self.is_sampled:
self._breakdown.counter("transaction.breakdown.count", reset_on_collect=True, **labels).inc()
self._breakdown.timer(
"span.self_time",
reset_on_collect=True,
**{"span.type": "app", "transaction.name": self.name, "transaction.type": self.transaction_type}
).update(self.duration - self._child_durations.duration)
def transform(value, stack=None, context=None):
# TODO: make this extendable
if context is None:
context = {}
if stack is None:
stack = []
objid = id(value)
if objid in context:
return "<...>"
context[objid] = 1
transform_rec = lambda o: transform(o, stack + [value], context)
if any(value is s for s in stack):
ret = "cycle"
elif isinstance(value, (tuple, list, set, frozenset)):
try:
ret = type(value)(transform_rec(o) for o in value)
except Exception:
# We may be dealing with a namedtuple
class value_type(list):
__name__ = type(value).__name__
ret = value_type(transform_rec(o) for o in value)
elif isinstance(value, uuid.UUID):
ret = repr(value)
elif isinstance(value, dict):
ret = dict((to_unicode(k), transform_rec(v))
for k, v in compat.iteritems(value))
elif isinstance(value, compat.text_type):
ret = to_unicode(value)
elif isinstance(value, compat.binary_type):
ret = to_string(value)
elif not isinstance(value,
compat.class_types) and _has_zuqa_metadata(value):
ret = transform_rec(value.__zuqa__())
elif isinstance(value, bool):
ret = bool(value)
elif isinstance(value, float):
ret = float(value)
elif isinstance(value, int):
ret = int(value)
elif compat.PY2 and isinstance(value, long): # noqa F821
ret = long(value) # noqa F821
elif value is not None:
try:
ret = transform(repr(value))
except Exception:
# It's common case that a model's __unicode__ definition may try to query the database
# which if it was not cleaned up correctly, would hit a transaction aborted exception
ret = u"<BadRepr: %s>" % type(value)
else:
ret = None
del context[objid]
return ret
def get_headers(environ):
"""
Returns only proper HTTP headers.
"""
for key, value in compat.iteritems(environ):
key = str(key)
if key.startswith("HTTP_") and key not in ("HTTP_CONTENT_TYPE",
"HTTP_CONTENT_LENGTH"):
yield key[5:].replace("_", "-").lower(), value
elif key in ("CONTENT_TYPE", "CONTENT_LENGTH"):
yield key.replace("_", "-").lower(), value
def test_metrics_reset_after_collect(zuqa_client):
zuqa_client.begin_transaction("request")
with zuqa.capture_span("test",
span_type="db",
span_subtype="mysql",
duration=5):
pass
zuqa_client.end_transaction("test", "OK", duration=15)
breakdown = zuqa_client._metrics.get_metricset(
"zuqa.metrics.sets.breakdown.BreakdownMetricSet")
transaction_metrics = zuqa_client._metrics.get_metricset(
"zuqa.metrics.sets.transactions.TransactionsMetricSet")
for metricset in (breakdown, transaction_metrics):
for labels, c in compat.iteritems(metricset._counters):
assert c.val != 0
for labels, t in compat.iteritems(metricset._timers):
assert t.val != (0, 0)
list(metricset.collect())
for labels, c in compat.iteritems(metricset._counters):
assert c.val == 0
for labels, t in compat.iteritems(metricset._timers):
assert t.val == (0, 0)
def enforce_label_format(labels):
"""
Enforces label format:
* dots, double quotes or stars in keys are replaced by underscores
* string values are limited to a length of 1024 characters
* values can only be of a limited set of types
:param labels: a dictionary of labels
:return: a new dictionary with sanitized keys/values
"""
new = {}
for key, value in compat.iteritems(labels):
if not isinstance(value, LABEL_TYPES):
value = keyword_field(compat.text_type(value))
new[LABEL_RE.sub("_", compat.text_type(key))] = value
return new
def shorten(var, list_length=50, string_length=200, dict_length=50):
"""
Shorten a given variable based on configurable maximum lengths, leaving
breadcrumbs in the object to show that it was shortened.
For strings, truncate the string to the max length, and append "..." so
the user knows data was lost.
For lists, truncate the list to the max length, and append two new strings
to the list: "..." and "(<x> more elements)" where <x> is the number of
elements removed.
For dicts, truncate the dict to the max length (based on number of key/value
pairs) and add a new (key, value) pair to the dict:
("...", "(<x> more elements)") where <x> is the number of key/value pairs
removed.
:param var: Variable to be shortened
:param list_length: Max length (in items) of lists
:param string_length: Max length (in characters) of strings
:param dict_length: Max length (in key/value pairs) of dicts
:return: Shortened variable
"""
var = transform(var)
if isinstance(var, compat.string_types) and len(var) > string_length:
var = var[:string_length - 3] + "..."
elif isinstance(var,
(list, tuple, set, frozenset)) and len(var) > list_length:
# TODO: we should write a real API for storing some metadata with vars when
# we get around to doing ref storage
var = list(var)[:list_length] + [
"...", "(%d more elements)" % (len(var) - list_length, )
]
elif isinstance(var, dict) and len(var) > dict_length:
trimmed_tuples = [
(k, v)
for (k, v) in itertools.islice(compat.iteritems(var), dict_length)
]
if "<truncated>" not in var:
trimmed_tuples += [
("<truncated>",
"(%d more elements)" % (len(var) - dict_length))
]
var = dict(trimmed_tuples)
return var
def collect(self):
"""
Collect metrics from all registered metric sets and queues them for sending
:return:
"""
if self.collect_actively:
if self.client.config.is_recording:
logger.debug("Collecting metrics")
for _, metricset in compat.iteritems(self._metricsets):
for data in metricset.collect():
self.transaction_metrics_data.append(data)
elif len(self.transaction_metrics_data) > 0:
self.client.queue(constants.TRANSACTION_METRICSET, {
"url": self.last_transaction_name,
"metricsets": self.transaction_metrics_data
},
flush=True)
self.transaction_metrics_data = []
def varmap(func, var, context=None, name=None):
"""
Executes ``func(key_name, value)`` on all values,
recursively discovering dict and list scoped
values.
"""
if context is None:
context = set()
objid = id(var)
if objid in context:
return func(name, "<...>")
context.add(objid)
if isinstance(var, dict):
ret = func(
name,
dict((k, varmap(func, v, context, k))
for k, v in compat.iteritems(var)))
elif isinstance(var, (list, tuple)):
ret = func(name, [varmap(func, f, context, name) for f in var])
else:
ret = func(name, var)
context.remove(objid)
return ret
def _emit(self, record, **kwargs):
data = {}
for k, v in compat.iteritems(record.__dict__):
if "." not in k and k not in ("culprit",):
continue
data[k] = v
stack = getattr(record, "stack", None)
if stack is True:
stack = iter_stack_frames(config=self.client.config)
if stack:
frames = []
started = False
last_mod = ""
for item in stack:
if isinstance(item, (list, tuple)):
frame, lineno = item
else:
frame, lineno = item, item.f_lineno
if not started:
f_globals = getattr(frame, "f_globals", {})
module_name = f_globals.get("__name__", "")
if last_mod.startswith("logging") and not module_name.startswith("logging"):
started = True
else:
last_mod = module_name
continue
frames.append((frame, lineno))
stack = frames
custom = getattr(record, "data", {})
# Add in all of the data from the record that we aren't already capturing
for k in record.__dict__.keys():
if k in (
"stack",
"name",
"args",
"msg",
"levelno",
"exc_text",
"exc_info",
"data",
"created",
"levelname",
"msecs",
"relativeCreated",
):
continue
if k.startswith("_"):
continue
custom[k] = record.__dict__[k]
# If there's no exception being processed,
# exc_info may be a 3-tuple of None
# http://docs.python.org/library/sys.html#sys.exc_info
if record.exc_info and all(record.exc_info):
handler = self.client.get_handler("zuqa.events.Exception")
exception = handler.capture(self.client, exc_info=record.exc_info)
else:
exception = None
return self.client.capture(
"Message",
param_message={"message": compat.text_type(record.msg), "params": record.args},
stack=stack,
custom=custom,
exception=exception,
level=record.levelno,
logger_name=record.name,
**kwargs
)
def get_data_from_request(self, request, event_type):
result = {
"env": dict(get_environ(request.META)),
"method": request.method,
"socket": {
"remote_address": request.META.get("REMOTE_ADDR"),
"encrypted": request.is_secure()
},
"cookies": dict(request.COOKIES),
}
if self.config.capture_headers:
request_headers = dict(get_headers(request.META))
for key, value in request_headers.items():
if isinstance(value, (int, float)):
request_headers[key] = str(value)
result["headers"] = request_headers
if request.method in constants.HTTP_WITH_BODY:
capture_body = self.config.capture_body in ("all", event_type)
if not capture_body:
result["body"] = "[REDACTED]"
else:
content_type = request.META.get("CONTENT_TYPE")
if content_type == "application/x-www-form-urlencoded":
data = compat.multidict_to_dict(request.POST)
elif content_type and content_type.startswith(
"multipart/form-data"):
data = compat.multidict_to_dict(request.POST)
if request.FILES:
data["_files"] = {
field: file.name
for field, file in compat.iteritems(request.FILES)
}
else:
try:
data = request.body
except Exception as e:
self.logger.debug("Can't capture request body: %s",
compat.text_type(e))
data = "<unavailable>"
if data is not None:
result["body"] = data
if hasattr(request, "get_raw_uri"):
# added in Django 1.9
url = request.get_raw_uri()
else:
try:
# Requires host to be in ALLOWED_HOSTS, might throw a
# DisallowedHost exception
url = request.build_absolute_uri()
except DisallowedHost:
# We can't figure out the real URL, so we have to set it to
# DisallowedHost
result["url"] = {"full": "DisallowedHost"}
url = None
if url:
result["url"] = get_url_dict(url)
return result
def get_frame_info(
frame,
lineno,
with_locals=True,
library_frame_context_lines=None,
in_app_frame_context_lines=None,
include_paths_re=None,
exclude_paths_re=None,
locals_processor_func=None,
):
# Support hidden frames
f_locals = getattr(frame, "f_locals", {})
if _getitem_from_frame(f_locals, "__traceback_hide__"):
return None
f_globals = getattr(frame, "f_globals", {})
loader = f_globals.get("__loader__")
module_name = f_globals.get("__name__")
f_code = getattr(frame, "f_code", None)
if f_code:
abs_path = frame.f_code.co_filename
function = frame.f_code.co_name
else:
abs_path = None
function = None
# Try to pull a relative file path
# This changes /foo/site-packages/baz/bar.py into baz/bar.py
try:
base_filename = sys.modules[module_name.split(".", 1)[0]].__file__
filename = abs_path.split(base_filename.rsplit(os.path.sep, 2)[0],
1)[-1].lstrip(os.path.sep)
except Exception:
filename = abs_path
if not filename:
filename = abs_path
frame_result = {
"abs_path":
abs_path,
"filename":
filename,
"module":
module_name,
"function":
function,
"lineno":
lineno,
"library_frame":
is_library_frame(abs_path, include_paths_re, exclude_paths_re),
}
context_lines = library_frame_context_lines if frame_result[
"library_frame"] else in_app_frame_context_lines
if context_lines and lineno is not None and abs_path:
# context_metadata will be processed by zuqa.processors.add_context_lines_to_frames.
# This ensures that blocking operations (reading from source files) happens on the background
# processing thread.
frame_result["context_metadata"] = (abs_path, lineno,
int(context_lines / 2), loader,
module_name)
if with_locals:
if f_locals is not None and not isinstance(f_locals, dict):
# XXX: Genshi (and maybe others) have broken implementations of
# f_locals that are not actually dictionaries
try:
f_locals = to_dict(f_locals)
except Exception:
f_locals = "<invalid local scope>"
if locals_processor_func:
f_locals = {
varname: locals_processor_func(var)
for varname, var in compat.iteritems(f_locals)
}
frame_result["vars"] = transform(f_locals)
return frame_result
def auth_headers(self):
headers = super(Transport, self).auth_headers
return {k.encode("ascii"): v.encode("ascii") for k, v in compat.iteritems(headers)}
def _build_msg_for_logging(self,
event_type,
date=None,
context=None,
custom=None,
stack=None,
handled=True,
**kwargs):
"""
Captures, processes and serializes an event into a dict object
"""
transaction = execution_context.get_transaction()
span = execution_context.get_span()
if transaction:
transaction_context = deepcopy(transaction.context)
else:
transaction_context = {}
event_data = {}
if custom is None:
custom = {}
if date is not None:
warnings.warn(
"The date argument is no longer evaluated and will be removed in a future release",
DeprecationWarning)
date = time.time()
if stack is None:
stack = self.config.auto_log_stacks
if context:
transaction_context.update(context)
context = transaction_context
else:
context = transaction_context
event_data["context"] = context
if transaction and transaction.labels:
context["tags"] = deepcopy(transaction.labels)
# if '.' not in event_type:
# Assume it's a builtin
event_type = "zuqa.events.%s" % event_type
handler = self.get_handler(event_type)
result = handler.capture(self, **kwargs)
if self._filter_exception_type(result):
return
# data (explicit) culprit takes over auto event detection
culprit = result.pop("culprit", None)
if custom.get("culprit"):
culprit = custom.pop("culprit")
for k, v in compat.iteritems(result):
if k not in event_data:
event_data[k] = v
log = event_data.get("log", {})
if stack and "stacktrace" not in log:
if stack is True:
frames = stacks.iter_stack_frames(skip=3, config=self.config)
else:
frames = stack
frames = stacks.get_stack_info(
frames,
with_locals=self.config.collect_local_variables
in ("errors", "all"),
library_frame_context_lines=self.config.
source_lines_error_library_frames,
in_app_frame_context_lines=self.config.
source_lines_error_app_frames,
include_paths_re=self.include_paths_re,
exclude_paths_re=self.exclude_paths_re,
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
dict_length=self.config.local_var_dict_max_length,
),
local_var,
),
)
log["stacktrace"] = frames
if "stacktrace" in log and not culprit:
culprit = stacks.get_culprit(log["stacktrace"],
self.config.include_paths,
self.config.exclude_paths)
if "level" in log and isinstance(log["level"], compat.integer_types):
log["level"] = logging.getLevelName(log["level"]).lower()
if log:
event_data["log"] = log
if culprit:
event_data["culprit"] = culprit
if "custom" in context:
context["custom"].update(custom)
else:
context["custom"] = custom
# Make sure all data is coerced
event_data = transform(event_data)
if "exception" in event_data:
event_data["exception"]["handled"] = bool(handled)
event_data["timestamp"] = int(date * 1000000)
if transaction:
if transaction.trace_parent:
event_data["trace_id"] = transaction.trace_parent.trace_id
# parent id might already be set in the handler
event_data.setdefault("parent_id",
span.id if span else transaction.id)
event_data["transaction_id"] = transaction.id
event_data["transaction"] = {
"sampled": transaction.is_sampled,
"type": transaction.transaction_type
}
return event_data
def _labels_to_key(self, labels):
return tuple((k, compat.text_type(v))
for k, v in sorted(compat.iteritems(labels)))
