def test_drop_reason_buffer_full(self):
statsd = mock.Mock()
writer_metrics_reset = mock.Mock()
writer = AgentWriter(agent_url="http://asdf:1234",
buffer_size=5300,
dogstatsd=statsd,
report_metrics=False)
writer._metrics_reset = writer_metrics_reset
for i in range(10):
writer.write([
Span(tracer=None,
name="name",
trace_id=i,
span_id=j,
parent_id=j - 1 or None) for j in range(5)
])
writer.write([
Span(tracer=None,
name="a",
trace_id=i,
span_id=j,
parent_id=j - 1 or None) for j in range(5)
])
writer.stop()
writer.join()
writer_metrics_reset.assert_called_once()
assert 1 == writer._metrics["buffer.dropped.traces"]["count"]
assert ["reason:full"
] == writer._metrics["buffer.dropped.traces"]["tags"]
def test_metrics_trace_too_big(self):
statsd = mock.Mock()
writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=True)
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.write(
[Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)]
)
writer.stop()
writer.join()
statsd.distribution.assert_has_calls(
[
mock.call("datadog.tracer.buffer.accepted.traces", 10, tags=[]),
mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]),
mock.call("datadog.tracer.buffer.dropped.traces", 1, tags=["reason:t_too_big"]),
mock.call("datadog.tracer.buffer.dropped.bytes", AnyInt(), tags=["reason:t_too_big"]),
mock.call("datadog.tracer.http.requests", writer.RETRY_ATTEMPTS, tags=[]),
mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]),
mock.call("datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]),
],
any_order=True,
)
def test_metrics_bad_endpoint(self):
statsd = mock.Mock()
writer = AgentWriter(dogstatsd=statsd,
report_metrics=True,
hostname="asdf",
port=1234)
for i in range(10):
writer.write([
Span(tracer=None,
name="name",
trace_id=i,
span_id=j,
parent_id=j - 1 or None) for j in range(5)
])
writer.stop()
writer.join()
statsd.increment.assert_has_calls([
mock.call("datadog.tracer.http.requests"),
])
statsd.distribution.assert_has_calls(
[
mock.call("datadog.tracer.buffer.accepted.traces", 10,
tags=[]),
mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]),
mock.call("datadog.tracer.http.requests", 1, tags=[]),
mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]),
mock.call(
"datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]),
],
any_order=True,
)
def test_write_sync(self):
statsd = mock.Mock()
writer = AgentWriter(agent_url="http://asdf:1234",
dogstatsd=statsd,
report_metrics=True,
sync_mode=True)
writer.write([
Span(tracer=None,
name="name",
trace_id=1,
span_id=j,
parent_id=j - 1 or None) for j in range(5)
])
statsd.distribution.assert_has_calls(
[
mock.call("datadog.tracer.buffer.accepted.traces", 1, tags=[]),
mock.call("datadog.tracer.buffer.accepted.spans", 5, tags=[]),
mock.call("datadog.tracer.http.requests",
writer.RETRY_ATTEMPTS,
tags=[]),
mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]),
mock.call(
"datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]),
],
any_order=True,
)
def test_drop_reason_encoding_error(self):
n_traces = 10
statsd = mock.Mock()
writer_encoder = mock.Mock()
writer_encoder.__len__ = (
lambda *args: n_traces).__get__(writer_encoder)
writer_metrics_reset = mock.Mock()
writer_encoder.encode.side_effect = Exception
writer = AgentWriter(agent_url="http://asdf:1234",
dogstatsd=statsd,
report_metrics=False)
writer._encoder = writer_encoder
writer._metrics_reset = writer_metrics_reset
for i in range(n_traces):
writer.write([
Span(tracer=None,
name="name",
trace_id=i,
span_id=j,
parent_id=j - 1 or None) for j in range(5)
])
writer.stop()
writer.join()
writer_metrics_reset.assert_called_once()
assert 10 == writer._metrics["encoder.dropped.traces"]["count"]
def test_double_stop():
# Ensure double stopping doesn't result in an exception.
writer = AgentWriter(agent_url="http://dne:1234")
writer.write([])
assert writer.status == service.ServiceStatus.RUNNING
writer.stop()
assert writer.status == service.ServiceStatus.STOPPED
writer.stop()
assert writer.status == service.ServiceStatus.STOPPED
def test_double_stop():
# Ensure double stopping doesn't result in an exception.
writer = AgentWriter(agent_url="http://dne:1234")
writer.write([])
assert writer.started
writer.stop()
assert writer.started
assert not writer.is_alive()
writer.stop()
assert writer.started
assert not writer.is_alive()
def test_flush_queue_raise():
writer = AgentWriter(agent_url="http://dne:1234")
# Should not raise
writer.write([])
writer.flush_queue(raise_exc=False)
error = OSError if PY3 else IOError
with pytest.raises(error):
writer.write([])
writer.flush_queue(raise_exc=True)
def test_flush_log(caplog):
caplog.set_level(logging.INFO)
writer = AgentWriter(agent.get_trace_url())
with mock.patch("ddtrace.internal.writer.log") as log:
writer.write([])
writer.flush_queue(raise_exc=True)
calls = [
mock.call(logging.DEBUG, "sent %s in %.5fs", AnyStr(), AnyFloat())
]
log.log.assert_has_calls(calls)
def test_metrics_disabled(self):
statsd = mock.Mock()
writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False)
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.stop()
writer.join()
statsd.increment.assert_not_called()
statsd.distribution.assert_not_called()
def create_worker(self, filters=None, api_class=DummyAPI):
self.dogstatsd = mock.Mock()
worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters)
self.api = api_class()
worker.api = self.api
for i in range(self.N_TRACES):
worker.write([
Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None)
for j in range(7)
])
worker.stop()
worker.join()
return worker
def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False):
with self.override_global_config(dict(health_metrics_enabled=enable_stats)):
self.dogstatsd = mock.Mock()
worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters)
worker._STATS_EVERY_INTERVAL = 1
self.api = api_class()
worker.api = self.api
for i in range(self.N_TRACES):
worker.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(7)]
)
worker.stop()
worker.join()
return worker
def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False):
self.dogstatsd = mock.Mock()
worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters)
worker._ENABLE_STATS = enable_stats
worker._STATS_EVERY_INTERVAL = 1
self.api = api_class()
worker.api = self.api
for i in range(self.N_TRACES):
worker.write([
Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None)
for j in range(7)
])
worker.stop()
worker.join()
return worker
def create_worker(self, filters):
worker = AgentWriter(filters=filters)
self.api = DummmyAPI()
worker.api = self.api
for i in range(self.N_TRACES):
worker.write([
Span(tracer=None,
name='name',
trace_id=i,
span_id=j,
parent_id=j - 1 or None) for j in range(7)
])
worker.stop()
worker.join()
return worker
def test_drop_reason_bad_endpoint(self):
statsd = mock.Mock()
writer_metrics_reset = mock.Mock()
writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False)
writer._metrics_reset = writer_metrics_reset
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.stop()
writer.join()
writer_metrics_reset.assert_called_once()
assert 1 == writer._metrics["http.errors"]["count"]
assert 10 == writer._metrics["http.dropped.traces"]["count"]
def test_drop_reason_trace_too_big(self):
statsd = mock.Mock()
writer_metrics_reset = mock.Mock()
writer = AgentWriter(dogstatsd=statsd, report_metrics=False, hostname="asdf", port=1234)
writer._metrics_reset = writer_metrics_reset
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.write(
[Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)]
)
writer.stop()
writer.join()
writer_metrics_reset.assert_called_once()
assert 1 == writer._metrics["buffer.dropped.traces"]["count"]
assert ["reason:t_too_big"] == writer._metrics["buffer.dropped.traces"]["tags"]
def test_drop_reason_encoding_error(self):
statsd = mock.Mock()
writer_encoder = mock.Mock()
writer_metrics_reset = mock.Mock()
writer_encoder.encode_trace.side_effect = Exception
writer = AgentWriter(dogstatsd=statsd, report_metrics=False, hostname="asdf", port=1234)
writer._encoder = writer_encoder
writer._metrics_reset = writer_metrics_reset
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.stop()
writer.join()
writer_metrics_reset.assert_called_once()
assert 10 == writer._metrics["encoder.dropped.traces"]["count"]
def test_flush_log(caplog, encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
caplog.set_level(logging.INFO)
writer = AgentWriter(agent.get_trace_url())
with mock.patch("ddtrace.internal.writer.log") as log:
writer.write([])
writer.flush_queue(raise_exc=True)
calls = [
mock.call(
logging.DEBUG,
"sent %s in %.5fs to %s",
AnyStr(),
AnyFloat(),
writer.agent_url,
)
]
log.log.assert_has_calls(calls)
def test_flush_log(caplog, encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
caplog.set_level(logging.INFO)
writer = AgentWriter(agent.get_trace_url())
with mock.patch("ddtrace.internal.writer.log") as log:
writer.write([])
writer.flush_queue(raise_exc=True)
# for latest agent, default to v0.3 since no priority sampler is set
expected_encoding = "v0.3" if AGENT_VERSION == "v5" else (encoding
or "v0.3")
calls = [
mock.call(
logging.DEBUG,
"sent %s in %.5fs to %s",
AnyStr(),
AnyFloat(),
"{}/{}/traces".format(writer.agent_url, expected_encoding),
)
]
log.log.assert_has_calls(calls)
def test_keep_rate(self):
statsd = mock.Mock()
writer_run_periodic = mock.Mock()
writer_put = mock.Mock()
writer_put.return_value = Response(status=200)
writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False)
writer.run_periodic = writer_run_periodic
writer._put = writer_put
traces = [
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
for i in range(4)
]
traces_too_big = [
[Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)]
for i in range(4)
]
# 1. We write 4 traces successfully.
for trace in traces:
writer.write(trace)
writer.flush_queue()
payload = msgpack.unpackb(writer_put.call_args.args[0])
# No previous drops.
assert 0.0 == writer._drop_sma.get()
# 4 traces written.
assert 4 == len(payload)
# 100% of traces kept (refers to the past).
# No traces sent before now so 100% kept.
for trace in payload:
assert 1.0 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
# 2. We fail to write 4 traces because of size limitation.
for trace in traces_too_big:
writer.write(trace)
writer.flush_queue()
# 50% of traces were dropped historically.
# 4 successfully written before and 4 dropped now.
assert 0.5 == writer._drop_sma.get()
# put not called since no new traces are available.
writer_put.assert_called_once()
# 3. We write 2 traces successfully.
for trace in traces[:2]:
writer.write(trace)
writer.flush_queue()
payload = msgpack.unpackb(writer_put.call_args.args[0])
# 40% of traces were dropped historically.
assert 0.4 == writer._drop_sma.get()
# 2 traces written.
assert 2 == len(payload)
# 50% of traces kept (refers to the past).
# We had 4 successfully written and 4 dropped.
for trace in payload:
assert 0.5 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
# 4. We write 1 trace successfully and fail to write 3.
writer.write(traces[0])
for trace in traces_too_big[:3]:
writer.write(trace)
writer.flush_queue()
payload = msgpack.unpackb(writer_put.call_args.args[0])
# 50% of traces were dropped historically.
assert 0.5 == writer._drop_sma.get()
# 1 trace written.
assert 1 == len(payload)
# 60% of traces kept (refers to the past).
# We had 4 successfully written, then 4 dropped, then 2 written.
for trace in payload:
assert 0.6 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
def collect(tracer):
"""Collect system and library information into a serializable dict."""
# The tracer doesn't actually maintain a hostname/port, instead it stores
# it on the possibly None writer which actually stores it on an API object.
# Note that the tracer DOES have hostname and port attributes that it
# sets to the defaults and ignores afterwards.
if tracer.writer and isinstance(tracer.writer, LogWriter):
agent_url = "AGENTLESS"
agent_error = None
else:
if isinstance(tracer.writer, AgentWriter):
writer = tracer.writer
else:
writer = AgentWriter()
agent_url = writer.agent_url
try:
writer.write([])
writer.flush_queue(raise_exc=True)
except Exception as e:
agent_error = "Agent not reachable at %s. Exception raised: %s" % (
agent_url, str(e))
else:
agent_error = None
is_venv = in_venv()
packages_available = {
p.project_name: p.version
for p in pkg_resources.working_set
}
integration_configs = {}
for module, enabled in ddtrace.monkey.PATCH_MODULES.items():
# TODO: this check doesn't work in all cases... we need a mapping
# between the module and the library name.
module_available = module in packages_available
module_instrumented = module in ddtrace.monkey._PATCHED_MODULES
module_imported = module in sys.modules
if enabled:
# Note that integration configs aren't added until the integration
# module is imported. This typically occurs as a side-effect of
# patch().
# This also doesn't load work in all cases since we don't always
# name the configuration entry the same as the integration module
# name :/
config = ddtrace.config._config.get(module, "N/A")
else:
config = None
if module_available:
integration_configs[module] = dict(
enabled=enabled,
instrumented=module_instrumented,
module_available=module_available,
module_version=packages_available[module],
module_imported=module_imported,
config=config,
)
else:
# Use N/A here to avoid the additional clutter of an entire
# config dictionary for a module that isn't available.
integration_configs[module] = "N/A"
pip_version = packages_available.get("pip", "N/A")
return dict(
# Timestamp UTC ISO 8601
date=datetime.datetime.utcnow().isoformat(),
# eg. "Linux", "Darwin"
os_name=platform.system(),
# eg. 12.5.0
os_version=platform.release(),
is_64_bit=sys.maxsize > 2**32,
architecture=platform.architecture()[0],
vm=platform.python_implementation(),
version=ddtrace.__version__,
lang="python",
lang_version=platform.python_version(),
pip_version=pip_version,
in_virtual_env=is_venv,
agent_url=agent_url,
agent_error=agent_error,
env=ddtrace.config.env or "",
is_global_tracer=tracer == ddtrace.tracer,
enabled_env_setting=os.getenv("DATADOG_TRACE_ENABLED"),
tracer_enabled=tracer.enabled,
sampler_type=type(tracer.sampler).__name__
if tracer.sampler else "N/A",
priority_sampler_type=type(tracer.priority_sampler).__name__
if tracer.priority_sampler else "N/A",
service=ddtrace.config.service or "",
debug=ddtrace.tracer.log.isEnabledFor(logging.DEBUG),
enabled_cli="ddtrace" in os.getenv("PYTHONPATH", ""),
analytics_enabled=ddtrace.config.analytics_enabled,
log_injection_enabled=ddtrace.config.logs_injection,
health_metrics_enabled=ddtrace.config.health_metrics_enabled,
dd_version=ddtrace.config.version or "",
priority_sampling_enabled=tracer.priority_sampler is not None,
global_tags=os.getenv("DD_TAGS", ""),
tracer_tags=tags_to_str(tracer.tags),
integrations=integration_configs,
)
