def test_metrics_bad_endpoint(self):
statsd = mock.Mock()
writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=True)
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.stop()
writer.join()
statsd.distribution.assert_has_calls(
[
mock.call("datadog.tracer.buffer.accepted.traces", 10, tags=[]),
mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]),
mock.call("datadog.tracer.http.requests", writer.RETRY_ATTEMPTS, tags=[]),
mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]),
mock.call("datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]),
],
any_order=True,
)
def test_payload_too_large():
t = Tracer()
# Make sure a flush doesn't happen partway through.
t.configure(writer=AgentWriter(processing_interval=1000))
with mock.patch("ddtrace.internal.writer.log") as log:
for i in range(100000):
with t.trace("operation") as s:
s.set_tag(str(i), "b" * 190)
s.set_tag(str(i), "a" * 190)
t.shutdown()
calls = [
mock.call(
"trace buffer (%s traces %db/%db) cannot fit trace of size %db, dropping",
AnyInt(),
AnyInt(),
AnyInt(),
AnyInt(),
)
]
log.warning.assert_has_calls(calls)
log.error.assert_not_called()
def test_flush_log(caplog):
caplog.set_level(logging.INFO)
writer = AgentWriter(agent.get_trace_url())
with mock.patch("ddtrace.internal.writer.log") as log:
writer.write([])
writer.flush_queue(raise_exc=True)
calls = [
mock.call(
logging.DEBUG,
"sent %s in %.5fs to %s",
AnyStr(),
AnyFloat(),
writer.agent_url,
)
]
log.log.assert_has_calls(calls)
def __init__(self, app, service='atlas-api', sqlalchemy_engine=None, datadog_agent=None,
span_callback=None, ignored_paths: List[str] = None, sql_service=None):
self.app = app
trace.set_tracer_provider(TracerProvider())
self.tracer = trace.get_tracer(__name__)
if datadog_agent:
from ddtrace.internal.writer import AgentWriter
from opentelemetry.exporter.datadog import DatadogExportSpanProcessor, \
DatadogSpanExporter
exporter = DatadogSpanExporter(agent_url=datadog_agent, service=service)
exporter._agent_writer = AgentWriter(datadog_agent)
span_processor = DatadogExportSpanProcessor(exporter)
trace.get_tracer_provider().add_span_processor(span_processor)
AtlasFastAPIInstrumentor.instrument_app(app, span_callback=span_callback,
ignored_paths=ignored_paths)
RequestsInstrumentor().instrument()
BotocoreInstrumentor().instrument(tracer_provider=trace.get_tracer_provider())
BotoInstrumentor().instrument(tracer_provider=trace.get_tracer_provider())
RedisInstrumentor().instrument(tracer_provider=trace.get_tracer_provider())
if sqlalchemy_engine:
sqlalch_service_name = service if not sql_service else sql_service
SQLAlchemyInstrumentor().instrument(engine=sqlalchemy_engine,
service=sqlalch_service_name)
def test_flush_log(caplog, encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
caplog.set_level(logging.INFO)
writer = AgentWriter(agent.get_trace_url())
with mock.patch("ddtrace.internal.writer.log") as log:
writer.write([])
writer.flush_queue(raise_exc=True)
# for latest agent, default to v0.3 since no priority sampler is set
expected_encoding = "v0.3" if AGENT_VERSION == "v5" else (encoding
or "v0.3")
calls = [
mock.call(
logging.DEBUG,
"sent %s in %.5fs to %s",
AnyStr(),
AnyFloat(),
"{}/{}/traces".format(writer.agent_url, expected_encoding),
)
]
log.log.assert_has_calls(calls)
def test_flush_connection_uds(endpoint_uds_server):
writer = AgentWriter(_HOST,
2019,
uds_path=endpoint_uds_server.server_address)
writer._send_payload("foobar", 12)
def test_flush_connection_uds(endpoint_uds_server):
writer = AgentWriter(agent_url="unix://%s" % endpoint_uds_server.server_address)
writer._buffer.put(b"foobar")
writer.flush_queue(raise_exc=True)
def test_flush_connection_timeout(endpoint_test_timeout_server):
writer = AgentWriter(agent_url="http://%s:%s" % (_HOST, _TIMEOUT_PORT))
with pytest.raises(socket.timeout):
writer._buffer.put(b"foobar")
writer.flush_queue(raise_exc=True)
def test_agent_url_path(endpoint_assert_path):
# test without base path
endpoint_assert_path("/v0.")
writer = AgentWriter(agent_url="http://%s:%s/" % (_HOST, _PORT))
writer._buffer.put(b"foobar")
writer.flush_queue(raise_exc=True)
# test without base path nor trailing slash
writer = AgentWriter(agent_url="http://%s:%s" % (_HOST, _PORT))
writer._buffer.put(b"foobar")
writer.flush_queue(raise_exc=True)
# test with a base path
endpoint_assert_path("/test/v0.")
writer = AgentWriter(agent_url="http://%s:%s/test/" % (_HOST, _PORT))
writer._buffer.put(b"foobar")
writer.flush_queue(raise_exc=True)
def test_keep_rate(self):
statsd = mock.Mock()
writer_run_periodic = mock.Mock()
writer_put = mock.Mock()
writer_put.return_value = Response(status=200)
writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False)
writer.run_periodic = writer_run_periodic
writer._put = writer_put
traces = [
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
for i in range(4)
]
traces_too_big = [
[Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)]
for i in range(4)
]
# 1. We write 4 traces successfully.
for trace in traces:
writer.write(trace)
writer.flush_queue()
payload = msgpack.unpackb(writer_put.call_args.args[0])
# No previous drops.
assert 0.0 == writer._drop_sma.get()
# 4 traces written.
assert 4 == len(payload)
# 100% of traces kept (refers to the past).
# No traces sent before now so 100% kept.
for trace in payload:
assert 1.0 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
# 2. We fail to write 4 traces because of size limitation.
for trace in traces_too_big:
writer.write(trace)
writer.flush_queue()
# 50% of traces were dropped historically.
# 4 successfully written before and 4 dropped now.
assert 0.5 == writer._drop_sma.get()
# put not called since no new traces are available.
writer_put.assert_called_once()
# 3. We write 2 traces successfully.
for trace in traces[:2]:
writer.write(trace)
writer.flush_queue()
payload = msgpack.unpackb(writer_put.call_args.args[0])
# 40% of traces were dropped historically.
assert 0.4 == writer._drop_sma.get()
# 2 traces written.
assert 2 == len(payload)
# 50% of traces kept (refers to the past).
# We had 4 successfully written and 4 dropped.
for trace in payload:
assert 0.5 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
# 4. We write 1 trace successfully and fail to write 3.
writer.write(traces[0])
for trace in traces_too_big[:3]:
writer.write(trace)
writer.flush_queue()
payload = msgpack.unpackb(writer_put.call_args.args[0])
# 50% of traces were dropped historically.
assert 0.5 == writer._drop_sma.get()
# 1 trace written.
assert 1 == len(payload)
# 60% of traces kept (refers to the past).
# We had 4 successfully written, then 4 dropped, then 2 written.
for trace in payload:
assert 0.6 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
def test_drop_reason_buffer_full(self):
statsd = mock.Mock()
writer_metrics_reset = mock.Mock()
writer = AgentWriter(agent_url="http://asdf:1234", buffer_size=5300, dogstatsd=statsd, report_metrics=False)
writer._metrics_reset = writer_metrics_reset
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.write([Span(tracer=None, name="a", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)])
writer.stop()
writer.join()
writer_metrics_reset.assert_called_once()
assert 1 == writer._metrics["buffer.dropped.traces"]["count"]
assert ["reason:full"] == writer._metrics["buffer.dropped.traces"]["tags"]
def collect(tracer):
"""Collect system and library information into a serializable dict."""
# The tracer doesn't actually maintain a hostname/port, instead it stores
# it on the possibly None writer which actually stores it on an API object.
# Note that the tracer DOES have hostname and port attributes that it
# sets to the defaults and ignores afterwards.
if tracer.writer and isinstance(tracer.writer, LogWriter):
agent_url = "AGENTLESS"
agent_error = None
else:
if isinstance(tracer.writer, AgentWriter):
writer = tracer.writer
else:
writer = AgentWriter()
agent_url = writer.agent_url
try:
writer.write([])
writer.flush_queue(raise_exc=True)
except Exception as e:
agent_error = "Agent not reachable at %s. Exception raised: %s" % (
agent_url, str(e))
else:
agent_error = None
is_venv = in_venv()
packages_available = {
p.project_name: p.version
for p in pkg_resources.working_set
}
integration_configs = {}
for module, enabled in ddtrace.monkey.PATCH_MODULES.items():
# TODO: this check doesn't work in all cases... we need a mapping
# between the module and the library name.
module_available = module in packages_available
module_instrumented = module in ddtrace.monkey._PATCHED_MODULES
module_imported = module in sys.modules
if enabled:
# Note that integration configs aren't added until the integration
# module is imported. This typically occurs as a side-effect of
# patch().
# This also doesn't load work in all cases since we don't always
# name the configuration entry the same as the integration module
# name :/
config = ddtrace.config._config.get(module, "N/A")
else:
config = None
if module_available:
integration_configs[module] = dict(
enabled=enabled,
instrumented=module_instrumented,
module_available=module_available,
module_version=packages_available[module],
module_imported=module_imported,
config=config,
)
else:
# Use N/A here to avoid the additional clutter of an entire
# config dictionary for a module that isn't available.
integration_configs[module] = "N/A"
pip_version = packages_available.get("pip", "N/A")
return dict(
# Timestamp UTC ISO 8601
date=datetime.datetime.utcnow().isoformat(),
# eg. "Linux", "Darwin"
os_name=platform.system(),
# eg. 12.5.0
os_version=platform.release(),
is_64_bit=sys.maxsize > 2**32,
architecture=platform.architecture()[0],
vm=platform.python_implementation(),
version=ddtrace.__version__,
lang="python",
lang_version=platform.python_version(),
pip_version=pip_version,
in_virtual_env=is_venv,
agent_url=agent_url,
agent_error=agent_error,
env=ddtrace.config.env or "",
is_global_tracer=tracer == ddtrace.tracer,
enabled_env_setting=os.getenv("DATADOG_TRACE_ENABLED"),
tracer_enabled=tracer.enabled,
sampler_type=type(tracer.sampler).__name__
if tracer.sampler else "N/A",
priority_sampler_type=type(tracer.priority_sampler).__name__
if tracer.priority_sampler else "N/A",
service=ddtrace.config.service or "",
debug=ddtrace.tracer.log.isEnabledFor(logging.DEBUG),
enabled_cli="ddtrace" in os.getenv("PYTHONPATH", ""),
analytics_enabled=ddtrace.config.analytics_enabled,
log_injection_enabled=ddtrace.config.logs_injection,
health_metrics_enabled=ddtrace.config.health_metrics_enabled,
dd_version=ddtrace.config.version or "",
priority_sampling_enabled=tracer.priority_sampler is not None,
global_tags=os.getenv("DD_TAGS", ""),
tracer_tags=tags_to_str(tracer.tags),
integrations=integration_configs,
)
def test_sampling(writer, tracer):
if writer == "sync":
writer = AgentWriter(
tracer.writer.agent_url,
priority_sampler=tracer.priority_sampler,
sync_mode=True,
)
# Need to copy the headers which contain the test token to associate
# traces with this test case.
writer._headers = tracer.writer._headers
else:
writer = tracer.writer
tracer.configure(writer=writer)
with tracer.trace("trace1"):
with tracer.trace("child"):
pass
sampler = DatadogSampler(default_sample_rate=1.0)
tracer.configure(sampler=sampler, writer=writer)
with tracer.trace("trace2"):
with tracer.trace("child"):
pass
sampler = DatadogSampler(default_sample_rate=0.000001)
tracer.configure(sampler=sampler, writer=writer)
with tracer.trace("trace3"):
with tracer.trace("child"):
pass
sampler = DatadogSampler(default_sample_rate=1, rules=[SamplingRule(1.0)])
tracer.configure(sampler=sampler, writer=writer)
with tracer.trace("trace4"):
with tracer.trace("child"):
pass
sampler = DatadogSampler(default_sample_rate=1, rules=[SamplingRule(0)])
tracer.configure(sampler=sampler, writer=writer)
with tracer.trace("trace5"):
with tracer.trace("child"):
pass
sampler = DatadogSampler(default_sample_rate=1)
tracer.configure(sampler=sampler, writer=writer)
with tracer.trace("trace6"):
with tracer.trace("child") as span:
span.set_tag(MANUAL_DROP_KEY)
sampler = DatadogSampler(default_sample_rate=1)
tracer.configure(sampler=sampler, writer=writer)
with tracer.trace("trace7"):
with tracer.trace("child") as span:
span.set_tag(MANUAL_KEEP_KEY)
sampler = RateSampler(0.0000000001)
tracer.configure(sampler=sampler, writer=writer)
# This trace should not appear in the snapshot
with tracer.trace("trace8"):
with tracer.trace("child"):
pass
tracer.shutdown()
def test_metrics_multi(self):
statsd = mock.Mock()
writer = AgentWriter(dogstatsd=statsd,
report_metrics=True,
hostname="asdf",
port=1234)
for i in range(10):
writer.write([
Span(tracer=None,
name="name",
trace_id=i,
span_id=j,
parent_id=j - 1 or None) for j in range(5)
])
writer.flush_queue()
statsd.increment.assert_has_calls([
mock.call("datadog.tracer.http.requests"),
])
statsd.distribution.assert_has_calls(
[
mock.call("datadog.tracer.buffer.accepted.traces", 10,
tags=[]),
mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]),
mock.call("datadog.tracer.http.requests", 1, tags=[]),
mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]),
mock.call(
"datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]),
],
any_order=True,
)
statsd.reset_mock()
for i in range(10):
writer.write([
Span(tracer=None,
name="name",
trace_id=i,
span_id=j,
parent_id=j - 1 or None) for j in range(5)
])
writer.stop()
writer.join()
statsd.increment.assert_has_calls([
mock.call("datadog.tracer.http.requests"),
])
statsd.distribution.assert_has_calls(
[
mock.call("datadog.tracer.buffer.accepted.traces", 10,
tags=[]),
mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]),
mock.call("datadog.tracer.http.requests", 1, tags=[]),
mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]),
mock.call(
"datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]),
],
any_order=True,
)
def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False):
self.dogstatsd = mock.Mock()
worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters)
worker._ENABLE_STATS = enable_stats
worker._STATS_EVERY_INTERVAL = 1
self.api = api_class()
worker.api = self.api
for i in range(self.N_TRACES):
worker.write([
Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None)
for j in range(7)
])
worker.stop()
worker.join()
return worker
def test_bad_encoding(monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", "foo")
with pytest.raises(ValueError):
AgentWriter(agent_url="http://localhost:9126")
def test_additional_headers():
with override_env(dict(_DD_TRACE_WRITER_ADDITIONAL_HEADERS="additional-header:additional-value,header2:value2")):
writer = AgentWriter(agent_url="http://localhost:9126")
assert writer._headers["additional-header"] == "additional-value"
assert writer._headers["header2"] == "value2"
def test_flush_connection_uds(endpoint_uds_server):
writer = AgentWriter(agent_url="unix://%s" % endpoint_uds_server.server_address)
writer._encoder.put([Span(None, "foobar")])
writer.flush_queue(raise_exc=True)
def test_synchronous_writer_shutdown():
tracer = Tracer()
tracer.configure(
writer=AgentWriter(tracer.writer.agent_url, sync_mode=True))
# Ensure this doesn't raise.
tracer.shutdown()
def test_flush_connection_timeout(endpoint_test_timeout_server):
writer = AgentWriter(agent_url="http://%s:%s" % (_HOST, _TIMEOUT_PORT))
with pytest.raises(socket.timeout):
writer._send_payload("foobar", 12)
def test_flush_connection_uds(endpoint_uds_server):
writer = AgentWriter(agent_url="unix://%s" % endpoint_uds_server.server_address)
writer._send_payload("foobar", 12)
def test_drop_reason_trace_too_big(self):
statsd = mock.Mock()
writer_metrics_reset = mock.Mock()
writer = AgentWriter(dogstatsd=statsd, report_metrics=False, hostname="asdf", port=1234)
writer._metrics_reset = writer_metrics_reset
for i in range(10):
writer.write(
[Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]
)
writer.write(
[Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)]
)
writer.stop()
writer.join()
writer_metrics_reset.assert_called_once()
assert 1 == writer._metrics["buffer.dropped.traces"]["count"]
assert ["reason:t_too_big"] == writer._metrics["buffer.dropped.traces"]["tags"]
