def test_payload_too_large(encoding, monkeypatch):
SIZE = 1 << 12 # 4KB
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
monkeypatch.setenv("DD_TRACE_WRITER_BUFFER_SIZE_BYTES", str(SIZE))
monkeypatch.setenv("DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES", str(SIZE))
t = Tracer()
assert t.writer._max_payload_size == SIZE
assert t.writer._buffer_size == SIZE
# Make sure a flush doesn't happen partway through.
t.configure(
writer=AgentWriter(agent.get_trace_url(), processing_interval=1000))
with mock.patch("ddtrace.internal.writer.log") as log:
for i in range(100000 if encoding == "v0.5" else 1000):
with t.trace("operation") as s:
s.set_tag(str(i), "b" * 190)
s.set_tag(str(i), "a" * 190)
t.shutdown()
calls = [
mock.call(
"trace buffer (%s traces %db/%db) cannot fit trace of size %db, dropping",
AnyInt(),
AnyInt(),
AnyInt(),
AnyInt(),
)
]
log.warning.assert_has_calls(calls)
log.error.assert_not_called()
def test_metrics():
with override_global_config(dict(health_metrics_enabled=True)):
t = Tracer()
statsd_mock = mock.Mock()
t.writer.dogstatsd = statsd_mock
assert t.writer._report_metrics
with mock.patch("ddtrace.internal.writer.log") as log:
for _ in range(5):
spans = []
for i in range(3000):
spans.append(t.trace("op"))
for s in spans:
s.finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
statsd_mock.distribution.assert_has_calls(
[
mock.call("datadog.tracer.buffer.accepted.traces", 5, tags=[]),
mock.call(
"datadog.tracer.buffer.accepted.spans", 15000, tags=[]),
mock.call("datadog.tracer.http.requests", 1, tags=[]),
mock.call("datadog.tracer.http.sent.bytes", AnyInt()),
],
any_order=True,
)
def test_partial_flush_log(run_python_code_in_subprocess):
partial_flush_min_spans = 2
t = Tracer()
t.configure(
partial_flush_enabled=True,
partial_flush_min_spans=partial_flush_min_spans,
)
s1 = t.trace("1")
s2 = t.trace("2")
s3 = t.trace("3")
t_id = s3.trace_id
with mock.patch("ddtrace.internal.processor.trace.log") as log:
s3.finish()
s2.finish()
calls = [
mock.call("trace %d has %d spans, %d finished", t_id, 3, 1),
mock.call("Partially flushing %d spans for trace %d",
partial_flush_min_spans, t_id),
]
log.debug.assert_has_calls(calls)
s1.finish()
t.shutdown()
def test_priority_sampling_response(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
# Send the data once because the agent doesn't respond with them on the
# first payload.
t = Tracer()
s = t.trace("operation", service="my-svc")
s.set_tag("env", "my-env")
s.finish()
assert "service:my-svc,env:my-env" not in t.writer._priority_sampler._by_service_samplers
t.shutdown()
# For some reason the agent doesn't start returning the service information
# immediately
import time
time.sleep(5)
t = Tracer()
s = t.trace("operation", service="my-svc")
s.set_tag("env", "my-env")
s.finish()
assert "service:my-svc,env:my-env" not in t.writer._priority_sampler._by_service_samplers
t.shutdown()
assert "service:my-svc,env:my-env" in t.writer._priority_sampler._by_service_samplers
def test_bad_payload_log_payload(monkeypatch):
monkeypatch.setenv("_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS", "true")
t = Tracer()
class BadEncoder:
def __len__(self):
return 0
def put(self, trace):
pass
def encode(self):
return b"bad_payload"
def encode_traces(self, traces):
return b"bad_payload"
t.writer._encoder = BadEncoder()
with mock.patch("ddtrace.internal.writer.log") as log:
t.trace("asdf").finish()
t.shutdown()
calls = [
mock.call(
"failed to send traces to Datadog Agent at %s: HTTP error status %s, reason %s, payload %s",
"http://localhost:8126/v0.4/traces",
400,
"Bad Request",
"6261645f7061796c6f6164",
)
]
log.error.assert_has_calls(calls)
def test_bad_payload():
t = Tracer()
class BadEncoder:
def __len__(self):
return 0
def put(self, trace):
pass
def encode(self):
return ""
def encode_traces(self, traces):
return ""
t.writer._encoder = BadEncoder()
with mock.patch("ddtrace.internal.writer.log") as log:
t.trace("asdf").finish()
t.shutdown()
calls = [
mock.call(
"failed to send traces to Datadog Agent at %s: HTTP error status %s, reason %s",
"http://localhost:8126",
400,
"Bad Request",
)
]
log.error.assert_has_calls(calls)
def test_bad_encoder():
t = Tracer()
class BadEncoder:
def __len__(self):
return 0
def put(self, trace):
pass
def encode(self):
raise Exception()
def encode_traces(self, traces):
raise Exception()
t.writer._encoder = BadEncoder()
with mock.patch("ddtrace.internal.writer.log") as log:
t.trace("asdf").finish()
t.shutdown()
calls = [
mock.call("failed to encode trace with encoder %r",
t.writer._encoder,
exc_info=True)
]
log.error.assert_has_calls(calls)
def test_wrong_span_name_type_refused_by_agent(self):
"""Span names should be a text type."""
tracer = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
with tracer.trace(123):
pass
tracer.shutdown()
self._assert_bad_trace_refused_by_agent(log)
def test_single_trace_uds():
t = Tracer()
sockdir = "/tmp/ddagent/trace.sock"
t.configure(uds_path=sockdir)
with mock.patch("ddtrace.internal.writer.log") as log:
t.trace("client.testing").finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_uds_wrong_socket_path():
t = Tracer()
t.configure(uds_path="/tmp/ddagent/nosockethere")
with mock.patch("ddtrace.internal.writer.log") as log:
t.trace("client.testing").finish()
t.shutdown()
calls = [
mock.call("failed to send traces to Datadog Agent at %s", "unix:///tmp/ddagent/nosockethere", exc_info=True)
]
log.error.assert_has_calls(calls)
def test_bad_endpoint():
t = Tracer()
t.writer._endpoint = "/bad"
with mock.patch("ddtrace.internal.writer.log") as log:
s = t.trace("operation", service="my-svc")
s.set_tag("env", "my-env")
s.finish()
t.shutdown()
calls = [mock.call("unsupported endpoint '%s': received response %s from Datadog Agent", "/bad", 404)]
log.error.assert_has_calls(calls)
def test_downgrade():
t = Tracer()
t.writer._downgrade(None, None)
assert t.writer._endpoint == "/v0.3/traces"
with mock.patch("ddtrace.internal.writer.log") as log:
s = t.trace("operation", service="my-svc")
s.finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_simple_trace_accepted_by_agent(self):
tracer = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
with tracer.trace("root"):
for _ in range(999):
with tracer.trace("child"):
pass
tracer.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_trace_bad_url():
t = Tracer()
t.configure(hostname="bad", port=1111)
with mock.patch("ddtrace.internal.writer.log") as log:
with t.trace("op"):
pass
t.shutdown()
calls = [mock.call("failed to send traces to Datadog Agent at %s", "http://bad:1111", exc_info=True)]
log.error.assert_has_calls(calls)
def test_trace_with_wrong_metrics_types_refused_by_agent(self, metrics):
tracer = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
with tracer.trace("root") as root:
root.metrics = metrics
for _ in range(499):
with tracer.trace("child") as child:
child.metrics = metrics
tracer.shutdown()
self._assert_bad_trace_refused_by_agent(log)
def test_single_trace_too_large():
t = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
with t.trace("huge"):
for i in range(100000):
with tracer.trace("operation") as s:
s.set_tag("a" * 10, "b" * 10)
t.shutdown()
calls = [mock.call("trace (%db) larger than payload limit (%db), dropping", AnyInt(), AnyInt())]
log.warning.assert_has_calls(calls)
log.error.assert_not_called()
def test_single_trace_uds(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
sockdir = "/tmp/ddagent/trace.sock"
t.configure(uds_path=sockdir)
with mock.patch("ddtrace.internal.writer.log") as log:
t.trace("client.testing").finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_trace_with_meta_accepted_by_agent(self, tags):
"""Meta tags should be text types."""
tracer = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
with tracer.trace("root", service="test_encoding", resource="test_resource") as root:
root.set_tags(tags)
for _ in range(999):
with tracer.trace("child") as child:
child.set_tags(tags)
tracer.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_trace_with_metrics_accepted_by_agent(self, metrics):
"""Metric tags should be numeric types - i.e. int, float, long (py3), and str numbers."""
tracer = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
with tracer.trace("root") as root:
root.set_metrics(metrics)
for _ in range(999):
with tracer.trace("child") as child:
child.set_metrics(metrics)
tracer.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_span_tags():
t = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
s = t.trace("operation", service="my-svc")
s.set_tag("env", "my-env")
s.set_metric("number", 123)
s.set_metric("number", 12.0)
s.set_metric("number", "1")
s.finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_large_payload():
t = Tracer()
# Traces are approx. 275 bytes.
# 10,000*275 ~ 3MB
with mock.patch("ddtrace.internal.writer.log") as log:
for i in range(10000):
with t.trace("operation"):
pass
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_child_spans():
t = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
spans = []
for i in range(10000):
spans.append(t.trace("op"))
for s in spans:
s.finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_large_payload(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
# Traces are approx. 275 bytes.
# 10,000*275 ~ 3MB
with mock.patch("ddtrace.internal.writer.log") as log:
for i in range(10000):
with t.trace("operation"):
pass
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_child_spans(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
spans = []
for i in range(10000):
spans.append(t.trace("op"))
for s in spans:
s.finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_span_tags(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
s = t.trace("operation", service="my-svc")
s.set_tag("env", "my-env")
s.set_metric("number", 123)
s.set_metric("number", 12.0)
s.set_metric("number", "1")
s.finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_downgrade(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
t.writer._downgrade(None, None)
assert t.writer._endpoint == {
"v0.5": "v0.4/traces",
"v0.4": "v0.3/traces"
}[encoding or "v0.4"]
with mock.patch("ddtrace.internal.writer.log") as log:
s = t.trace("operation", service="my-svc")
s.finish()
t.shutdown()
log.warning.assert_not_called()
log.error.assert_not_called()
def test_trace_bad_url(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
t.configure(hostname="bad", port=1111)
with mock.patch("ddtrace.internal.writer.log") as log:
with t.trace("op"):
pass
t.shutdown()
calls = [
mock.call("failed to send traces to Datadog Agent at %s",
"http://bad:1111",
exc_info=True)
]
log.error.assert_has_calls(calls)
def test_uds_wrong_socket_path(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
t.configure(uds_path="/tmp/ddagent/nosockethere")
with mock.patch("ddtrace.internal.writer.log") as log:
t.trace("client.testing").finish()
t.shutdown()
calls = [
mock.call(
"failed to send traces to Datadog Agent at %s",
"unix:///tmp/ddagent/nosockethere/{}/traces".format(
encoding if encoding else "v0.4"),
exc_info=True,
)
]
log.error.assert_has_calls(calls)
def test_single_trace_too_large(encoding, monkeypatch):
monkeypatch.setenv("DD_TRACE_API_VERSION", encoding)
t = Tracer()
with mock.patch("ddtrace.internal.writer.log") as log:
with t.trace("huge"):
for i in range(200000):
with t.trace("operation") as s:
s.set_tag("a" * 10, "b" * 10)
t.shutdown()
calls = [
mock.call(
"trace (%db) larger than payload buffer limit (%db), dropping",
AnyInt(), AnyInt())
]
log.warning.assert_has_calls(calls)
log.error.assert_not_called()
def test_tracer_trace_across_fork():
"""
When a trace is started in a parent process and a child process is spawned
The trace should be continued in the child process
"""
tracer = Tracer()
def task(tracer):
with tracer.trace("child"):
pass
tracer.shutdown()
with tracer.trace("parent"):
p = multiprocessing.Process(target=task, args=(tracer, ))
p.start()
p.join()
tracer.shutdown()