def test_metrics_bad_endpoint(self): statsd = mock.Mock() writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=True) for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.stop() writer.join() statsd.distribution.assert_has_calls( [ mock.call("datadog.tracer.buffer.accepted.traces", 10, tags=[]), mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]), mock.call("datadog.tracer.http.requests", writer.RETRY_ATTEMPTS, tags=[]), mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]), mock.call("datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]), ], any_order=True, )
def test_payload_too_large(): t = Tracer() # Make sure a flush doesn't happen partway through. t.configure(writer=AgentWriter(processing_interval=1000)) with mock.patch("ddtrace.internal.writer.log") as log: for i in range(100000): with t.trace("operation") as s: s.set_tag(str(i), "b" * 190) s.set_tag(str(i), "a" * 190) t.shutdown() calls = [ mock.call( "trace buffer (%s traces %db/%db) cannot fit trace of size %db, dropping", AnyInt(), AnyInt(), AnyInt(), AnyInt(), ) ] log.warning.assert_has_calls(calls) log.error.assert_not_called()
def test_flush_log(caplog): caplog.set_level(logging.INFO) writer = AgentWriter(agent.get_trace_url()) with mock.patch("ddtrace.internal.writer.log") as log: writer.write([]) writer.flush_queue(raise_exc=True) calls = [ mock.call( logging.DEBUG, "sent %s in %.5fs to %s", AnyStr(), AnyFloat(), writer.agent_url, ) ] log.log.assert_has_calls(calls)
def __init__(self, app, service='atlas-api', sqlalchemy_engine=None, datadog_agent=None, span_callback=None, ignored_paths: List[str] = None, sql_service=None): self.app = app trace.set_tracer_provider(TracerProvider()) self.tracer = trace.get_tracer(__name__) if datadog_agent: from ddtrace.internal.writer import AgentWriter from opentelemetry.exporter.datadog import DatadogExportSpanProcessor, \ DatadogSpanExporter exporter = DatadogSpanExporter(agent_url=datadog_agent, service=service) exporter._agent_writer = AgentWriter(datadog_agent) span_processor = DatadogExportSpanProcessor(exporter) trace.get_tracer_provider().add_span_processor(span_processor) AtlasFastAPIInstrumentor.instrument_app(app, span_callback=span_callback, ignored_paths=ignored_paths) RequestsInstrumentor().instrument() BotocoreInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) BotoInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) RedisInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) if sqlalchemy_engine: sqlalch_service_name = service if not sql_service else sql_service SQLAlchemyInstrumentor().instrument(engine=sqlalchemy_engine, service=sqlalch_service_name)
def test_flush_log(caplog, encoding, monkeypatch): monkeypatch.setenv("DD_TRACE_API_VERSION", encoding) caplog.set_level(logging.INFO) writer = AgentWriter(agent.get_trace_url()) with mock.patch("ddtrace.internal.writer.log") as log: writer.write([]) writer.flush_queue(raise_exc=True) # for latest agent, default to v0.3 since no priority sampler is set expected_encoding = "v0.3" if AGENT_VERSION == "v5" else (encoding or "v0.3") calls = [ mock.call( logging.DEBUG, "sent %s in %.5fs to %s", AnyStr(), AnyFloat(), "{}/{}/traces".format(writer.agent_url, expected_encoding), ) ] log.log.assert_has_calls(calls)
def test_flush_connection_uds(endpoint_uds_server): writer = AgentWriter(_HOST, 2019, uds_path=endpoint_uds_server.server_address) writer._send_payload("foobar", 12)
def test_flush_connection_uds(endpoint_uds_server): writer = AgentWriter(agent_url="unix://%s" % endpoint_uds_server.server_address) writer._buffer.put(b"foobar") writer.flush_queue(raise_exc=True)
def test_flush_connection_timeout(endpoint_test_timeout_server): writer = AgentWriter(agent_url="http://%s:%s" % (_HOST, _TIMEOUT_PORT)) with pytest.raises(socket.timeout): writer._buffer.put(b"foobar") writer.flush_queue(raise_exc=True)
def test_agent_url_path(endpoint_assert_path): # test without base path endpoint_assert_path("/v0.") writer = AgentWriter(agent_url="http://%s:%s/" % (_HOST, _PORT)) writer._buffer.put(b"foobar") writer.flush_queue(raise_exc=True) # test without base path nor trailing slash writer = AgentWriter(agent_url="http://%s:%s" % (_HOST, _PORT)) writer._buffer.put(b"foobar") writer.flush_queue(raise_exc=True) # test with a base path endpoint_assert_path("/test/v0.") writer = AgentWriter(agent_url="http://%s:%s/test/" % (_HOST, _PORT)) writer._buffer.put(b"foobar") writer.flush_queue(raise_exc=True)
def test_keep_rate(self): statsd = mock.Mock() writer_run_periodic = mock.Mock() writer_put = mock.Mock() writer_put.return_value = Response(status=200) writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False) writer.run_periodic = writer_run_periodic writer._put = writer_put traces = [ [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] for i in range(4) ] traces_too_big = [ [Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)] for i in range(4) ] # 1. We write 4 traces successfully. for trace in traces: writer.write(trace) writer.flush_queue() payload = msgpack.unpackb(writer_put.call_args.args[0]) # No previous drops. assert 0.0 == writer._drop_sma.get() # 4 traces written. assert 4 == len(payload) # 100% of traces kept (refers to the past). # No traces sent before now so 100% kept. for trace in payload: assert 1.0 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1) # 2. We fail to write 4 traces because of size limitation. for trace in traces_too_big: writer.write(trace) writer.flush_queue() # 50% of traces were dropped historically. # 4 successfully written before and 4 dropped now. assert 0.5 == writer._drop_sma.get() # put not called since no new traces are available. writer_put.assert_called_once() # 3. We write 2 traces successfully. for trace in traces[:2]: writer.write(trace) writer.flush_queue() payload = msgpack.unpackb(writer_put.call_args.args[0]) # 40% of traces were dropped historically. assert 0.4 == writer._drop_sma.get() # 2 traces written. assert 2 == len(payload) # 50% of traces kept (refers to the past). # We had 4 successfully written and 4 dropped. for trace in payload: assert 0.5 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1) # 4. We write 1 trace successfully and fail to write 3. writer.write(traces[0]) for trace in traces_too_big[:3]: writer.write(trace) writer.flush_queue() payload = msgpack.unpackb(writer_put.call_args.args[0]) # 50% of traces were dropped historically. assert 0.5 == writer._drop_sma.get() # 1 trace written. assert 1 == len(payload) # 60% of traces kept (refers to the past). # We had 4 successfully written, then 4 dropped, then 2 written. for trace in payload: assert 0.6 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
def test_drop_reason_buffer_full(self): statsd = mock.Mock() writer_metrics_reset = mock.Mock() writer = AgentWriter(agent_url="http://asdf:1234", buffer_size=5300, dogstatsd=statsd, report_metrics=False) writer._metrics_reset = writer_metrics_reset for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.write([Span(tracer=None, name="a", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)]) writer.stop() writer.join() writer_metrics_reset.assert_called_once() assert 1 == writer._metrics["buffer.dropped.traces"]["count"] assert ["reason:full"] == writer._metrics["buffer.dropped.traces"]["tags"]
def collect(tracer): """Collect system and library information into a serializable dict.""" # The tracer doesn't actually maintain a hostname/port, instead it stores # it on the possibly None writer which actually stores it on an API object. # Note that the tracer DOES have hostname and port attributes that it # sets to the defaults and ignores afterwards. if tracer.writer and isinstance(tracer.writer, LogWriter): agent_url = "AGENTLESS" agent_error = None else: if isinstance(tracer.writer, AgentWriter): writer = tracer.writer else: writer = AgentWriter() agent_url = writer.agent_url try: writer.write([]) writer.flush_queue(raise_exc=True) except Exception as e: agent_error = "Agent not reachable at %s. Exception raised: %s" % ( agent_url, str(e)) else: agent_error = None is_venv = in_venv() packages_available = { p.project_name: p.version for p in pkg_resources.working_set } integration_configs = {} for module, enabled in ddtrace.monkey.PATCH_MODULES.items(): # TODO: this check doesn't work in all cases... we need a mapping # between the module and the library name. module_available = module in packages_available module_instrumented = module in ddtrace.monkey._PATCHED_MODULES module_imported = module in sys.modules if enabled: # Note that integration configs aren't added until the integration # module is imported. This typically occurs as a side-effect of # patch(). # This also doesn't load work in all cases since we don't always # name the configuration entry the same as the integration module # name :/ config = ddtrace.config._config.get(module, "N/A") else: config = None if module_available: integration_configs[module] = dict( enabled=enabled, instrumented=module_instrumented, module_available=module_available, module_version=packages_available[module], module_imported=module_imported, config=config, ) else: # Use N/A here to avoid the additional clutter of an entire # config dictionary for a module that isn't available. integration_configs[module] = "N/A" pip_version = packages_available.get("pip", "N/A") return dict( # Timestamp UTC ISO 8601 date=datetime.datetime.utcnow().isoformat(), # eg. "Linux", "Darwin" os_name=platform.system(), # eg. 12.5.0 os_version=platform.release(), is_64_bit=sys.maxsize > 2**32, architecture=platform.architecture()[0], vm=platform.python_implementation(), version=ddtrace.__version__, lang="python", lang_version=platform.python_version(), pip_version=pip_version, in_virtual_env=is_venv, agent_url=agent_url, agent_error=agent_error, env=ddtrace.config.env or "", is_global_tracer=tracer == ddtrace.tracer, enabled_env_setting=os.getenv("DATADOG_TRACE_ENABLED"), tracer_enabled=tracer.enabled, sampler_type=type(tracer.sampler).__name__ if tracer.sampler else "N/A", priority_sampler_type=type(tracer.priority_sampler).__name__ if tracer.priority_sampler else "N/A", service=ddtrace.config.service or "", debug=ddtrace.tracer.log.isEnabledFor(logging.DEBUG), enabled_cli="ddtrace" in os.getenv("PYTHONPATH", ""), analytics_enabled=ddtrace.config.analytics_enabled, log_injection_enabled=ddtrace.config.logs_injection, health_metrics_enabled=ddtrace.config.health_metrics_enabled, dd_version=ddtrace.config.version or "", priority_sampling_enabled=tracer.priority_sampler is not None, global_tags=os.getenv("DD_TAGS", ""), tracer_tags=tags_to_str(tracer.tags), integrations=integration_configs, )
def test_sampling(writer, tracer): if writer == "sync": writer = AgentWriter( tracer.writer.agent_url, priority_sampler=tracer.priority_sampler, sync_mode=True, ) # Need to copy the headers which contain the test token to associate # traces with this test case. writer._headers = tracer.writer._headers else: writer = tracer.writer tracer.configure(writer=writer) with tracer.trace("trace1"): with tracer.trace("child"): pass sampler = DatadogSampler(default_sample_rate=1.0) tracer.configure(sampler=sampler, writer=writer) with tracer.trace("trace2"): with tracer.trace("child"): pass sampler = DatadogSampler(default_sample_rate=0.000001) tracer.configure(sampler=sampler, writer=writer) with tracer.trace("trace3"): with tracer.trace("child"): pass sampler = DatadogSampler(default_sample_rate=1, rules=[SamplingRule(1.0)]) tracer.configure(sampler=sampler, writer=writer) with tracer.trace("trace4"): with tracer.trace("child"): pass sampler = DatadogSampler(default_sample_rate=1, rules=[SamplingRule(0)]) tracer.configure(sampler=sampler, writer=writer) with tracer.trace("trace5"): with tracer.trace("child"): pass sampler = DatadogSampler(default_sample_rate=1) tracer.configure(sampler=sampler, writer=writer) with tracer.trace("trace6"): with tracer.trace("child") as span: span.set_tag(MANUAL_DROP_KEY) sampler = DatadogSampler(default_sample_rate=1) tracer.configure(sampler=sampler, writer=writer) with tracer.trace("trace7"): with tracer.trace("child") as span: span.set_tag(MANUAL_KEEP_KEY) sampler = RateSampler(0.0000000001) tracer.configure(sampler=sampler, writer=writer) # This trace should not appear in the snapshot with tracer.trace("trace8"): with tracer.trace("child"): pass tracer.shutdown()
def test_metrics_multi(self): statsd = mock.Mock() writer = AgentWriter(dogstatsd=statsd, report_metrics=True, hostname="asdf", port=1234) for i in range(10): writer.write([ Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5) ]) writer.flush_queue() statsd.increment.assert_has_calls([ mock.call("datadog.tracer.http.requests"), ]) statsd.distribution.assert_has_calls( [ mock.call("datadog.tracer.buffer.accepted.traces", 10, tags=[]), mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]), mock.call("datadog.tracer.http.requests", 1, tags=[]), mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]), mock.call( "datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]), ], any_order=True, ) statsd.reset_mock() for i in range(10): writer.write([ Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5) ]) writer.stop() writer.join() statsd.increment.assert_has_calls([ mock.call("datadog.tracer.http.requests"), ]) statsd.distribution.assert_has_calls( [ mock.call("datadog.tracer.buffer.accepted.traces", 10, tags=[]), mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]), mock.call("datadog.tracer.http.requests", 1, tags=[]), mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]), mock.call( "datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]), ], any_order=True, )
def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False): self.dogstatsd = mock.Mock() worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters) worker._ENABLE_STATS = enable_stats worker._STATS_EVERY_INTERVAL = 1 self.api = api_class() worker.api = self.api for i in range(self.N_TRACES): worker.write([ Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(7) ]) worker.stop() worker.join() return worker
def test_bad_encoding(monkeypatch): monkeypatch.setenv("DD_TRACE_API_VERSION", "foo") with pytest.raises(ValueError): AgentWriter(agent_url="http://localhost:9126")
def test_additional_headers(): with override_env(dict(_DD_TRACE_WRITER_ADDITIONAL_HEADERS="additional-header:additional-value,header2:value2")): writer = AgentWriter(agent_url="http://localhost:9126") assert writer._headers["additional-header"] == "additional-value" assert writer._headers["header2"] == "value2"
def test_flush_connection_uds(endpoint_uds_server): writer = AgentWriter(agent_url="unix://%s" % endpoint_uds_server.server_address) writer._encoder.put([Span(None, "foobar")]) writer.flush_queue(raise_exc=True)
def test_synchronous_writer_shutdown(): tracer = Tracer() tracer.configure( writer=AgentWriter(tracer.writer.agent_url, sync_mode=True)) # Ensure this doesn't raise. tracer.shutdown()
def test_flush_connection_timeout(endpoint_test_timeout_server): writer = AgentWriter(agent_url="http://%s:%s" % (_HOST, _TIMEOUT_PORT)) with pytest.raises(socket.timeout): writer._send_payload("foobar", 12)
def test_flush_connection_uds(endpoint_uds_server): writer = AgentWriter(agent_url="unix://%s" % endpoint_uds_server.server_address) writer._send_payload("foobar", 12)
def test_drop_reason_trace_too_big(self): statsd = mock.Mock() writer_metrics_reset = mock.Mock() writer = AgentWriter(dogstatsd=statsd, report_metrics=False, hostname="asdf", port=1234) writer._metrics_reset = writer_metrics_reset for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.write( [Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)] ) writer.stop() writer.join() writer_metrics_reset.assert_called_once() assert 1 == writer._metrics["buffer.dropped.traces"]["count"] assert ["reason:t_too_big"] == writer._metrics["buffer.dropped.traces"]["tags"]