def test_drop_reason_buffer_full(self): statsd = mock.Mock() writer_metrics_reset = mock.Mock() writer = AgentWriter(agent_url="http://asdf:1234", buffer_size=5300, dogstatsd=statsd, report_metrics=False) writer._metrics_reset = writer_metrics_reset for i in range(10): writer.write([ Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5) ]) writer.write([ Span(tracer=None, name="a", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5) ]) writer.stop() writer.join() writer_metrics_reset.assert_called_once() assert 1 == writer._metrics["buffer.dropped.traces"]["count"] assert ["reason:full" ] == writer._metrics["buffer.dropped.traces"]["tags"]
def test_metrics_trace_too_big(self): statsd = mock.Mock() writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=True) for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.write( [Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)] ) writer.stop() writer.join() statsd.distribution.assert_has_calls( [ mock.call("datadog.tracer.buffer.accepted.traces", 10, tags=[]), mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]), mock.call("datadog.tracer.buffer.dropped.traces", 1, tags=["reason:t_too_big"]), mock.call("datadog.tracer.buffer.dropped.bytes", AnyInt(), tags=["reason:t_too_big"]), mock.call("datadog.tracer.http.requests", writer.RETRY_ATTEMPTS, tags=[]), mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]), mock.call("datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]), ], any_order=True, )
def test_metrics_bad_endpoint(self): statsd = mock.Mock() writer = AgentWriter(dogstatsd=statsd, report_metrics=True, hostname="asdf", port=1234) for i in range(10): writer.write([ Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5) ]) writer.stop() writer.join() statsd.increment.assert_has_calls([ mock.call("datadog.tracer.http.requests"), ]) statsd.distribution.assert_has_calls( [ mock.call("datadog.tracer.buffer.accepted.traces", 10, tags=[]), mock.call("datadog.tracer.buffer.accepted.spans", 50, tags=[]), mock.call("datadog.tracer.http.requests", 1, tags=[]), mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]), mock.call( "datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]), ], any_order=True, )
def test_write_sync(self): statsd = mock.Mock() writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=True, sync_mode=True) writer.write([ Span(tracer=None, name="name", trace_id=1, span_id=j, parent_id=j - 1 or None) for j in range(5) ]) statsd.distribution.assert_has_calls( [ mock.call("datadog.tracer.buffer.accepted.traces", 1, tags=[]), mock.call("datadog.tracer.buffer.accepted.spans", 5, tags=[]), mock.call("datadog.tracer.http.requests", writer.RETRY_ATTEMPTS, tags=[]), mock.call("datadog.tracer.http.errors", 1, tags=["type:err"]), mock.call( "datadog.tracer.http.dropped.bytes", AnyInt(), tags=[]), ], any_order=True, )
def test_drop_reason_encoding_error(self): n_traces = 10 statsd = mock.Mock() writer_encoder = mock.Mock() writer_encoder.__len__ = ( lambda *args: n_traces).__get__(writer_encoder) writer_metrics_reset = mock.Mock() writer_encoder.encode.side_effect = Exception writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False) writer._encoder = writer_encoder writer._metrics_reset = writer_metrics_reset for i in range(n_traces): writer.write([ Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5) ]) writer.stop() writer.join() writer_metrics_reset.assert_called_once() assert 10 == writer._metrics["encoder.dropped.traces"]["count"]
def test_double_stop(): # Ensure double stopping doesn't result in an exception. writer = AgentWriter(agent_url="http://dne:1234") writer.write([]) assert writer.status == service.ServiceStatus.RUNNING writer.stop() assert writer.status == service.ServiceStatus.STOPPED writer.stop() assert writer.status == service.ServiceStatus.STOPPED
def test_double_stop(): # Ensure double stopping doesn't result in an exception. writer = AgentWriter(agent_url="http://dne:1234") writer.write([]) assert writer.started writer.stop() assert writer.started assert not writer.is_alive() writer.stop() assert writer.started assert not writer.is_alive()
def test_flush_queue_raise(): writer = AgentWriter(agent_url="http://dne:1234") # Should not raise writer.write([]) writer.flush_queue(raise_exc=False) error = OSError if PY3 else IOError with pytest.raises(error): writer.write([]) writer.flush_queue(raise_exc=True)
def test_flush_log(caplog): caplog.set_level(logging.INFO) writer = AgentWriter(agent.get_trace_url()) with mock.patch("ddtrace.internal.writer.log") as log: writer.write([]) writer.flush_queue(raise_exc=True) calls = [ mock.call(logging.DEBUG, "sent %s in %.5fs", AnyStr(), AnyFloat()) ] log.log.assert_has_calls(calls)
def test_metrics_disabled(self): statsd = mock.Mock() writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False) for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.stop() writer.join() statsd.increment.assert_not_called() statsd.distribution.assert_not_called()
def create_worker(self, filters=None, api_class=DummyAPI): self.dogstatsd = mock.Mock() worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters) self.api = api_class() worker.api = self.api for i in range(self.N_TRACES): worker.write([ Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(7) ]) worker.stop() worker.join() return worker
def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False): with self.override_global_config(dict(health_metrics_enabled=enable_stats)): self.dogstatsd = mock.Mock() worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters) worker._STATS_EVERY_INTERVAL = 1 self.api = api_class() worker.api = self.api for i in range(self.N_TRACES): worker.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(7)] ) worker.stop() worker.join() return worker
def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False): self.dogstatsd = mock.Mock() worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters) worker._ENABLE_STATS = enable_stats worker._STATS_EVERY_INTERVAL = 1 self.api = api_class() worker.api = self.api for i in range(self.N_TRACES): worker.write([ Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(7) ]) worker.stop() worker.join() return worker
def create_worker(self, filters): worker = AgentWriter(filters=filters) self.api = DummmyAPI() worker.api = self.api for i in range(self.N_TRACES): worker.write([ Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(7) ]) worker.stop() worker.join() return worker
def test_drop_reason_bad_endpoint(self): statsd = mock.Mock() writer_metrics_reset = mock.Mock() writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False) writer._metrics_reset = writer_metrics_reset for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.stop() writer.join() writer_metrics_reset.assert_called_once() assert 1 == writer._metrics["http.errors"]["count"] assert 10 == writer._metrics["http.dropped.traces"]["count"]
def test_drop_reason_trace_too_big(self): statsd = mock.Mock() writer_metrics_reset = mock.Mock() writer = AgentWriter(dogstatsd=statsd, report_metrics=False, hostname="asdf", port=1234) writer._metrics_reset = writer_metrics_reset for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.write( [Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)] ) writer.stop() writer.join() writer_metrics_reset.assert_called_once() assert 1 == writer._metrics["buffer.dropped.traces"]["count"] assert ["reason:t_too_big"] == writer._metrics["buffer.dropped.traces"]["tags"]
def test_drop_reason_encoding_error(self): statsd = mock.Mock() writer_encoder = mock.Mock() writer_metrics_reset = mock.Mock() writer_encoder.encode_trace.side_effect = Exception writer = AgentWriter(dogstatsd=statsd, report_metrics=False, hostname="asdf", port=1234) writer._encoder = writer_encoder writer._metrics_reset = writer_metrics_reset for i in range(10): writer.write( [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] ) writer.stop() writer.join() writer_metrics_reset.assert_called_once() assert 10 == writer._metrics["encoder.dropped.traces"]["count"]
def test_flush_log(caplog, encoding, monkeypatch): monkeypatch.setenv("DD_TRACE_API_VERSION", encoding) caplog.set_level(logging.INFO) writer = AgentWriter(agent.get_trace_url()) with mock.patch("ddtrace.internal.writer.log") as log: writer.write([]) writer.flush_queue(raise_exc=True) calls = [ mock.call( logging.DEBUG, "sent %s in %.5fs to %s", AnyStr(), AnyFloat(), writer.agent_url, ) ] log.log.assert_has_calls(calls)
def test_flush_log(caplog, encoding, monkeypatch): monkeypatch.setenv("DD_TRACE_API_VERSION", encoding) caplog.set_level(logging.INFO) writer = AgentWriter(agent.get_trace_url()) with mock.patch("ddtrace.internal.writer.log") as log: writer.write([]) writer.flush_queue(raise_exc=True) # for latest agent, default to v0.3 since no priority sampler is set expected_encoding = "v0.3" if AGENT_VERSION == "v5" else (encoding or "v0.3") calls = [ mock.call( logging.DEBUG, "sent %s in %.5fs to %s", AnyStr(), AnyFloat(), "{}/{}/traces".format(writer.agent_url, expected_encoding), ) ] log.log.assert_has_calls(calls)
def test_keep_rate(self): statsd = mock.Mock() writer_run_periodic = mock.Mock() writer_put = mock.Mock() writer_put.return_value = Response(status=200) writer = AgentWriter(agent_url="http://asdf:1234", dogstatsd=statsd, report_metrics=False) writer.run_periodic = writer_run_periodic writer._put = writer_put traces = [ [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(5)] for i in range(4) ] traces_too_big = [ [Span(tracer=None, name="a" * 5000, trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(2 ** 10)] for i in range(4) ] # 1. We write 4 traces successfully. for trace in traces: writer.write(trace) writer.flush_queue() payload = msgpack.unpackb(writer_put.call_args.args[0]) # No previous drops. assert 0.0 == writer._drop_sma.get() # 4 traces written. assert 4 == len(payload) # 100% of traces kept (refers to the past). # No traces sent before now so 100% kept. for trace in payload: assert 1.0 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1) # 2. We fail to write 4 traces because of size limitation. for trace in traces_too_big: writer.write(trace) writer.flush_queue() # 50% of traces were dropped historically. # 4 successfully written before and 4 dropped now. assert 0.5 == writer._drop_sma.get() # put not called since no new traces are available. writer_put.assert_called_once() # 3. We write 2 traces successfully. for trace in traces[:2]: writer.write(trace) writer.flush_queue() payload = msgpack.unpackb(writer_put.call_args.args[0]) # 40% of traces were dropped historically. assert 0.4 == writer._drop_sma.get() # 2 traces written. assert 2 == len(payload) # 50% of traces kept (refers to the past). # We had 4 successfully written and 4 dropped. for trace in payload: assert 0.5 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1) # 4. We write 1 trace successfully and fail to write 3. writer.write(traces[0]) for trace in traces_too_big[:3]: writer.write(trace) writer.flush_queue() payload = msgpack.unpackb(writer_put.call_args.args[0]) # 50% of traces were dropped historically. assert 0.5 == writer._drop_sma.get() # 1 trace written. assert 1 == len(payload) # 60% of traces kept (refers to the past). # We had 4 successfully written, then 4 dropped, then 2 written. for trace in payload: assert 0.6 == trace[0]["metrics"].get(KEEP_SPANS_RATE_KEY, -1)
def collect(tracer): """Collect system and library information into a serializable dict.""" # The tracer doesn't actually maintain a hostname/port, instead it stores # it on the possibly None writer which actually stores it on an API object. # Note that the tracer DOES have hostname and port attributes that it # sets to the defaults and ignores afterwards. if tracer.writer and isinstance(tracer.writer, LogWriter): agent_url = "AGENTLESS" agent_error = None else: if isinstance(tracer.writer, AgentWriter): writer = tracer.writer else: writer = AgentWriter() agent_url = writer.agent_url try: writer.write([]) writer.flush_queue(raise_exc=True) except Exception as e: agent_error = "Agent not reachable at %s. Exception raised: %s" % ( agent_url, str(e)) else: agent_error = None is_venv = in_venv() packages_available = { p.project_name: p.version for p in pkg_resources.working_set } integration_configs = {} for module, enabled in ddtrace.monkey.PATCH_MODULES.items(): # TODO: this check doesn't work in all cases... we need a mapping # between the module and the library name. module_available = module in packages_available module_instrumented = module in ddtrace.monkey._PATCHED_MODULES module_imported = module in sys.modules if enabled: # Note that integration configs aren't added until the integration # module is imported. This typically occurs as a side-effect of # patch(). # This also doesn't load work in all cases since we don't always # name the configuration entry the same as the integration module # name :/ config = ddtrace.config._config.get(module, "N/A") else: config = None if module_available: integration_configs[module] = dict( enabled=enabled, instrumented=module_instrumented, module_available=module_available, module_version=packages_available[module], module_imported=module_imported, config=config, ) else: # Use N/A here to avoid the additional clutter of an entire # config dictionary for a module that isn't available. integration_configs[module] = "N/A" pip_version = packages_available.get("pip", "N/A") return dict( # Timestamp UTC ISO 8601 date=datetime.datetime.utcnow().isoformat(), # eg. "Linux", "Darwin" os_name=platform.system(), # eg. 12.5.0 os_version=platform.release(), is_64_bit=sys.maxsize > 2**32, architecture=platform.architecture()[0], vm=platform.python_implementation(), version=ddtrace.__version__, lang="python", lang_version=platform.python_version(), pip_version=pip_version, in_virtual_env=is_venv, agent_url=agent_url, agent_error=agent_error, env=ddtrace.config.env or "", is_global_tracer=tracer == ddtrace.tracer, enabled_env_setting=os.getenv("DATADOG_TRACE_ENABLED"), tracer_enabled=tracer.enabled, sampler_type=type(tracer.sampler).__name__ if tracer.sampler else "N/A", priority_sampler_type=type(tracer.priority_sampler).__name__ if tracer.priority_sampler else "N/A", service=ddtrace.config.service or "", debug=ddtrace.tracer.log.isEnabledFor(logging.DEBUG), enabled_cli="ddtrace" in os.getenv("PYTHONPATH", ""), analytics_enabled=ddtrace.config.analytics_enabled, log_injection_enabled=ddtrace.config.logs_injection, health_metrics_enabled=ddtrace.config.health_metrics_enabled, dd_version=ddtrace.config.version or "", priority_sampling_enabled=tracer.priority_sampler is not None, global_tags=os.getenv("DD_TAGS", ""), tracer_tags=tags_to_str(tracer.tags), integrations=integration_configs, )