def test_asbool(self): # ensure the value is properly cast self.assertTrue(asbool("True")) self.assertTrue(asbool("true")) self.assertTrue(asbool("1")) self.assertFalse(asbool("False")) self.assertFalse(asbool("false")) self.assertFalse(asbool(None)) self.assertFalse(asbool("")) self.assertTrue(asbool(True)) self.assertFalse(asbool(False))
def _get_default_heap_sample_size( default_heap_sample_size=512 * 1024, # type: int ): # type: (...) -> int heap_sample_size = os.environ.get("DD_PROFILING_HEAP_SAMPLE_SIZE") if heap_sample_size is not None: return int(heap_sample_size) if not formats.asbool(os.environ.get("DD_PROFILING_HEAP_ENABLED", "1")): return 0 try: from ddtrace.vendor import psutil total_mem = psutil.swap_memory().total + psutil.virtual_memory().total except Exception: LOG.warning( "Unable to get total memory available, using default value of %d KB", default_heap_sample_size / 1024, exc_info=True, ) return default_heap_sample_size # This is TRACEBACK_ARRAY_MAX_COUNT max_samples = 2**16 return max(math.ceil(total_mem / max_samples), default_heap_sample_size)
def update_patched_modules(): modules_to_patch = os.getenv("DD_PATCH_MODULES") if not modules_to_patch: return modules = parse_tags_str(modules_to_patch) for module, should_patch in modules.items(): EXTRA_PATCHED_MODULES[module] = asbool(should_patch)
class _ProfilerInstance(service.Service): """A instance of the profiler. Each process must manage its own instance. """ # User-supplied values url = attr.ib(default=None) service = attr.ib(factory=_get_service_name) tags = attr.ib(factory=dict) env = attr.ib(factory=lambda: os.environ.get("DD_ENV")) version = attr.ib(factory=lambda: os.environ.get("DD_VERSION")) tracer = attr.ib(default=ddtrace.tracer) api_key = attr.ib(factory=lambda: os.environ.get("DD_API_KEY"), type=Optional[str]) agentless = attr.ib(factory=lambda: formats.asbool( os.environ.get("DD_PROFILING_AGENTLESS", "False")), type=bool) _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) _scheduler = attr.ib(init=False, default=None) ENDPOINT_TEMPLATE = "https://intake.profile.{}" def _build_default_exporters(self): # type: (...) -> List[exporter.Exporter] _OUTPUT_PPROF = os.environ.get("DD_PROFILING_OUTPUT_PPROF") if _OUTPUT_PPROF: return [ file.PprofFileExporter(_OUTPUT_PPROF), ] if self.url is not None: endpoint = self.url elif self.agentless: LOG.warning( "Agentless uploading is currently for internal usage only and not officially supported. " "You should not enable it unless somebody at Datadog instructed you to do so." ) endpoint = self.ENDPOINT_TEMPLATE.format( os.environ.get("DD_SITE", "datadoghq.com")) else: if isinstance(self.tracer.writer, writer.AgentWriter): endpoint = self.tracer.writer.agent_url else: endpoint = agent.get_trace_url() if self.agentless: endpoint_path = "/v1/input" else: # Agent mode # path is relative because it is appended # to the agent base path. endpoint_path = "profiling/v1/input" return [ http.PprofHTTPExporter( service=self.service, env=self.env, tags=self.tags, version=self.version, api_key=self.api_key, endpoint=endpoint, endpoint_path=endpoint_path, ), ] def __attrs_post_init__(self): r = self._recorder = recorder.Recorder( max_events={ # Allow to store up to 10 threads for 60 seconds at 100 Hz stack.StackSampleEvent: 10 * 60 * 100, stack.StackExceptionSampleEvent: 10 * 60 * 100, # (default buffer size / interval) * export interval memalloc.MemoryAllocSampleEvent: int((memalloc.MemoryCollector._DEFAULT_MAX_EVENTS / memalloc.MemoryCollector._DEFAULT_INTERVAL) * 60), # Do not limit the heap sample size as the number of events is relative to allocated memory anyway memalloc.MemoryHeapSampleEvent: None, }, default_max_events=int( os.environ.get("DD_PROFILING_MAX_EVENTS", recorder.Recorder._DEFAULT_MAX_EVENTS)), ) self._collectors = [ stack.StackCollector(r, tracer=self.tracer), memalloc.MemoryCollector(r), threading.LockCollector(r, tracer=self.tracer), ] exporters = self._build_default_exporters() if exporters: self._scheduler = scheduler.Scheduler( recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) def _collectors_snapshot(self): for c in self._collectors: try: snapshot = c.snapshot() if snapshot: for events in snapshot: self._recorder.push_events(events) except Exception: LOG.error("Error while snapshoting collector %r", c, exc_info=True) def copy(self): return self.__class__(service=self.service, env=self.env, version=self.version, tracer=self.tracer, tags=self.tags) def _start_service(self): # type: ignore[override] # type: (...) -> None """Start the profiler.""" collectors = [] for col in self._collectors: try: col.start() except collector.CollectorUnavailable: LOG.debug("Collector %r is unavailable, disabling", col) except Exception: LOG.error("Failed to start collector %r, disabling.", col, exc_info=True) else: collectors.append(col) self._collectors = collectors if self._scheduler is not None: self._scheduler.start() def _stop_service( # type: ignore[override] self, flush=True # type: bool ): # type: (...) -> None """Stop the profiler. :param flush: Flush a last profile. """ if self._scheduler is not None: self._scheduler.stop() # Wait for the export to be over: export might need collectors (e.g., for snapshot) so we can't stop # collectors before the possibly running flush is finished. self._scheduler.join() if flush: # Do not stop the collectors before flushing, they might be needed (snapshot) self._scheduler.flush() for col in reversed(self._collectors): try: col.stop() except service.ServiceStatusError: # It's possible some collector failed to start, ignore failure to stop pass for col in reversed(self._collectors): col.join()
from ddtrace.tracer import DD_LOG_FORMAT # noqa from ddtrace.tracer import debug_mode from ddtrace.vendor.debtcollector import deprecate if config.logs_injection: # immediately patch logging if trace id injected from ddtrace import patch patch(logging=True) # DEV: Once basicConfig is called here, future calls to it cannot be used to # change the formatter since it applies the formatter to the root handler only # upon initializing it the first time. # See https://github.com/python/cpython/blob/112e4afd582515fcdcc0cde5012a4866e5cfda12/Lib/logging/__init__.py#L1550 # Debug mode from the tracer will do a basicConfig so only need to do this otherwise call_basic_config = asbool(os.environ.get("DD_CALL_BASIC_CONFIG", "false")) if not debug_mode and call_basic_config: deprecate( "ddtrace.tracer.logging.basicConfig", message= "`logging.basicConfig()` should be called in a user's application." " ``DD_CALL_BASIC_CONFIG`` will be removed in a future version.", ) if config.logs_injection: logging.basicConfig(format=DD_LOG_FORMAT) else: logging.basicConfig() log = get_logger(__name__) EXTRA_PATCHED_MODULES = {
from .. import trace_utils log = get_logger(__name__) config._add( "django", dict( _default_service="django", cache_service_name=get_env("django", "cache_service_name") or "django", database_service_name_prefix=get_env("django", "database_service_name_prefix", default=""), database_service_name=get_env("django", "database_service_name", default=""), trace_fetch_methods=asbool( get_env("django", "trace_fetch_methods", default=False)), distributed_tracing_enabled=True, instrument_middleware=asbool( get_env("django", "instrument_middleware", default=True)), instrument_databases=True, instrument_caches=True, analytics_enabled= None, # None allows the value to be overridden by the global config analytics_sample_rate=None, trace_query_string=None, # Default to global config include_user_name=True, use_handler_resource_format=asbool( get_env("django", "use_handler_resource_format", default=False)), use_legacy_resource_format=asbool( get_env("django", "use_legacy_resource_format", default=False)), ),
def main(): parser = argparse.ArgumentParser( description=USAGE, prog="ddtrace-run", usage="ddtrace-run <your usual python command>", formatter_class=argparse.RawTextHelpFormatter, ) parser.add_argument("command", nargs=argparse.REMAINDER, type=str, help="Command string to execute.") parser.add_argument("-d", "--debug", help="enable debug mode (disabled by default)", action="store_true") parser.add_argument( "-i", "--info", help= ("print library info useful for debugging. Only reflects configurations made via environment " "variables, not those made in code."), action="store_true", ) parser.add_argument("-p", "--profiling", help="enable profiling (disabled by default)", action="store_true") parser.add_argument("-v", "--version", action="version", version="%(prog)s " + ddtrace.__version__) parser.add_argument("-nc", "--colorless", help="print output of command without color", action="store_true") args = parser.parse_args() if args.profiling: os.environ["DD_PROFILING_ENABLED"] = "true" debug_mode = args.debug or asbool(get_env("trace", "debug", default=False)) if debug_mode: logging.basicConfig(level=logging.DEBUG) os.environ["DD_TRACE_DEBUG"] = "true" if args.info: # Inline imports for performance. from ddtrace.internal.debug import pretty_collect print(pretty_collect(ddtrace.tracer, color=not args.colorless)) sys.exit(0) root_dir = os.path.dirname(ddtrace.__file__) log.debug("ddtrace root: %s", root_dir) bootstrap_dir = os.path.join(root_dir, "bootstrap") log.debug("ddtrace bootstrap: %s", bootstrap_dir) _add_bootstrap_to_pythonpath(bootstrap_dir) log.debug("PYTHONPATH: %s", os.environ["PYTHONPATH"]) log.debug("sys.path: %s", sys.path) if not args.command: parser.print_help() sys.exit(1) # Find the executable path executable = spawn.find_executable(args.command[0]) if not executable: print("ddtrace-run: failed to find executable '%s'.\n" % args.command[0]) parser.print_usage() sys.exit(1) log.debug("program executable: %s", executable) if os.path.basename(executable) == "uwsgi": print(( "ddtrace-run has known compatibility issues with uWSGI where the " "tracer is not started properly in uWSGI workers which can cause " "broken behavior. It is recommended you remove ddtrace-run and " "update your uWSGI configuration following " "https://ddtrace.readthedocs.io/en/stable/advanced_usage.html#uwsgi." )) try: os.execl(executable, executable, *args.command[1:]) except PermissionError: print("ddtrace-run: permission error while launching '%s'" % executable) print("Did you mean `ddtrace-run python %s`?" % executable) sys.exit(1) except Exception: print("ddtrace-run: error launching '%s'" % executable) raise sys.exit(0)
from ddtrace.internal.utils.formats import parse_tags_str from ddtrace.tracer import DD_LOG_FORMAT # noqa from ddtrace.tracer import debug_mode if config.logs_injection: # immediately patch logging if trace id injected from ddtrace import patch patch(logging=True) # DEV: Once basicConfig is called here, future calls to it cannot be used to # change the formatter since it applies the formatter to the root handler only # upon initializing it the first time. # See https://github.com/python/cpython/blob/112e4afd582515fcdcc0cde5012a4866e5cfda12/Lib/logging/__init__.py#L1550 # Debug mode from the tracer will do a basicConfig so only need to do this otherwise call_basic_config = asbool(os.environ.get("DD_CALL_BASIC_CONFIG", "true")) if not debug_mode and call_basic_config: if config.logs_injection: logging.basicConfig(format=DD_LOG_FORMAT) else: logging.basicConfig() log = get_logger(__name__) EXTRA_PATCHED_MODULES = { "bottle": True, "django": True, "falcon": True, "flask": True, "pylons": True, "pyramid": True,
from ..trace_utils import unwrap from ..trace_utils import with_traced_module as with_traced_module_sync from ..trace_utils import wrap from ..trace_utils_async import with_traced_module log = get_logger(__name__) # Server config config._add( "aiohttp", dict(distributed_tracing=True), ) config._add( "aiohttp_client", dict(distributed_tracing=asbool( os.getenv("DD_AIOHTTP_CLIENT_DISTRIBUTED_TRACING", True)), ), ) class _WrappedConnectorClass(wrapt.ObjectProxy): def __init__(self, obj, pin): super().__init__(obj) pin.onto(self) async def connect(self, req, *args, **kwargs): pin = Pin.get_from(self) with pin.tracer.trace("%s.connect" % self.__class__.__name__): result = await self.__wrapped__.connect(req, *args, **kwargs) return result async def _create_connection(self, req, *args, **kwargs):
from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.wrappers import unwrap as _u from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator from ddtrace.vendor.wrapt import wrap_function_wrapper as _w if typing.TYPE_CHECKING: from ddtrace import Span from ddtrace.vendor.wrapt import BoundFunctionWrapper config._add( "httpx", { "distributed_tracing": asbool(os.getenv("DD_HTTPX_DISTRIBUTED_TRACING", default=True)), "split_by_domain": asbool(os.getenv("DD_HTTPX_SPLIT_BY_DOMAIN", default=False)), }, ) def _url_to_str(url): # type: (httpx.URL) -> str """ Helper to convert the httpx.URL parts from bytes to a str """ scheme, host, port, raw_path = url.raw url = scheme + b"://" + host if port is not None: url += b":" + ensure_binary(str(port)) url += raw_path
import mariadb from ddtrace import Pin from ddtrace import config from ddtrace.contrib.dbapi import TracedConnection from ddtrace.ext import db from ddtrace.ext import net from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import get_env from ddtrace.internal.utils.wrappers import unwrap from ddtrace.vendor import wrapt config._add( "mariadb", dict( trace_fetch_methods=asbool( get_env("mariadb", "trace_fetch_methods", default=False)), _default_service="mariadb", ), ) def patch(): if getattr(mariadb, "_datadog_patch", False): return setattr(mariadb, "_datadog_patch", True) wrapt.wrap_function_wrapper("mariadb", "connect", _connect) def unpatch(): if getattr(mariadb, "_datadog_patch", False): setattr(mariadb, "_datadog_patch", False)
import mariadb from ddtrace import Pin from ddtrace import config from ddtrace.contrib.dbapi import TracedConnection from ddtrace.ext import db from ddtrace.ext import net from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.wrappers import unwrap from ddtrace.vendor import wrapt config._add( "mariadb", dict( trace_fetch_methods=asbool( os.getenv("DD_MARIADB_TRACE_FETCH_METHODS", default=False)), _default_service="mariadb", _dbapi_span_name_prefix="mariadb", ), ) def patch(): if getattr(mariadb, "_datadog_patch", False): return setattr(mariadb, "_datadog_patch", True) wrapt.wrap_function_wrapper("mariadb", "connect", _connect) def unpatch(): if getattr(mariadb, "_datadog_patch", False):
import ddtrace.internal.forksafe as forksafe from ddtrace.internal.utils.formats import asbool from .app import patch_app from .app import unpatch_app from .constants import PRODUCER_SERVICE from .constants import WORKER_SERVICE forksafe._soft = True # Celery default settings config._add( "celery", { "distributed_tracing": asbool(os.getenv("DD_CELERY_DISTRIBUTED_TRACING", default=False)), "producer_service_name": os.getenv("DD_CELERY_PRODUCER_SERVICE_NAME", default=PRODUCER_SERVICE), "worker_service_name": os.getenv("DD_CELERY_WORKER_SERVICE_NAME", default=WORKER_SERVICE), }, ) def patch(): """Instrument Celery base application and the `TaskRegistry` so that any new registered task is automatically instrumented. In the case of Django-Celery integration, also the `@shared_task` decorator must be instrumented because Django doesn't use the Celery registry. """ patch_app(celery.Celery)
from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import get_env from ddtrace.internal.utils.wrappers import unwrap as _u from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator from ddtrace.vendor.wrapt import wrap_function_wrapper as _w if typing.TYPE_CHECKING: from ddtrace import Span from ddtrace.vendor.wrapt import BoundFunctionWrapper config._add( "httpx", { "distributed_tracing": asbool(get_env("httpx", "distributed_tracing", default=True)), "split_by_domain": asbool(get_env("httpx", "split_by_domain", default=False)), }, ) def _url_to_str(url): # type: (httpx.URL) -> str """ Helper to convert the httpx.URL parts from bytes to a str """ scheme, host, port, raw_path = url.raw url = scheme + b"://" + host if port is not None: url += b":" + ensure_binary(str(port))