def _test_memory_ignore(ignore):
r = recorder.Recorder()
# Start a stack collector so it allocates memory
with stack.StackCollector(r) as sc:
r = recorder.Recorder()
c = memory.MemoryCollector(r, ignore_profiler=ignore, capture_pct=100)
with c as mc:
while not r.events[memory.MemorySampleEvent]:
_ = _alloc()
# Allow gevent to switch to the memory collector thread
time.sleep(0)
sc.join()
mc.join()
events = r.events[memory.MemorySampleEvent]
files = {
frame.filename
for event in events for trace in event.snapshot.traces
for frame in trace.traceback
}
return files
def test_before_flush_failure(caplog):
def call_me():
raise Exception("LOL")
r = recorder.Recorder()
s = scheduler.Scheduler(r, [exporter.NullExporter()], before_flush=call_me)
r.push_events([event.Event()] * 10)
s.flush()
assert caplog.record_tuples == [
(("ddtrace.profiling.scheduler", logging.ERROR, "Scheduler before_flush hook failed"))
]
def test_patch():
r = recorder.Recorder()
lock = threading.Lock
collector = collector_threading.LockCollector(r)
collector.start()
assert lock == collector.original
# wrapt makes this true
assert lock == threading.Lock
collector.stop()
assert lock == threading.Lock
assert collector.original == threading.Lock
def test_before_flush():
x = {}
def call_me():
x["OK"] = True
r = recorder.Recorder()
s = scheduler.Scheduler(r, [exporter.NullExporter()], before_flush=call_me)
r.push_events([event.Event()] * 10)
s.flush()
assert x["OK"]
def _test_restart(collector, **kwargs):
r = recorder.Recorder()
c = collector(r, **kwargs)
c.start()
c.stop()
c.join()
c.start()
with pytest.raises(RuntimeError):
c.start()
c.stop()
c.join()
def test_collect_truncate():
r = recorder.Recorder()
c = stack.StackCollector(r, nframes=5)
c.start()
func1()
while not r.events[stack.StackSampleEvent]:
pass
c.stop()
e = r.events[stack.StackSampleEvent][0]
assert e.nframes > c.nframes
assert len(e.frames) == c.nframes
def test_new_interval():
r = recorder.Recorder()
c = stack.StackCollector(r, max_time_usage_pct=2)
new_interval = c._compute_new_interval(1000000)
assert new_interval == 0.049
new_interval = c._compute_new_interval(2000000)
assert new_interval == 0.098
c = stack.StackCollector(r, max_time_usage_pct=10)
new_interval = c._compute_new_interval(200000)
assert new_interval == 0.01
new_interval = c._compute_new_interval(1)
assert new_interval == c.min_interval_time
def _test_collector_collect(collector, event_type, fn=None, **kwargs):
r = recorder.Recorder()
c = collector(r, **kwargs)
c.start()
thread_id = c._worker.ident
while not r.events[event_type]:
if fn is not None:
_ = fn()
# Sleep so gevent can switch to the other thread
time.sleep(0)
c.stop()
assert len(r.events[event_type]) >= 1
return r, c, thread_id
def test_filter_remove():
r = recorder.Recorder()
def filter_all(events):
return []
r.add_event_filter(event.Event, filter_all)
r.push_event(event.Event())
assert len(r.events[event.Event]) == 0
r.remove_event_filter(event.Event, filter_all)
r.push_event(event.Event())
assert len(r.events[event.Event]) == 1
def test_collect_once():
r = recorder.Recorder()
s = stack.StackCollector(r)
# Start the collector as we need to have a start time set
with s:
all_events = s.collect()
assert len(all_events) == 2
e = all_events[0][0]
assert e.thread_id > 0
assert e.thread_name == "MainThread"
assert len(e.frames) >= 1
assert e.frames[0][0].endswith(".py")
assert e.frames[0][1] > 0
assert isinstance(e.frames[0][2], str)
def test_collect_truncate():
r = recorder.Recorder()
c = stack.StackCollector(r, nframes=5)
c.start()
func1()
while not r.events[stack.StackSampleEvent]:
pass
c.stop()
for e in r.events[stack.StackSampleEvent]:
if e.thread_name == "MainThread":
assert len(e.frames) <= c.nframes
break
else:
pytest.fail("Unable to find the main thread")
def test_collect_once():
r = recorder.Recorder()
s = stack.StackCollector(r)
s._init()
all_events = s._collect()
assert len(all_events) == 2
e = all_events[0][0]
assert e.thread_id > 0
# Thread name is None with gevent
assert isinstance(e.thread_name, (str, type(None)))
assert len(e.frames) >= 1
assert e.frames[0][0].endswith(".py")
assert e.frames[0][1] > 0
assert isinstance(e.frames[0][2], str)
def test_lock_gevent_tasks():
r = recorder.Recorder()
def play_with_lock():
lock = threading.Lock()
lock.acquire()
lock.release()
with collector_threading.LockCollector(r, capture_pct=100):
t = threading.Thread(name="foobar", target=play_with_lock)
t.start()
t.join()
assert len(r.events[collector_threading.LockAcquireEvent]) >= 1
assert len(r.events[collector_threading.LockReleaseEvent]) >= 1
for event in r.events[collector_threading.LockAcquireEvent]:
if event.lock_name == "test_threading.py:199":
assert event.thread_id == nogevent.main_thread_id
assert event.wait_time_ns >= 0
assert event.task_id == t.ident
assert event.task_name == "foobar"
# It's called through pytest so I'm sure it's gonna be that long, right?
assert len(event.frames) > 3
assert event.nframes > 3
assert event.frames[0] == (__file__, 200, "play_with_lock")
assert event.sampling_pct == 100
assert event.task_id == t.ident
assert event.task_name == "foobar"
break
else:
pytest.fail("Lock event not found")
for event in r.events[collector_threading.LockReleaseEvent]:
if event.lock_name == "test_threading.py:199":
assert event.thread_id == nogevent.main_thread_id
assert event.locked_for_ns >= 0.1
assert event.task_id == t.ident
assert event.task_name == "foobar"
# It's called through pytest so I'm sure it's gonna be that long, right?
assert len(event.frames) > 3
assert event.nframes > 3
assert event.frames[0] == (__file__, 201, "play_with_lock")
assert event.sampling_pct == 100
assert event.task_id == t.ident
assert event.task_name == "foobar"
break
else:
pytest.fail("Lock event not found")
def test_stress_threads():
NB_THREADS = 20
threads = []
for i in range(NB_THREADS):
t = threading.Thread(target=_f0) # noqa: E149,F821
t.start()
threads.append(t)
s = stack.StackCollector(recorder=recorder.Recorder())
number = 10000
with s:
exectime = timeit.timeit(s.collect, number=number)
print("%.3f ms per call" % (1000.0 * exectime / number))
for t in threads:
t.join()
def test_lock_acquire_events():
r = recorder.Recorder()
with collector_threading.LockCollector(r, capture_pct=100):
lock = threading.Lock()
lock.acquire()
assert len(r.events[collector_threading.LockAcquireEvent]) == 1
assert len(r.events[collector_threading.LockReleaseEvent]) == 0
event = r.events[collector_threading.LockAcquireEvent][0]
assert event.lock_name == "test_threading.py:59"
assert event.thread_id == _thread.get_ident()
assert event.wait_time_ns > 0
# It's called through pytest so I'm sure it's gonna be that long, right?
assert len(event.frames) > 3
assert event.nframes > 3
assert event.frames[0] == (__file__, 60, "test_lock_acquire_events")
assert event.sampling_pct == 100
def test_lock_release_events():
r = recorder.Recorder()
with collector_threading.ThreadingLockCollector(r, capture_pct=100):
lock = threading.Lock()
lock.acquire()
lock.release()
assert len(r.events[collector_threading.ThreadingLockAcquireEvent]) == 1
assert len(r.events[collector_threading.ThreadingLockReleaseEvent]) == 1
event = r.events[collector_threading.ThreadingLockReleaseEvent][0]
assert event.lock_name == "test_threading.py:178"
assert event.thread_id == nogevent.thread_get_ident()
assert event.locked_for_ns >= 0.1
# It's called through pytest so I'm sure it's gonna be that long, right?
assert len(event.frames) > 3
assert event.nframes > 3
assert event.frames[0] == (__file__, 180, "test_lock_release_events")
assert event.sampling_pct == 100
def test_collect_once():
r = recorder.Recorder()
s = stack.StackCollector(r)
s._init()
all_events = s.collect()
assert len(all_events) == 2
stack_events = all_events[0]
for e in stack_events:
if e.thread_name == "MainThread":
assert e.thread_id > 0
assert len(e.frames) >= 1
assert e.frames[0][0].endswith(".py")
assert e.frames[0][1] > 0
assert isinstance(e.frames[0][2], str)
break
else:
pytest.fail("Unable to find MainThread")
def test_stress_threads_run_as_thread():
NB_THREADS = 40
threads = []
for i in range(NB_THREADS):
t = threading.Thread(target=_f0) # noqa: E149,F821
t.start()
threads.append(t)
r = recorder.Recorder()
s = stack.StackCollector(recorder=r)
# This mainly check nothing bad happens when we collect a lot of threads and store the result in the Recorder
with s:
time.sleep(3)
assert r.events[stack.StackSampleEvent]
for t in threads:
t.join()
def __attrs_post_init__(self):
r = self._recorder = recorder.Recorder(
max_events={
# Allow to store up to 10 threads for 60 seconds at 100 Hz
stack.StackSampleEvent:
10 * 60 * 100,
stack.StackExceptionSampleEvent:
10 * 60 * 100,
# This can generate one event every 0.1s if 100% are taken — though we take 5% by default.
# = (60 seconds / 0.1 seconds)
memory.MemorySampleEvent:
int(60 / 0.1),
# (default buffer size / interval) * export interval
memalloc.MemoryAllocSampleEvent:
int((memalloc.MemoryCollector._DEFAULT_MAX_EVENTS /
memalloc.MemoryCollector._DEFAULT_INTERVAL) * 60),
# Do not limit the heap sample size as the number of events is relative to allocated memory anyway
memalloc.MemoryHeapSampleEvent:
None,
},
default_max_events=int(
os.environ.get("DD_PROFILING_MAX_EVENTS",
recorder.Recorder._DEFAULT_MAX_EVENTS)),
)
if formats.asbool(os.environ.get("DD_PROFILING_MEMALLOC", "true")):
mem_collector = memalloc.MemoryCollector(r)
else:
mem_collector = memory.MemoryCollector(r)
self._collectors = [
stack.StackCollector(r, tracer=self.tracer),
mem_collector,
threading.LockCollector(r, tracer=self.tracer),
]
exporters = self._build_default_exporters(self.tracer, self.url,
self.tags, self.service,
self.env, self.version)
if exporters:
self._scheduler = scheduler.Scheduler(
recorder=r,
exporters=exporters,
before_flush=self._collectors_snapshot)
def test_stress_threads():
NB_THREADS = 10
threads = []
for i in range(NB_THREADS):
t = threading.Thread(target=_f0) # noqa: E149,F821
t.start()
threads.append(t)
s = stack.StackCollector(recorder=recorder.Recorder())
# Make sure that the collector thread does not interfere with the test
s.MIN_INTERVAL_TIME = 60
number = 20000
s._init()
exectime = timeit.timeit(s.collect, number=number)
# Threads are fake threads with gevent, so result is actually for one thread, not NB_THREADS
print("%.3f ms per call" % (1000.0 * exectime / number))
for t in threads:
t.join()
def test_memory_collector_ignore_profiler(ignore_profiler):
r = recorder.Recorder()
mc = memalloc.MemoryCollector(r, ignore_profiler=ignore_profiler)
with mc:
object()
# Make sure we collect at least once
mc.periodic()
ok = False
for event in r.events[memalloc.MemoryAllocSampleEvent]:
for frame in event.frames:
if ignore_profiler:
assert frame[0] != _periodic.__file__
elif frame[0] == _periodic.__file__:
ok = True
break
if not ignore_profiler:
assert ok
def test_with_override():
seen = {"seen": False}
def myhook(exctype, value, traceback):
seen["seen"] = True
sys.excepthook = myhook
r = recorder.Recorder()
c = exceptions.UncaughtExceptionCollector(r)
c.start()
sys.excepthook(ValueError, ValueError(), None)
c.stop()
assert seen["seen"]
seen["seen"] = False
sys.excepthook(ValueError, ValueError(), None)
assert seen["seen"]
events = r.events[exceptions.UncaughtExceptionEvent]
assert len(events) == 1
_check_event(events[0])
def test_exception_collection():
r = recorder.Recorder()
c = stack.StackCollector(r)
with c:
try:
raise ValueError("hello")
except Exception:
nogevent.sleep(1)
exception_events = r.events[stack.StackExceptionSampleEvent]
assert len(exception_events) >= 1
e = exception_events[0]
assert e.timestamp > 0
assert e.sampling_period > 0
assert e.thread_id == nogevent.thread_get_ident()
assert e.thread_name == "MainThread"
assert e.frames == [(__file__, 321, "test_exception_collection")]
assert e.nframes == 1
assert e.exc_type == ValueError
def test_memory_collector_ignore_profiler(ignore_profiler):
r = recorder.Recorder()
mc = memalloc.MemoryCollector(r, ignore_profiler=ignore_profiler)
with mc:
thread_id = mc._worker.ident
object()
# Make sure we collect at least once
mc.periodic()
ok = False
for event in r.events[memalloc.MemoryAllocSampleEvent]:
if ignore_profiler:
assert event.thread_id != thread_id
elif event.thread_id == thread_id:
ok = True
break
if not ignore_profiler:
assert ok
def test_wrapper():
r = recorder.Recorder()
collector = collector_threading.LockCollector(r)
with collector:
class Foobar(object):
lock_class = threading.Lock
def __init__(self):
lock = self.lock_class()
assert lock.acquire()
lock.release()
# Try to access the attribute
lock = Foobar.lock_class()
assert lock.acquire()
lock.release()
# Try this way too
Foobar()
def test_heap_collector():
heap_sample_size = 1024
r = recorder.Recorder()
mc = memalloc.MemoryCollector(r, heap_sample_size=heap_sample_size)
with mc:
keep_me = _allocate_1k()
events = mc.snapshot()
assert len(events) == 1
assert len(events[0]) >= 1
del keep_me
for event in events[0]:
assert 0 < len(event.frames) <= mc.max_nframe
assert event.nframes >= len(event.frames)
assert event.sample_size == heap_sample_size
assert len(event.frames) >= 1
assert event.size > 0
assert event.thread_id > 0
assert isinstance(event.thread_name, str)
def test_exception_collection():
r = recorder.Recorder()
c = stack.StackCollector(r)
c.start()
try:
raise ValueError("hello")
except Exception:
sleep(1)
c.stop()
exception_events = r.events[stack.StackExceptionSampleEvent]
assert len(exception_events) >= 1
e = exception_events[0]
assert e.timestamp > 0
assert e.sampling_period > 0
if not TESTING_GEVENT:
assert e.thread_id == _thread.get_ident()
assert e.thread_name == "MainThread"
assert e.frames == [(__file__, 209, "test_exception_collection")]
assert e.nframes == 1
assert e.exc_type == ValueError
def __attrs_post_init__(self):
r = recorder.Recorder(
max_events={
# Allow to store up to 10 threads for 60 seconds at 100 Hz
stack.StackSampleEvent:
10 * 60 * 100,
stack.StackExceptionSampleEvent:
10 * 60 * 100,
# This can generate one event every 0.1s if 100% are taken — though we take 5% by default.
# = (60 seconds / 0.1 seconds)
memory.MemorySampleEvent:
int(60 / 0.1),
# (default buffer size / interval) * export interval
memalloc.MemoryAllocSampleEvent:
int((64 / 0.5) * 60),
},
default_max_events=int(
os.environ.get("DD_PROFILING_MAX_EVENTS",
recorder.Recorder._DEFAULT_MAX_EVENTS)),
)
if formats.asbool(os.environ.get("DD_PROFILING_MEMALLOC", "false")):
mem_collector = memalloc.MemoryCollector(r)
else:
mem_collector = memory.MemoryCollector(r)
self._collectors = [
stack.StackCollector(r, tracer=self.tracer),
mem_collector,
exceptions.UncaughtExceptionCollector(r),
threading.LockCollector(r),
]
exporters = self._build_default_exporters(self.service, self.env,
self.version)
if exporters:
self._scheduler = scheduler.Scheduler(recorder=r,
exporters=exporters)
def test_exception_collection_trace(tracer):
r = recorder.Recorder()
c = stack.StackCollector(r, tracer=tracer)
with c:
with tracer.trace("test123") as span:
try:
raise ValueError("hello")
except Exception:
nogevent.sleep(1)
exception_events = r.events[stack.StackExceptionSampleEvent]
assert len(exception_events) >= 1
e = exception_events[0]
assert e.timestamp > 0
assert e.sampling_period > 0
assert e.thread_id == nogevent.thread_get_ident()
assert e.thread_name == "MainThread"
assert e.frames == [(__file__, 344, "test_exception_collection_trace")]
assert e.nframes == 1
assert e.exc_type == ValueError
assert e.span_id == span.span_id
assert e.trace_id == span.trace_id
def test_collect_once():
r = recorder.Recorder()
s = stack.StackCollector(r)
s._init()
all_events = s.collect()
assert len(all_events) == 2
stack_events = all_events[0]
for e in stack_events:
if e.thread_name == "MainThread":
if TESTING_GEVENT and stack.FEATURES["gevent-tasks"]:
assert e.task_id > 0
assert e.task_name == e.thread_name
else:
assert e.task_id is None
assert e.task_name is None
assert e.thread_id > 0
assert len(e.frames) >= 1
assert e.frames[0][0].endswith(".py")
assert e.frames[0][1] > 0
assert isinstance(e.frames[0][2], str)
break
else:
pytest.fail("Unable to find MainThread")
