def run( self ):
self.file = open( self.fname, "a" )
print >> self.file, "Heartbeat for pid %d thread started at %s" % ( self.pid, time.asctime() )
print >> self.file
self.file_nonsleeping = open ( self.fname_nonsleeping, "a" )
print >> self.file_nonsleeping, "Non-Sleeping-threads for pid %d thread started at %s" % ( self.pid, time.asctime() )
print >> self.file_nonsleeping
try:
while not self.should_stop:
# Print separator with timestamp
print >> self.file, "Traceback dump for all threads at %s:" % time.asctime()
print >> self.file
# Print the thread states
threads = get_current_thread_object_dict()
for thread_id, frame in threadframe.dict().iteritems():
if thread_id in threads:
object = repr( threads[thread_id] )
else:
object = "<No Thread object>"
print >> self.file, "Thread %s, %s:" % ( thread_id, object )
print >> self.file
traceback.print_stack( frame, file=self.file )
print >> self.file
print >> self.file, "End dump"
print >> self.file
self.file.flush()
self.print_nonsleeping(threads)
# Sleep for a bit
self.wait_event.wait( self.period )
finally:
print >> self.file, "Heartbeat for pid %d thread stopped at %s" % ( self.pid, time.asctime() )
print >> self.file
# Cleanup
self.file.close()
self.file_nonsleeping.close()
def run(self):
self.file = open(self.fname, "a")
print >> self.file, "Heartbeat for pid %d thread started at %s" % (
self.pid, time.asctime())
print >> self.file
try:
while not self.should_stop:
# Print separator with timestamp
print >> self.file, "Traceback dump for all threads at %s:" % time.asctime(
)
print >> self.file
# Print the thread states
threads = get_current_thread_object_dict()
for thread_id, frame in threadframe.dict().iteritems():
if thread_id in threads:
object = repr(threads[thread_id])
else:
object = "<No Thread object>"
print >> self.file, "Thread %s, %s:" % (thread_id,
object)
print >> self.file
traceback.print_stack(frame, file=self.file)
print >> self.file
print >> self.file, "End dump"
print >> self.file
self.file.flush()
# Sleep for a bit
self.wait_event.wait(self.period)
finally:
print >> self.file, "Heartbeat for pid %d thread stopped at %s" % (
self.pid, time.asctime())
print >> self.file
# Cleanup
self.file.close()
def print_nonsleeping(self, threads_object_dict):
print >> self.file_nonsleeping, "Non-Sleeping threads at %s:" % time.asctime(
)
print >> self.file_nonsleeping
all_threads_are_sleeping = True
threads = get_current_thread_object_dict()
for thread_id, frame in threadframe.dict().iteritems():
if thread_id in threads:
object = repr(threads[thread_id])
else:
object = "<No Thread object>"
tb = traceback.extract_stack(frame)
if self.thread_is_sleeping(tb[-1]):
if thread_id in self.nonsleeping_heartbeats:
del self.nonsleeping_heartbeats[thread_id]
continue
# Count non-sleeping thread heartbeats
if thread_id in self.nonsleeping_heartbeats:
self.nonsleeping_heartbeats[thread_id] += 1
else:
self.nonsleeping_heartbeats[thread_id] = 1
good_frame = self.get_interesting_stack_frame(tb)
print >> self.file_nonsleeping, "Thread %s\t%s\tnon-sleeping for %d heartbeat(s)\n File %s:%d\n Function \"%s\"\n %s" % \
( thread_id, object, self.nonsleeping_heartbeats[thread_id], good_frame[0], good_frame[1], good_frame[2], good_frame[3] )
all_threads_are_sleeping = False
if all_threads_are_sleeping:
print >> self.file_nonsleeping, "All threads are sleeping."
print >> self.file_nonsleeping
self.file_nonsleeping.flush()
def print_nonsleeping( self, threads_object_dict ):
print >> self.file_nonsleeping, "Non-Sleeping threads at %s:" % time.asctime()
print >> self.file_nonsleeping
all_threads_are_sleeping = True
threads = get_current_thread_object_dict()
for thread_id, frame in threadframe.dict().iteritems():
if thread_id in threads:
object = repr( threads[thread_id] )
else:
object = "<No Thread object>"
tb = traceback.extract_stack(frame)
if self.thread_is_sleeping(tb[-1]):
if thread_id in self.nonsleeping_heartbeats:
del self.nonsleeping_heartbeats[thread_id]
continue
# Count non-sleeping thread heartbeats
if thread_id in self.nonsleeping_heartbeats:
self.nonsleeping_heartbeats[thread_id] += 1
else:
self.nonsleeping_heartbeats[thread_id]=1
good_frame = self.get_interesting_stack_frame(tb)
print >> self.file_nonsleeping, "Thread %s\t%s\tnon-sleeping for %d heartbeat(s)\n File %s:%d\n Function \"%s\"\n %s" % \
( thread_id, object, self.nonsleeping_heartbeats[thread_id], good_frame[0], good_frame[1], good_frame[2], good_frame[3] )
all_threads_are_sleeping = False
if all_threads_are_sleeping:
print >> self.file_nonsleeping, "All threads are sleeping."
print >> self.file_nonsleeping
self.file_nonsleeping.flush()
def print_stacks():
while True:
print '=' * 72
for id, frame in threadframe.dict().iteritems():
print '----- [%s] ------' % id
print_stack(frame)
time.sleep(10)
def print_stacks():
while True:
print '=' * 72
for id,frame in threadframe.dict().iteritems():
print '----- [%s] ------' % id
print_stack(frame)
time.sleep(10)
def dump(self):
try:
stacks = sys._current_frames()
except AttributeError:
try:
stacks = threadframe.dict()
except:
self.logger.debug("Stack dumping not supported")
return
for (id, stack) in stacks.items():
msg = "Stack trace for thread %i\n" % id
msg = msg + ''.join(traceback.format_stack(stack))
self.logger.debug(msg)
def dump(self):
try:
stacks = sys._current_frames()
except AttributeError:
try:
stacks = threadframe.dict()
except:
self.logger.debug("Stack dumping not supported")
return
for (id, stack) in stacks.items():
msg = "Stack trace for thread %i\n" % id
msg = msg + ''.join(traceback.format_stack(stack))
self.logger.debug(msg)
def dump_threads():
"""Dump running threads
Returns a string with the tracebacks.
"""
res = []
frames = threadframe.dict()
this_thread_id = thread.get_ident()
for thread_id, frame in frames.iteritems():
if thread_id == this_thread_id:
continue
# Find request in frame
reqinfo = ['']
f = frame
while f is not None:
co = f.f_code
if co.co_filename.endswith('Publish.py'):
request = f.f_locals.get('request')
if request is not None:
method = request.get('REQUEST_METHOD', '')
path = request.get('PATH_INFO', '')
url = request.get('URL', '')
agent = request.get('HTTP_USER_AGENT', '')
query_string = request.get('QUERY_STRING')
query = 'QUERY: %s %s' % (method, path)
if query_string is not None:
query += '?%s' % query_string
reqinfo.append(query)
reqinfo.append('URL: %s' % url)
reqinfo.append('HTTP_USER_AGENT: %s' % agent)
break
f = f.f_back
output = StringIO()
traceback.print_stack(frame, file=output)
output = output.getvalue()
lines = [line.strip() for line in output.split('\n') if line.strip() != '']
zeo_marker = os.path.join('ZEO', 'zrpc', 'connection')
acquire_marker = 'l.acquire()'
if len(lines) > 1 and (zeo_marker in lines[-2] or acquire_marker in lines[-1]):
output = None
res.append((thread_id, reqinfo, output))
return res
def dump_threads():
"""Dump running threads
Returns a string with the tracebacks.
"""
frames = threadframe.dict()
this_thread_id = thread.get_ident()
now = time.strftime("%Y-%m-%d %H:%M:%S")
res = ["Threads traceback dump at %s\n" % now]
for thread_id, frame in frames.iteritems():
if thread_id == this_thread_id:
continue
# Find request in frame
reqinfo = ''
f = frame
while f is not None:
co = f.f_code
if (co.co_name == 'publish' and
co.co_filename.endswith('/ZPublisher/Publish.py')):
request = f.f_locals.get('request')
if request is not None:
reqinfo = (request.get('REQUEST_METHOD', '') + ' ' +
request.get('PATH_INFO', ''))
qs = request.get('QUERY_STRING')
if qs:
reqinfo += '?'+qs
break
f = f.f_back
if reqinfo:
reqinfo = " (%s)" % reqinfo
output = StringIO()
traceback.print_stack(frame, file=output)
res.append("Thread %s%s:\n%s" %
(thread_id, reqinfo, output.getvalue()))
frames = None
res.append("End of dump")
result = '\n'.join(res)
if isinstance(result, unicode):
result = result.encode('utf-8')
return result
def threadDump(signum, frame): """Signal handler for dumping thread stack frames to stdout.""" print print "App server has been signaled to attempt a thread dump." print print "Thread stack frame dump at", asclocaltime() sys.stdout.flush() frames = threadframe.dict() items = frames.items() items.sort() print print "-" * 79 print for threadId, frame in items: print "Thread ID: %d (reference count = %d)" % ( threadId, sys.getrefcount(frame)) print ''.join(traceback.format_list(traceback.extract_stack(frame))) items.sort() print "-" * 79 sys.stdout.flush()
def dumpThreads(self):
"""Display all different traceback of Threads for debugging.
Require threadframe module."""
import threadframe
stacks = {}
frames = threadframe.dict()
for thread_id, frame in frames.iteritems():
stack = ''.join(traceback.format_stack(frame))
stacks[stack] = stacks.setdefault(stack, []) + [thread_id]
def sort_stack(x, y):
"""sort stack by number of thread."""
return cmp(len(x[1]), len(y[1]))
stacks = stacks.items()
stacks.sort(sort_stack)
for stack, thread_ids in stacks:
trace('=' * 72 + '\n')
trace('%i threads : %s\n' % (len(thread_ids), str(thread_ids)))
trace('-' * 72 + '\n')
trace(stack + '\n')
def __call__(self):
frames = threadframe.dict()
this_thread_id = thread.get_ident()
now = time.strftime("%Y-%m-%d %H:%M:%S")
res = ["Threads traceback dump at %s\n" % now]
for thread_id, frame in frames.iteritems():
if thread_id == this_thread_id:
continue
# Find request in frame
reqinfo = ''
f = frame
while f is not None:
co = f.f_code
if co.co_name == 'publish':
if co.co_filename.endswith('/publisher/publish.py') or \
co.co_filename.endswith('/ZPublisher/Publish.py'):
request = f.f_locals.get('request')
if request is not None:
reqinfo += (request.get('REQUEST_METHOD', '') + ' ' +
request.get('PATH_INFO', ''))
qs = request.get('QUERY_STRING')
if qs:
reqinfo += '?'+qs
break
f = f.f_back
if reqinfo:
reqinfo = " (%s)" % reqinfo
output = StringIO()
traceback.print_stack(frame, file=output)
res.append("Thread %s%s:\n%s" %
(thread_id, reqinfo, output.getvalue()))
frames = None
res.append("End of dump")
return '\n'.join(res)
def dumpThreads(self):
"""Display all different traceback of Threads for debugging.
Require threadframe module."""
import threadframe
stacks = {}
frames = threadframe.dict()
for thread_id, frame in frames.iteritems():
stack = ''.join(traceback.format_stack(frame))
stacks[stack] = stacks.setdefault(stack, []) + [thread_id]
def sort_stack(x, y):
"""sort stack by number of thread."""
return cmp(len(x[1]), len(y[1]))
stacks = stacks.items()
stacks.sort(sort_stack)
for stack, thread_ids in stacks:
trace('=' * 72 + '\n')
trace('%i threads : %s\n' % (len(thread_ids), str(thread_ids)))
trace('-' * 72 + '\n')
trace(stack + '\n')
def teardown_test_environment():
"""
This method is called by nose_runner when
the tests all finish. This helps track
down when tests aren't cleaning up after
themselves and leaving threads hanging around.
"""
import threading
# We should shut down all relevant threads by test completion.
threads = list(threading.enumerate())
try:
import threadframe
import traceback
if len(threads) > 1:
for v in threadframe.dict().values():
traceback.print_stack(v)
finally:
# threadframe is only available in the dev build.
pass
assert 1 == len(threads), threads
django_mako.render_to_string = django_mako.render_to_string_normal
def teardown_test_environment():
"""
This method is called by nose_runner when
the tests all finish. This helps track
down when tests aren't cleaning up after
themselves and leaving threads hanging around.
"""
import threading
# We should shut down all relevant threads by test completion.
threads = list(threading.enumerate())
try:
import threadframe
import traceback
if len(threads) > 1:
for v in threadframe.dict().values():
traceback.print_stack(v)
finally:
# threadframe is only available in the dev build.
pass
assert 1 == len(threads), threads
django_mako.render_to_string = django_mako.render_to_string_normal
import thread
import threading
from django.conf import settings
from django.core.handlers.wsgi import WSGIRequest
from perftools.middleware import Base
from perftools.utils import get_culprit
try:
# Available from Python >= 2.5
from sys import _current_frames as threadframe
except ImportError:
import threadframe as _threadframe
# Wrapper to provide the same interface as the one from Python >= 2.5
threadframe = lambda: _threadframe.dict()
class SlowRequestLoggingMiddleware(Base):
def __init__(self,
application,
threshold=1,
stacks=True,
logger=None,
**kwargs):
self.application = application
self.threshold = float(threshold) / 1000
self.stacks = stacks
self.logger = logger or logging.getLogger(__name__)
super(SlowRequestLoggingMiddleware,
self).__init__(application, **kwargs)
print 'done'
print 'launching self-deadlocking thread...',
U().start()
print 'done'
print 'launching thread that will die before the end...',
v = V()
v.start()
print 'done'
time.sleep(5)
# Python 2.2 does not support threadframe.dict()
if sys.hexversion < 0x02030000:
frames = threadframe.threadframe()
else:
frames = threadframe.dict()
# signal the thread V to die, then wait for it to oblige
V_event.set()
v.join()
if sys.hexversion < 0x02030000:
for frame in frames:
print '-' * 72
print 'frame ref count = %d' % sys.getrefcount(frame)
traceback.print_stack(frame)
else:
for thread_id, frame in frames.iteritems():
print '-' * 72
print '[%s] %d' % (thread_id, sys.getrefcount(frame))
traceback.print_stack(frame)
def _threads(): import threadframe return threadframe.dict().iteritems()
print 'done'
print 'launching self-deadlocking thread...',
U().start()
print 'done'
print 'launching thread that will die before the end...',
v = V()
v.start()
print 'done'
time.sleep(5)
# Python 2.2 does not support threadframe.dict()
if sys.hexversion < 0x02030000:
frames = threadframe.threadframe()
else:
frames = threadframe.dict()
# signal the thread V to die, then wait for it to oblige
V_event.set()
v.join()
if sys.hexversion < 0x02030000:
for frame in frames:
print '-' * 72
print 'frame ref count = %d' % sys.getrefcount(frame)
traceback.print_stack(frame)
else:
for thread_id, frame in frames.iteritems():
print '-' * 72
print '[%s] %d' % (thread_id, sys.getrefcount(frame))
traceback.print_stack(frame)
import thread
import traceback
import time
from cStringIO import StringIO
try:
# Available from Python >= 2.5
from sys import _current_frames as threadframe
except ImportError:
import threadframe as _threadframe
# Wrapper to provide the same interface as the one from Python >= 2.5
threadframe = lambda: _threadframe.dict()
import webob
def dump_threads():
"""Dump running threads
Returns a string with the tracebacks.
"""
frames = threadframe()
this_thread_id = thread.get_ident()
now = time.strftime("%Y-%m-%d %H:%M:%S")
res = ["Threads traceback dump at %s\n" % now]
for thread_id, frame in frames.iteritems():
if thread_id == this_thread_id:
continue
# Find request in frame
reqinfo = ''
def _threads(): import threadframe return threadframe.dict().iteritems()