def test_no_new_reference_cycles(self):
# Similar to https://github.com/mgedmin/objgraph/pull/22 but for
# typestats()
gc.disable()
x = type('MyClass', (), {})()
self.assertEqual(len(gc.get_referrers(x)), 1)
objgraph.typestats()
self.assertEqual(len(gc.get_referrers(x)), 1)
def test_no_new_reference_cycles(self):
# Similar to https://github.com/mgedmin/objgraph/pull/22 but for
# typestats()
gc.disable()
x = type('MyClass', (), {})()
self.assertEqual(len(gc.get_referrers(x)), 1)
objgraph.typestats()
self.assertEqual(len(gc.get_referrers(x)), 1)
def get_memory_leak_report(options):
import objgraph
import pympler.asizeof
original_path = options.module_name
if os.path.sep in original_path:
module_name = original_path.replace(".py", "").replace(".pyc", "")
module_name = module_name.replace(os.path.sep, ".")
else:
module_name = original_path
args = [original_path] + options.args.split()
sys.argv = args
module = importlib.import_module(module_name)
current_stats = objgraph.typestats(shortnames=True)
for _ in range(options.init_step_count):
module.main()
# Get the baseline
current_stats = objgraph.typestats(shortnames=True)
total = 0
growths = []
for i in range(options.leak_step_count):
logger.debug("Step %s", i)
# Call the API again, and check the growth.
# Ideally there is no growth after each call
module.main()
growth = get_growth_from_stats(current_stats)
adj_growth = []
for type_, new_count, inc in growth:
fn = "%s-leak-step-%s-%s.png" % (options.short_name, i, type_)
fn = os.path.join(options.destination, fn)
leaked_objects = objgraph.by_type(type_)[-inc:]
# Generate a graph for the leaked objects
# up to a maximum of 100
objgraph.show_backrefs(leaked_objects[:50], filename=fn)
# Also get the size of the objects leaked
size = pympler.asizeof.asized(leaked_objects).size / 2 ** 10
total += size
logger.debug("+%s: %r grew to %s (+%s KB)",
inc, type_, new_count, size)
adj_growth.append((type_, new_count, inc, size))
adj_growth.sort(key=lambda x: x[3], reverse=True)
growths.append(adj_growth)
# Update the current stats
current_stats = objgraph.typestats(shortnames=True)
return growths, total
def test_without_filter(self):
MyClass = type('MyClass', (), {'__module__': 'mymodule'}) # noqa
x, y = MyClass(), MyClass()
x.magic_attr = True
y.magic_attr = False
stats = objgraph.typestats(shortnames=False)
self.assertEqual(2, stats['mymodule.MyClass'])
def report_growth(label='', limit=20, peak_stats={}):
"""Using the objgraph module, report the growth of objects since the last call."""
# pylint:disable=dangerous-default-value
# pylint 'Dangerous default value {} as argument'
if LOG.isEnabledFor(logging.DEBUG3) and _ObjGraph:
# Tried and failed to redirect stdout to get this output in the
# log, so copying the entire show_growth() function here just to
# write to the log.
LOG.debug3("object growth {}".format(label))
gc.collect()
# Have to get the objects ourselves and free them, otherwise typestats() leaks.
all_objects = gc.get_objects()
try:
stats = objgraph.typestats(all_objects)
deltas = {}
for name, count in stats.items():
old_count = peak_stats.get(name, 0)
if count > old_count:
deltas[name] = count - old_count
peak_stats[name] = count
deltas = sorted(deltas.items(),
key=operator.itemgetter(1),
reverse=True)
if limit:
deltas = deltas[:limit]
if deltas:
width = max(len(name) for name, count in deltas)
for name, delta in deltas:
LOG.debug3('%-*s%9d %+9d' %
(width, name, stats[name], delta))
finally:
del all_objects
def get_new_objects(lst, fn, *args, **kwargs):
"""
Collect types and numbers of new objects left over after the given function is called.
If lst is not empty after the call, this MAY indicate a memory leak, but not necessarily,
since some functions are intended to create new objects for later use.
Parameters
----------
lst : list
List used to collect objects and deltas.
fn : function
The function being checked for possible memory leaks.
*args : tuple
Positional args passed to fn.
**kwargs : dict
Named args to be passed to fn.
Returns
-------
object
The object returned by the call to fn.
"""
gc.collect()
start_objs = objgraph.typestats()
start_objs['frame'] += 1
start_objs['function'] += 1
start_objs['builtin_function_or_method'] += 1
start_objs['cell'] += 1
ret = fn(*args, **kwargs)
lst.extend([(str(o), delta) for o, _, delta in objgraph.growth(peak_stats=start_objs)])
return ret
def test_without_filter(self):
MyClass = type('MyClass', (), {'__module__': 'mymodule'}) # noqa
x, y = MyClass(), MyClass()
x.magic_attr = True
y.magic_attr = False
stats = objgraph.typestats(shortnames=False)
self.assertEqual(2, stats['mymodule.MyClass'])
def main():
a1 = dict()
a2 = dict()
a3 = dict()
a1['a2'] = a2
a2['a3'] = a3
a3['a1'] = a1
b = dict()
b['b'] = b
c = dict()
c['c'] = 'c'
import gc
import types
print(len(_bfs([b], gc.get_referrers)))
sccs = tarjan([a1, b, c], gc.get_referrers)
show_cycles(sccs, joined=True)
print(sccs)
del sccs
gc.collect()
sccs = tarjan([d, e, f])
show_cycles(sccs, joined=True)
return
sccs = tarjan(gc.get_objects(), gc.get_referrers)
print([len(i) for i in sccs])
import objgraph
objs = objgraph.at_addrs(sccs[0])
print(objgraph.typestats(objs))
def OBJGRAPH_DELTAS():
from collections import OrderedDict
try:
import objgraph
import gc
import operator
peak_stats = cache.ram('peak_stats', lambda: {}, time_expire=None)
last_checked = cache.ram('last_checked',
lambda: request.now,
time_expire=None)
gc.collect()
#cobbled together from https://github.com/mgedmin/objgraph/blob/master/objgraph.py (show_growth())
stats = objgraph.typestats(shortnames=False)
deltas = {}
for name, count in stats.items():
old_count = peak_stats.get(name, 0)
if count > old_count:
deltas[name] = count - old_count
peak_stats[name] = count
peak_stats = cache.ram('peak_stats', lambda: peak_stats, time_expire=0)
cache.ram('last_checked', lambda: request.now, time_expire=0)
deltas = sorted(deltas.items(),
key=operator.itemgetter(1),
reverse=True)
return ({
'growth': deltas,
'last_checked': last_checked,
'time_period': "{}".format(request.now - last_checked)
})
except Exception as e:
return ({'error': e})
def wrapper(*args, **kwargs):
start_objs = objgraph.typestats()
start_objs['frame'] += 1
start_objs['cell'] += 1
ret = fn(*args, **kwargs)
for obj, _, delta_objs in objgraph.growth(peak_stats=start_objs):
print(str(fn), "added %s %+d" % (obj, delta_objs))
return ret
def set_type_count_checkpoint():
import objgraph
global checkpoint_type_stats
checkpoint_type_stats = objgraph.typestats()
print '====>\t', 'checkpoint', datetime.strftime(datetime.now(), '%X')
print '\n'.join(('\t%s: %d' % item) for item in
reversed(sorted(checkpoint_type_stats.iteritems(), key=itemgetter(1)))), \
'\n', '-'*40
def test_with_filter(self):
MyClass = type('MyClass', (), {'__module__': 'mymodule'}) # noqa
x, y = MyClass(), MyClass()
x.magic_attr = True
y.magic_attr = False
stats = objgraph.typestats(
shortnames=False,
filter=lambda e: isinstance(e, MyClass) and e.magic_attr)
self.assertEqual(1, stats['mymodule.MyClass'])
def test_with_filter(self):
MyClass = type('MyClass', (), {'__module__': 'mymodule'}) # noqa
x, y = MyClass(), MyClass()
x.magic_attr = True
y.magic_attr = False
stats = objgraph.typestats(
shortnames=False,
filter=lambda e: isinstance(e, MyClass) and e.magic_attr)
self.assertEqual(1, stats['mymodule.MyClass'])
def sample_objects(timestamp, stream):
# instead of keeping the count_per_type in memory, stream the data to a file
# to save memory
count_per_type = objgraph.typestats()
# add the timestamp for plotting
data = [timestamp, count_per_type]
data_pickled = pickle.dumps(data)
stream.write(data_pickled)
def sample_objects(timestamp, stream):
# instead of keeping the count_per_type in memory, stream the data to a file
# to save memory
count_per_type = objgraph.typestats()
# add the timestamp for plotting
data = [timestamp, count_per_type]
data_pickled = pickle.dumps(data)
stream.write(data_pickled)
def _tick(self):
print("TICK")
if self.request is not None:
return
print(objgraph.typestats())
self.request = QtNetwork.QNetworkRequest()
self.request.setUrl(QtCore.QUrl(self.url))
self.request.setRawHeader(b"User-Agent", b"Test")
self.reply = self.manager.get(self.request)
self.reply.finished.connect(self._finished)
def _trace_call(frame, arg, stack, context):
"""
This is called after we have matched based on glob pattern and isinstance check.
"""
global time0
if time0 is None:
time0 = time.time()
(qual_cache, method_counts, class_counts, id2count, verbose, memory, leaks,
stream, show_ptrs) = context
funcname = find_qualified_name(frame.f_code.co_filename,
frame.f_code.co_firstlineno, qual_cache)
self = frame.f_locals['self']
try:
pname = "(%s)" % self.pathname
except AttributeError:
pname = ""
cname = self.__class__.__name__
my_id = id(self)
if my_id in id2count:
id_count = id2count[my_id]
else:
class_counts[cname] += 1
id2count[my_id] = id_count = class_counts[cname]
sname = "%s#%d%s" % (self.__class__.__name__, id_count, pname)
fullname = '.'.join((sname, funcname))
method_counts[fullname] += 1
indent = tab * (len(stack) - 1)
if verbose:
_printer("%s--> %s (%d)" % (indent, fullname, method_counts[fullname]))
_indented_print(frame.f_locals,
frame.f_locals,
len(stack) - 1,
show_ptrs=show_ptrs)
else:
_printer("%s-->%s" % (indent, fullname))
if memory is not None:
memory.append(mem_usage())
if leaks is not None:
stats = objgraph.typestats()
stats['frame'] += 1
stats['cell'] += 1
stats['list'] += 1
leaks.append(stats)
stream.flush()
def __call__(self, request, *args, **kwargs):
Metrics.RequestCounter.labels(method=request.method).inc()
response = self.get_response(request, *args, **kwargs)
Metrics.ResponseCounter.labels(response.status_code).inc()
gc.collect()
stats = objgraph.typestats()
for obj, cnt in stats.items():
Metrics.ObjectsInMemory.labels(obj).set(cnt)
return response
def memory_content(interactive=False):
typestats = dict(objgraph.typestats())
while True:
types = []
for itype, type_name in enumerate(sorted(typestats.keys())):
print '%d %s: %d' % (itype, type_name, typestats[type_name])
types.append(type_name)
cls = eval(type_name)
if hasattr(cls, '__len__'):
for obj in objgraph.by_type(type_name):
if len(obj) > 1000:
print ' Large object (n=%d) [%s ...]' % (len(obj),
str(obj[0]))
if not interactive:
break
print 'Inspect? (0-%d or q):' % itype
response = sys.stdin.readline().strip()
if response == 'q':
break
try:
itype = int(response)
objs = objgraph.by_type(types[itype])
except:
print 'Unrecognized input %s' % response
continue
while True:
print 'Item #? (0-%d or q):' % len(objs)
response = sys.stdin.readline()
if response == 'q':
break
try:
iobj = int(response)
obj = objs[iobj]
except:
print 'Unrecognized input %s' % response
continue
while True:
print 'Expression? (use "obj", e.g., len(obj), or q):'
response = sys.stdin.readline()
if response == 'q':
break
try:
print eval(response)
except:
print 'Unrecognized input %s' % response
continue
def memory_content(interactive = False):
typestats = dict(objgraph.typestats())
while True:
types = []
for itype, type_name in enumerate(sorted(typestats.keys())):
print '%d %s: %d' % (itype, type_name, typestats[type_name])
types.append(type_name)
cls = eval(type_name)
if hasattr(cls, '__len__'):
for obj in objgraph.by_type(type_name):
if len(obj) > 1000:
print ' Large object (n=%d) [%s ...]' % (len(obj), str(obj[0]))
if not interactive:
break
print 'Inspect? (0-%d or q):' % itype
response = sys.stdin.readline().strip()
if response == 'q':
break
try:
itype = int(response)
objs = objgraph.by_type(types[itype])
except:
print 'Unrecognized input %s' % response
continue
while True:
print 'Item #? (0-%d or q):' % len(objs)
response = sys.stdin.readline()
if response == 'q':
break
try:
iobj = int(response)
obj = objs[iobj]
except:
print 'Unrecognized input %s' % response
continue
while True:
print 'Expression? (use "obj", e.g., len(obj), or q):'
response = sys.stdin.readline()
if response == 'q':
break
try:
print eval(response)
except:
print 'Unrecognized input %s' % response
continue
def dump_type_count_checkpoint_diff():
import objgraph
type_stats = objgraph.typestats()
diff = {}
for key, val in type_stats.iteritems():
countdiff = val - checkpoint_type_stats.get(key, 0)
if countdiff:
diff[key] = countdiff
print '====>\t', 'checkpoint diff', datetime.strftime(datetime.now(), '%X')
print '\n'.join(('\t%s: %d' % item) for item in
reversed(sorted(diff.iteritems(), key=itemgetter(1)))), \
'\n', '-'*40
def show_cycles(sccs, joined=False):
import objgraph
a = sccs
if joined:
a = []
for scc in sccs:
a.extend(scc)
a = [a]
for scc in a:
objs = objgraph.at_addrs(scc)
print(objgraph.typestats(objs))
objgraph.show_backrefs(objs, max_depth=len(scc) + 5,
filter=lambda x: id(x) in scc)
def get_growth_from_stats(current_stats):
import objgraph
new_stats = objgraph.typestats(shortnames=True)
growth = []
for type_, new_count in new_stats.items():
if type_ == "frame":
# This is an object that objgraph adds.
# Skip it.
continue
old_count = current_stats.get(type_, 0)
increment = new_count - old_count
if increment > 0:
growth.append((type_, new_count, increment))
growth.sort(key=lambda x: x[2], reverse=True)
return growth
def show_cycles(sccs, joined=False):
import objgraph
a = sccs
if joined:
a = []
for scc in sccs:
a.extend(scc)
a = [a]
for scc in a:
objs = objgraph.at_addrs(scc)
print(objgraph.typestats(objs))
objgraph.show_backrefs(objs,
max_depth=len(scc) + 5,
filter=lambda x: id(x) in scc)
def object_growth():
"""
Shows changes in allocations, like objgraph.show_growth(), except:
- show_growth() prints to stdout, this is flask view.
- this saves the peaks in the session, so that each user sees the changes
between their last page load, not some global.
- this function is commented :)
"""
# We don't want our numbers crudded up by a GC cycle that hasn't run yet,
# so force GC before we gather stats.
gc.collect()
# `typestats() `returns a dict of {type-name: count-of-allocations}. We'll
# compare the current count for each type to the previous count, stored
# in the session as `peak_stats`, and save the changes into `deltas`.
peak_stats = flask.session.get('peak_stats', {})
stats = objgraph.typestats()
deltas = {}
# For each type, look it the old count in `peak_stats`, defaulting to 0.
# We're really only interested in *growth* -- remember, we're looking for
# memory leaks -- so if the current count is greater than the peak count,
# we want to return that change in `deltas` and also note the new peak
# for later.
for name, count in stats.iteritems():
old_count = peak_stats.get(name, 0)
if count > old_count:
deltas[name] = count - old_count
peak_stats[name] = count
# We have to remember to store `peak_stats` back in the session, otherwise
# Flask won't notice that it's changed.
flask.session['peak_stats'] = peak_stats
# Create (type-name, delta) tuples, sorted by objects with the biggest growth.
deltas = sorted(deltas.items(), key=operator.itemgetter(1), reverse=True)
return flask.render_template(
'growth.html',
growth=deltas,
)
def memory_delta():
typestats = dict(objgraph.typestats())
names = set(snapshot.keys()) | set(typestats.keys())
for name in names:
try:
now = typestats[name]
except:
now = 0
try:
before = snapshot[name]
except:
before = 0
print ' %s: %+d' % (name, (now - before))
snapshot = typestats
def memory_delta():
typestats = dict(objgraph.typestats())
names = set(snapshot.keys()) | set(typestats.keys())
for name in names:
try:
now = typestats[name]
except:
now = 0
try:
before = snapshot[name]
except:
before = 0
print ' %s: %+d' % (name, (now - before))
snapshot = typestats
def check_leaks(self, suppress=False, limit=20):
stats = objgraph.typestats(shortnames=False)
deltas = {}
for name, count in stats.items():
old_count = self.peak.get(name, 0)
if count > old_count:
deltas[name] = count - old_count
self.peak[name] = count
deltas = sorted(deltas.items(), key=operator.itemgetter(1), reverse=True)
deltas = deltas[:limit]
if not suppress:
if deltas:
self.logger.info("Peak memory usage change:")
width = max(len(name) for name, count in deltas)
for name, delta in deltas:
self.logger.info(' %-*s%9d %+9d' % (width, name, stats[name], delta))
def object_growth():
"""
Shows changes in allocations, like objgraph.show_growth(), except:
- show_growth() prints to stdout, this is flask view.
- this saves the peaks in the session, so that each user sees the changes
between their last page load, not some global.
- this function is commented :)
"""
# We don't want our numbers crudded up by a GC cycle that hasn't run yet,
# so force GC before we gather stats.
gc.collect()
# `typestats() `returns a dict of {type-name: count-of-allocations}. We'll
# compare the current count for each type to the previous count, stored
# in the session as `peak_stats`, and save the changes into `deltas`.
peak_stats = flask.session.get('peak_stats', {})
stats = objgraph.typestats()
deltas = {}
# For each type, look it the old count in `peak_stats`, defaulting to 0.
# We're really only interested in *growth* -- remember, we're looking for
# memory leaks -- so if the current count is greater than the peak count,
# we want to return that change in `deltas` and also note the new peak
# for later.
for name, count in stats.iteritems():
old_count = peak_stats.get(name, 0)
if count > old_count:
deltas[name] = count - old_count
peak_stats[name] = count
# We have to remember to store `peak_stats` back in the session, otherwise
# Flask won't notice that it's changed.
flask.session['peak_stats'] = peak_stats
# Create (type-name, delta) tuples, sorted by objects with the biggest growth.
deltas = sorted(deltas.items(), key=operator.itemgetter(1), reverse=True)
return flask.render_template('growth.html',
growth = deltas,
)
def check_iter_leaks(niter, func, *args, **kwargs):
"""
Run func niter times and collect info on new objects left over after each iteration.
Parameters
----------
niter : int
Number of times to run func.
func : function
A function that takes no arguments.
*args : tuple
Positional args passed to func.
**kwargs : dict
Named args to be passed to func.
Returns
-------
set
set of tuples of the form (typename, count)
"""
if niter < 2:
raise RuntimeError(
"Must run the function at least twice, but niter={}".format(
niter))
iters = []
gc.collect()
start_objs = objgraph.typestats()
if 'frame' in start_objs:
start_objs['frame'] += 1
start_objs['function'] += 1
start_objs['builtin_function_or_method'] += 1
start_objs['cell'] += 1
for i in range(niter):
func(*args, **kwargs)
gc.collect()
lst = [(str(o), delta)
for o, _, delta in objgraph.growth(peak_stats=start_objs)]
iters.append(lst)
set1 = set(iters[-2])
set2 = set(iters[-1])
return set2 - set1
def main():
a1 = dict()
a2 = dict()
a3 = dict()
a1['a2'] = a2
a2['a3'] = a3
a3['a1'] = a1
b = dict()
b['b'] = b
c = dict()
c['c'] = 'c'
import gc
import types
print(len(
_bfs([b],
gc.get_referrers
)
))
sccs = tarjan([a1, b, c], gc.get_referrers)
show_cycles(sccs, joined=True)
print(sccs)
del sccs
gc.collect()
sccs = tarjan([d, e, f])
show_cycles(sccs, joined=True)
return
sccs = tarjan(gc.get_objects(), gc.get_referrers)
print([len(i) for i in sccs])
import objgraph
objs = objgraph.at_addrs(sccs[0])
print(objgraph.typestats(objs))
def OBJGRAPH_DELTAS():
from collections import OrderedDict
try:
import objgraph
import gc
import operator
peak_stats = cache.ram('peak_stats', lambda: {}, time_expire=None)
last_checked = cache.ram('last_checked', lambda: request.now, time_expire=None)
gc.collect()
#cobbled together from https://github.com/mgedmin/objgraph/blob/master/objgraph.py (show_growth())
stats = objgraph.typestats(shortnames=False)
deltas = {}
for name, count in stats.items():
old_count = peak_stats.get(name, 0)
if count > old_count:
deltas[name] = count - old_count
peak_stats[name] = count
peak_stats = cache.ram('peak_stats', lambda: peak_stats, time_expire=0)
cache.ram('last_checked', lambda: request.now, time_expire=0)
deltas = sorted(deltas.items(), key=operator.itemgetter(1), reverse=True)
return({'growth':deltas, 'last_checked': last_checked, 'time_period': "{}".format(request.now -last_checked)})
except Exception as e:
return({'error':e})
def test_long_type_names(self):
x = type('MyClass', (), {'__module__': 'mymodule'})() # noqa
stats = objgraph.typestats(shortnames=False)
self.assertEqual(1, stats['mymodule.MyClass'])
def test_long_type_names(self):
x = type('MyClass', (), {'__module__': 'mymodule'})() # noqa
stats = objgraph.typestats(shortnames=False)
self.assertEqual(1, stats['mymodule.MyClass'])
def obj_log(self):
self.obj_file.write('%s %s\n' % (
datetime.utcnow().isoformat(),
json.dumps(objgraph.typestats())
))
self.obj_file.flush()
def obj_log(self):
self.obj_file.write(
'%s %s\n' %
(datetime.utcnow().isoformat(), json.dumps(objgraph.typestats())))
self.obj_file.flush()
def do_update(self):
leaked = objgraph.get_leaking_objects()
self.g.update()
leaked = objgraph.get_leaking_objects()
ts = objgraph.typestats(leaked)
self.assertFalse(ts)