def test_load_defaults_to_computing_and_collapsing(self):
manager = loader.load(_instance_dump, show_prog=False, collapse=False)
instance_obj = manager[1]
self.assertEqual([2, 3], instance_obj.children)
manager = loader.load(_instance_dump, show_prog=False)
instance_obj = manager[1]
self.assertEqual([4, 5, 6, 7, 9, 10, 11, 12, 3], instance_obj.children)
def test_load_defaults_to_computing_and_collapsing(self):
manager = loader.load(_instance_dump, show_prog=False, collapse=False)
instance_obj = manager[1]
self.assertEqual([2, 3], instance_obj.children)
manager = loader.load(_instance_dump, show_prog=False)
instance_obj = manager[1]
self.assertEqual([4, 5, 6, 7, 9, 10, 11, 12, 3], instance_obj.children)
def dump_objects(self):
cmd = ';'.join([
"import os, shutil", "from meliae import scanner",
"tmp = '/tmp/%d' % os.getpid()",
"scanner.dump_all_objects(tmp + '.json')",
"shutil.move(tmp + '.json', tmp + '.objects')"
])
output = self.proc.cmd(cmd)
if 'No module named meliae' in output:
log.error('Error: %s is unable to import `meliae`' %
self.proc.title.strip())
return
self.update_progress(0.35)
# Clear previous model
self.obj_store.clear()
self.update_progress(0.4, "Loading object dump")
try:
objects = loader.load('/tmp/%d.objects' % self.proc.pid,
show_prog=False)
except NameError:
log.debug("Meliae not available, continuing...")
return
except:
log.debug("Falling back to slower meliae object dump loader")
objects = loader.load('/tmp/%d.objects' % self.proc.pid,
show_prog=False,
using_json=False)
objects.compute_referrers()
self.update_progress(0.45)
summary = objects.summarize()
self.update_progress(0.47)
def intify(x):
try:
return int(x)
except:
return x
for i, line in enumerate(str(summary).split('\n')):
if i == 0:
self.obj_totals.set_text(line)
elif i == 1:
continue # column headers
else:
obj = summary.summaries[i - 2]
self.obj_store.append([str(obj.max_address)] +
map(intify,
line.split()[1:]))
os.unlink('/tmp/%d.objects' % self.proc.pid)
def dump_objects(self, proc):
cmd = ';'.join(["import os, shutil", "from meliae import scanner",
"tmp = '/tmp/%d' % os.getpid()",
"scanner.dump_all_objects(tmp + '.json')",
"shutil.move(tmp + '.json', tmp + '.objects')"])
output = proc.cmd(cmd)
if 'No module named meliae' in output:
log.error('Error: %s is unable to import `meliae`' %
proc.title.strip())
return
self.update_progress(0.35)
# Clear previous model
self.obj_store.clear()
self.update_progress(0.4, "Loading object dump")
try:
objects = loader.load('/tmp/%d.objects' % proc.pid,
show_prog=False)
except NameError:
log.debug("Meliae not available, continuing...")
return
except:
log.debug("Falling back to slower meliae object dump loader")
objects = loader.load('/tmp/%d.objects' % proc.pid,
show_prog=False, using_json=False)
objects.compute_referrers()
self.update_progress(0.45)
summary = objects.summarize()
self.update_progress(0.47)
def intify(x):
try:
return int(x)
except:
return x
for i, line in enumerate(str(summary).split('\n')):
if i == 0:
self.obj_totals.set_text(line)
elif i == 1:
continue # column headers
else:
obj = summary.summaries[i - 2]
self.obj_store.append([str(obj.max_address)] +
map(intify, line.split()[1:]))
os.unlink('/tmp/%d.objects' % proc.pid)
def test_load_without_simplejson(self):
objs = loader.load([
b'{"address": 1234, "type": "int", "size": 12, "value": 10'
b', "refs": []}',
b'{"address": 2345, "type": "module", "size": 60, "name": "mymod"'
b', "refs": [1234]}',
('{"address": 4567, "type": "%s", "size": 150, "len": 126'
', "value": "Test \\/whoami\\/\\u000a\\"Your name\\""'
', "refs": []}' % bytes.__name__).encode('UTF-8'),
('{"address": 5678, "type": "%s", "size": 150, "len": 126'
', "value": "Test \\/whoami\\/\\u000a\\"Your name\\""'
', "refs": []}' % six.text_type.__name__).encode('UTF-8'),
],
using_json=False,
show_prog=False).objs
keys = sorted(objs.keys())
self.assertEqual([1234, 2345, 4567, 5678], keys)
obj = objs[1234]
self.assertTrue(isinstance(obj, _loader._MemObjectProxy))
# The address should be exactly the same python object as the key in
# the objs dictionary.
self.assertTrue(keys[0] is obj.address)
self.assertEqual(10, obj.value)
obj = objs[2345]
self.assertEqual("module", obj.type_str)
self.assertEqual(b"mymod", obj.value)
obj = objs[4567]
self.assertTrue(isinstance(obj.value, bytes))
self.assertEqual(b"Test \\/whoami\\/\\u000a\\\"Your name\\\"",
obj.value)
obj = objs[5678]
self.assertTrue(isinstance(obj.value, six.text_type))
self.assertEqual(u"Test \\/whoami\\/\\u000a\\\"Your name\\\"",
obj.value)
def test_summarize_refs(self):
manager = loader.load(_example_dump, show_prog=False)
summary = manager.summarize(manager[8])
# Note that the module is included in the summary
self.assertEqual(['int', 'module', 'str', 'tuple'],
sorted(summary.type_summaries.keys()))
self.assertEqual(257, summary.total_size)
def main():
if len(sys.argv) != 2:
print("[ pyrasite memory viewer ]\n")
print("Usage: %s <pid> <objects.json>" % sys.argv[0])
print("\n pid - the running process id")
print("")
sys.exit(1)
pid = int(sys.argv[1])
payload = abspath(join(dirname(__file__), '..',
'payloads', 'dump_memory.py'))
pyrasite.inject(pid, payload)
filename = '/tmp/pyrasite-%d-objects.json' % pid
# Work around bug caused by meliae dumping unicode strings:
# https://bugs.launchpad.net/meliae/+bug/876810
with open(filename) as sample_file, open(filename + '.tmp', 'w') as output_file:
pattern = re.compile(r"(?<!\\)\\u([dD][0-9a-fA-F]{3,3})")
for line in sample_file:
output_file.write(pattern.sub("#S\g<1>", line))
os.rename(filename + '.tmp', filename)
objects = loader.load(filename)
objects.compute_referrers()
PyrasiteMemoryViewer(pid=pid, objects=objects).main()
def test_collapse_instance_dicts(self):
manager = loader.load(_instance_dump, show_prog=False, collapse=False)
# This should collapse all of the references from the instance's dict
# @2 into the instance @1
instance = manager.objs[1]
self.assertEqual(32, instance.size)
self.assertEqual([2, 3], instance.children)
inst_dict = manager.objs[2]
self.assertEqual(140, inst_dict.size)
self.assertEqual([4, 5, 6, 7, 9, 10, 11, 12], inst_dict.children)
mod = manager.objs[14]
self.assertEqual([15], mod.children)
mod_dict = manager.objs[15]
self.assertEqual([5, 6, 9, 6], mod_dict.children)
manager.compute_parents()
tpl = manager.objs[12]
self.assertEqual([2], tpl.parents)
self.assertEqual([1], inst_dict.parents)
self.assertEqual([14], mod_dict.parents)
manager.collapse_instance_dicts()
# The instance dict has been removed
self.assertEqual([4, 5, 6, 7, 9, 10, 11, 12, 3], instance.children)
self.assertEqual(172, instance.size)
self.assertFalse(2 in manager.objs)
self.assertEqual([1], tpl.parents)
self.assertEqual([5, 6, 9, 6], mod.children)
self.assertFalse(15 in manager.objs)
def test_load_without_simplejson(self):
objs = loader.load([
'{"address": 1234, "type": "int", "size": 12, "value": 10'
', "refs": []}',
'{"address": 2345, "type": "module", "size": 60, "name": "mymod"'
', "refs": [1234]}',
'{"address": 4567, "type": "str", "size": 150, "len": 126'
', "value": "Test \\\'whoami\\\'\\u000a\\"Your name\\""'
', "refs": []}'
],
using_json=False,
show_prog=False).objs
keys = sorted(objs.keys())
self.assertEqual([1234, 2345, 4567], keys)
obj = objs[1234]
self.assertTrue(isinstance(obj, _loader._MemObjectProxy))
# The address should be exactly the same python object as the key in
# the objs dictionary.
self.assertTrue(keys[0] is obj.address)
self.assertEqual(10, obj.value)
obj = objs[2345]
self.assertEqual("module", obj.type_str)
self.assertEqual("mymod", obj.value)
obj = objs[4567]
self.assertEqual("Test \\'whoami\\'\\u000a\\\"Your name\\\"",
obj.value)
def test_summarize_excluding(self):
manager = loader.load(_example_dump, show_prog=False)
summary = manager.summarize(manager[8], excluding=[4, 5])
# No ints when they are explicitly filtered
self.assertEqual(['module', 'str', 'tuple'],
sorted(summary.type_summaries.keys()))
self.assertEqual(233, summary.total_size)
def test_compute_referrers(self):
# Deprecated
logged = []
def log_warn(msg, klass, stacklevel=None):
logged.append((msg, klass, stacklevel))
old_func = warn.trap_warnings(log_warn)
try:
manager = loader.load(_example_dump, show_prog=False)
manager.compute_referrers()
self.assertEqual([
('.compute_referrers is deprecated.'
' Use .compute_parents instead.', DeprecationWarning, 3),
], logged)
objs = manager.objs
finally:
warn.trap_warnings(old_func)
self.assertEqual((), objs[1].parents)
self.assertEqual([1, 3], objs[3].parents)
self.assertEqual([3, 7, 8], sorted(objs[4].parents))
self.assertEqual([3, 7, 8], sorted(objs[5].parents))
self.assertEqual([8], objs[6].parents)
self.assertEqual([8], objs[7].parents)
self.assertEqual((), objs[8].parents)
def test_summarize_refs(self):
manager = loader.load(_example_dump, show_prog=False)
summary = manager.summarize(manager[8])
# Note that the module is included in the summary
self.assertEqual(['int', 'module', 'str', 'tuple'],
sorted(summary.type_summaries.keys()))
self.assertEqual(257, summary.total_size)
def test_compute_parents_many_parents(self):
content = [
'{"address": 2, "type": "str", "size": 25, "len": 1, "value": "a", "refs": []}',
]
for x in xrange(200):
content.append('{"address": %d, "type": "tuple", "size": 20,'
' "len": 2, "refs": [2, 2]}' % (x+100))
# By default, we only track 100 parents
manager = loader.load(content, show_prog=False)
self.assertEqual(100, manager[2].num_parents)
manager = loader.load(content, show_prog=False, max_parents=0)
self.assertEqual(0, manager[2].num_parents)
manager = loader.load(content, show_prog=False, max_parents=-1)
self.assertEqual(200, manager[2].num_parents)
manager = loader.load(content, show_prog=False, max_parents=10)
self.assertEqual(10, manager[2].num_parents)
def test_collapse_instance_dicts(self):
manager = loader.load(_instance_dump, show_prog=False, collapse=False)
# This should collapse all of the references from the instance's dict
# @2 into the instance @1
instance = manager.objs[1]
self.assertEqual(32, instance.size)
self.assertEqual([2, 3], instance.children)
inst_dict = manager.objs[2]
self.assertEqual(140, inst_dict.size)
self.assertEqual([4, 5, 6, 7, 9, 10, 11, 12], inst_dict.children)
mod = manager.objs[14]
self.assertEqual([15], mod.children)
mod_dict = manager.objs[15]
self.assertEqual([5, 6, 9, 6], mod_dict.children)
manager.compute_parents()
tpl = manager.objs[12]
self.assertEqual([2], tpl.parents)
self.assertEqual([1], inst_dict.parents)
self.assertEqual([14], mod_dict.parents)
manager.collapse_instance_dicts()
# The instance dict has been removed
self.assertEqual([4, 5, 6, 7, 9, 10, 11, 12, 3], instance.children)
self.assertEqual(172, instance.size)
self.assertFalse(2 in manager.objs)
self.assertEqual([1], tpl.parents)
self.assertEqual([5, 6, 9, 6], mod.children)
self.assertFalse(15 in manager.objs)
def test_summarize_excluding(self):
manager = loader.load(_example_dump, show_prog=False)
summary = manager.summarize(manager[8], excluding=[4, 5])
# No ints when they are explicitly filtered
self.assertEqual(['module', 'str', 'tuple'],
sorted(summary.type_summaries.keys()))
self.assertEqual(233, summary.total_size)
def test_load_without_simplejson(self):
objs = loader.load([
'{"address": 1234, "type": "int", "size": 12, "value": 10'
', "refs": []}',
'{"address": 2345, "type": "module", "size": 60, "name": "mymod"'
', "refs": [1234]}',
'{"address": 4567, "type": "str", "size": 150, "len": 126'
', "value": "Test \\\'whoami\\\'\\u000a\\"Your name\\"'
', "refs": []}'
],
using_json=False,
show_prog=False).objs
keys = sorted(objs.keys())
self.assertEqual([1234, 2345, 4567], keys)
obj = objs[1234]
self.assertTrue(isinstance(obj, _loader._MemObjectProxy))
# The address should be exactly the same python object as the key in
# the objs dictionary.
self.assertTrue(keys[0] is obj.address)
self.assertEqual(10, obj.value)
obj = objs[2345]
self.assertEqual("module", obj.type_str)
self.assertEqual("mymod", obj.value)
obj = objs[4567]
# Known failure? We don't unescape properly, also, I'm surprised this
# works. " should exit the " string, but \" seems to leave it. But the
# '\' is also left verbatim because it is a raw string...
self.assertEqual(r"Test \'whoami\'\u000a\"Your name\"", obj.value)
def print_memorydump(filename):
from meliae import loader
om = loader.load(filename, collapse=True)
om.remove_expensive_references()
print(om.summarize())
return om
def test_compute_parents_many_parents(self):
content = [
'{"address": 2, "type": "str", "size": 25, "len": 1, "value": "a", "refs": []}',
]
for x in range(200):
content.append('{"address": %d, "type": "tuple", "size": 20,'
' "len": 2, "refs": [2, 2]}' % (x+100))
# By default, we only track 100 parents
manager = loader.load(content, show_prog=False)
self.assertEqual(100, manager[2].num_parents)
manager = loader.load(content, show_prog=False, max_parents=0)
self.assertEqual(0, manager[2].num_parents)
manager = loader.load(content, show_prog=False, max_parents=-1)
self.assertEqual(200, manager[2].num_parents)
manager = loader.load(content, show_prog=False, max_parents=10)
self.assertEqual(10, manager[2].num_parents)
def test_compute_parents_no_parents(self):
manager = loader.load(_intern_dict_dump,
show_prog=False,
max_parents=0)
str_5 = manager[5]
# Each of these refers to str_5 multiple times, but they should only
# show up 1 time in the parent list.
self.assertEqual([], sorted(str_5.parents))
def load_meliae(filename):
try:
om = loader.load(filename)
except Exception, e:
# Sometimes this collapse_instance_dicts fails with:
# KeyError: 'address 140088038122672 not present'
logging.debug('Failed to collapse_instance_dicts: "%s"' % e)
return
def test_summarize_excluding(self):
manager = loader.load(_example_dump, show_prog=False)
summary = manager.summarize(manager[9], excluding=[4, 5])
# No ints when they are explicitly filtered
self.assertEqual(
sorted(['module', bytes.__name__, six.text_type.__name__,
'tuple']), sorted(summary.type_summaries.keys()))
self.assertEqual(321, summary.total_size)
def print_leaks(self, prefix=''):
if not USE_MELIAE:
return
objgraph.show_growth()
tmp = tempfile.mkstemp(prefix='pcp-test')[1]
scanner.dump_all_objects(tmp)
leakreporter = loader.load(tmp)
summary = leakreporter.summarize()
print('{0}: {1}'.format(prefix, summary))
def print_leaks(self, prefix=''):
if not USE_MELIAE:
return
objgraph.show_growth()
tmp = tempfile.mkstemp(prefix='pcp-test')[1]
scanner.dump_all_objects(tmp)
leakreporter = loader.load(tmp)
summary = leakreporter.summarize()
print('{0}: {1}'.format(prefix, summary))
def test_compute_parents(self):
manager = loader.load(_example_dump, show_prog=False)
objs = manager.objs
self.assertEqual((), objs[1].parents)
self.assertEqual([1, 3], objs[3].parents)
self.assertEqual([3, 7, 8], sorted(objs[4].parents))
self.assertEqual([3, 7, 8], sorted(objs[5].parents))
self.assertEqual([8], objs[6].parents)
self.assertEqual([8], objs[7].parents)
self.assertEqual((), objs[8].parents)
def _cmd_meliae(app, *args):
""" 使用meliae分析内存 """
from meliae import loader
om = loader.load(args[0])
om.compute_parents()
om.collapse_instance_dicts()
log(om.summarize())
local = locals()
exec meliae_funcs in local
_shell(local=local)
def test_compute_total_size_missing_ref(self):
lines = list(_example_dump)
# 999 isn't in the dump, not sure how we get these in real life, but
# they exist. we should live with references that can't be resolved.
lines[-1] = ('{"address": 8, "type": "tuple", "size": 16, "len": 1'
', "refs": [999]}')
manager = loader.load(lines, show_prog=False)
obj = manager[8]
manager.compute_total_size(obj)
self.assertEqual(16, obj.total_size)
def test_summarize_refs(self):
manager = loader.load(_example_dump, show_prog=False)
summary = manager.summarize(manager[9])
# Note that the module is included in the summary
self.assertEqual(
sorted([
'int', 'module', bytes.__name__, six.text_type.__name__,
'tuple'
]), sorted(summary.type_summaries.keys()))
self.assertEqual(345, summary.total_size)
def test_compute_total_size_missing_ref(self):
lines = list(_example_dump)
# 999 isn't in the dump, not sure how we get these in real life, but
# they exist. we should live with references that can't be resolved.
lines[-1] = ('{"address": 8, "type": "tuple", "size": 16, "len": 1'
', "refs": [999]}')
manager = loader.load(lines, show_prog=False)
obj = manager[8]
manager.compute_total_size(obj)
self.assertEqual(16, obj.total_size)
def test_compute_parents(self):
manager = loader.load(_example_dump, show_prog=False)
objs = manager.objs
self.assertEqual((), objs[1].parents)
self.assertEqual([1, 3], objs[3].parents)
self.assertEqual([3, 7, 8], sorted(objs[4].parents))
self.assertEqual([3, 7, 8], sorted(objs[5].parents))
self.assertEqual([8], objs[6].parents)
self.assertEqual([8], objs[7].parents)
self.assertEqual((), objs[8].parents)
def test_load_one(self):
objs = loader.load([
'{"address": 1234, "type": "int", "size": 12, "value": 10'
', "refs": []}'], show_prog=False).objs
keys = list(objs.keys())
self.assertEqual([1234], keys)
obj = objs[1234]
self.assertTrue(isinstance(obj, _loader._MemObjectProxy))
# The address should be exactly the same python object as the key in
# the objs dictionary.
self.assertTrue(keys[0] is obj.address)
def test_sar(self):
"""Parses all the sar files and creates the pdf outputs"""
for example in self.sar_files:
print("Parsing: {0}".format(example))
grapher = SarGrapher([example])
stats = SarStats(grapher)
usage = resource.getrusage(resource.RUSAGE_SELF)
if USE_MELIAE:
objgraph.show_growth()
tmp = tempfile.mkstemp(prefix='sar-test')[1]
scanner.dump_all_objects(tmp)
leakreporter = loader.load(tmp)
summary = leakreporter.summarize()
print(
"SAR parsing: {0} usertime={1} systime={2} mem={3} MB".format(
end_of_path(example), usage[0], usage[1],
(usage[2] / 1024.0)))
if USE_PROFILER:
self.profile.disable()
str_io = StringIO.StringIO()
sortby = 'cumulative'
pstat = pstats.Stats(self.profile,
stream=str_io).sort_stats(sortby)
pstat.print_stats(TOP_PROFILED_FUNCTIONS)
print("\nProfiling of sar.parse()")
print(str_io.getvalue())
# Set up profiling for pdf generation
self.profile.enable()
out = "{0}.pdf".format(example)
stats.graph(example, [], out)
if USE_PROFILER:
self.profile.disable()
str_io = StringIO.StringIO()
sortby = 'cumulative'
pstat = pstats.Stats(self.profile,
stream=str_io).sort_stats(sortby)
pstat.print_stats(TOP_PROFILED_FUNCTIONS)
print("\nProfiling of sarstats.graph()")
print(str_io.getvalue())
print("Wrote: {0}".format(out))
os.remove(out)
grapher.close()
del grapher
del stats
usage = resource.getrusage(resource.RUSAGE_SELF)
print(
"SAR graphing: {0} usertime={1} systime={2} mem={3} MB".format(
end_of_path(example), usage[0], usage[1],
(usage[2] / 1024.0)))
def test_load_one(self):
objs = loader.load([
'{"address": 1234, "type": "int", "size": 12, "value": 10'
', "refs": []}'], show_prog=False).objs
keys = objs.keys()
self.assertEqual([1234], keys)
obj = objs[1234]
self.assertTrue(isinstance(obj, _loader._MemObjectProxy))
# The address should be exactly the same python object as the key in
# the objs dictionary.
self.assertTrue(keys[0] is obj.address)
def view(filename, objectname):
om = loader.load(filename)
om.compute_parents()
om.collapse_instance_dicts()
print om.summarize()
p = om.get_all(objectname)
# dump the first object
print p[0]
# dump all references of the first object
print p[0].c
# dump all referrers of the first object
print p[0].p
def test_expand_refs_as_dict(self):
# TODO: This test fails if simplejson is not installed, because the
# regex extractor does not cast to integers (they stay as
# strings). We could fix the test, or fix the extractor.
manager = loader.load(_instance_dump, show_prog=False, collapse=False)
as_dict = manager.refs_as_dict(manager[15])
self.assertEqual({1: 'c', 'b': 'c'}, as_dict)
manager.compute_parents()
manager.collapse_instance_dicts()
self.assertEqual({1: 'c', 'b': 'c'}, manager.refs_as_dict(manager[14]))
self.assertEqual({'a': 1, 'c': manager[7], 'b': 'string',
'd': manager[12]}, manager.refs_as_dict(manager[1]))
def view(filename, objectname):
om = loader.load(filename)
om.compute_parents()
om.collapse_instance_dicts()
print om.summarize()
p = om.get_all(objectname)
# dump the first object
print p[0]
# dump all references of the first object
print p[0].c
# dump all referrers of the first object
print p[0].p
def test_expand_refs_as_dict(self):
# TODO: This test fails if simplejson is not installed, because the
# regex extractor does not cast to integers (they stay as
# strings). We could fix the test, or fix the extractor.
manager = loader.load(_instance_dump, show_prog=False, collapse=False)
as_dict = manager.refs_as_dict(manager[15])
self.assertEqual({1: 'c', 'b': 'c'}, as_dict)
manager.compute_parents()
manager.collapse_instance_dicts()
self.assertEqual({1: 'c', 'b': 'c'}, manager.refs_as_dict(manager[14]))
self.assertEqual({'a': 1, 'c': manager[7], 'b': 'string',
'd': manager[12]}, manager.refs_as_dict(manager[1]))
def test_load_smoketest(self):
test_dict = {1:2, None:'a string'}
t = tempfile.TemporaryFile(prefix='meliae-')
# On some platforms TemporaryFile returns a wrapper object with 'file'
# being the real object, on others, the returned object *is* the real
# file object
t_file = getattr(t, 'file', t)
scanner.dump_all_referenced(t_file, test_dict)
t_file.seek(0)
manager = loader.load(t_file, show_prog=False)
test_dict_id = id(test_dict)
self.assertTrue(test_dict_id in manager.objs,
'%s not found in %s' % (test_dict_id, manager.objs.keys()))
def create_mem_stat(f):
"""create Memory stats from file
@param f file to load
"""
om = loader.load(f)
om.summarize()
thesummary = loader._ObjSummary()
objs = om.objs.itervalues()
for obj in objs:
thesummary._add(obj)
if thesummary.summaries is None:
thesummary.by_size()
return thesummary
def test_load_smoketest(self):
test_dict = {1:2, None:'a string'}
t = tempfile.TemporaryFile(prefix='meliae-')
# On some platforms TemporaryFile returns a wrapper object with 'file'
# being the real object, on others, the returned object *is* the real
# file object
t_file = getattr(t, 'file', t)
scanner.dump_all_referenced(t_file, test_dict)
t_file.seek(0)
manager = loader.load(t_file, show_prog=False)
test_dict_id = id(test_dict)
self.assertTrue(test_dict_id in manager.objs,
'%s not found in %s' % (test_dict_id, list(manager.objs.keys())))
def test_to_json(self):
manager = loader.load(_example_dump, show_prog=False, collapse=False)
objs = manager.objs.values()
objs.sort(key=lambda x:x.address)
expected = [
'{"address": 1, "type": "tuple", "size": 20, "refs": [2, 3]}',
'{"address": 2, "type": "dict", "size": 124, "refs": [4, 5, 6, 7]}',
'{"address": 3, "type": "list", "size": 44, "refs": [3, 4, 5]}',
'{"address": 4, "type": "int", "size": 12, "value": 2, "refs": []}',
'{"address": 5, "type": "int", "size": 12, "value": 1, "refs": []}',
'{"address": 6, "type": "str", "size": 29, "value": "a str", "refs": []}',
'{"address": 7, "type": "tuple", "size": 20, "refs": [4, 5]}',
'{"address": 8, "type": "module", "size": 60, "value": "mymod", "refs": [2]}',
]
self.assertEqual(expected, [obj.to_json() for obj in objs])
def main():
if len(sys.argv) != 3:
print "[ pyrasite memory viewer ]\n"
print "Usage: %s <pid> <objects.json>" % sys.argv[0]
print "\n pid - the running process id"
print " objects.json - the output of the dump-memory payload"
print
sys.exit(1)
pid = int(sys.argv[1])
filename = sys.argv[2]
objects = loader.load(filename)
objects.compute_referrers()
PyrasiteMemoryViewer(pid=pid, objects=objects).main()
def test_to_json(self):
manager = loader.load(_example_dump, show_prog=False, collapse=False)
objs = list(manager.objs.values())
objs.sort(key=lambda x:x.address)
expected = [
'{"address": 1, "type": "tuple", "size": 20, "refs": [2, 3]}',
'{"address": 2, "type": "dict", "size": 124, "refs": [4, 5, 6, 7]}',
'{"address": 3, "type": "list", "size": 44, "refs": [3, 4, 5]}',
'{"address": 4, "type": "int", "size": 12, "value": 2, "refs": []}',
'{"address": 5, "type": "int", "size": 12, "value": 1, "refs": []}',
'{"address": 6, "type": "str", "size": 29, "value": "a str", "refs": []}',
'{"address": 7, "type": "tuple", "size": 20, "refs": [4, 5]}',
'{"address": 8, "type": "module", "size": 60, "value": "mymod", "refs": [2]}',
]
self.assertEqual(expected, [obj.to_json() for obj in objs])
def main():
if len(sys.argv) != 3:
print "[ pyrasite memory viewer ]\n"
print "Usage: %s <pid> <objects.json>" % sys.argv[0]
print "\n pid - the running process id"
print " objects.json - the output of the dump-memory payload"
print
sys.exit(1)
pid = int(sys.argv[1])
filename = sys.argv[2]
objects = loader.load(filename)
objects.compute_referrers()
PyrasiteMemoryViewer(pid=pid, objects=objects).main()
def test_load_compressed(self):
# unfortunately NamedTemporaryFile's cannot be re-opened on Windows
fd, name = tempfile.mkstemp(prefix='meliae-')
f = os.fdopen(fd, 'wb')
try:
content = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=f)
for line in _example_dump:
content.write(line + '\n')
content.flush()
content.close()
del content
f.close()
objs = loader.load(name, show_prog=False).objs
objs[1]
finally:
f.close()
os.remove(name)
def test_load_compressed(self):
# unfortunately NamedTemporaryFile's cannot be re-opened on Windows
fd, name = tempfile.mkstemp(prefix='meliae-')
f = os.fdopen(fd, 'wb')
try:
content = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=f)
for line in _example_dump:
content.write(line + '\n')
content.flush()
content.close()
del content
f.close()
objs = loader.load(name, show_prog=False).objs
objs[1]
finally:
f.close()
os.remove(name)
def main():
if len(sys.argv) != 2:
print("[ pyrasite memory viewer ]\n")
print("Usage: %s <pid> <objects.json>" % sys.argv[0])
print("\n pid - the running process id")
print("")
sys.exit(1)
pid = int(sys.argv[1])
payload = abspath(join(dirname(__file__), '..',
'payloads', 'dump_memory.py'))
pyrasite.inject(pid, payload)
filename = '/tmp/pyrasite-%d-objects.json' % pid
objects = loader.load(filename)
objects.compute_referrers()
PyrasiteMemoryViewer(pid=pid, objects=objects).main()
def main():
if len(sys.argv) != 2:
print("[ pyrasite memory viewer ]\n")
print("Usage: %s <pid> <objects.json>" % sys.argv[0])
print("\n pid - the running process id")
print("")
sys.exit(1)
pid = int(sys.argv[1])
payload = abspath(
join(dirname(__file__), '..', 'payloads', 'dump_memory.py'))
pyrasite.inject(pid, payload)
filename = '/tmp/pyrasite-%d-objects.json' % pid
objects = loader.load(filename)
objects.compute_referrers()
PyrasiteMemoryViewer(pid=pid, objects=objects).main()
def test_collapse_old_instance_dicts(self):
manager = loader.load(_old_instance_dump, show_prog=False,
collapse=False)
instance = manager.objs[1]
self.assertEqual('instance', instance.type_str)
self.assertEqual(36, instance.size)
self.assertEqual([2, 3], instance.children)
inst_dict = manager[3]
self.assertEqual(140, inst_dict.size)
self.assertEqual([4, 5, 6, 7], inst_dict.children)
manager.compute_parents()
manager.collapse_instance_dicts()
# The instance dict has been removed, and its references moved into the
# instance, further, the type has been updated from generic 'instance'
# to being 'OldStyle'.
self.assertFalse(3 in manager.objs)
self.assertEqual(176, instance.size)
self.assertEqual([4, 5, 6, 7, 2], instance.children)
self.assertEqual('OldStyle', instance.type_str)
def test_collapse_old_instance_dicts(self):
manager = loader.load(_old_instance_dump, show_prog=False,
collapse=False)
instance = manager.objs[1]
self.assertEqual('instance', instance.type_str)
self.assertEqual(36, instance.size)
self.assertEqual([2, 3], instance.children)
inst_dict = manager[3]
self.assertEqual(140, inst_dict.size)
self.assertEqual([4, 5, 6, 7], inst_dict.children)
manager.compute_parents()
manager.collapse_instance_dicts()
# The instance dict has been removed, and its references moved into the
# instance, further, the type has been updated from generic 'instance'
# to being 'OldStyle'.
self.assertFalse(3 in manager.objs)
self.assertEqual(176, instance.size)
self.assertEqual([4, 5, 6, 7, 2], instance.children)
self.assertEqual('OldStyle', instance.type_str)
def measure_memory(self, _id):
from meliae import scanner, loader
scanner.dump_all_objects(self.MEMORY_DUMP % _id)
om = loader.load(self.MEMORY_DUMP % _id)
om.remove_expensive_references()
summary = om.summarize()
print summary
#print('runsnakemem %s' % self.MEMORY_DUMP)
usage = resource.getrusage(resource.RUSAGE_SELF)
print 'maximum resident set size', usage.ru_maxrss
print 'shared memory size', usage.ru_ixrss
print 'unshared memory size', usage.ru_idrss
print 'unshared stack size', usage.ru_isrss
import psutil
self_pid = psutil.Process()
# pylint: disable=E1101
print self_pid.memory_info()
def test_remove_expensive_references(self):
lines = list(_example_dump)
lines.pop(-1) # Remove the old module
lines.append('{"address": 8, "type": "module", "size": 12'
', "name": "mymod", "refs": [9]}')
lines.append('{"address": 9, "type": "dict", "size": 124'
', "refs": [10, 11]}')
lines.append('{"address": 10, "type": "module", "size": 12'
', "name": "mod2", "refs": [12]}')
lines.append('{"address": 11, "type": "str", "size": 27'
', "value": "boo", "refs": []}')
lines.append('{"address": 12, "type": "dict", "size": 124'
', "refs": []}')
manager = loader.load(lines, show_prog=False, collapse=False)
mymod_dict = manager.objs[9]
self.assertEqual([10, 11], mymod_dict.children)
manager.remove_expensive_references()
self.assertTrue(0 in manager.objs)
null_obj = manager.objs[0]
self.assertEqual(0, null_obj.address)
self.assertEqual('<ex-reference>', null_obj.type_str)
self.assertEqual([11, 0], mymod_dict.children)
def test_load_without_simplejson(self):
objs = loader.load([
'{"address": 1234, "type": "int", "size": 12, "value": 10'
', "refs": []}',
'{"address": 2345, "type": "module", "size": 60, "name": "mymod"'
', "refs": [1234]}',
'{"address": 4567, "type": "str", "size": 150, "len": 126'
', "value": "Test \\\'whoami\\\'\\u000a\\"Your name\\""'
', "refs": []}'
], using_json=False, show_prog=False).objs
keys = sorted(objs.keys())
self.assertEqual([1234, 2345, 4567], keys)
obj = objs[1234]
self.assertTrue(isinstance(obj, _loader._MemObjectProxy))
# The address should be exactly the same python object as the key in
# the objs dictionary.
self.assertTrue(keys[0] is obj.address)
self.assertEqual(10, obj.value)
obj = objs[2345]
self.assertEqual("module", obj.type_str)
self.assertEqual("mymod", obj.value)
obj = objs[4567]
self.assertEqual("Test \\'whoami\\'\\u000a\\\"Your name\\\"", obj.value)
def test_compute_referrers(self):
# Deprecated
logged = []
def log_warn(msg, klass, stacklevel=None):
logged.append((msg, klass, stacklevel))
old_func = warn.trap_warnings(log_warn)
try:
manager = loader.load(_example_dump, show_prog=False)
manager.compute_referrers()
self.assertEqual([('.compute_referrers is deprecated.'
' Use .compute_parents instead.',
DeprecationWarning, 3),
], logged)
objs = manager.objs
finally:
warn.trap_warnings(old_func)
self.assertEqual((), objs[1].parents)
self.assertEqual([1, 3], objs[3].parents)
self.assertEqual([3, 7, 8], sorted(objs[4].parents))
self.assertEqual([3, 7, 8], sorted(objs[5].parents))
self.assertEqual([8], objs[6].parents)
self.assertEqual([8], objs[7].parents)
self.assertEqual((), objs[8].parents)
from meliae import loader
om = loader.load('memory.json')
om.compute_parents()
om.collapse_instance_dicts()
s = om.summarize()
print s
'''
p = om.get_all('list')
print p[0]
print p[0].c
print p[0].p
'''
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
om.remove_expensive_references()
om.collapse_instance_dicts()
om.compute_referrers()
om.compute_total_size()
#print om.summarize()
from pprint import pprint as pp
def larger(x, y):
return om[y].total_size - om[x].total_size
def larger(x, y):
return int(y.total_size - x.total_size)
objs = sorted(iter(om.objs.values()), larger)
def load(filename):
om = loader.load(filename)
om.compute_parents()
om.collapse_instance_dicts()
print om.summarize()
def test_guess_intern_dict(self):
manager = loader.load(_intern_dict_dump, show_prog=False)
obj = manager.guess_intern_dict()
self.assertEqual(8, obj.address)
def test_expand_refs_as_list(self):
# TODO: This test fails if simplejson is not installed, because the
# regex extractor does not cast to integers (they stay as
# strings). We could fix the test, or fix the extractor.
manager = loader.load(_instance_dump, show_prog=False)
self.assertEqual([2], manager.refs_as_list(manager[12]))
