def test_stracker_create_summary(self):
"""Test that a summary is created correctly.
This can only be done heuristically, e.g that most recent objects are
included.
Also check that summaries managed by the tracker are excluded if
ignore_self is enabled.
"""
# at the beginning, there should not be an indicator object listed
tmp_tracker = tracker.SummaryTracker()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), None)
# now an indicator object should be listed
o = self._get_indicator()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), 1)
# with ignore_self enabled a second summary should not list the first
# summary
sn = tmp_tracker.create_summary()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.assertEqual(len(tmp), 0)
# but with ignore_self turned off, there should be some difference
tmp_tracker = tracker.SummaryTracker(ignore_self=False)
sn = tmp_tracker.create_summary()
tmp_tracker.new_obj = self._get_indicator()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.failIfEqual(len(tmp), 0)
def test_stracker_create_summary(self):
"""Test that a summary is created correctly.
This can only be done heuristically, e.g that most recent objects are
included.
Also check that summaries managed by the tracker are excluded if
ignore_self is enabled.
"""
# at the beginning, there should not be an indicator object listed
tmp_tracker = tracker.SummaryTracker()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), None)
# now an indicator object should be listed
o = self._get_indicator()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), 1)
# with ignore_self enabled a second summary should not list the first
# summary
sn = tmp_tracker.create_summary()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.assertEqual(len(tmp), 0)
# but with ignore_self turned off, there should be some difference
tmp_tracker = tracker.SummaryTracker(ignore_self=False)
sn = tmp_tracker.create_summary()
tmp_tracker.new_obj = self._get_indicator()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.failIfEqual(len(tmp), 0)
def test_sweep(self):
"""Test that all and only empty entries are removed from a summary."""
objects = ['the', 'quick', 'brown', 'fox', 1298, 123, 234, [], {}]
summ = summary.summarize(objects)
# correct removal of rows when sizes are empty
summary._subtract(summ, {})
summary._subtract(summ, [])
summ = summary._sweep(summ)
found_dict = found_tuple = False
for row in summ:
if row[0] == "<type 'dict'>":
found_dict = True
if row[0] == "<type 'tuple'>":
found_tuple = True
self.assert_(found_dict == False)
self.assert_(found_tuple == False)
# do not remove row if one of the sizes is not empty
# e.g. if the number of objects of a type did not change, but the
# total size did
summ = summary._subtract(summ, 'the')
summ = summary._subtract(summ, 'quick')
summ = summary._subtract(summ, 'brown')
summ = summary._subtract(summ, '42')
summ = summary._sweep(summ)
found_string = False
for row in summ:
if row[0] == summary._repr(''):
found_string = True
self.assert_(row[1] == 0)
totalsize = getsizeof('fox') - getsizeof('42')
self.assert_(row[2] == totalsize)
self.assert_(found_string == True)
def test_sweep(self):
"""Test that all and only empty entries are removed from a summary."""
objects = ['the', 'quick', 'brown', 'fox', 1298, 123, 234, [], {}]
summ = summary.summarize(objects)
# correct removal of rows when sizes are empty
summary._subtract(summ, {})
summary._subtract(summ, [])
summ = summary._sweep(summ)
found_dict = found_tuple = False
for row in summ:
if row[0] == "<type 'dict'>":
found_dict = True
if row[0] == "<type 'tuple'>":
found_tuple = True
self.assert_(found_dict == False)
self.assert_(found_tuple == False)
# do not remove row if one of the sizes is not empty
# e.g. if the number of objects of a type did not change, but the
# total size did
summ = summary._subtract(summ, 'the')
summ = summary._subtract(summ, 'quick')
summ = summary._subtract(summ, 'brown')
summ = summary._subtract(summ, '42')
summ = summary._sweep(summ)
found_string = False
for row in summ:
if row[0] == summary._repr(''):
found_string = True
self.assert_(row[1] == 0)
totalsize = _getsizeof('fox') - _getsizeof('42')
self.assert_(row[2] == totalsize)
self.assert_(found_string == True)
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
def diff(self, summary1=None, summary2=None):
"""Compute diff between to summaries.
If no summary is provided, the diff from the last to the current
summary is used. If summary1 is provided the diff from summary1
to the current summary is used. If summary1 and summary2 are
provided, the diff between these two is used.
"""
res = None
if summary2 is None:
self.s1 = self.create_summary()
if summary1 is None:
res = summary.get_diff(self.s0, self.s1)
else:
res = summary.get_diff(summary1, self.s1)
self.s0 = self.s1
else:
if summary1 is not None:
res = summary.get_diff(summary1, summary2)
else:
raise ValueError("You cannot provide summary2 without summary1.")
return summary._sweep(res)
def diff(self, summary1=None, summary2=None):
"""Compute diff between to summaries.
If no summary is provided, the diff from the last to the current
summary is used. If summary1 is provided the diff from summary1
to the current summary is used. If summary1 and summary2 are
provided, the diff between these two is used.
"""
res = None
if summary2 is None:
self.s1 = self.create_summary()
if summary1 is None:
res = summary.get_diff(self.s0, self.s1)
else:
res = summary.get_diff(summary1, self.s1)
self.s0 = self.s1
else:
if summary1 is not None:
res = summary.get_diff(summary1, summary2)
else:
raise ValueError(
"You cannot provide summary2 without summary1.")
return summary._sweep(res)
def _get_usage(function, *args):
"""Test if more memory is used after the function has been called.
The function will be invoked twice and only the second measurement will be
considered. Thus, memory used in initialisation (e.g. loading modules)
will not be included in the result. The goal is to identify memory leaks
caused by functions which use more and more memory.
Any arguments next to the function will be passed on to the function
on invocation.
Note that this function is currently experimental, because it is not
tested thoroughly and performs poorly.
"""
# The usage of a function is calculated by creating one summary of all
# objects before the function is invoked and afterwards. These summaries
# are compared and the diff is returned.
# This function works in a 2-steps process. Before the actual function is
# invoked an empty dummy function is measurement to identify the overhead
# involved in the measuring process. This overhead then is subtracted from
# the measurement performed on the passed function. The result reflects the
# actual usage of a function call.
# Also, a measurement is performed twice, allowing the adjustment to
# initializing things, e.g. modules
res = None
def _get_summaries(function, *args):
"""Get a 2-tuple containing one summary from before, and one summary
from after the function has been invoked.
"""
s_before = summary.summarize(get_objects())
function(*args)
s_after = summary.summarize(get_objects())
return (s_before, s_after)
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
# calibrate; twice for initialization
def noop():
pass
offset = _get_usage(noop)
offset = _get_usage(noop)
# perform operation twice to handle objects possibly used in
# initialisation
tmp = _get_usage(function, *args)
tmp = _get_usage(function, *args)
tmp = summary.get_diff(offset, tmp)
tmp = summary._sweep(tmp)
if len(tmp) != 0:
res = tmp
return res
def _get_usage(function, *args):
"""Test if more memory is used after the function has been called.
The function will be invoked twice and only the second measurement will be
considered. Thus, memory used in initialisation (e.g. loading modules)
will not be included in the result. The goal is to identify memory leaks
caused by functions which use more and more memory.
Any arguments next to the function will be passed on to the function
on invocation.
Note that this function is currently experimental, because it is not
tested thoroughly and performs poorly.
"""
# The usage of a function is calculated by creating one summary of all
# objects before the function is invoked and afterwards. These summaries
# are compared and the diff is returned.
# This function works in a 2-steps process. Before the actual function is
# invoked an empty dummy function is measurement to identify the overhead
# involved in the measuring process. This overhead then is subtracted from
# the measurement performed on the passed function. The result reflects the
# actual usage of a function call.
# Also, a measurement is performed twice, allowing the adjustment to
# initializing things, e.g. modules
res = None
def _get_summaries(function, *args):
"""Get a 2-tuple containing one summary from before, and one summary
from after the function has been invoked.
"""
s_before = summary.summarize(get_objects())
function(*args)
s_after = summary.summarize(get_objects())
return (s_before, s_after)
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
# calibrate; twice for initialization
def noop():
pass
offset = _get_usage(noop)
offset = _get_usage(noop)
# perform operation twice to handle objects possibly used in
# initialisation
tmp = _get_usage(function, *args)
tmp = _get_usage(function, *args)
tmp = summary.get_diff(offset, tmp)
tmp = summary._sweep(tmp)
if len(tmp) != 0:
res = tmp
return res