def profile_memory():
from pympler.classtracker import ClassTracker
from pympler.asizeof import asizeof
from soap.common import Flyweight, _cache_map
from soap.expression import Expr
tracker = ClassTracker()
tracker.track_object(Flyweight._cache)
tracker.track_class(Expr)
yield
tracker.create_snapshot()
tracker.stats.print_summary()
print('Flyweight cache size', asizeof(Flyweight._cache))
print('Global cache size', asizeof(_cache_map))
class TrackObjectTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.detach_all()
def test_track_object(self):
"""Test object registration.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.assert_(id(foo) in self.tracker.objects)
self.assert_(id(bar) in self.tracker.objects)
self.assert_('Foo' in self.tracker.index)
self.assert_('Bar' in self.tracker.index)
self.assertEqual(self.tracker.objects[id(foo)].ref(),foo)
self.assertEqual(self.tracker.objects[id(bar)].ref(),bar)
def test_type_errors(self):
"""Test intrackable objects.
"""
i = 42
j = 'Foobar'
k = [i,j]
l = {i: j}
self.assertRaises(TypeError, self.tracker.track_object, i)
self.assertRaises(TypeError, self.tracker.track_object, j)
self.assertRaises(TypeError, self.tracker.track_object, k)
self.assertRaises(TypeError, self.tracker.track_object, l)
self.assert_(id(i) not in self.tracker.objects)
self.assert_(id(j) not in self.tracker.objects)
self.assert_(id(k) not in self.tracker.objects)
self.assert_(id(l) not in self.tracker.objects)
def test_track_by_name(self):
"""Test registering objects by name.
"""
foo = Foo()
self.tracker.track_object(foo, name='Foobar')
self.assert_('Foobar' in self.tracker.index )
self.assertEqual(self.tracker.index['Foobar'][0].ref(),foo)
def test_keep(self):
"""Test lifetime of tracked objects.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo, keep=1)
self.tracker.track_object(bar)
idfoo = id(foo)
idbar = id(bar)
del foo
del bar
self.assert_(self.tracker.objects[idfoo].ref() is not None)
self.assert_(self.tracker.objects[idbar].ref() is None)
def test_mixed_tracking(self):
"""Test mixed instance and class tracking.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.track_class(Foo)
objs = []
for _ in range(10):
objs.append(Foo())
self.tracker.create_snapshot()
def test_recurse(self):
"""Test recursive sizing and saving of referents.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=1)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref),1)
dref = dref[0]
self.assert_(dref.size > 0, dref.size)
self.assert_(dref.flat > 0, dref.flat)
self.assertEqual(dref.refs,())
# Test track_change and more fine-grained resolution
self.tracker.track_change(foo, resolution_level=2)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref),1)
dref = dref[0]
namerefs = [r.name for r in dref.refs]
self.assert_('[K] foo' in namerefs, namerefs)
self.assert_("[V] foo: 'foo'" in namerefs, namerefs)
class SnapshotTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_timestamp(self):
"""Test timestamp of snapshots.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.tracker.create_snapshot()
self.tracker.create_snapshot()
self.tracker.create_snapshot()
refts = [fp.timestamp for fp in self.tracker.snapshots]
for to in self.tracker.objects.values():
ts = [t for (t,sz) in to.snapshots[1:]]
self.assertEqual(ts,refts)
def test_snapshot_members(self):
"""Test existence and value of snapshot members.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.create_snapshot(compute_total=True)
fp = self.tracker.snapshots[0]
fp_with_total = self.tracker.snapshots[1]
self.assert_(fp.overhead > 0, fp.overhead)
self.assert_(fp.tracked_total > 0, fp.tracked_total)
self.assertEqual(fp.asizeof_total, 0)
self.assert_(fp_with_total.asizeof_total > 0, fp_with_total.asizeof_total)
self.assert_(fp_with_total.asizeof_total >= fp_with_total.tracked_total)
if pympler.process.is_available():
procmem = fp.system_total
self.assertEqual(fp.total, procmem.vsz)
self.assert_(procmem.vsz > 0, procmem)
self.assert_(procmem.rss > 0, procmem)
self.assertTrue(procmem.vsz >= procmem.rss, procmem)
self.assert_(procmem.vsz > fp.overhead, procmem)
self.assert_(procmem.vsz > fp.tracked_total, procmem)
self.assert_(fp_with_total.system_total.vsz > fp_with_total.asizeof_total)
else:
self.assertEqual(fp_with_total.total, fp_with_total.asizeof_total)
self.assertEqual(fp.total, fp.tracked_total)
def test_desc(self):
"""Test snapshot label.
"""
self.tracker.create_snapshot()
self.tracker.create_snapshot('alpha')
self.tracker.create_snapshot(description='beta')
self.tracker.create_snapshot(42)
self.assertEqual(len(self.tracker.snapshots), 4)
self.assertEqual(self.tracker.snapshots[0].desc, '')
self.assertEqual(self.tracker.snapshots[1].desc, 'alpha')
self.assertEqual(self.tracker.snapshots[2].desc, 'beta')
self.assertEqual(self.tracker.snapshots[3].desc, '42')
snapshot = self.tracker.snapshots[0]
self.assertEqual(snapshot.label, '%.3fs' % snapshot.timestamp)
snapshot = self.tracker.snapshots[1]
self.assertEqual(snapshot.label, 'alpha (%.3fs)' % snapshot.timestamp)
snapshot = self.tracker.snapshots[3]
self.assertEqual(snapshot.label, '42 (%.3fs)' % snapshot.timestamp)
def test_background_monitoring(self):
"""Test background monitoring.
"""
self.tracker.start_periodic_snapshots(0.1)
self.assertEqual(self.tracker._periodic_thread.interval, 0.1)
self.assertEqual(self.tracker._periodic_thread.getName(), 'BackgroundMonitor')
for x in range(10): # try to interfere
self.tracker.create_snapshot(str(x))
time.sleep(0.5)
self.tracker.start_periodic_snapshots(0.2)
self.assertEqual(self.tracker._periodic_thread.interval, 0.2)
self.tracker.stop_periodic_snapshots()
self.assert_(self.tracker._periodic_thread is None)
self.assert_(len(self.tracker.snapshots) > 10)
class GlobalContainer(box.BoxLayout):
graph_list = prop.ListProperty([])
active_graph = prop.ObjectProperty(None)
name_manager = NameManager.Instance()
show_sidepanel = prop.BooleanProperty(True)
modestr = prop.StringProperty('insert')
itemstr = prop.StringProperty('atom')
modes = {'insert': asp.Mode.INSERT, 'select': asp.Mode.SELECT}
items = {
'atom': asp.Item.ATOM,
'ellipse': asp.Item.ELLIPSE,
'rectangle': asp.Item.SQUARE
}
def __init__(self, **kwargs):
super(GlobalContainer, self).__init__(**kwargs)
self._keyboard = None
self.request_keyboard()
self.working_dir = './'
self.tutorial = None
self.popup_stack = []
window.Window.bind(on_resize=self.on_resize)
if DEBUG:
self.tracker = ClassTracker()
self.tracker.track_object(MenuButton)
self.all_objects = muppy.get_objects()
def request_keyboard(self):
self._keyboard = window.Window.request_keyboard(
self._keyboard_release, self)
self._keyboard.bind(on_key_down=self._on_keyboard_down)
def _keyboard_catch(self):
self._keyboard.bind(on_key_down=self._on_keyboard_down)
def _keyboard_release(self):
self._keyboard.unbind(on_key_down=self._on_keyboard_down)
#self._keyboard = None
def _on_keyboard_down(self, keyboard, keycode, text, modifiers):
if keycode[1] == 'escape':
base.stopTouchApp()
elif keycode[1] == 'd':
self.set_mode('insert')
elif keycode[1] == 's':
self.set_mode('select')
elif keycode[1] == 'w':
self.set_item('atom')
elif keycode[1] == 'e':
self.set_item('ellipse')
elif keycode[1] == 'r':
self.set_item('rectangle')
elif keycode[1] == 'p':
self.active_graph.show_tree(0)
elif keycode[1] == 'o':
self.view_symbolic_formula()
elif keycode[1] == 'n':
self.show_gringo_query()
elif keycode[1] == 'g':
self.show_save()
elif keycode[1] == 'l':
self.show_load()
elif keycode[1] == 'y':
print self.name_manager.get_all()
print asp.Line.get_all_lines()
elif keycode[1] == 't':
self.toggle_sidepanel()
elif keycode[1] == 'tab':
if not self.show_sidepanel:
self.toggle_sidepanel()
self.ids.atom_input.focus = True
elif keycode[1] == '.':
if DEBUG:
rb = refbrowser.InteractiveBrowser(self)
rb.main()
# self.all_objects = muppy.get_objects()
# sum1 = summary.summarize(self.all_objects)
# summary.print_(sum1)
# self.tracker.create_snapshot()
# self.tracker.stats.print_summary()
return True
def _focus_name_list(self, value):
if value and (asp.AtomWidget.active_atom is not None):
previous_name = asp.AtomWidget.active_atom.name
for button in self.ids.name_list.children:
if (button.text == previous_name) and (button.state != 'down'):
button.trigger_action()
elif value:
if self.ids.name_list.children:
self.ids.name_list.children[-1].trigger_action()
else:
for button in self.ids.name_list.children:
if button.state == 'down':
button.trigger_action()
def on_resize(self, window, width, height):
if self.show_sidepanel:
self.ids.sidepanel.width = self.width * .15
self.active_graph.size = self.ids.stencilview.size
def toggle_sidepanel(self):
self.show_sidepanel = not self.show_sidepanel
if self.show_sidepanel:
width = self.width * .15
else:
width = 0
anim.Animation(width=width, d=.15,
t='out_quart').start(self.ids.sidepanel)
if not self.show_sidepanel:
self.ids.sidepanel.focus = False
# Also release keyboard
#self.update_sourcecode()
def update_atom_editor(self, name):
editor = self.ids.atom_editor
if name == '':
editor.disabled = True
else:
if editor.disabled:
editor.disabled = False
atom = self.name_manager.get(name)
editor.update(atom)
def update_atom(self,
name,
new_name='',
new_hook_points=[],
is_constant=False):
atom = self.name_manager.get(name)
if new_name <> '':
atom.name = new_name
if len(new_hook_points) == 4:
atom.hook_points = new_hook_points
# print id(atom.hook_points)
atom.is_constant = is_constant
def register_atom(self,
name,
hooks=[False, False, False, False],
is_constant=False):
if name == '':
return
children = self.ids.name_list.children
new_button = None
name_to_insert = name
i = len(children) - 1
# If the name doesn't exist, register atom
if self.name_manager.get(name) is None:
self.name_manager.register(name, asp.Atom(name, hooks,
is_constant))
# print id(self.name_manager.get(name).hook_points)
# Insert in name_list sorted by name
while i >= 0:
if children[i].text < name_to_insert:
#print children[i].text, '<', name_to_insert
pass
elif children[i].text == name_to_insert:
# Already exists
if children[i].state != 'down':
children[i].trigger_action()
return
elif children[i].text > name_to_insert:
#print children[i].text, '>', name_to_insert
temp = children[i].text
children[i].text = name_to_insert
name_to_insert = temp
if new_button == None:
new_button = children[i]
i -= 1
self.ids.name_list.add_widget(AtomSelectionButton(text=name_to_insert))
if new_button == None:
new_button = children[0]
if new_button.state == 'down':
asp.AtomWidget.active_atom = self.name_manager.get(name)
self.update_atom_editor(name)
self.set_item('atom')
else:
new_button.trigger_action()
def rename_atom(self, new_name):
old_name = ''
for button in self.ids.name_list.children:
if button.state == 'down':
old_name = button.text
selected_button = button
if (old_name != '') and (new_name != ''):
try:
atom = self.name_manager.get(old_name)
exists = self.name_manager.get(new_name)
assert atom is not None
assert exists is None
except AssertionError:
#self.show_error('Name already exists.')
print 'Name already exists.'
return
selected_button.text = new_name
atom.name = new_name
self.name_manager.unregister(old_name)
self.name_manager.register(new_name, atom)
self.update_atom_editor(new_name)
def delete_atom(self):
for button in self.ids.name_list.children:
if button.state == 'down':
self.name_manager.unregister(button.text)
self.active_graph.delete_atom(button.text)
asp.AtomWidget.active_atom = None
self.ids.name_list.remove_widget(button)
self.update_atom_editor('')
def clear_atoms(self):
self.ids.name_list.clear_widgets()
self.name_manager.clear()
self.update_atom_editor('')
asp.AtomWidget.active_atom = None
def new_graph(self):
# TODO: Migrate to tab system
asp.Line.clear_lines()
self.clear_atoms()
if self.active_graph is None:
g = asp.RootWidget()
self.graph_list.append(g)
self.active_graph = g
self.ids.stencilview.add_widget(g)
else:
self.active_graph.delete_tree()
def close_graph(self):
if self.active_graph is not None:
asp.Line.clear_lines()
self.active_graph.delete_tree()
self.active_graph.delete_root()
self.clear_atoms()
self.graph_list.pop()
self.active_graph = None
def set_mode(self, mode):
try:
prev_mode = self.active_graph.mode
new_mode = self.modes[mode]
if new_mode == asp.Mode.INSERT:
self.active_graph.show_hooks()
else:
if prev_mode == asp.Mode.INSERT:
self.active_graph.hide_hooks()
self.active_graph.mode = new_mode
self.modestr = mode
except KeyError as err:
print 'ERROR: Invalid mode {0} requested.'.format(str(err))
def set_item(self, item):
try:
self.active_graph.item = self.items[item]
self.itemstr = item
except KeyError as err:
print 'ERROR: Invalid item {0} requested.'.format(str(err))
except AttributeError:
pass
if item == 'atom':
self._focus_name_list(True)
else:
self._focus_name_list(False)
def push_popup(self, popup):
self.popup_stack.append(popup)
popup.open()
def dismiss_popup(self):
popup = self.popup_stack.pop()
popup.dismiss()
def show_rename_atom(self):
content = TextInputDialog(caption="Enter new name",
validate_callback=self.rename_atom,
cancel=self.dismiss_popup)
p = CustomPopup(self,
title="Rename atom",
content=content,
size_hint=(0.4, 0.25))
self.push_popup(p)
def show_load(self):
content = LoadDialog(load=self.load, cancel=self.dismiss_popup)
content.ids.filechooser.path = self.working_dir
p = CustomPopup(self,
title="Load file",
content=content,
size_hint=(0.9, 0.9))
self.push_popup(p)
def show_save(self):
content = SaveDialog(save=self.save, cancel=self.dismiss_popup)
content.ids.filechooser.path = self.working_dir
p = CustomPopup(self,
title="Save file",
content=content,
size_hint=(0.9, 0.9))
self.push_popup(p)
def show_export(self):
content = ExportDialog(export=self.export, cancel=self.dismiss_popup)
content.ids.filechooser.path = self.working_dir
p = CustomPopup(self,
title="Export file",
content=content,
size_hint=(0.9, 0.9))
self.push_popup(p)
def show_gringo_query(self):
caption = "Enter desired predicates separated by commas"
content = TextInputDialog(caption=caption,
validate_callback=self.gringo_query,
dismiss_on_validate=False,
focus=False,
cancel=self.dismiss_popup)
p = CustomPopup(self,
title="Output predicates",
content=content,
size_hint=(0.4, 0.25))
self.push_popup(p)
def show_stable_models(self, solver):
models = solver.get_models()
content = None
if len(models) == 0:
content = ErrorDialog('Unsatisfiable', cancel=self.dismiss_popup)
else:
solver.generate_graph(models[0])
content = StableModelDialog(solver, cancel=self.dismiss_popup)
content.ids.img.reload()
p = CustomPopup(self,
catch_keyboard=False,
title="Stable Models",
content=content,
size_hint=(0.9, 0.9))
self.push_popup(p)
def show_error(self, err_str):
content = ErrorDialog(err_str, cancel=self.dismiss_popup)
p = CustomPopup(self,
catch_keyboard=False,
title="Error",
content=content,
size_hint=(0.4, 0.3))
self.push_popup(p)
def show_about(self):
content = AboutDialog(cancel=self.dismiss_popup)
p = CustomPopup(self,
catch_keyboard=False,
title="About " + __title__,
content=content,
size_hint=(0.5, 0.5))
self.push_popup(p)
def load(self, path, filename):
self.close_graph()
self.working_dir = path
try:
f = os.path.join(path, filename[0])
# Temporal line storage. Its contents are arranged as follows:
# { line_id: (graph, hook_list) , ... }
lines = {}
with open(f, 'r') as stream:
for line in stream:
if line.startswith(NameParser.TOKENS['name']):
name, hooks, is_constant = NameParser.parse_name(line)
self.register_atom(name, hooks, is_constant)
if line.startswith(NameParser.TOKENS['line']):
line_id, graph = NameParser.parse_line(line)
lines[line_id] = (graph, [None] * len(graph))
new_graph = lang.Builder.load_file(f)
self.ids.stencilview.add_widget(new_graph)
self.active_graph = new_graph
#self.graph_list.pop()
self.graph_list.append(new_graph)
for w in self.active_graph.walk(restrict=True):
if isinstance(w, asp.AtomWidget):
w._deferred_init()
for i in w.get_line_info():
line_id = i[0]
hook_index = i[1]
lines[line_id][1][hook_index] = i[2]
elif isinstance(w, asp.NexusWidget):
for i in w.line_info:
line_id = i[0]
hook_index = i[1]
lines[line_id][1][hook_index] = w
for line, info in lines.iteritems():
print line, info
asp.Line.build_from_graph(info[0], info[1])
self.set_mode(self.modestr)
self.set_item(self.itemstr)
self.dismiss_popup()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.dismiss_popup()
self.close_graph()
self.new_graph()
self.show_error('Corrupted file.')
def save(self, path, filename):
self.working_dir = path
with open(os.path.join(path, filename), 'w') as stream:
stream.write('#:kivy 1.0.9\n\n')
for (name, atom) in self.name_manager.get_all():
stream.write(
NameParser.get_name_str(name, atom.hook_points,
atom.is_constant))
for line in asp.Line.get_all_lines():
stream.write(
NameParser.get_line_str(line.line_id,
line.get_full_graph()))
stream.write('\n')
stream.write(self.active_graph.get_tree(0))
self.dismiss_popup()
def export(self, path, filename):
_, ext = os.path.splitext(filename)
if ext == '.png':
self.active_graph.export_to_png(os.path.join(path, filename))
elif ext == '.lp':
rpn = self.active_graph.get_formula_RPN()
constants = self.active_graph.get_constants()
solver = eg_solver.Solver()
solver.set_formula(rpn, constants)
rules = solver.generate_asp_rules()
with open(os.path.join(path, filename), 'w') as stream:
for r in rules:
stream.write(r)
stream.write('\n')
else:
error_str = 'File extension not supported.'
print error_str
self.show_error(error_str)
return
self.dismiss_popup()
def highlight_variables(self):
self.active_graph.highlight_variables()
def view_symbolic_formula(self):
print self.active_graph.get_formula()
def gringo_query(self, show_predicates):
def generate_show_statements(predicates):
pred_list = predicates.split(',')
pred_list = map(lambda s: s.strip(), pred_list)
n_args = (lambda name: self.name_manager.get(name).hook_points.
count(True))
pred_list = [p + '/' + str(n_args(p)) for p in pred_list]
show_list = ['#show {0}.'.format(p) for p in pred_list]
return show_list
self.dismiss_popup()
rpn = self.active_graph.get_formula_RPN()
rpn = norm.LIT.TRUE if rpn == '' else rpn
constants = self.active_graph.get_constants()
print 80 * '-'
print 'RPN formula:\n', rpn
solver = eg_solver.Solver()
result = ''
try:
show_statements = []
if show_predicates:
try:
show_statements = generate_show_statements(show_predicates)
except Exception:
pass
solver.set_formula(rpn, constants)
result = solver.solve(show=show_statements)
except norm.MalformedFormulaError:
self.show_error('Malformed formula.')
return
except RuntimeError, e:
print e
self.show_error(str(e))
return
self.show_stable_models(solver)
class TrackObjectTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.detach_all()
def test_track_object(self):
"""Test object registration.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.assertTrue(id(foo) in self.tracker.objects)
self.assertTrue(id(bar) in self.tracker.objects)
self.assertTrue('Foo' in self.tracker.index)
self.assertTrue('Bar' in self.tracker.index)
self.assertEqual(self.tracker.objects[id(foo)].ref(), foo)
self.assertEqual(self.tracker.objects[id(bar)].ref(), bar)
def test_type_errors(self):
"""Test intrackable objects.
"""
i = 42
j = 'Foobar'
k = [i, j]
l = {i: j}
self.assertRaises(TypeError, self.tracker.track_object, i)
self.assertRaises(TypeError, self.tracker.track_object, j)
self.assertRaises(TypeError, self.tracker.track_object, k)
self.assertRaises(TypeError, self.tracker.track_object, l)
self.assertTrue(id(i) not in self.tracker.objects)
self.assertTrue(id(j) not in self.tracker.objects)
self.assertTrue(id(k) not in self.tracker.objects)
self.assertTrue(id(l) not in self.tracker.objects)
def test_track_by_name(self):
"""Test registering objects by name.
"""
foo = Foo()
self.tracker.track_object(foo, name='Foobar')
self.assertTrue('Foobar' in self.tracker.index)
self.assertEqual(self.tracker.index['Foobar'][0].ref(), foo)
def test_keep(self):
"""Test lifetime of tracked objects.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo, keep=1)
self.tracker.track_object(bar)
idfoo = id(foo)
idbar = id(bar)
del foo
del bar
self.assertTrue(self.tracker.objects[idfoo].ref() is not None)
self.assertTrue(self.tracker.objects[idbar].ref() is None)
def test_mixed_tracking(self):
"""Test mixed instance and class tracking.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.track_class(Foo)
objs = []
for _ in range(10):
objs.append(Foo())
self.tracker.create_snapshot()
def test_recurse(self):
"""Test recursive sizing and saving of referents.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=1)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref), 1)
dref = dref[0]
self.assertTrue(dref.size > 0, dref.size)
self.assertTrue(dref.flat > 0, dref.flat)
self.assertEqual(dref.refs, ())
# Test track_change and more fine-grained resolution
self.tracker.track_change(foo, resolution_level=2)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref), 1)
dref = dref[0]
namerefs = [r.name for r in dref.refs]
self.assertTrue('[K] foo' in namerefs, namerefs)
self.assertTrue("[V] foo: 'foo'" in namerefs, namerefs)
class SnapshotTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_timestamp(self):
"""Test timestamp of snapshots.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.tracker.create_snapshot()
self.tracker.create_snapshot()
self.tracker.create_snapshot()
refts = [fp.timestamp for fp in self.tracker.snapshots]
for to in self.tracker.objects.values():
ts = [t for (t, sz) in to.snapshots[1:]]
self.assertEqual(ts, refts)
def test_snapshot_members(self):
"""Test existence and value of snapshot members.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.create_snapshot(compute_total=True)
fp = self.tracker.snapshots[0]
fp_with_total = self.tracker.snapshots[1]
self.assertTrue(fp.overhead > 0, fp.overhead)
self.assertTrue(fp.tracked_total > 0, fp.tracked_total)
self.assertEqual(fp.asizeof_total, 0)
self.assertTrue(fp_with_total.asizeof_total > 0,
fp_with_total.asizeof_total)
self.assertTrue(
fp_with_total.asizeof_total >= fp_with_total.tracked_total)
if pympler.process.is_available():
procmem = fp.system_total
self.assertEqual(fp.total, procmem.vsz)
self.assertTrue(procmem.vsz > 0, procmem)
self.assertTrue(procmem.rss > 0, procmem)
self.assertTrue(procmem.vsz >= procmem.rss, procmem)
self.assertTrue(procmem.vsz > fp.overhead, procmem)
self.assertTrue(procmem.vsz > fp.tracked_total, procmem)
self.assertTrue(
fp_with_total.system_total.vsz > fp_with_total.asizeof_total)
else:
self.assertEqual(fp_with_total.total, fp_with_total.asizeof_total)
self.assertEqual(fp.total, fp.tracked_total)
def test_desc(self):
"""Test snapshot label.
"""
self.tracker.create_snapshot()
self.tracker.create_snapshot('alpha')
self.tracker.create_snapshot(description='beta')
self.tracker.create_snapshot(42)
self.assertEqual(len(self.tracker.snapshots), 4)
self.assertEqual(self.tracker.snapshots[0].desc, '')
self.assertEqual(self.tracker.snapshots[1].desc, 'alpha')
self.assertEqual(self.tracker.snapshots[2].desc, 'beta')
self.assertEqual(self.tracker.snapshots[3].desc, '42')
snapshot = self.tracker.snapshots[0]
self.assertEqual(snapshot.label, '%.3fs' % snapshot.timestamp)
snapshot = self.tracker.snapshots[1]
self.assertEqual(snapshot.label, 'alpha (%.3fs)' % snapshot.timestamp)
snapshot = self.tracker.snapshots[3]
self.assertEqual(snapshot.label, '42 (%.3fs)' % snapshot.timestamp)
def test_background_monitoring(self):
"""Test background monitoring.
"""
self.tracker.start_periodic_snapshots(0.1)
self.assertEqual(self.tracker._periodic_thread.interval, 0.1)
self.assertEqual(self.tracker._periodic_thread.getName(),
'BackgroundMonitor')
for x in range(10): # try to interfere
self.tracker.create_snapshot(str(x))
time.sleep(0.5)
self.tracker.start_periodic_snapshots(0.2)
self.assertEqual(self.tracker._periodic_thread.interval, 0.2)
self.tracker.stop_periodic_snapshots()
self.assertTrue(self.tracker._periodic_thread is None)
self.assertTrue(len(self.tracker.snapshots) > 10)
class LogTestCase(unittest.TestCase):
def setUp(self):
self.out = StringIO()
self.tracker = ClassTracker(stream=self.out)
@property
def output(self):
"""Return output recorded in `ClassTracker` output stream."""
return self.out.getvalue()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_dump(self):
"""Test serialization of log data.
"""
foo = Foo()
foo.data = range(1000)
bar = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.track_object(bar)
self.tracker.create_snapshot('Footest')
f1 = StringIO()
f2 = StringIO()
ConsoleStats(tracker=self.tracker, stream=f1).print_stats()
tmp = BytesIO()
Stats(tracker=self.tracker).dump_stats(tmp, close=False)
self.tracker.clear()
stats = ConsoleStats(stream=f2)
self.assertEqual(stats.index, None)
self.assertEqual(stats.snapshots, None)
tmp.seek(0)
stats.load_stats(tmp)
tmp.close()
self.assert_('Foo' in stats.index)
stats.print_stats()
self.assertEqual(f1.getvalue(), f2.getvalue())
# Test partial printing
stats.stream = f3 = StringIO()
stats.sort_stats()
tolen = len(stats.sorted)
stats.print_stats(clsname='Bar')
self.assertEqual(len(stats.sorted), tolen)
stats.print_summary()
clsname = f3.getvalue().split('\n')[0]
self.assertNotEqual(re.search('Bar', clsname), None, clsname)
f1.close()
f2.close()
f3.close()
def test_sort_stats(self):
"""Test sort_stats and reverse_order.
"""
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
foo.data = list(range(1000))
bar1 = Bar()
bar2 = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.create_snapshot()
stats = self.tracker.stats
# Test sort_stats and reverse_order
self.assertEqual(stats.sort_stats('size'), stats)
self.assertEqual(stats.sorted[0].classname, 'Foo')
stats.reverse_order()
self.assertEqual(stats.sorted[0].classname, 'Bar')
stats.sort_stats('classname', 'birth')
self.assertEqual(stats.sorted[0].classname, 'Bar')
self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn')
stats.sort_stats('classname')
def test_dump_load_with_filename(self):
"""Test serialization with filename.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=2)
self.tracker.create_snapshot()
fhandle, fname = mkstemp(prefix='pympler_test_dump')
os.close(fhandle)
try:
self.tracker.stats.dump_stats(fname)
output = StringIO()
stats = ConsoleStats(filename=fname, stream=output)
stats.print_stats()
self.assertTrue('<Foo>' in output.getvalue(), output.getvalue())
# Check if a Stats loaded from a dump can be dumped again
stats.dump_stats(fname)
finally:
os.unlink(fname)
def test_tracked_classes(self):
"""Test listing tracked classes.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
self.tracker.create_snapshot()
bar = Bar()
self.tracker.create_snapshot()
foo = FooNew()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
stats = self.tracker.stats
self.assertEqual(stats.tracked_classes, ['Bar', 'Foo', 'FooNew'])
stats.print_summary()
def test_print_stats(self):
"""Test printing class-filtered statistics.
"""
self.tracker.track_class(Foo, name='Foo', trace=True)
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
bar = Bar()
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(clsname='Foo')
self.assertTrue('Foo' in self.output, self.output)
self.assertFalse('Bar' in self.output, self.output)
self.assertTrue('foo = Foo()' in self.output, self.output)
def test_print_stats_limit(self):
"""Test printing limited statistics.
"""
self.tracker.track_class(Foo, name='Foo')
foo = [Foo() for _ in range(10)]
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(limit=3)
self.assertEqual(self.output.count('<Foo>'), 3)
self.out.seek(0)
self.out.truncate()
stats.print_stats(limit=0.5)
self.assertEqual(self.output.count('<Foo>'), 5)
def test_snapshots(self):
"""Test multiple snapshots.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
self.tracker.track_class(FooNew, name='FooNew')
self.tracker.create_snapshot()
f1 = Foo()
self.tracker.create_snapshot()
f2 = Foo()
f3 = FooNew()
self.tracker.create_snapshot()
b = Bar()
del b
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats()
stats.print_summary()
def test_merge(self):
"""Test merging of reference trees.
"""
self.tracker.track_class(FooNew, name='Foo', resolution_level=2)
f1 = FooNew()
f1.a = list(range(1000))
f2 = FooNew()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
sizer = Asizer()
sz1 = sizer.asized(f1)
sz2 = sizer.asized(f2)
stats = self.tracker.stats
for fp in stats.snapshots:
if fp.desc == 'Merge test':
stats.annotate_snapshot(fp)
self.assert_(hasattr(fp, 'classes'))
classes = fp.classes
stats.annotate_snapshot(fp)
self.assertEqual(fp.classes, classes)
self.assert_('Foo' in fp.classes, fp.classes)
self.assert_('merged' in fp.classes['Foo'])
fm = fp.classes['Foo']['merged']
self.assertEqual(fm.size, sz1.size + sz2.size, (fm.size, str(sz1), str(sz2)))
refs = {}
for ref in fm.refs:
refs[ref.name] = ref
self.assert_('__dict__' in refs.keys(), refs.keys())
refs2 = {}
for ref in refs['__dict__'].refs:
refs2[ref.name] = ref
self.assert_('[V] a' in refs2.keys(), refs2.keys())
self.assert_('[V] b' in refs2.keys(), refs2.keys())
self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a))
def test_html(self):
"""Test emitting HTML statistics."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
self.tracker.track_class(Bar, name='Bar', trace=True)
f1 = Foo()
f1.a = list(range(100000))
f2 = Foo()
f2.a = list(range(1000))
f2.b = 'This is some stupid spam.'
f1 = Bar()
self.tracker.create_snapshot('Merge test')
stats = HtmlStats(tracker=self.tracker)
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'footest.html')
stats.create_html(output)
source = open(output).read()
# Ensure relative links are used
fname = os.path.join('footest_files', 'Foo.html')
self.assertTrue('<a href="%s">' % fname in source, (fname, source))
finally:
rmtree(target)
def test_charts(self):
"""Test emitting graphic charts."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
f1 = Foo()
f1.a = list(range(1000))
f2 = Foo()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
from pympler import charts
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'timespace.png')
charts.tracker_timespace(output, self.tracker.stats)
finally:
rmtree(target)
class KademliaCrawlerProtocol(kademlia.KademliaProtocol):
logger = logging.getLogger("KademliaCrawlerProtocol")
def __init__(self, node: kademlia.Node, wire: 'DiscoveryProtocol') -> None:
super(KademliaCrawlerProtocol, self).__init__(node, wire)
self.logger.debug("Kademlia Crawler Created")
self.tracker = ClassTracker()
self.myTree = btree.Tree(256)
self.tracker.track_object(self.myTree)
self.MAX_CONCURRENT = 64
async def bootstrap(self, bootstrap_nodes: List[kademlia.Node]) -> None:
bonded = await asyncio.gather(*[self.bond(n) for n in bootstrap_nodes])
if not any(bonded):
self.logger.info("Failed to bond with bootstrap nodes {}".format(
bootstrap_nodes))
return
await self.lookup(self.this_node.id)
async def lookup(self,
node_id: int,
use_k_bucket=True) -> List[kademlia.Node]:
"""Lookup performs a network search for nodes close to the given target.
It approaches the target by querying nodes that are closer to it on each
iteration. The given target does not need to be an actual node identifier.
"""
self.logger.info("Kademlia Crawler Lookup Initiated")
nodes_asked = set() # type: Set[Node]
nodes_seen = set() # type: Set[Node]
async def _find_node(node_id, remote, semaphore):
async with semaphore:
self.wire.send_find_node(remote, node_id)
candidates = await self.wait_neighbours(remote)
if len(candidates) == 0:
self.logger.debug(
"got no candidates from {}, returning".format(remote))
return candidates
candidates = [c for c in candidates if c not in nodes_seen]
self.logger.debug("got {} new candidates".format(
len(candidates)))
# Add new candidates to nodes_seen so that we don't attempt to
# bond with failing ones in the future.
nodes_seen.update(candidates)
bonded = await asyncio.gather(
*[self.bond(c) for c in candidates])
self.logger.debug("bonded with {} candidates".format(
bonded.count(True)))
return [c for c in candidates if bonded[candidates.index(c)]]
def _exclude_if_asked(nodes):
nodes_to_ask = list(set(nodes).difference(nodes_asked))
return kademlia.sort_by_distance(
nodes_to_ask, node_id)[:kademlia.k_find_concurrency]
# Get all of the nodes in our list, not just the closest k_bucket nodes
closest = self.routing.neighbours(node_id)
self.logger.debug(
"starting lookup; initial neighbours: {}".format(closest))
nodes_to_ask = _exclude_if_asked(closest)
mySem = asyncio.Semaphore(value=self.MAX_CONCURRENT)
while nodes_to_ask:
#summary.print_(summary.summarize(muppy.get_objects()))
#print('Tasks count: ', len(asyncio.Task.all_tasks()))
#print('Active tasks count: ', len(
#[task for task in asyncio.Task.all_tasks() if not task.done()]))
self.logger.debug("node lookup; querying {}".format(nodes_to_ask))
nodes_asked.update(nodes_to_ask)
tasks = [_find_node(node_id, n, mySem) for n in nodes_to_ask]
results = await asyncio.gather(*tasks)
for candidates in results:
closest.extend(candidates)
# Here we're keeping all of the nodes that were returned so that we
# can get a better view of the entire network.
if use_k_bucket:
closest = kademlia.sort_by_distance(
closest, node_id)[:kademlia.k_bucket_size]
else:
closest = kademlia.sort_by_distance(closest, node_id)
nodes_to_ask = _exclude_if_asked(closest)
self.logger.info("lookup finished for {}: {} nodes discovered".format(
node_id, len(closest)))
return closest
async def crawl(self):
"""
Continue to look up random nodes until no new nodes are discovered.
"""
def _extract_new_nodes(
nodes: List[kademlia.Node]) -> List[kademlia.Node]:
"""
Returns a list of nodes that we havne't previously encountered
"""
return [node.id for node in nodes if node.id not in known_nodes]
known_nodes = [] # Will be a list of node ids
# Populate our known nodes with nodes near us
new_nodes = await self.lookup(self.this_node.id, False)
print("Size of new nodes pre strip: " +
str(asizeof.asizeof(new_nodes)))
new_nodes = _extract_new_nodes(
new_nodes) # Reduced to list of node ids
print("Size of new nodes post strip: " +
str(asizeof.asizeof(new_nodes)))
self.myTree.add_list(new_nodes)
print("Size of myTree: " + str(asizeof.asizeof(self.myTree)))
known_nodes += new_nodes
while new_nodes is not []:
#summary.print_(summary.summarize(muppy.get_objects()))
#self.tracker.create_snapshot()
#self.tracker.stats.print_summary()
#print("Size of known nodes: " + str(asizeof.asizeof(known_nodes)))
self.logger.debug(
"In crawl loop... {} new nodes discovered...".format(
len(new_nodes)))
self.logger.info("{} total nodes discovered...".format(
len(known_nodes)))
dump_new_nodes(known_nodes)
#summary.print_(summary.summarize(muppy.get_objects()))
try:
new_nodes = await self.lookup(self.myTree.path_to_stump(),
False)
new_nodes = _extract_new_nodes(new_nodes)
self.myTree.add_list(new_nodes)
known_nodes += new_nodes
except evm.p2p.kademlia.AlreadyWaiting as e:
self.logger.error(e)
self.logger.info("Crawled newtork... {} nodes found...".format(
len(known_nodes)))
return known_nodes
async def targeted_crawl(self,
remote: kademlia.Node) -> List[kademlia.Node]:
def print_neighbours_list(low, searched, high):
def _get_neighbour(result, index):
if index >= len(result["neighbours"]):
return ""
else:
node = list(result["neighbours"])[index]
#return str(node) + " (" + str(hex(_xor_distance(node.id, remote.id))) + ")"
return str(node)
result_lengths = [
len(x["neighbours"]) for x in [low, searched, high]
]
print(header_string("Search Result"))
print("Low" + " " * 29 + "\t Searched " + " " * 29 + "\t High")
for index in range(0, max(result_lengths) - 1):
low_n = _get_neighbour(low, index)
searched_n = _get_neighbour(searched, index)
high_n = _get_neighbour(high, index)
print(low_n + "\t" + searched_n + "\t" + high_n)
print()
async def _find_node_at_distance(distance: int):
"""
Sends a find_node request to the remote node at a given distance from
the node we're targeting. Returns all of the nodes given in the response.
"""
node_id = _node_at_distance(remote, distance)
self.logger.debug("Searching for node: %d", node_id)
self.wire.send_find_node(remote, node_id)
candidates = await self.wait_neighbours(remote)
if len(candidates) == 0:
self.logger.warning(
"got no candidates from {}, returning".format(remote))
return candidates
# candidates = [c for c in candidates if c not in nodes_seen]
# self.logger.debug("got {} new candidates".format(len(candidates)))
# # Add new candidates to nodes_seen so that we don't attempt to bond with failing ones
# # in the future.
# nodes_seen.update(candidates)
# bonded = await asyncio.gather(*[self.bond(c) for c in candidates])
# self.logger.debug("bonded with {} candidates".format(bonded.count(True)))
# return [c for c in candidates if bonded[candidates.index(c)]]
#def recurse_step(interest_low, interest_high):
async def recurse_step(interest_low: 'DistanceResult',
interest_high: 'DistanceResult'):
"""
{
"distance": int,
"neighbours": Set([kademlia.Node])
}
:d_start: The closest distance to the remote node
:interest_high: The furthest distance from the remote node
"""
d_interest = int(
(interest_low["distance"] + interest_high["distance"]) // 2)
self.logger.info(
"Performing Recursive Search...\n" +
"Max Distance: {}\n".format(hex(interest_high["distance"])) +
"Mid Distance: {}\n".format(hex(d_interest)) +
"Min Distance: {}".format(hex(interest_low["distance"])))
if (interest_high["distance"] == d_interest):
self.logger.warning("Error... high = mid\n")
print_neighbours_list(interest_low, interest_high,
interest_high)
_compare_nodes_at_distance(remote, interest_low["distance"],
interest_high["distance"])
return interest_high["neighbours"]
elif (interest_low["distance"] == d_interest):
self.logger.warning("Error... low = mid\n")
print_neighbours_list(interest_low, interest_low,
interest_high)
_compare_nodes_at_distance(remote, interest_low["distance"],
interest_high["distance"])
return interest_low["neighbours"]
interest_result = make_distanceResult(
d_interest, await _find_node_at_distance(d_interest))
print_neighbours_list(interest_low, interest_result, interest_high)
disjoint_up = interest_result["neighbours"].isdisjoint(
interest_high["neighbours"])
disjoint_down = interest_result["neighbours"].isdisjoint(
interest_low["neighbours"])
if not disjoint_up and not disjoint_down:
# This is a BASE CASE
# We overlap completely with both bounds. return this iteraiton.
self.logger.info("Hit base case!\n")
return interest_result["neighbours"]
elif disjoint_up and not disjoint_down:
# Overlapped bottom, but not top: check top
# Check [d_interst, interest_high]
self.logger.info(
"Overlapped bottom, but not top - Search up!\n")
interest_result["neighbours"].update(\
await recurse_step(interest_result, interest_high))
return interest_result["neighbours"]
elif not disjoint_up and disjoint_down:
# Overlapped top, but not bottom: check bottom
# Check [interest_low, interest_result]
self.logger.info(
"Overlapped top, but not bottom - Search down!\n")
interest_result["neighbours"].update(\
await recurse_step(interest_low, interest_result))
return interest_result["neighbours"]
else:
# Overlapped neither. search both.
#recurse_tasks = [
# recurse_step(interest_low, interest_result),\
# recurse_step(interest_result, interest_high)]
#responses = await asyncio.gather(*recurse_tasks)
self.logger.info("No Overalp: search both\n")
responses = await recurse_step(interest_low, interest_result)
responses.update(await recurse_step(interest_result,
interest_high))
interest_result["neighbours"].update(responses)
return interest_result["neighbours"]
furthest_node = make_distanceResult(
_max_distance_to_node(), await
_find_node_at_distance(_max_distance_to_node()))
closest_node = make_distanceResult(0, await _find_node_at_distance(0))
try:
addr_book = await recurse_step(closest_node, furthest_node)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
traceback.print_tb(exc_tb)
print(e)
finally:
addr_book = set()
return addr_book
class LogTestCase(unittest.TestCase):
def setUp(self):
self.out = StringIO()
self.tracker = ClassTracker(stream=self.out)
@property
def output(self):
"""Return output recorded in `ClassTracker` output stream."""
return self.out.getvalue()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_dump(self):
"""Test serialization of log data.
"""
foo = Foo()
foo.data = range(1000)
bar = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.track_object(bar)
self.tracker.create_snapshot('Footest')
f1 = StringIO()
f2 = StringIO()
ConsoleStats(tracker=self.tracker, stream=f1).print_stats()
tmp = BytesIO()
Stats(tracker=self.tracker).dump_stats(tmp, close=False)
self.tracker.clear()
stats = ConsoleStats(stream=f2)
self.assertEqual(stats.index, None)
self.assertEqual(stats.snapshots, None)
tmp.seek(0)
stats.load_stats(tmp)
tmp.close()
self.assert_('Foo' in stats.index)
stats.print_stats()
self.assertEqual(f1.getvalue(), f2.getvalue())
# Test partial printing
stats.stream = f3 = StringIO()
stats.sort_stats()
tolen = len(stats.sorted)
stats.print_stats(clsname='Bar')
self.assertEqual(len(stats.sorted), tolen)
stats.print_summary()
clsname = f3.getvalue().split('\n')[0]
self.assertNotEqual(re.search('\.Bar', clsname), None, clsname)
f1.close()
f2.close()
f3.close()
def test_sort_stats(self):
"""Test sort_stats and reverse_order.
"""
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
foo.data = list(range(1000))
bar1 = Bar()
bar2 = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.create_snapshot()
stats = self.tracker.stats
# Test sort_stats and reverse_order
self.assertEqual(stats.sort_stats('size'), stats)
self.assertEqual(stats.sorted[0].classname, 'Foo')
stats.reverse_order()
self.assertEqual(stats.sorted[0].classname, 'Bar')
stats.sort_stats('classname', 'birth')
self.assertEqual(stats.sorted[0].classname, 'Bar')
self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn')
stats.sort_stats('classname')
def test_dump_load_with_filename(self):
"""Test serialization with filename.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=2)
self.tracker.create_snapshot()
fhandle, fname = mkstemp(prefix='pympler_test_dump')
os.close(fhandle)
try:
self.tracker.stats.dump_stats(fname)
output = StringIO()
stats = ConsoleStats(filename=fname, stream=output)
stats.print_stats()
self.assertTrue('<Foo>' in output.getvalue(), output.getvalue())
# Check if a Stats loaded from a dump can be dumped again
stats.dump_stats(fname)
finally:
os.unlink(fname)
def test_tracked_classes(self):
"""Test listing tracked classes.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
self.tracker.create_snapshot()
bar = Bar()
self.tracker.create_snapshot()
foo = FooNew()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
stats = self.tracker.stats
self.assertEqual(stats.tracked_classes, ['Bar', 'Foo', 'FooNew'])
stats.print_summary()
def test_print_stats(self):
"""Test printing class-filtered statistics.
"""
self.tracker.track_class(Foo, name='Foo', trace=True)
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
bar = Bar()
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(clsname='Foo')
self.assertTrue('Foo' in self.output, self.output)
self.assertFalse('Bar' in self.output, self.output)
self.assertTrue('foo = Foo()' in self.output, self.output)
def test_print_stats_limit(self):
"""Test printing limited statistics.
"""
self.tracker.track_class(Foo, name='Foo')
foo = [Foo() for _ in range(10)]
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(limit=3)
self.assertEqual(self.output.count('<Foo>'), 3)
self.out.seek(0)
self.out.truncate()
stats.print_stats(limit=0.5)
self.assertEqual(self.output.count('<Foo>'), 5)
def test_snapshots(self):
"""Test multiple snapshots.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
self.tracker.track_class(FooNew, name='FooNew')
self.tracker.create_snapshot()
f1 = Foo()
self.tracker.create_snapshot()
f2 = Foo()
f3 = FooNew()
self.tracker.create_snapshot()
b = Bar()
del b
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats()
stats.print_summary()
def test_merge(self):
"""Test merging of reference trees.
"""
self.tracker.track_class(FooNew, name='Foo', resolution_level=2)
f1 = FooNew()
f1.a = list(range(1000))
f2 = FooNew()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
sizer = Asizer()
sz1 = sizer.asized(f1)
sz2 = sizer.asized(f2)
stats = self.tracker.stats
for fp in stats.snapshots:
if fp.desc == 'Merge test':
stats.annotate_snapshot(fp)
self.assert_(hasattr(fp, 'classes'))
classes = fp.classes
stats.annotate_snapshot(fp)
self.assertEqual(fp.classes, classes)
self.assert_('Foo' in fp.classes, fp.classes)
self.assert_('merged' in fp.classes['Foo'])
fm = fp.classes['Foo']['merged']
self.assertEqual(fm.size, sz1.size + sz2.size,
(fm.size, str(sz1), str(sz2)))
refs = {}
for ref in fm.refs:
refs[ref.name] = ref
self.assert_('__dict__' in refs.keys(), refs.keys())
refs2 = {}
for ref in refs['__dict__'].refs:
refs2[ref.name] = ref
self.assert_('[V] a' in refs2.keys(), refs2.keys())
self.assert_('[V] b' in refs2.keys(), refs2.keys())
self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a))
def test_html(self):
"""Test emitting HTML statistics."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
self.tracker.track_class(Bar, name='Bar', trace=True)
f1 = Foo()
f1.a = list(range(100000))
f2 = Foo()
f2.a = list(range(1000))
f2.b = 'This is some stupid spam.'
f1 = Bar()
self.tracker.create_snapshot('Merge test')
stats = HtmlStats(tracker=self.tracker)
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'footest.html')
stats.create_html(output)
source = open(output).read()
# Ensure relative links are used
fname = os.path.join('footest_files', 'Foo.html')
self.assertTrue('<a href="%s">' % fname in source, (fname, source))
finally:
rmtree(target)
def test_charts(self):
"""Test emitting graphic charts."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
f1 = Foo()
f1.a = list(range(1000))
f2 = Foo()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
from pympler import charts
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'timespace.png')
charts.tracker_timespace(output, self.tracker.stats)
finally:
rmtree(target)
class GlobalContainer(box.BoxLayout):
graph_list = prop.ListProperty([])
active_graph = prop.ObjectProperty(None)
name_manager = NameManager.Instance()
show_sidepanel = prop.BooleanProperty(True)
modestr = prop.StringProperty('insert')
itemstr = prop.StringProperty('atom')
modes = {'insert': asp.Mode.INSERT,
'select': asp.Mode.SELECT}
items = {'atom': asp.Item.ATOM,
'ellipse': asp.Item.ELLIPSE,
'rectangle': asp.Item.SQUARE}
def __init__(self, **kwargs):
super(GlobalContainer, self).__init__(**kwargs)
self._keyboard = None
self.request_keyboard()
self.working_dir = './'
self.tutorial = None
self.popup_stack = []
window.Window.bind(on_resize=self.on_resize)
if DEBUG:
self.tracker = ClassTracker()
self.tracker.track_object(MenuButton)
self.all_objects = muppy.get_objects()
def request_keyboard(self):
self._keyboard = window.Window.request_keyboard(self._keyboard_release,
self)
self._keyboard.bind(on_key_down=self._on_keyboard_down)
def _keyboard_catch(self):
self._keyboard.bind(on_key_down=self._on_keyboard_down)
def _keyboard_release(self):
self._keyboard.unbind(on_key_down=self._on_keyboard_down)
#self._keyboard = None
def _on_keyboard_down(self, keyboard, keycode, text, modifiers):
if keycode[1] == 'escape':
base.stopTouchApp()
elif keycode[1] == 'd':
self.set_mode('insert')
elif keycode[1] == 's':
self.set_mode('select')
elif keycode[1] == 'w':
self.set_item('atom')
elif keycode[1] == 'e':
self.set_item('ellipse')
elif keycode[1] == 'r':
self.set_item('rectangle')
elif keycode[1] == 'p':
self.active_graph.show_tree(0)
elif keycode[1] == 'o':
self.view_symbolic_formula()
elif keycode[1] == 'n':
self.show_gringo_query()
elif keycode[1] == 'g':
self.show_save()
elif keycode[1] == 'l':
self.show_load()
elif keycode[1] == 'y':
print self.name_manager.get_all()
print asp.Line.get_all_lines()
elif keycode[1] == 't':
self.toggle_sidepanel()
elif keycode[1] == 'tab':
if not self.show_sidepanel:
self.toggle_sidepanel()
self.ids.atom_input.focus = True
elif keycode[1] == '.':
if DEBUG:
rb = refbrowser.InteractiveBrowser(self)
rb.main()
# self.all_objects = muppy.get_objects()
# sum1 = summary.summarize(self.all_objects)
# summary.print_(sum1)
# self.tracker.create_snapshot()
# self.tracker.stats.print_summary()
return True
def _focus_name_list(self, value):
if value and (asp.AtomWidget.active_atom is not None):
previous_name = asp.AtomWidget.active_atom.name
for button in self.ids.name_list.children:
if (button.text == previous_name) and (button.state != 'down'):
button.trigger_action()
elif value:
if self.ids.name_list.children:
self.ids.name_list.children[-1].trigger_action()
else:
for button in self.ids.name_list.children:
if button.state == 'down':
button.trigger_action()
def on_resize(self, window, width, height):
if self.show_sidepanel:
self.ids.sidepanel.width = self.width * .15
self.active_graph.size = self.ids.stencilview.size
def toggle_sidepanel(self):
self.show_sidepanel = not self.show_sidepanel
if self.show_sidepanel:
width = self.width * .15
else:
width = 0
anim.Animation(width=width, d=.15, t='out_quart').start(
self.ids.sidepanel)
if not self.show_sidepanel:
self.ids.sidepanel.focus = False
# Also release keyboard
#self.update_sourcecode()
def update_atom_editor(self, name):
editor = self.ids.atom_editor
if name == '':
editor.disabled = True
else:
if editor.disabled:
editor.disabled = False
atom = self.name_manager.get(name)
editor.update(atom)
def update_atom(self, name, new_name='', new_hook_points=[], is_constant=False):
atom = self.name_manager.get(name)
if new_name <> '':
atom.name = new_name
if len(new_hook_points) == 4:
atom.hook_points = new_hook_points
# print id(atom.hook_points)
atom.is_constant = is_constant
def register_atom(self, name,
hooks=[False, False, False, False], is_constant=False):
if name == '':
return
children = self.ids.name_list.children
new_button = None
name_to_insert = name
i = len(children) - 1
# If the name doesn't exist, register atom
if self.name_manager.get(name) is None:
self.name_manager.register(name, asp.Atom(name, hooks, is_constant))
# print id(self.name_manager.get(name).hook_points)
# Insert in name_list sorted by name
while i >= 0:
if children[i].text < name_to_insert:
#print children[i].text, '<', name_to_insert
pass
elif children[i].text == name_to_insert:
# Already exists
if children[i].state != 'down':
children[i].trigger_action()
return
elif children[i].text > name_to_insert:
#print children[i].text, '>', name_to_insert
temp = children[i].text
children[i].text = name_to_insert
name_to_insert = temp
if new_button == None:
new_button = children[i]
i -= 1
self.ids.name_list.add_widget(AtomSelectionButton(text=name_to_insert))
if new_button == None:
new_button = children[0]
if new_button.state == 'down':
asp.AtomWidget.active_atom = self.name_manager.get(name)
self.update_atom_editor(name)
self.set_item('atom')
else:
new_button.trigger_action()
def rename_atom(self, new_name):
old_name = ''
for button in self.ids.name_list.children:
if button.state == 'down':
old_name = button.text
selected_button = button
if (old_name != '') and (new_name != ''):
try:
atom = self.name_manager.get(old_name)
exists = self.name_manager.get(new_name)
assert atom is not None
assert exists is None
except AssertionError:
#self.show_error('Name already exists.')
print 'Name already exists.'
return
selected_button.text = new_name
atom.name = new_name
self.name_manager.unregister(old_name)
self.name_manager.register(new_name, atom)
self.update_atom_editor(new_name)
def delete_atom(self):
for button in self.ids.name_list.children:
if button.state == 'down':
self.name_manager.unregister(button.text)
self.active_graph.delete_atom(button.text)
asp.AtomWidget.active_atom = None
self.ids.name_list.remove_widget(button)
self.update_atom_editor('')
def clear_atoms(self):
self.ids.name_list.clear_widgets()
self.name_manager.clear()
self.update_atom_editor('')
asp.AtomWidget.active_atom = None
def new_graph(self):
# TODO: Migrate to tab system
asp.Line.clear_lines()
self.clear_atoms()
if self.active_graph is None:
g = asp.RootWidget()
self.graph_list.append(g)
self.active_graph = g
self.ids.stencilview.add_widget(g)
else:
self.active_graph.delete_tree()
def close_graph(self):
if self.active_graph is not None:
asp.Line.clear_lines()
self.active_graph.delete_tree()
self.active_graph.delete_root()
self.clear_atoms()
self.graph_list.pop()
self.active_graph = None
def set_mode(self, mode):
try:
prev_mode = self.active_graph.mode
new_mode = self.modes[mode]
if new_mode == asp.Mode.INSERT:
self.active_graph.show_hooks()
else:
if prev_mode == asp.Mode.INSERT:
self.active_graph.hide_hooks()
self.active_graph.mode = new_mode
self.modestr = mode
except KeyError as err:
print 'ERROR: Invalid mode {0} requested.'.format(str(err))
def set_item(self, item):
try:
self.active_graph.item = self.items[item]
self.itemstr = item
except KeyError as err:
print 'ERROR: Invalid item {0} requested.'.format(str(err))
except AttributeError:
pass
if item == 'atom':
self._focus_name_list(True)
else:
self._focus_name_list(False)
def push_popup(self, popup):
self.popup_stack.append(popup)
popup.open()
def dismiss_popup(self):
popup = self.popup_stack.pop()
popup.dismiss()
def show_rename_atom(self):
content = TextInputDialog(caption="Enter new name",
validate_callback=self.rename_atom,
cancel=self.dismiss_popup)
p = CustomPopup(self, title="Rename atom", content=content,
size_hint=(0.4, 0.25))
self.push_popup(p)
def show_load(self):
content = LoadDialog(load=self.load, cancel=self.dismiss_popup)
content.ids.filechooser.path = self.working_dir
p = CustomPopup(self, title="Load file", content=content,
size_hint=(0.9, 0.9))
self.push_popup(p)
def show_save(self):
content = SaveDialog(save=self.save, cancel=self.dismiss_popup)
content.ids.filechooser.path = self.working_dir
p = CustomPopup(self, title="Save file", content=content,
size_hint=(0.9, 0.9))
self.push_popup(p)
def show_export(self):
content = ExportDialog(export=self.export, cancel=self.dismiss_popup)
content.ids.filechooser.path = self.working_dir
p = CustomPopup(self, title="Export file", content=content,
size_hint=(0.9, 0.9))
self.push_popup(p)
def show_gringo_query(self):
caption = "Enter desired predicates separated by commas"
content = TextInputDialog(caption=caption,
validate_callback=self.gringo_query,
dismiss_on_validate=False,
focus=False,
cancel=self.dismiss_popup)
p = CustomPopup(self, title="Output predicates", content=content,
size_hint=(0.4, 0.25))
self.push_popup(p)
def show_stable_models(self, solver):
models = solver.get_models()
content = None
if len(models) == 0:
content = ErrorDialog('Unsatisfiable', cancel=self.dismiss_popup)
else:
solver.generate_graph(models[0])
content = StableModelDialog(solver, cancel=self.dismiss_popup)
content.ids.img.reload()
p = CustomPopup(self, catch_keyboard=False, title="Stable Models",
content=content, size_hint=(0.9, 0.9))
self.push_popup(p)
def show_error(self, err_str):
content = ErrorDialog(err_str, cancel=self.dismiss_popup)
p = CustomPopup(self, catch_keyboard=False, title="Error",
content=content, size_hint=(0.4, 0.3))
self.push_popup(p)
def show_about(self):
content = AboutDialog(cancel=self.dismiss_popup)
p = CustomPopup(self, catch_keyboard=False, title="About "+__title__,
content=content, size_hint=(0.5, 0.5))
self.push_popup(p)
def load(self, path, filename):
self.close_graph()
self.working_dir = path
try:
f = os.path.join(path, filename[0])
# Temporal line storage. Its contents are arranged as follows:
# { line_id: (graph, hook_list) , ... }
lines = {}
with open(f, 'r') as stream:
for line in stream:
if line.startswith(NameParser.TOKENS['name']):
name, hooks, is_constant = NameParser.parse_name(line)
self.register_atom(name, hooks, is_constant)
if line.startswith(NameParser.TOKENS['line']):
line_id, graph = NameParser.parse_line(line)
lines[line_id] = (graph, [None] * len(graph))
new_graph = lang.Builder.load_file(f)
self.ids.stencilview.add_widget(new_graph)
self.active_graph = new_graph
#self.graph_list.pop()
self.graph_list.append(new_graph)
for w in self.active_graph.walk(restrict=True):
if isinstance(w, asp.AtomWidget):
w._deferred_init()
for i in w.get_line_info():
line_id = i[0]
hook_index = i[1]
lines[line_id][1][hook_index] = i[2]
elif isinstance(w, asp.NexusWidget):
for i in w.line_info:
line_id = i[0]
hook_index = i[1]
lines[line_id][1][hook_index] = w
for line, info in lines.iteritems():
print line, info
asp.Line.build_from_graph(info[0], info[1])
self.set_mode(self.modestr)
self.set_item(self.itemstr)
self.dismiss_popup()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.dismiss_popup()
self.close_graph()
self.new_graph()
self.show_error('Corrupted file.')
def save(self, path, filename):
self.working_dir = path
with open(os.path.join(path, filename), 'w') as stream:
stream.write('#:kivy 1.0.9\n\n')
for (name, atom) in self.name_manager.get_all():
stream.write(NameParser.get_name_str(name, atom.hook_points,
atom.is_constant))
for line in asp.Line.get_all_lines():
stream.write(NameParser.get_line_str(line.line_id,
line.get_full_graph()))
stream.write('\n')
stream.write(self.active_graph.get_tree(0))
self.dismiss_popup()
def export(self, path, filename):
_, ext = os.path.splitext(filename)
if ext == '.png':
self.active_graph.export_to_png(os.path.join(path, filename))
elif ext == '.lp':
rpn = self.active_graph.get_formula_RPN()
constants = self.active_graph.get_constants()
solver = eg_solver.Solver()
solver.set_formula(rpn, constants)
rules = solver.generate_asp_rules()
with open(os.path.join(path, filename), 'w') as stream:
for r in rules:
stream.write(r)
stream.write('\n')
else:
error_str = 'File extension not supported.'
print error_str
self.show_error(error_str)
return
self.dismiss_popup()
def highlight_variables(self):
self.active_graph.highlight_variables()
def view_symbolic_formula(self):
print self.active_graph.get_formula()
def gringo_query(self, show_predicates):
def generate_show_statements(predicates):
pred_list = predicates.split(',')
pred_list = map(lambda s: s.strip(), pred_list)
n_args = (lambda name:
self.name_manager.get(name).hook_points.count(True))
pred_list = [p + '/' + str(n_args(p)) for p in pred_list]
show_list = ['#show {0}.'.format(p) for p in pred_list]
return show_list
self.dismiss_popup()
rpn = self.active_graph.get_formula_RPN()
rpn = norm.LIT.TRUE if rpn == '' else rpn
constants = self.active_graph.get_constants()
print 80 * '-'
print 'RPN formula:\n', rpn
solver = eg_solver.Solver()
result = ''
try:
show_statements = []
if show_predicates:
try:
show_statements = generate_show_statements(show_predicates)
except Exception:
pass
solver.set_formula(rpn, constants)
result = solver.solve(show=show_statements)
except norm.MalformedFormulaError:
self.show_error('Malformed formula.')
return
except RuntimeError, e:
print e
self.show_error(str(e))
return
self.show_stable_models(solver)
