def test_restrict_on_nested_blocks(self):
b1 = Block('a=0;b=a+1')
b2 = Block('x=99;z=x-1')
composite = Block([b1, b2])
self.assertSimilar(Block('a=0'), composite.restrict(outputs=['a']))
self.assertSimilar(Block('x=99'), composite.restrict(outputs=['x']))
def test_imports(self):
'restrict blocks containing imports'
# Test 'from' syntax
b = Block('from math import sin, pi\n'\
'b=sin(pi/a)\n' \
'd = c * 3.3')
sub_block = b.restrict(inputs=('a'))
self.assertEqual(sub_block.inputs, set(['a']))
self.assertEqual(sub_block.outputs, set(['b']))
self.assertEqual(sub_block.fromimports, set(['pi', 'sin']))
context = {'a': 2, 'c': 0.0}
sub_block.execute(context)
self.assertTrue('b' in context)
self.assertEqual(context['b'], 1.0)
# Test 'import' syntax
b = Block('import math\n'\
'b=math.sin(math.pi/a)\n' \
'd = c * 3.3')
sub_block = b.restrict(inputs=('a'))
self.assertEqual(sub_block.inputs, set(['a']))
self.assertEqual(sub_block.outputs, set(['b']))
self.assertEqual(sub_block.fromimports, set(['math']))
context = {'a': 2, 'c': 0.0}
sub_block.execute(context)
self.assertTrue('b' in context)
self.assertEqual(context['b'], 1.0)
def test_caching_code(self):
"Caching: '_code'"
b, c = Block('a=2'), {}
b._code
b.ast = Block('a=3').ast
b.execute(c)
self.assertEqual(c['a'], 3)
b, c = Block('a=2'), {}
b._code
b.sub_blocks = [Block('a=3')]
b.execute(c)
self.assertEqual(c['a'], 3)
b, c = Block('a=3; a=2'), {}
b._code
b.sub_blocks.pop()
b.execute(c)
self.assertEqual(c['a'], 3)
b, c = Block(''), {}
b._code
b.sub_blocks.append(Block('a=3'))
b.sub_blocks = [Block('a=3')]
b.execute(c)
assert 'a' in c
def test_block_composition(self):
'Composing Blocks'
self.assertSimilar(Block('a; b; c'), Block(('a', 'b', 'c')))
self.assertSimilar(Block('a'), Block(('a')))
self.assertSimilar(Block(''), Block(()))
self.assertSimilar(
Block('if t: a = b\nc = f(a)'), Block(('if t: a = b', 'c = f(a)')))
def test_ast_policy(self):
'Policy: Keep tidy ASTs'
a = Discard(Name('a'))
empty = Stmt([])
self.assertEqual(empty, Block('').ast)
self.assertEqual(empty, Block(empty).ast)
self.assertEqual(empty, Block(Module(None, empty)).ast)
self.assertEqual(a, Block('a').ast)
self.assertEqual(a, Block(a).ast)
self.assertEqual(a, Block(Stmt([a])).ast)
self.assertEqual(a, Block(Module(None, Stmt([a]))).ast)
# Similar, except we don't use strings since Block does its own parsing
b = Block()
b.ast = empty
self.assertEqual(b.ast, empty)
b.ast = Module(None, empty)
self.assertEqual(b.ast, empty)
b.ast = a
self.assertEqual(b.ast, a)
b.ast = Stmt([a])
self.assertEqual(b.ast, a)
b.ast = Module(None, Stmt([a]))
self.assertEqual(b.ast, a)
def __init__(self, **kwtraits):
if 'external_block' in kwtraits:
self.external_block = kwtraits.pop('external_block')
super(FormulaExecutingContext, self).__init__(**kwtraits)
self._regenerate_expression_block()
if self.external_block is None:
self._external_code_changed(self.external_code)
else:
self.external_block = Block(
[self.external_block,
Block(self.external_code)])
self._regenerate_composite_block()
def _regenerate_expression_block(self):
exprs = [
'%s = %s' % (var, expr)
for var, expr in list(self._expressions.items())
]
expression_code = '\n'.join(exprs) + '\n'
self._expression_block = Block(expression_code)
def test_impure_execute():
code = """
import os # module and function names are discarded by default.
def ff():
global y # will not be retained but will be available in the code block.
y = a + x
b.append(4)
x = a
b.append(3)
ff()
z = y
_x = x # names beginning with underscore are discarded by default
a = 99
"""
context = DataContext(subcontext=dict(a=1, b=[2]))
block = Block(code)
# by default, clean shadow after execution:
shadow = block.execute_impure(context)
assert_equal(set(context.keys()), set(['a', 'b'])) # names unchanged
assert_equal(context['b'],
[2, 3, 4]) # mutable object was changed in context
assert_equal(set(shadow.keys()), set(['x', 'z', 'a']))
assert_equal(context['a'], 1) # original mutable object does not change,
assert_equal(shadow['a'], 99) # but the new object is in the shadow dict.
# do not clean shadow after execution:
shadow = block.execute_impure(context, clean_shadow=False)
assert_equal(set(shadow.keys()), set(['x', 'z', 'a', '_x', 'os', 'ff']))
def test_intermediate_inputs(self):
""" restrict blocks with inputs which are intermediates """
block = Block('c = a + b\n'\
'd = c * 3')
sub_block = block.restrict(inputs=('c'))
self.assertEqual(sub_block.inputs, set(['c']))
self.assertEqual(sub_block.outputs, set(['d']))
context = {'a': 1, 'b': 2}
block.execute(context)
self.assertEqual(context['c'], 3)
self.assertEqual(context['d'], 9)
context = {'c': 10}
sub_block.execute(context)
self.assertEqual(context['c'], 10)
self.assertEqual(context['d'], 30)
context = {'d': 15}
sub_block = block.restrict(inputs=('d'))
self.assertEqual(sub_block.inputs, set([]))
self.assertEqual(sub_block.outputs, set([]))
sub_block.execute(context)
self.assertEqual(context['d'], 15)
def test_intermediate_inputs_with_highly_connected_graph(self):
""" restrict blocks with inputs which are intermediates on a highly connected graph"""
code = "c = a + b\n" \
"d = c * 3\n" \
"e = a * c\n" \
"f = d + e\n" \
"g = e + c\n" \
"h = a * 3"
block = Block(code)
sub_block = block.restrict(inputs=('c'))
self.assertEqual(sub_block.inputs, set(['a', 'c']))
self.assertEqual(sub_block.outputs, set(['d', 'e', 'f', 'g']))
context = {'a': 1, 'b': 2}
block.execute(context)
self.assertEqual(context['c'], 3)
self.assertEqual(context['d'], 9)
self.assertEqual(context['e'], 3)
self.assertEqual(context['f'], 12)
self.assertEqual(context['g'], 6)
context = {'a': 1, 'c': 10}
sub_block.execute(context)
self.assertEqual(context['c'], 10)
self.assertEqual(context['d'], 30)
self.assertEqual(context['e'], 10)
self.assertEqual(context['f'], 40)
self.assertEqual(context['g'], 20)
def __getitem__(self, key):
if key in self.underlying_context.keys():
return self.underlying_context[key]
else:
try:
# FIXME imports need to be more configurable
# FIXME we may want to have cache rules on sizes so that we
# don't keep around huge values that aren't needed anymore
# however, there is a time/space tradeoff that really should be
# used. We currently support del so that is available if the user
# wants to manually manage this.
eval_globals = {}
for lib in (__builtins__, __import__('numpy')):
for sym in dir(lib):
if not key.startswith('__'):
eval_globals[sym] = getattr(lib, sym)
result = eval(key, eval_globals, self.underlying_context)
self._expressions[key] = result
for dep in Block(key).inputs:
self._dependencies.setdefault(dep, list())
self._dependencies[dep].append(key)
return result
except:
return None
def test_intermediate_inputs_and_outputs(self):
""" restrict blocks with inputs and outputs which are intermediates """
code = "c = a + b\n" \
"d = c * 3\n" \
"e = a * c\n" \
"f = d + e\n" \
"g = e + c\n" \
"h = a * 3"
block = Block(code)
sub_block = block.restrict(inputs=('c'), outputs=('e', 'g'))
self.assertEqual(sub_block.inputs, set(['a', 'c']))
self.assertEqual(sub_block.outputs, set(['e', 'g']))
context = {'a': 1, 'b': 2}
block.execute(context)
self.assertEqual(context['c'], 3)
self.assertEqual(context['e'], 3)
self.assertEqual(context['g'], 6)
context = {'a': 1, 'c': 10}
sub_block.execute(context)
self.assertEqual(context['c'], 10)
self.assertEqual(context['e'], 10)
self.assertEqual(context['g'], 20)
def test_tracebacks(self):
'Tracebacks have correct file names and line numbers'
# If we want tracebacks to make sense, then the reported file names and
# line numbers need to associate with the code being executed
# regardless which block represents and executes the code.
def test(tb, lineno, filename):
self.assertEqual(tb.tb_lineno, lineno+1)
self.assertEqual(tb.tb_frame.f_code.co_filename, filename)
def tracebacks():
"A list of the current exception's traceback objects."
tb = sys.exc_info()[2]
l = [tb]
while tb.tb_next is not None:
tb = tb.tb_next
l.append(tb)
return l
class File(StringIO, object):
"Extend StringIO with a 'name' attribute."
def __init__(self, name, *args, **kw):
super(File, self).__init__(*args, **kw)
self.name = name
a = Block(File('foo/a.py', 'y = x'))
try:
a.execute({})
except NameError, e:
test(tracebacks()[-1], 1, 'foo/a.py')
def test_block_name_replacer(self):
""" Does BlockNameReplacer work?
"""
code = "x = x(x, x)\nx\n"
desired = "y = y(y, y)\ny\n"
b = Block(code)
rename_variable(b.ast, 'x', 'y')
self.assertEqual(desired, unparse(b.ast))
def test_inputs_are_dependent_outputs(self):
""" restrict blocks with inputs which are intermediates and outputs"""
code = "t2 = b * 2\n" \
"t3 = t2 + 3\n"
block = Block(code)
sub_block = block.restrict(inputs=['t2', 't3'])
def test_dep_graph_exists_for_line_of_code(self):
""" Does block treat 1 func blocks like multi-func blocks.
It doesn't appear that simple blocks are forcing updates
to the dep_graph. One func graphs Should be the same as
multi-line ones (I think). Without this, we have to
always check for None and special case that code path in
all the processing tools.
fixme: I (eric) haven't examined this very deeply, it
just cropped up in some of my code. This test
is a reminder that we need to either fix it or
verify that we don't want to fix it.
"""
block = Block('b = foo(a)\nc=bar(b)\n')
self.assertTrue(block._dep_graph is not None)
block = Block('b = foo(a)\n')
self.assertTrue(block._dep_graph is not None)
def test_import_and_rename(self):
code = "from blockcanvas.debug.my_operator import add as add1\n" \
"a = add1(1,2)\n" \
"b = add1(a,3)"
foo_block = Block(code)
info = find_functions(foo_block.ast)
foo_call = FunctionCall.from_ast(foo_block.sub_blocks[1].ast, info)
desired = 'result = add(1, 2)'
self.assertEqual(foo_call.call_signature, desired)
def test_local_function_preprocessing(self):
code = "def foo(a,b):\n" \
"\tx,y=a,b\n" \
"\treturn x,y\n" \
"i,j = foo(1,2)\n"
foo_block = Block(code)
info = find_functions(foo_block.ast)
assert 'foo' in info
assert info['foo']
def test_function_names(self):
""" Does function_names find all the functions in an AST?
"""
code = "x = func1(func2(1),2)\n" \
"y = func3()\n"
desired = ["func1", "func2", "func3"]
b = Block(code)
actual = sorted(function_names(b))
self.assertEqual(actual, desired)
def test_cyclic_dep_graph(self):
"Regression: Don't make cyclic dep graphs"
# When an input and output name are the same, we used to represent them
# identically in the dep graph, making it cyclic (and wrong anyway)
try:
# (';' appends an empty 'Discard' statement to the block, which
# gives it multiple sub-blocks, which forces 'restrict' to run
# non-trivially)
Block('a = a;').restrict(inputs=['a'])
except graph.CyclicGraph:
self.fail()
def __delitem__(self, key):
"""Delete an item from the ExpressionContext -- either in the underlying context,
or if an already cached expression, delete it."""
# if item is an expression, delete it from the list of dependencies, otherwise pass it down
# to underlying
if key in self._expressions:
self._expressions.remove(key)
for dep in list(Block(key).inputs):
self._dependencies[dep].remove(key)
else:
del self.underlying_context[key]
def test_restrict_outputs():
"""Test a basic use of the restrict(outputs=(...)) method."""
code = 'x = a + b\ny = b - c\nz = c**2'
b = Block(code)
br = b.restrict(outputs=('z', ))
names = dict(c=5)
br.execute(names)
assert_equal(sorted(names), ['c', 'z'])
assert_equal(names['c'], 5)
assert_equal(names['z'], 25)
def test_local_def(self):
code = "def foo(a):\n" \
" b = a\n" \
" return b\n" \
"y = foo(2)\n"
foo_block = Block(code)
info = find_functions(foo_block.ast)
foo_call = FunctionCall.from_ast(foo_block.sub_blocks[-1].ast, info)
desired = 'b = foo(2)'
self.assertEqual(foo_call.call_signature, desired)
def _base(self, code, inputs, outputs, *results):
# Convert results to a string if necessary
def try_join(x):
if not isinstance(x, basestring):
return '\n'.join(x)
else:
return x
results = map(try_join, results)
# Avoid empty discard statements at the end of 'results'
results = [ re.sub(';*$', '', r) for r in results ]
# Make sure code's sub-block is one of the given results (restrict
# isn't deterministic on parallelizable code)
restricted = Block(code).restrict(inputs=inputs, outputs=outputs)
if results == ['']:
self.assertSimilar(restricted, Block(()))
else:
self.assertTrue(restricted.ast in [Block(r).ast for r in results])
def _base(self, code, expected):
context = default_context()
Block(code).execute(context)
for k in expected:
self.assert_(k in context)
if isinstance(expected[k], ndarray) or \
isinstance(context[k], ndarray):
self.assert_(all(expected[k] == context[k]),
'expected = %s, dict(context) = %s' % \
(expected, dict(context)))
else:
self.assertEqual(expected[k], context[k])
def test_basic_02():
"""Another test of the basic use of a Block."""
code = 'y = x + 1'
b = Block(code)
assert_equal(b.inputs, set(['x']))
assert_equal(b.outputs, set(['y']))
names = dict(x=100)
b.execute(names)
assert_equal(sorted(names), ['x', 'y'])
assert_equal(names['x'], 100)
assert_equal(names['y'], 101)
def test_errors(self):
'Errors for block restriction'
# Note: 'a' can be passed as a input, which should return
# a trivial subblock. This is due to supporting intermediate
# inputs
b = Block('a = b')
self.assertRaises(ValueError, b.restrict, outputs='b')
self.assertRaises(ValueError, b.restrict, inputs='z')
self.assertRaises(ValueError, b.restrict, outputs='z')
self.assertRaises(ValueError, b.restrict)
def test_bound_inputs(self):
code = "c = a * b\n" \
"x = 3 + c\n"
block = Block(code)
context = DataContext()
context['a'] = 1
context['b'] = 2
sub_block = block.restrict(inputs=('c'))
self.assertEqual(sub_block.inputs, set(['c']))
def setUp(self):
code = "from blockcanvas.debug.my_operator import add, mul\n" \
"c = add(a,b)\n" \
"d = mul(c, 2)\n" \
"e = mul(c, 3)\n" \
"f = add(d,e)"
self.block = Block(code)
# Context setup.
self.context = MultiContext(DataContext(name='Data'), {})
self.context['a'] = 1
self.context['b'] = 2
def test_pickle(self):
'Pickling'
strings = [
''
'a=b',
'a=b;c=d',
'a=f(z); b=g(y); c=h(a,b); d=k(b)',
]
for s in strings:
b = Block(s)
self.assertEqual(loads(dumps(b)), b, '(where s = %r)' % s)
