def test_impure_execute():
raise SkipTest
code = """
import os # module and function names are discarded by default.
def ff():
global y # will not be retained but will be available in the code block.
y = a + x
b.append(4)
x = a
b.append(3)
ff()
z = y
_x = x # names beginning with underscore are discarded by default
a = 99
"""
context = DataContext(subcontext=dict(a=1, b=[2]))
block = Block(code)
# by default, clean shadow after execution:
shadow = block.execute_impure(context)
assert_equal(set(context.keys()), set(['a', 'b'])) # names unchanged
assert_equal(context['b'],
[2, 3, 4]) # mutable object was changed in context
assert_equal(set(shadow.keys()), set(['x', 'z', 'a']))
assert_equal(context['a'], 1) # original mutable object does not change,
assert_equal(shadow['a'], 99) # but the new object is in the shadow dict.
# do not clean shadow after execution:
shadow = block.execute_impure(context, clean_shadow=False)
assert_equal(set(shadow.keys()), set(['x', 'z', 'a', '_x', 'os', 'ff']))
def test_impure_execute():
code="""
import os # module and function names are discarded by default.
def ff():
global y # will not be retained but will be available in the code block.
y = a + x
b.append(4)
x = a
b.append(3)
ff()
z = y
_x = x # names beginning with underscore are discarded by default
a = 99
"""
context = DataContext(subcontext=dict(a=1,b=[2]))
block = Block(code)
# by default, clean shadow after execution:
shadow = block.execute_impure(context)
assert_equal(set(context.keys()), set(['a', 'b'])) # names unchanged
assert_equal(context['b'], [2,3,4]) # mutable object was changed in context
assert_equal(set(shadow.keys()), set(['x', 'z', 'a']))
assert_equal(context['a'], 1) # original mutable object does not change,
assert_equal(shadow['a'], 99) # but the new object is in the shadow dict.
# do not clean shadow after execution:
shadow = block.execute_impure(context, clean_shadow=False)
assert_equal(set(shadow.keys()), set(['x', 'z', 'a', '_x', 'os', 'ff']))
def test_load_and_resave_project(self):
""" Does load and resaving a project work correctly ?
"""
project_name = create_unique_project_name(self.dir_name,
'test_load_save_project')
project_dir = os.path.join(self.dir_name, project_name+'_files')
project_file = os.path.join(self.dir_name, project_name+'.prj')
script_path = os.path.join(project_dir, project_name+'.py')
context_path = os.path.join(project_dir, project_name+'.pickle')
layout_path = os.path.join(project_dir, 'layout.pickle')
script = '\n'.join(('SCRIPT_PATH = '+script_path,
'CONTEXT_PATH = '+context_path,
'LAYOUT_PATH = '+layout_path))
os.makedirs(project_dir)
file_object = open(project_file, 'wb')
file_object.write(script)
file_object.close()
file_object = open(script_path, 'w')
file_object.write(self.code)
file_object.close()
d = DataContext(name = 'dummy_context',
_bindings = {'a': 1, 'b':2, 'e': 0.01})
d.save_context_to_file(context_path)
# Check if the setup is correct
self.assertTrue([os.path.exists(project_file),
os.path.exists(project_dir),
os.path.exists(script_path),
os.path.exists(context_path)])
# Load the project from the file.
self.app.block_unit = BlockUnit()
self.app.load_project_from_file(project_file)
self.app.block_unit.codeblock.code = self.app.block_unit.codeblock.code + '\nd = add(a,c)'
self.app.save_loaded_project()
# The key 'context' should get removed while saving the project.
expected_keys = set(['a', 'b', 'c', 'd', 'e'])
self.assertEqual(set(self.app.block_unit.data_context.keys()),
expected_keys)
self.assertEqual(self.app.block_unit.data_context['d'], 4)
# Check if the new code contains the new line
file_object = open(script_path, 'r')
code = file_object.read()
a,b = 1,2
exec code
file_object.close()
self.assertEqual(d, 4)
# Clean up after loading
os.remove(project_file)
os.remove(script_path)
os.remove(context_path)
os.remove(layout_path)
os.rmdir(project_dir)
def new_main():
from numpy import linspace
from codetools.contexts.api import DataContext
from blockcanvas.block_display.block_unit_variables import BlockUnitVariableList
from codetools.contexts.api import DataContext
from blockcanvas.app.experiment import Experiment
code = "y = a*x*x + b*x + c"
context = DataContext(name='Data')
exp = Experiment(code=code, shared_context=context)
context.update(dict(a=1.0, b=1.0, c=0.0))
context['x'] = linspace(-5., 5., 60)
# FIXME: Shouldn't have to manually call this now
exp.context.execute_for_names(["a", "b", "c"])
block = exp.exec_model.block
# Use the experiment's local context as the shadow context
exp.context.shadows = [exp._local_context]
vars = BlockUnitVariableList(block=block, context=exp.context)
config_interactor = InteractorConfig(vars=vars.variables)
interactor = ConfigurableInteractor(context=exp.context,
block=block,
interactor_config=config_interactor)
interactor.configure_traits()
def new_main():
from numpy import linspace
from codetools.contexts.api import DataContext
from blockcanvas.block_display.block_unit_variables import BlockUnitVariableList
from codetools.contexts.api import DataContext
from blockcanvas.app.experiment import Experiment
code = "y = a*x*x + b*x + c"
context = DataContext(name="Data")
exp = Experiment(code=code, shared_context=context)
context.update(dict(a=1.0, b=1.0, c=0.0))
context["x"] = linspace(-5.0, 5.0, 60)
# FIXME: Shouldn't have to manually call this now
exp.context.execute_for_names(["a", "b", "c"])
block = exp.exec_model.block
# Use the experiment's local context as the shadow context
exp.context.shadows = [exp._local_context]
vars = BlockUnitVariableList(block=block, context=exp.context)
config_interactor = InteractorConfig(vars=vars.variables)
interactor = ConfigurableInteractor(context=exp.context, block=block, interactor_config=config_interactor)
interactor.configure_traits()
def test_save_new_project(self):
""" Does saving a new project to file work correctly ?
"""
context = DataContext(name='dummy_context', _bindings={'a': 1, 'b': 2})
self.app.block_unit = BlockUnit(code=self.code, data_context=context)
project_name = create_unique_project_name(self.dir_name,
'test_save_project')
project_file = os.path.join(self.dir_name, project_name + '.prj')
self.app.save_project_to_file(project_file)
# Check if the new files exist
project_dir = os.path.join(self.dir_name, project_name + '_files')
context_file = os.path.join(project_dir, project_name + '.pickle')
script_file = os.path.join(project_dir, project_name + '.py')
layout_file = os.path.join(project_dir, 'layout.pickle')
# print project_file, project_dir, context_file, script_file
self.assertTrue([
os.path.exists(project_file),
os.path.exists(project_dir),
os.path.exists(context_file),
os.path.exists(script_file)
])
# Check if the code was saved out correctly
file_object = open(script_file, 'r')
check_code = file_object.read()
file_object.close()
a, b = 1, 2
exec check_code
self.assertTrue(c, 3)
# Check if the project file was written out correctly
file_object = open(project_file, 'rb')
check_code = file_object.read()
file_object.close()
actual_result = [
line[line.find('=') + 1:].strip()
for line in check_code.split('\n')
]
self.assertTrue(
actual_result == [script_file, context_file, layout_file])
# Check if the context is correct
retrieved_context = DataContext.load_context_from_file(context_file)
expected_keys = set(['a', 'b', 'c'])
self.assertEqual(retrieved_context.name, 'dummy_context')
self.assertEqual(set(retrieved_context.keys()), expected_keys)
self.assertEqual(retrieved_context['c'], 3)
# Cleanup
os.remove(project_file)
os.remove(script_file)
os.remove(context_file)
os.remove(layout_file)
os.rmdir(project_dir)
def test_load_project_from_file(self):
""" Does loading a project from file work correctly ?
"""
# Preparing the test case
# Make project.py, project.pickle, project.prj
project_name = create_unique_project_name(self.dir_name,
'test_load_project')
project_dir = os.path.join(self.dir_name, project_name + '_files')
project_file = os.path.join(self.dir_name, project_name + '.prj')
script_path = os.path.join(project_dir, project_name + '.py')
context_path = os.path.join(project_dir, project_name + '.pickle')
script = '\n'.join(
('SCRIPT_PATH = ' + script_path, 'CONTEXT_PATH = ' + context_path))
os.makedirs(project_dir)
file_object = open(project_file, 'wb')
file_object.write(script)
file_object.close()
file_object = open(script_path, 'w')
file_object.write(self.code)
file_object.close()
d = DataContext(name='dummy_context',
_bindings={
'a': 1,
'b': 2,
'd': 0.01
})
d.save_context_to_file(context_path)
# Check if the setup is correct
self.assertTrue([
os.path.exists(project_file),
os.path.exists(project_dir),
os.path.exists(script_path),
os.path.exists(context_path)
])
# Load the project from the file.
self.app.block_unit = BlockUnit()
self.app.load_project_from_file(project_file)
# Clean up after loading
os.remove(project_file)
os.remove(script_path)
os.remove(context_path)
os.rmdir(project_dir)
expected_keys = set(['a', 'b', 'c', 'd', 'context'])
self.assertEqual(set(self.app.block_unit.data_context.keys()),
expected_keys)
self.assertTrue(self.app.block_unit.data_context.has_key('c'))
self.assertEqual(self.app.block_unit.data_context['c'], 3)
def test_load_project_from_file(self):
""" Does loading a project from file work correctly ?
"""
# Preparing the test case
# Make project.py, project.pickle, project.prj
project_name = create_unique_project_name(self.dir_name,
'test_load_project')
project_dir = os.path.join(self.dir_name, project_name+'_files')
project_file = os.path.join(self.dir_name, project_name+'.prj')
script_path = os.path.join(project_dir, project_name+'.py')
context_path = os.path.join(project_dir, project_name+'.pickle')
script = '\n'.join(('SCRIPT_PATH = '+script_path,
'CONTEXT_PATH = '+context_path))
os.makedirs(project_dir)
file_object = open(project_file, 'wb')
file_object.write(script)
file_object.close()
file_object = open(script_path, 'w')
file_object.write(self.code)
file_object.close()
d = DataContext(name = 'dummy_context',
_bindings = {'a': 1, 'b':2, 'd': 0.01})
d.save_context_to_file(context_path)
# Check if the setup is correct
self.assertTrue([os.path.exists(project_file),
os.path.exists(project_dir),
os.path.exists(script_path),
os.path.exists(context_path)])
# Load the project from the file.
self.app.block_unit = BlockUnit()
self.app.load_project_from_file(project_file)
# Clean up after loading
os.remove(project_file)
os.remove(script_path)
os.remove(context_path)
os.rmdir(project_dir)
expected_keys = set(['a', 'b', 'c', 'd', 'context'])
self.assertEqual(set(self.app.block_unit.data_context.keys()),
expected_keys)
self.assertTrue(self.app.block_unit.data_context.has_key('c'))
self.assertEqual(self.app.block_unit.data_context['c'], 3)
def __init__(self, code=None, shared_context=None, *args, **kwargs):
super(Experiment, self).__init__(*args, **kwargs)
if code is None:
self.exec_model = ExecutionModel()
else:
self.exec_model = ExecutionModel.from_code(code)
self.controller = BlockGraphController(execution_model=self.exec_model)
self.canvas = BlockCanvas(graph_controller=self.controller)
self.controller.canvas = self.canvas
self._shared_context = shared_context
self._local_context = DataContext(name=self._LOCAL_NAME_TEMPLATE())
self._update_exec_context()
def create_simple_experiment():
""" Returns a simple experiment with a basic context in it """
ctx = DataContext(name="main")
ctx["b"] = 3
ctx["c"] = 5
return Experiment(code=SAMPLE_CODE_1, shared_context=ctx)
def test_eval(self):
"""Test most straightforward evaluation functionality"""
d = DataContext()
d['a'] = 10
d['b'] = 20
ec = ExpressionContext(d)
self.assertEqual(200, ec['a*b'])
def setUp(self):
self.events = []
self.exceptions = []
d = DataContext()
d['a'] = 1
d['b'] = 2
self.ec = AsyncExecutingContext(subcontext=d, executable=ce)
def test_simple_output_reduction(self):
code = 'from blockcanvas.debug.my_operator import add, mul\n' \
'c = add(a,b)\n' \
'd = mul(a,16)\n'
block = Block(code)
context = DataContext()
context['a'] = 1
context['b'] = 2
sub_block = block.restrict(outputs=('d'))
self.assertEqual(sub_block.inputs, set(['a']))
sub_block.execute(context)
self.assertTrue(context.has_key('d'))
self.assertEqual(context['d'], 16)
def test_simple_output_reduction(self):
code = 'from blockcanvas.debug.my_operator import add, mul\n' \
'c = add(a,b)\n' \
'd = mul(a,16)\n'
block = Block(code)
context = DataContext()
context['a'] = 1
context['b'] = 2
sub_block = block.restrict(outputs=('d'))
self.assertEqual(sub_block.inputs, set(['a']))
sub_block.execute(context)
self.assertTrue(context.has_key('d'))
self.assertEqual(context['d'], 16)
def setUp(self):
code = "from blockcanvas.debug.my_operator import add\n"\
"c = add(a,b)"
self.block = Block(code)
# Context setup
self.context = MultiContext(DataContext(name='Data'), {})
self.context['a'] = 1
self.context['b'] = 2
def context_factory(self):
""" Return the type of context we are testing.
"""
data_context = DataContext()
context = AdaptedDataContext(context=data_context)
name = self.key_name()
getitem_units = {name: meters / second}
adapter = UnitConversionAdapter(getitem_units=getitem_units)
context.push_adapter(adapter)
return context
def test_save_new_project(self):
""" Does saving a new project to file work correctly ?
"""
context = DataContext(name = 'dummy_context',
_bindings = {'a':1, 'b':2})
self.app.block_unit = BlockUnit(code=self.code, data_context = context)
project_name = create_unique_project_name(self.dir_name,
'test_save_project')
project_file = os.path.join(self.dir_name,project_name+'.prj')
self.app.save_project_to_file(project_file)
# Check if the new files exist
project_dir = os.path.join(self.dir_name,project_name+'_files')
context_file = os.path.join(project_dir, project_name+'.pickle')
script_file = os.path.join(project_dir, project_name+'.py')
layout_file = os.path.join(project_dir, 'layout.pickle')
# print project_file, project_dir, context_file, script_file
self.assertTrue([os.path.exists(project_file),
os.path.exists(project_dir),
os.path.exists(context_file),
os.path.exists(script_file)])
# Check if the code was saved out correctly
file_object = open(script_file, 'r')
check_code = file_object.read()
file_object.close()
a, b = 1,2
exec check_code
self.assertTrue(c, 3)
# Check if the project file was written out correctly
file_object = open(project_file, 'rb')
check_code = file_object.read()
file_object.close()
actual_result = [line[line.find('=')+1:].strip() for line in
check_code.split('\n')]
self.assertTrue(actual_result==[script_file, context_file, layout_file])
# Check if the context is correct
retrieved_context = DataContext.load_context_from_file(context_file)
expected_keys = set(['a', 'b', 'c'])
self.assertEqual(retrieved_context.name, 'dummy_context')
self.assertEqual(set(retrieved_context.keys()), expected_keys)
self.assertEqual(retrieved_context['c'], 3)
# Cleanup
os.remove(project_file)
os.remove(script_file)
os.remove(context_file)
os.remove(layout_file)
os.rmdir(project_dir)
def test_use_as_context(self):
# Can we use the context as the namespace of a python calculation?
from codetools.contexts.api import DataContext, MultiContext
with tables.open_file('test.h5') as table:
hdf_context = Hdf5Context(file_object=table,
path=['root', 'root.group1', 'root.group2'])
results = DataContext()
context = MultiContext(results, hdf_context)
exec_('array2_plus_one = [a2 + 1 for a2 in array2]', {}, context)
assert_equal(context['array2_plus_one'], [2, 3, 4, 5])
def test_exception(self):
ce = CodeExecutable(code="c = a + b")
d = DataContext()
d['a'] = 1
d['b'] = 2
ec = AsyncExecutingContext(subcontext=d, executable=ce)
ec.on_trait_change(self._handle_exceptions, 'exception')
ec.code = '1/0'
ec._wait()
self.assertEqual(len(self.exceptions), 1)
self.assertEqual(ZeroDivisionError, type(self.exceptions[0]))
def test_events(self):
self.last_event = None
self.event_count = 0
d = DataContext()
d['a'] = 10
d['b'] = 20
ec = ExpressionContext(d)
ec['a*b']
ec.on_trait_change(self._event_handler, 'items_modified')
ec['a'] = 30
assert 'a' in self.last_event.modified
assert 'a*b' in self.last_event.modified
def gen_context():
""" Create a test context.
"""
dc = DataContext()
dc['foo_int'] = 1
dc['foo_float'] = 1.5
dc['foo_array'] = np.arange(10)
dc['foo_str'] = 'a string'
dc['bar_int'] = 2
dc['bar_float'] = 2.5
return dc
def test_bound_inputs(self):
code = "c = a * b\n" \
"x = 3 + c\n"
block = Block(code)
context = DataContext()
context['a'] = 1
context['b'] = 2
sub_block = block.restrict(inputs=('c'))
self.assertEqual(sub_block.inputs, set(['c']))
def test_unbound_inputs(self):
code = 'from blockcanvas.debug.my_operator import add, mul\n' \
"c = add(a,b)\n" \
"d = mul(c,a)\n" \
"z = add(b, 2)"
block = Block(code)
context = DataContext()
context['a'] = 2
context['b'] = 1
sub_block = block.restrict(inputs=('a'))
self.assertEqual(sub_block.inputs, set(['a', 'b']))
self.assertEqual(sub_block.outputs, set(['c', 'd', 'add', 'mul']))
sub_block.execute(context)
self.assertTrue(context.has_key('c'))
self.assertEqual(context['c'], 3)
self.assertTrue(context.has_key('d'))
self.assertEqual(context['d'], 6)
def replace_context(self, parameter_s=''):
"""Replace the IPython namespace with a DataContext.
"""
ipshell = self.shell
if hasattr(ipshell.user_ns, 'subcontext'):
# Toggle back to plain dict.
user_ns = ipshell.user_ns.subcontext
else:
from codetools.contexts.api import DataContext
user_ns = DataContext(subcontext=ipshell.user_ns)
ipshell.user_ns = user_ns
def test_unbound_inputs(self):
code = 'from blockcanvas.debug.my_operator import add, mul\n' \
"c = add(a,b)\n" \
"d = mul(c,a)\n" \
"z = add(b, 2)"
block = Block(code)
context = DataContext()
context['a'] = 2
context['b'] = 1
sub_block = block.restrict(inputs=('a'))
self.assertEqual(sub_block.inputs, set(['a', 'b']))
self.assertEqual(sub_block.outputs, set(['c', 'd', 'add', 'mul']))
sub_block.execute(context)
self.assertTrue(context.has_key('c'))
self.assertEqual(context['c'], 3)
self.assertTrue(context.has_key('d'))
self.assertEqual(context['d'], 6)
def create_multi_experiment_proj():
""" Returns a project with two contexts and two experiments.
"""
ctx = DataContext(name="main")
ctx["a"] = 5
ctx["b"] = 7
ctx["c"] = 11
ctx2 = DataContext(name="secondary")
ctx2["a"] = 4
ctx2["b"] = 16
ctx2["c"] = 32
ctx2["m"] = 64
ctx2["n"] = 128
proj = Project(contexts=[ctx, ctx2])
e1 = Experiment(code=SAMPLE_CODE_1, shared_context=ctx)
e2 = Experiment(code=SAMPLE_CODE_2, shared_context=ctx)
proj.experiments = [e1, e2]
return proj
def new_main():
from codetools.contexts.api import DataContext
from blockcanvas.app.experiment import Experiment
code2 = "from blockcanvas.debug.my_operator import add, mul\n" \
"c = mul(a,b)\n" \
"d = mul(c, 2)\n" \
"e = mul(c, 3)\n" \
"f = add(d,e)"
context = DataContext(name='data')
context.update(dict(a=2, b=3))
exp = Experiment(code=code2, shared_context=context)
interactor = ShadowInteractor(inputs=exp.exec_model.block.inputs, context=exp.context)
interactor.configure_traits()
from pprint import pprint
print "***** Experiment executing context *****"
pprint(exp.context.items())
print "\n***** Shared context *****"
pprint(context.items())
def test_execute(self):
d = DataContext()
d['a'] = 1
d['b'] = 3
d['g'] = 5
rce = RestrictingCodeExecutable(code=CODE)
ec = AsyncExecutingContext(subcontext=d, executable=rce)
ec.on_trait_change(self._items_modified_fired, 'items_modified')
ec.execute()
ec._wait()
self.assertEqual(self.events[0].added, ['c', 'd', 'f'])
self.assertEqual(len(self.events), 1)
def setUp(self):
unittest.TestCase.setUp(self)
# Create the contexts
self.context = AdaptedDataContext(subcontext=DataContext())
self.raw_context = self.context.subcontext
# Add data (before creating the adapter)
self.context.update(fun=1, bar=2, baz=3, not_mapped=4)
# Add an adapter
self.adapter = NameAdapter(map={"foo": "fun", "bar": "baz"})
self.context.push_adapter(self.adapter)
def create_simple_project():
""" Returns a simple project with a single experiment and a single context """
ctx = DataContext(name="main")
ctx["a"] = 5
ctx["b"] = 7
ctx["c"] = 11
proj = Project(contexts=[ctx])
exp = Experiment(code=SAMPLE_CODE_1, shared_context=ctx)
proj.add_experiment(exp)
return proj
def setUp(self):
code = "from blockcanvas.debug.my_operator import add, mul\n" \
"c = add(a,b)\n" \
"d = mul(c, 2)\n" \
"e = mul(c, 3)\n" \
"f = add(d,e)"
self.block = Block(code)
# Context setup.
self.context = MultiContext(DataContext(name='Data'), {})
self.context['a'] = 1
self.context['b'] = 2
def test_load_block_from_file(self):
""" Does loading script from file work correctly ?
"""
# Preparing the test case
file_path = os.path.join(self.dir_name, 'test_load_block.py')
file_object = open(file_path, 'w')
file_object.write(self.code)
file_object.close()
# Load the script from the file.
self.app.block_unit = BlockUnit()
self.app.load_block_from_file(file_path)
# Cleanup after loading the file
os.remove(file_path)
# Test the block unit
context = DataContext(name='dummy_context', _bindings={'a': 1, 'b': 2})
self.app.block_unit.codeblock.execute(context)
self.assertTrue(context.has_key('c'))
self.assertEqual(context['c'], 3)
def reset(self):
""" Reset the Application to its initial state.
Clean all old attributes and instanciate new ones.
"""
for context in self.project.contexts:
self.project.remove_context(context)
data_context = DataContext(name='data')
self.project.add_context(data_context)
exp = Experiment(shared_context=data_context)
self.project.active_experiment = exp
self.context_viewer = ContextVariableList(context=exp.context)
def test_execute_for_names(self):
"""`execute_for_names()` is part of the ExecutingContext interface """
d = DataContext()
d['a'] = 1
d['b'] = 3
d['g'] = 5
rce = RestrictingCodeExecutable(code=CODE)
ec = AsyncExecutingContext(subcontext=d, executable=rce)
ec.on_trait_change(self._items_modified_fired, 'items_modified')
ec.execute_for_names(['g'])
ec._wait()
self.assertEqual(self.events[0].added, ['f'])
def load(self, dirname):
""" Loads the project from the given directory. The existing state
of the project is completely modified.
"""
# TODO: We should formalize this dependency at some point and move
# this import to the top level
from configobj import ConfigObj
if dirname == "":
raise IOError("Cannot load project from empty path.")
elif not os.path.isdir(dirname):
raise IOError("Cannot find directory: " + dirname)
filename = abspath(join(dirname, self.PROJECT_FILE_NAME))
if not os.path.isfile(filename):
raise IOError('Cannot load %s from project directory "%s".' % \
(self.PROJECT_FILE_NAME, dirname))
# Every project is saved as a text ConfigObj file that describes
# the name and location of the real data files.
configspec = ConfigObj(self.CONFIG_SPEC, list_values=False)
config = ConfigObj(filename, configspec=configspec)
contexts = []
for context_config in config["Contexts"].values():
ctx = DataContext.load(join(dirname, context_config["file"]))
ctx.name = context_config["name"]
contexts.append(ctx)
self.contexts = contexts
experiments = []
if hasattr(scripting, "app"):
app = scripting.app
else:
app = None
for exp_config in config["Experiments"].values():
experiment = Experiment()
exp_dir = join(dirname, exp_config["save_dir"])
experiment.load_from_config(exp_config, exp_dir, project=self)
experiments.append(experiment)
self.experiments = experiments
proj_config = config["Project"]
if proj_config.has_key("active_experiment"):
self.active_experiment = self.find_experiment(proj_config["active_experiment"])
# Update Project Save Path
self.project_save_path = dirname
return
def test_defer_execution():
""" Does deferring execution work?
"""
d = DataContext()
ec = ExecutingContext(subcontext=d, executable=ce)
ec.defer_execution = True
ec['a'] = 1
assert 'c' not in ec
ec['b'] = 2
assert 'c' not in ec
ec.defer_execution = False
assert 'c' in ec
assert ec['c'] == 3
def new_main():
from codetools.contexts.api import DataContext
from blockcanvas.app.experiment import Experiment
code2 = "from blockcanvas.debug.my_operator import add, mul\n" \
"c = mul(a,b)\n" \
"d = mul(c, 2)\n" \
"e = mul(c, 3)\n" \
"f = add(d,e)"
context = DataContext(name='data')
context.update(dict(a=2, b=3))
exp = Experiment(code=code2, shared_context=context)
interactor = ShadowInteractor(inputs=exp.exec_model.block.inputs,
context=exp.context)
interactor.configure_traits()
from pprint import pprint
print "***** Experiment executing context *****"
pprint(exp.context.items())
print "\n***** Shared context *****"
pprint(context.items())
def load_from_config(self, config, dirname, project=None):
""" Loads the experiment. The existing state of the experiment is
completely modified.
Parameters
----------
config: a dict-like object
The keys should correspond to the configspec for experiments as
defined in project_config_spec.
dirname: a string
the absolute path to the subdirectory of the project that
holds the experiment's saved data
project: Project instance
If provided, the project is used to hook up references to
the shared context.
"""
join = os.path.join
self.name = config.get("name", "")
if "code_file" in config:
# Clear out the old references
self.canvas = self.controller = self.exec_model = None
self.exec_model = ExecutionModel.from_file(
join(dirname, config["code_file"]))
self.controller = BlockGraphController(
execution_model=self.exec_model)
self.canvas = BlockCanvas(graph_controller=self.controller)
self.controller.canvas = self.canvas
if "layout_file" in config:
self.canvas.load_layout(join(dirname, config["layout_file"]))
if "local_context" in config:
self._local_context = DataContext.load(
join(dirname, config["local_context"]))
shared_context = None
if project is not None:
name = config.get("shared_context")
if name != "":
shared_context = project.find_context(name)
self._shared_context = shared_context
self._update_exec_context()
def __init__(self, code=None, shared_context=None, *args, **kwargs):
super(Experiment, self).__init__(*args, **kwargs)
if code is None:
self.exec_model = ExecutionModel()
else:
self.exec_model = ExecutionModel.from_code(code)
self.controller = BlockGraphController(execution_model = self.exec_model)
self.canvas = BlockCanvas(graph_controller=self.controller)
self.controller.canvas = self.canvas
self._shared_context = shared_context
self._local_context = DataContext(name = self._LOCAL_NAME_TEMPLATE())
self._update_exec_context()
def test_load_block_from_file(self):
""" Does loading script from file work correctly ?
"""
# Preparing the test case
file_path = os.path.join(self.dir_name, 'test_load_block.py')
file_object = open(file_path, 'w')
file_object.write(self.code)
file_object.close()
# Load the script from the file.
self.app.block_unit = BlockUnit()
self.app.load_block_from_file(file_path)
# Cleanup after loading the file
os.remove(file_path)
# Test the block unit
context = DataContext(name = 'dummy_context',
_bindings = {'a':1, 'b':2})
self.app.block_unit.codeblock.execute(context)
self.assertTrue(context.has_key('c'))
self.assertEqual(context['c'], 3)
def load_from_config(self, config, dirname, project=None):
""" Loads the experiment. The existing state of the experiment is
completely modified.
Parameters
----------
config: a dict-like object
The keys should correspond to the configspec for experiments as
defined in project_config_spec.
dirname: a string
the absolute path to the subdirectory of the project that
holds the experiment's saved data
project: Project instance
If provided, the project is used to hook up references to
the shared context.
"""
join = os.path.join
self.name = config.get("name", "")
if "code_file" in config:
# Clear out the old references
self.canvas = self.controller = self.exec_model = None
self.exec_model = ExecutionModel.from_file(join(dirname, config["code_file"]))
self.controller = BlockGraphController(execution_model = self.exec_model)
self.canvas = BlockCanvas(graph_controller=self.controller)
self.controller.canvas = self.canvas
if "layout_file" in config:
self.canvas.load_layout(join(dirname, config["layout_file"]))
if "local_context" in config:
self._local_context = DataContext.load(join(dirname, config["local_context"]))
shared_context = None
if project is not None:
name = config.get("shared_context")
if name != "":
shared_context = project.find_context(name)
self._shared_context = shared_context
self._update_exec_context()
class Experiment(HasTraits):
"""
An Experiment represents a particular workflow and its associated data.
It is associated with a particular context, but any data it produces
is placed into a shadowing "local" context.
It encapsulates an execution model, its associated canvas, and the
local context.
"""
# The name of the experiment.
name = String()
# The Execution Model object
exec_model = Instance(ExecutionModel)
# The Block Canvas
canvas = Instance(BlockCanvas)
# The controller between the canvas and execution model
controller = Instance(BlockGraphController)
# The execution context
context = Instance(ExecutingContext)
# A reference to a context that might be shared with other experiments.
# This is usually a context from the project's list of contexts. If None,
# then all of the data in the experiment is kept "locally".
#
# Note: For most purposes, you should use the **context** attribute
# and not this one, unless you really know what you are doing.
shared_context = Property(Instance(IListenableContext))
#---------------------------------------------------------------------
# Private Traits
#---------------------------------------------------------------------
# A shadow trait for **shared_context** property
_shared_context = Instance(IListenableContext, adapt='yes',
rich_compare=False)
# The context in which all of our execution model's generated values are
# stored. This context is exposed as part of self.context. This remains
# constant even as the _shared_context gets changed.
_local_context = Instance(IListenableContext, rich_compare=False)
# Class-level generator for the name of the local context; takes
# **self** as an argument.
_LOCAL_NAME_TEMPLATE = lambda x: "%s_local_%d" % (x.name, id(x))
#---------------------------------------------------------------------
# Public methods
#---------------------------------------------------------------------
def __init__(self, code=None, shared_context=None, *args, **kwargs):
super(Experiment, self).__init__(*args, **kwargs)
if code is None:
self.exec_model = ExecutionModel()
else:
self.exec_model = ExecutionModel.from_code(code)
self.controller = BlockGraphController(execution_model = self.exec_model)
self.canvas = BlockCanvas(graph_controller=self.controller)
self.controller.canvas = self.canvas
self._shared_context = shared_context
self._local_context = DataContext(name = self._LOCAL_NAME_TEMPLATE())
self._update_exec_context()
#---------------------------------------------------------------------
# Persistence
#---------------------------------------------------------------------
def load_code_from_file(self,filename):
# Clear out the old references
self.canvas = self.controller = self.exec_model = None
self.exec_model = ExecutionModel.from_file(filename)
self.controller = BlockGraphController(execution_model = self.exec_model)
self.canvas = BlockCanvas(graph_controller=self.controller)
self.controller.canvas = self.canvas
def load_from_config(self, config, dirname, project=None):
""" Loads the experiment. The existing state of the experiment is
completely modified.
Parameters
----------
config: a dict-like object
The keys should correspond to the configspec for experiments as
defined in project_config_spec.
dirname: a string
the absolute path to the subdirectory of the project that
holds the experiment's saved data
project: Project instance
If provided, the project is used to hook up references to
the shared context.
"""
join = os.path.join
self.name = config.get("name", "")
if "code_file" in config:
# Clear out the old references
self.canvas = self.controller = self.exec_model = None
self.exec_model = ExecutionModel.from_file(join(dirname, config["code_file"]))
self.controller = BlockGraphController(execution_model = self.exec_model)
self.canvas = BlockCanvas(graph_controller=self.controller)
self.controller.canvas = self.canvas
if "layout_file" in config:
self.canvas.load_layout(join(dirname, config["layout_file"]))
if "local_context" in config:
self._local_context = DataContext.load(join(dirname, config["local_context"]))
shared_context = None
if project is not None:
name = config.get("shared_context")
if name != "":
shared_context = project.find_context(name)
self._shared_context = shared_context
self._update_exec_context()
def save(self, basename, dirname):
""" Saves the experiment into a directory.
Parameters
----------
basename: string
the name of the project directory
dirname: string
the name of the experiment's subdirectory
Returns
-------
A dict representing the saved state for this object. All path
references in the dict will be relative to the given root directory.
"""
# Make the directory using the absolute path.
fullpath = join(basename, dirname)
if not os.path.isdir(fullpath):
os.mkdir(fullpath)
config = {}
config["name"] = self.name
config["save_dir"] = dirname
# Save out the canvas
config["layout_file"] = "layout.dat"
self.canvas.save_layout(join(fullpath, "layout.dat"))
# Save the execution model's code
config["code_file"] = "code.txt"
f = file(join(fullpath, "code.txt"), "w")
try:
f.write(self.exec_model.code)
finally:
f.close()
# Clean context
self.exec_model._clean_old_results_from_context(self.context.subcontext)
# Save the local context
config["local_context"] = "local_context.pickle"
self._local_context.save(join(fullpath, "local_context.pickle"))
# Store the name of the shared context
if self._shared_context is not None:
config["shared_context"] = self._shared_context.name
return config
def save_script(self,filename):
""" Save the execution model code as script. """
f = file(filename, "w")
try:
f.write(self.exec_model.code)
finally:
f.close()
def export_as_function(self,filename,func_name, mode="w"):
""" Export this experiment as a function for batch application.
It puts together the model (i.e. code) and the parameters
(i.e. context) to create a self-contained function to be used
in batch application.
"""
imports_and_locals = self.exec_model.imports_and_locals
sorted_statements = self.exec_model.sorted_statements
context = self.context
# Clean context
self.exec_model._clean_old_results_from_context(context.subcontext)
export_as_function(filename, func_name, \
imports_and_locals, sorted_statements, context, \
reserved_inputs=[], \
reserved_outputs=[], \
mode=mode)
def export_as_script(self,filename, mode="w"):
""" Export this experiment as a script for batch application.
It puts together the model (i.e. code) and the parameters
(i.e. context) to create a self-contained script to be used
in batch application.
"""
imports_and_locals = self.exec_model.imports_and_locals
sorted_statements = self.exec_model.sorted_statements
context = self.context
# Clean context
self.exec_model._clean_old_results_from_context(context.subcontext)
export_as_script(filename, \
imports_and_locals, sorted_statements, context, \
reserved_inputs=[], \
reserved_outputs=[], \
mode=mode)
#---------------------------------------------------------------------
# Trait Event Handlers
#---------------------------------------------------------------------
@on_trait_change('exec_model')
def _exec_model_changed(self, name, old, new):
""" Propagate the change to the objects under this one.
"""
if self.controller is not None:
self.controller.execution_model = new
if self.context is not None:
self.context.executable = new
#---------------------------------------------------------------------
# Property getters/setters
#---------------------------------------------------------------------
def _get_shared_context(self):
return self._shared_context
def _set_shared_context(self, newcontext):
subcontexts = self.context.subcontext.subcontexts
if self._shared_context is not None:
assert(self._shared_context in subcontexts)
if newcontext is None:
subcontexts.remove(self._shared_context)
self._shared_context = None
else:
# Swap out the new context for the old, in-place
ndx = subcontexts.index(self._shared_context)
subcontexts[ndx] = newcontext
self._shared_context = newcontext
elif newcontext is not None:
self._shared_context = newcontext
subcontexts.append(newcontext)
return
def _update_exec_context(self):
mc = MultiContext(
# Put the function filter in front so we don't dirty up the data
# context with function objects.
FunctionFilterContext(name='functions'),
self._local_context,
name='multi',
)
if self._shared_context is not None:
mc.subcontexts.append(self._shared_context)
self.context = ExecutingContext(
executable=self.exec_model,
subcontext=mc,
name='exec',
)
