def test_optimize_graph_false():
from dask.callbacks import Callback
d = {'x': 1, 'y': (inc, 'x'), 'z': (add, 10, 'y')}
keys = []
with Callback(pretask=lambda key, *args: keys.append(key)):
get(d, 'z', optimize_graph=False)
assert len(keys) == 2
def test_errors_propagate():
dsk = {'x': (bad,)}
try:
get(dsk, 'x')
except Exception as e:
assert isinstance(e, ValueError)
assert "12345" in str(e)
def test_unpicklable_results_generate_errors():
dsk = {'x': (make_bad_result,)}
try:
get(dsk, 'x')
except Exception as e:
# can't use type because pickle / cPickle distinction
assert type(e).__name__ in ('PicklingError', 'AttributeError')
def test_custom_context_ignored_elsewhere():
""" On Python 2/Windows, setting 'multiprocessing.context' doesn't explode.
Presumption is it's not used since unsupported, but mostly we care about
not breaking anything.
"""
assert get({'x': (inc, 1)}, 'x') == 2
with pytest.warns(UserWarning):
with dask.config.set({"multiprocessing.context": "forkserver"}):
assert get({'x': (inc, 1)}, 'x') == 2
def test_errors_propagate():
dsk = {"x": (bad,)}
try:
result = get(dsk, "x")
except Exception as e:
assert isinstance(e, ValueError)
assert "12345" in str(e)
def test_unpicklable_results_genreate_errors():
dsk = {"x": (make_bad_result,)}
try:
result = get(dsk, "x")
except Exception as e:
# can't use type because pickle / cPickle distinction
assert type(e).__name__ == "PicklingError"
def test_unpicklable_args_generate_errors():
a = NotUnpickleable()
def foo(a):
return 1
dsk = {'x': (foo, a)}
try:
get(dsk, 'x')
except Exception as e:
assert isinstance(e, ValueError)
dsk = {'x': (foo, 'a'),
'a': a}
try:
get(dsk, 'x')
except Exception as e:
assert isinstance(e, ValueError)
def test_custom_context_used_python3_posix():
""" The 'multiprocessing.context' config is used to create the pool.
We assume default is 'fork', and therefore test for 'spawn'. If default
context is changed this test will need to be modified to be different than
that.
"""
# We check for spawn by ensuring subprocess doesn't have modules only
# parent process should have:
def check_for_pytest():
import sys
return "FAKE_MODULE_FOR_TEST" in sys.modules
import sys
sys.modules["FAKE_MODULE_FOR_TEST"] = 1
try:
with dask.config.set({"multiprocessing.context": "spawn"}):
result = get({"x": (check_for_pytest,)}, "x")
assert not result
finally:
del sys.modules["FAKE_MODULE_FOR_TEST"]
def test_dumps_loads():
with set_options(func_dumps=pickle.dumps, func_loads=pickle.loads):
assert get({'x': 1, 'y': (add, 'x', 2)}, 'y') == 3
def test_lambda_results_with_cloudpickle():
dsk = {"x": (lambda_result, )}
f = get(dsk, "x")
assert f(2) == 3
def test_lambda_results_without_cloudpickle():
dsk = {"x": (lambda_result, )}
with pytest.raises(ModuleNotFoundError) as e:
get(dsk, "x")
assert "cloudpickle" in str(e.value)
def test_reuse_pool():
with multiprocessing.Pool() as pool:
with dask.config.set(pool=pool):
assert get({"x": (inc, 1)}, "x") == 2
assert get({"x": (inc, 1)}, "x") == 2
def test_lambda_with_cloudpickle():
dsk = {"x": 2, "y": (lambda x: x + 1, "x")}
assert get(dsk, "y") == 3
def test_reuse_pool():
pool = multiprocessing.Pool()
with set_options(pool=pool):
assert get({'x': (inc, 1)}, 'x') == 2
assert get({'x': (inc, 1)}, 'x') == 2
def test_fuse_doesnt_clobber_intermediates():
d = {"x": 1, "y": (inc, "x"), "z": (add, 10, "y")}
assert get(d, ["y", "z"]) == (2, 12)
def test_reuse_pool():
pool = multiprocessing.Pool()
with set_options(pool=pool):
assert get({'x': (inc, 1)}, 'x') == 2
assert get({'x': (inc, 1)}, 'x') == 2
def load(filename):
pass
'''TODOD: simple gui viewer for selecting files '''
...
def clean(data):
'''measure intensity time series for a time interval sample from tiffstack. write subroutines using import jax.'''
...
def analyze(sequence_of_data):
'''compute cross correlation functions of time series values along with measures of causality that could be scale invarient.'''
...
def store(result, print_file_name, initializeQ = False):
'''write results to a print_file_name.'''
with open(..., 'w') as f:
f.write(result)
dsk = {'load-1': (load, 'myfile.a.data'),
'load-2': (load, 'myfile.b.data'),
'load-3': (load, 'myfile.c.data'),
'clean-1': (clean, 'load-1'),
'clean-2': (clean, 'load-2'),
'clean-3': (clean, 'load-3'),
'analyze': (analyze, ['clean-%d' % i for i in [1, 2, 3]]),
'store': (store, 'analyze')}
#TODO: what's ^this?
from dask.multiprocessing import get
get(dsk, 'store') # executes in parallel
def test_errors_propagate():
dsk = {"x": (bad, )}
with pytest.raises(ValueError) as e:
get(dsk, "x")
assert "12345" in str(e.value)
def test_fuse_doesnt_clobber_intermediates():
d = {"x": 1, "y": (inc, "x"), "z": (add, 10, "y")}
assert get(d, ["y", "z"]) == (2, 12)
def test_dumps_loads():
with dask.config.set(func_dumps=pickle.dumps, func_loads=pickle.loads):
assert get({"x": 1, "y": (add, "x", 2)}, "y") == 3
def test_dumps_loads():
with set_options(func_dumps=pickle.dumps, func_loads=pickle.loads):
assert get({"x": 1, "y": (add, "x", 2)}, "y") == 3
def test_reuse_pool():
pool = multiprocessing.Pool()
with dask.config.set(pool=pool):
assert get({'x': (inc, 1)}, 'x') == 2
assert get({'x': (inc, 1)}, 'x') == 2
def test_fuse_doesnt_clobber_intermediates():
d = {'x': 1, 'y': (inc, 'x'), 'z': (add, 10, 'y')}
assert get(d, ['y', 'z']) == (2, 12)
def test_dumps_loads():
with dask.config.set(func_dumps=pickle.dumps, func_loads=pickle.loads):
assert get({'x': 1, 'y': (add, 'x', 2)}, 'y') == 3
def test_dumps_loads():
with set_options(func_dumps=pickle.dumps, func_loads=pickle.loads):
assert get({'x': 1, 'y': (add, 'x', 2)}, 'y') == 3
def test_fuse_doesnt_clobber_intermediates():
d = {'x': 1, 'y': (inc, 'x'), 'z': (add, 10, 'y')}
assert get(d, ['y', 'z']) == (2, 12)