def test_fuse_keys():
assert (fuse({
'a': 1,
'b': (inc, 'a'),
'c': (inc, 'b')
}, keys=['b']) == {
'b': (inc, 1),
'c': (inc, 'b')
})
assert fuse(
{
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
},
keys=['x', 'z']) == {
'w': (inc, 'x'),
'x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
def fuse_dag(dag, copy=False):
if copy:
dag = deepcopy(dag)
dag = dict(dag)
dsk, dep = cull(dag, dag.keys())
dsk, dep = fuse(dsk, ave_width=1, rename_keys=False)
return dsk
def test_fuse_keys():
assert (fuse({'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}, keys=['b'])
== {'b': (inc, 1), 'c': (inc, 'b')})
assert fuse({
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}, keys=['x', 'z']) == {
'w': (inc, 'x'),
'x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
def fuse2(*args, **kwargs):
"""Run both ``fuse`` and ``fuse_linear`` and compare results"""
rv1 = fuse_linear(*args, **kwargs)
if kwargs.get('rename_keys') is not False:
return rv1
rv2 = fuse(*args, **kwargs)
assert rv1 == rv2
return rv1
def fuse2(*args, **kwargs):
"""Run both ``fuse`` and ``fuse_linear`` and compare #1lab_results"""
rv1 = fuse_linear(*args, **kwargs)
if kwargs.get('rename_keys') is not False:
return rv1
rv2 = fuse(*args, **kwargs)
assert rv1 == rv2
return rv1
def test_fuse_keys():
fuse = fuse2 # tests both `fuse` and `fuse_linear`
d = {
'a': 1,
'b': (inc, 'a'),
'c': (inc, 'b'),
}
keys = ['b']
assert fuse(d, keys, rename_keys=False) == with_deps({
'b': (inc, 1),
'c': (inc, 'b'),
})
assert fuse(d, keys, rename_keys=True) == with_deps({
'a-b': (inc, 1),
'c': (inc, 'a-b'),
'b': 'a-b',
})
d = {
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
keys = ['x', 'z']
assert fuse(d, keys, rename_keys=False) == with_deps({
'w': (inc, 'x'),
'x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
})
assert fuse(d, keys, rename_keys=True) == with_deps({
'w': (inc, 'y-x'),
'y-x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a':
1,
'b':
2,
'x':
'y-x',
})
def optimize(dsk, keys, **kwargs):
flatkeys = list(flatten(keys)) if isinstance(keys, list) else [keys]
dsk, dependencies = cull(dsk, flatkeys)
dsk, dependencies = fuse(dsk,
keys,
dependencies=dependencies,
ave_width=_globals.get('fuse_ave_width', 1))
dsk, _ = cull(dsk, keys)
return dsk
def test_optimize_slicing():
dsk = {'a': (range, 10),
'b': (getarray, 'a', (slice(None, None, None),)),
'c': (getarray, 'b', (slice(None, None, None),)),
'd': (getarray, 'c', (slice(0, 5, None),)),
'e': (getarray, 'd', (slice(None, None, None),))}
expected = {'e': (getarray, (range, 10), (slice(0, 5, None),))}
result = optimize_slices(fuse(dsk, [])[0])
assert result == expected
# protect output keys
expected = {'c': (getarray, (range, 10), (slice(0, None, None),)),
'd': (getarray, 'c', (slice(0, 5, None),)),
'e': (getarray, 'd', (slice(None, None, None),))}
result = optimize_slices(fuse(dsk, ['c', 'd', 'e'])[0])
assert result == expected
def test_optimize_slicing():
dsk = {'a': (range, 10),
'b': (getter, 'a', (slice(None, None, None),)),
'c': (getter, 'b', (slice(None, None, None),)),
'd': (getter, 'c', (slice(0, 5, None),)),
'e': (getter, 'd', (slice(None, None, None),))}
expected = {'e': (getter, (range, 10), (slice(0, 5, None),))}
result = optimize_slices(fuse(dsk, [], rename_keys=False)[0])
assert result == expected
# protect output keys
expected = {'c': (getter, (range, 10), (slice(0, None, None),)),
'd': (getter, 'c', (slice(0, 5, None),)),
'e': (getter, 'd', (slice(None, None, None),))}
result = optimize_slices(fuse(dsk, ['c', 'd', 'e'], rename_keys=False)[0])
assert result == expected
def test_fuse_keys():
assert (fuse({'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}, keys=['b']) ==
({'b': (inc, 1), 'c': (inc, 'b')}, {'b': set(), 'c': set(['b'])}))
dsk, dependencies = fuse({
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}, keys=['x', 'z'])
assert dsk == {'w': (inc, 'x'),
'x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2 }
assert dependencies == {'a': set(), 'b': set(), 'z': set(['a', 'b']),
'x': set(['z']), 'w': set(['x'])}
def test_fuse_keys():
fuse = fuse2 # tests both `fuse` and `fuse_linear`
d = {
'a': 1,
'b': (inc, 'a'),
'c': (inc, 'b'),
}
keys = ['b']
assert fuse(d, keys, rename_keys=False) == with_deps({
'b': (inc, 1),
'c': (inc, 'b'),
})
assert fuse(d, keys, rename_keys=True) == with_deps({
'a-b': (inc, 1),
'c': (inc, 'a-b'),
'b': 'a-b',
})
d = {
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
keys = ['x', 'z']
assert fuse(d, keys, rename_keys=False) == with_deps({
'w': (inc, 'x'),
'x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2 ,
})
assert fuse(d, keys, rename_keys=True) == with_deps({
'w': (inc, 'y-x'),
'y-x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2 ,
'x': 'y-x',
})
def get(self,
dsk,
keys,
optimize_graph=True,
docker='lensa/dask.mesos',
params={},
threaded=True,
**kwargs): # should be mode instead: threaded / mesos / hybrid
""" Compute dask graph
Parameters
----------
dsk: dict
keys: object, or nested lists of objects
optimize_graph: bool
docker: string, default docker image for computations on mesos
params: dict, mesos options per dask key
threaded: bool, offload task without mesos parameters to threads
Examples
--------
>>> from operator import add # doctest: +SKIP
>>> e = MesosExecutor('127.0.0.1:8787') # doctest: +SKIP
>>> e.get({'x': (add, 1, 2)}, 'x') # doctest: +SKIP
3
See Also
--------
Executor.compute: Compute asynchronous collections
"""
# Optimize Dask
dsk2, dependencies = cull(dsk, keys)
if optimize_graph:
dsk3, dependencies = fuse(dsk2, keys, dependencies)
else:
dsk3 = dsk2
def apply_async(execute_task, args):
key = args[0]
if threaded and key not in params:
logging.info('Task `{}` is calculating in threads'.format(key))
return self.threadpool.submit(execute_task, *args)
options = params.get(key, {})
options['id'] = key
if 'docker' not in options:
options['docker'] = docker
logging.info('Task `{}` is calculating on mesos'.format(key))
return self.submit(execute_task, args, **options)
# Run
queue = Queue(self.zk, str(uuid4()))
result = get_async(apply_async, 1e4, dsk3, keys, queue=queue, **kwargs)
return result
def test_fuse_keys():
assert (fuse({
'a': 1,
'b': (inc, 'a'),
'c': (inc, 'b')
}, keys=['b']) == ({
'b': (inc, 1),
'c': (inc, 'b')
}, {
'b': set(),
'c': set(['b'])
}))
dsk, dependencies = fuse(
{
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
},
keys=['x', 'z'])
assert dsk == {
'w': (inc, 'x'),
'x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2
}
assert dependencies == {
'a': set(),
'b': set(),
'z': set(['a', 'b']),
'x': set(['z']),
'w': set(['x'])
}
def _optimize(dsk, keys, **kwargs):
dsk2, deps = fuse(dsk, keys)
return dsk2
def test_fuse():
dsk, dependencies = fuse({
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2
})
assert dsk == {'w': (inc, (inc, (inc, (add, 'a', 'b')))), 'a': 1, 'b': 2}
assert dependencies == {'a': set(), 'b': set(), 'w': set(['a', 'b'])}
assert (fuse({
'NEW': (inc, 'y'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2
}) == ({
'NEW': (inc, 'y'),
'w': (inc, (inc, 'y')),
'y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2
}, {
'a': set(),
'b': set(),
'y': set(['a', 'b']),
'w': set(['y']),
'NEW': set(['y'])
}))
assert (fuse({
'v': (inc, 'y'),
'u': (inc, 'w'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': (inc, 'c'),
'b': (inc, 'd'),
'c': 1,
'd': 2
}) == ({
'u': (inc, (inc, (inc, 'y'))),
'v': (inc, 'y'),
'y': (inc, (add, 'a', 'b')),
'a': (inc, 1),
'b': (inc, 2)
}, {
'a': set(),
'b': set(),
'y': set(['a', 'b']),
'v': set(['y']),
'u': set(['y'])
}))
assert (fuse({
'a': (inc, 'x'),
'b': (inc, 'x'),
'c': (inc, 'x'),
'd': (inc, 'c'),
'x': (inc, 'y'),
'y': 0
}) == ({
'a': (inc, 'x'),
'b': (inc, 'x'),
'd': (inc, (inc, 'x')),
'x': (inc, 0)
}, {
'x': set(),
'd': set(['x']),
'a': set(['x']),
'b': set(['x'])
}))
assert (fuse({
'a': 1,
'b': (inc, 'a'),
'c': (add, 'b', 'b')
}) == ({
'b': (inc, 1),
'c': (add, 'b', 'b')
}, {
'b': set(),
'c': set(['b'])
}))
def test_fuse_reductions_single_input():
def f(*args):
return args
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a', 'a'),
'c': (f, 'b1', 'b2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a': 1,
'c': (f, (f, 'a'), (f, 'a', 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-c': (f, (f, 'a'), (f, 'a', 'a')),
'c': 'b1-b2-c',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a', 'a'),
'b3': (f, 'a', 'a', 'a'),
'c': (f, 'b1', 'b2', 'b3'),
}
assert fuse(d, ave_width=2.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=2.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a': 1,
'c': (f, (f, 'a'), (f, 'a', 'a'), (f, 'a', 'a', 'a')),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-b3-c': (f, (f, 'a'), (f, 'a', 'a'), (f, 'a', 'a', 'a')),
'c': 'b1-b2-b3-c',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c': (f, 'a', 'b1', 'b2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a': 1,
'c': (f, 'a', (f, 'a'), (f, 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-c': (f, 'a', (f, 'a'), (f, 'a')),
'c': 'b1-b2-c',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c': (f, 'b1', 'b2'),
'd1': (f, 'c'),
'd2': (f, 'c'),
'e': (f, 'd1', 'd2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a': 1,
'c': (f, (f, 'a'), (f, 'a')),
'e': (f, (f, 'c'), (f, 'c')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-c': (f, (f, 'a'), (f, 'a')),
'd1-d2-e': (f, (f, 'c'), (f, 'c')),
'c': 'b1-b2-c',
'e': 'd1-d2-e',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'b4': (f, 'a'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b3', 'b4'),
'd': (f, 'c1', 'c2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
expected = with_deps({
'a': 1,
'c1': (f, (f, 'a'), (f, 'a')),
'c2': (f, (f, 'a'), (f, 'a')),
'd': (f, 'c1', 'c2'),
})
assert fuse(d, ave_width=2, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-c1': (f, (f, 'a'), (f, 'a')),
'b3-b4-c2': (f, (f, 'a'), (f, 'a')),
'd': (f, 'c1', 'c2'),
'c1': 'b1-b2-c1',
'c2': 'b3-b4-c2',
})
assert fuse(d, ave_width=2, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a': 1,
'd': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-b3-b4-c1-c2-d': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd': 'b1-b2-b3-b4-c1-c2-d',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'b4': (f, 'a'),
'b5': (f, 'a'),
'b6': (f, 'a'),
'b7': (f, 'a'),
'b8': (f, 'a'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b3', 'b4'),
'c3': (f, 'b5', 'b6'),
'c4': (f, 'b7', 'b8'),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'e': (f, 'd1', 'd2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
expected = with_deps({
'a': 1,
'c1': (f, (f, 'a'), (f, 'a')),
'c2': (f, (f, 'a'), (f, 'a')),
'c3': (f, (f, 'a'), (f, 'a')),
'c4': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'e': (f, 'd1', 'd2'),
})
assert fuse(d, ave_width=2, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-c1': (f, (f, 'a'), (f, 'a')),
'b3-b4-c2': (f, (f, 'a'), (f, 'a')),
'b5-b6-c3': (f, (f, 'a'), (f, 'a')),
'b7-b8-c4': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'e': (f, 'd1', 'd2'),
'c1': 'b1-b2-c1',
'c2': 'b3-b4-c2',
'c3': 'b5-b6-c3',
'c4': 'b7-b8-c4',
})
assert fuse(d, ave_width=2, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
expected = with_deps({
'a': 1,
'd1': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd2': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e': (f, 'd1', 'd2'),
})
assert fuse(d, ave_width=3, rename_keys=False) == expected
assert fuse(d, ave_width=4.6, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-b3-b4-c1-c2-d1': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b5-b6-b7-b8-c3-c4-d2': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e': (f, 'd1', 'd2'),
'd1': 'b1-b2-b3-b4-c1-c2-d1',
'd2': 'b5-b6-b7-b8-c3-c4-d2',
})
assert fuse(d, ave_width=3, rename_keys=True) == expected
assert fuse(d, ave_width=4.6, rename_keys=True) == expected
assert fuse(d, ave_width=4.7, rename_keys=False) == with_deps({
'a': 1,
'e': (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))))
})
assert fuse(d, ave_width=4.7, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e': (
f,
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))
),
'e': 'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'b4': (f, 'a'),
'b5': (f, 'a'),
'b6': (f, 'a'),
'b7': (f, 'a'),
'b8': (f, 'a'),
'b9': (f, 'a'),
'b10': (f, 'a'),
'b11': (f, 'a'),
'b12': (f, 'a'),
'b13': (f, 'a'),
'b14': (f, 'a'),
'b15': (f, 'a'),
'b16': (f, 'a'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b3', 'b4'),
'c3': (f, 'b5', 'b6'),
'c4': (f, 'b7', 'b8'),
'c5': (f, 'b9', 'b10'),
'c6': (f, 'b11', 'b12'),
'c7': (f, 'b13', 'b14'),
'c8': (f, 'b15', 'b16'),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'd3': (f, 'c5', 'c6'),
'd4': (f, 'c7', 'c8'),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
expected = with_deps({
'a': 1,
'c1': (f, (f, 'a'), (f, 'a')),
'c2': (f, (f, 'a'), (f, 'a')),
'c3': (f, (f, 'a'), (f, 'a')),
'c4': (f, (f, 'a'), (f, 'a')),
'c5': (f, (f, 'a'), (f, 'a')),
'c6': (f, (f, 'a'), (f, 'a')),
'c7': (f, (f, 'a'), (f, 'a')),
'c8': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'd3': (f, 'c5', 'c6'),
'd4': (f, 'c7', 'c8'),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
})
assert fuse(d, ave_width=2, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-c1': (f, (f, 'a'), (f, 'a')),
'b3-b4-c2': (f, (f, 'a'), (f, 'a')),
'b5-b6-c3': (f, (f, 'a'), (f, 'a')),
'b7-b8-c4': (f, (f, 'a'), (f, 'a')),
'b10-b9-c5': (f, (f, 'a'), (f, 'a')),
'b11-b12-c6': (f, (f, 'a'), (f, 'a')),
'b13-b14-c7': (f, (f, 'a'), (f, 'a')),
'b15-b16-c8': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'd3': (f, 'c5', 'c6'),
'd4': (f, 'c7', 'c8'),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
'c1': 'b1-b2-c1',
'c2': 'b3-b4-c2',
'c3': 'b5-b6-c3',
'c4': 'b7-b8-c4',
'c5': 'b10-b9-c5',
'c6': 'b11-b12-c6',
'c7': 'b13-b14-c7',
'c8': 'b15-b16-c8',
})
assert fuse(d, ave_width=2, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
expected = with_deps({
'a': 1,
'd1': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd2': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd3': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd4': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
})
assert fuse(d, ave_width=3, rename_keys=False) == expected
assert fuse(d, ave_width=4.6, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-b3-b4-c1-c2-d1': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b5-b6-b7-b8-c3-c4-d2': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b10-b11-b12-b9-c5-c6-d3': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b13-b14-b15-b16-c7-c8-d4': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
'd1': 'b1-b2-b3-b4-c1-c2-d1',
'd2': 'b5-b6-b7-b8-c3-c4-d2',
'd3': 'b10-b11-b12-b9-c5-c6-d3',
'd4': 'b13-b14-b15-b16-c7-c8-d4',
})
assert fuse(d, ave_width=3, rename_keys=True) == expected
assert fuse(d, ave_width=4.6, rename_keys=True) == expected
expected = with_deps({
'a': 1,
'e1': (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
'e2': (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
'f': (f, 'e1', 'e2'),
})
assert fuse(d, ave_width=4.7, rename_keys=False) == expected
assert fuse(d, ave_width=7.4, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e1': (
f,
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))
),
'b10-b11-b12-b13-b14-b15-b16-b9-c5-c6-c7-c8-d3-d4-e2': (
f,
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))
),
'f': (f, 'e1', 'e2'),
'e1': 'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e1',
'e2': 'b10-b11-b12-b13-b14-b15-b16-b9-c5-c6-c7-c8-d3-d4-e2',
})
assert fuse(d, ave_width=4.7, rename_keys=True) == expected
assert fuse(d, ave_width=7.4, rename_keys=True) == expected
assert fuse(d, ave_width=7.5, rename_keys=False) == with_deps({
'a': 1,
'f': (f, (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))))),
})
assert fuse(d, ave_width=7.5, rename_keys=True) == with_deps({
'a': 1,
'b1-b10-b11-b12-b13-b14-b15-b16-b2-b3-b4-b5-b6-b7-b8-b9-c1-c2-c3-c4-c5-c6-c7-c8-d1-d2-d3-d4-e1-e2-f': (
f,
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))))
),
'f': 'b1-b10-b11-b12-b13-b14-b15-b16-b2-b3-b4-b5-b6-b7-b8-b9-c1-c2-c3-c4-c5-c6-c7-c8-d1-d2-d3-d4-e1-e2-f',
})
d = {
'a': 1,
'b': (f, 'a'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'b': (f, 1)
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a-b': (f, 1),
'b': 'a-b',
})
d = {
'a': 1,
'b': (f, 'a'),
'c': (f, 'b'),
'd': (f, 'c'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'd': (f, (f, (f, 1)))
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a-b-c-d': (f, (f, (f, 1))),
'd': 'a-b-c-d',
})
d = {
'a': 1,
'b': (f, 'a'),
'c': (f, 'a', 'b'),
'd': (f, 'a', 'c'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'a': 1,
'd': (f, 'a', (f, 'a', (f, 'a'))),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a': 1,
'b-c-d': (f, 'a', (f, 'a', (f, 'a'))),
'd': 'b-c-d',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c1': (f, 'b1'),
'd1': (f, 'c1'),
'e1': (f, 'd1'),
'f': (f, 'e1', 'b2'),
}
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'e1': (f, (f, (f, (f, 'a')))),
'f': (f, 'e1', 'b2'),
})
assert fuse(d, ave_width=1, rename_keys=False) == expected
assert fuse(d, ave_width=1.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'b1-c1-d1-e1': (f, (f, (f, (f, 'a')))),
'f': (f, 'e1', 'b2'),
'e1': 'b1-c1-d1-e1',
})
assert fuse(d, ave_width=1, rename_keys=True) == expected
assert fuse(d, ave_width=1.9, rename_keys=True) == expected
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a': 1,
'f': (f, (f, (f, (f, (f, 'a')))), (f, 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-c1-d1-e1-f': (f, (f, (f, (f, (f, 'a')))), (f, 'a')),
'f': 'b1-b2-c1-d1-e1-f',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c1': (f, 'a', 'b1'),
'd1': (f, 'a', 'c1'),
'e1': (f, 'a', 'd1'),
'f': (f, 'a', 'e1', 'b2'),
}
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'e1': (f, 'a', (f, 'a', (f, 'a', (f, 'a')))),
'f': (f, 'a', 'e1', 'b2'),
})
assert fuse(d, ave_width=1, rename_keys=False) == expected
assert fuse(d, ave_width=1.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'b1-c1-d1-e1': (f, 'a', (f, 'a', (f, 'a', (f, 'a')))),
'f': (f, 'a', 'e1', 'b2'),
'e1': 'b1-c1-d1-e1',
})
assert fuse(d, ave_width=1, rename_keys=True) == expected
assert fuse(d, ave_width=1.9, rename_keys=True) == expected
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a': 1,
'f': (f, 'a', (f, 'a', (f, 'a', (f, 'a', (f, 'a')))), (f, 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a': 1,
'b1-b2-c1-d1-e1-f': (f, 'a', (f, 'a', (f, 'a', (f, 'a', (f, 'a')))), (f, 'a')),
'f': 'b1-b2-c1-d1-e1-f',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'c1': (f, 'b1'),
'c2': (f, 'b2'),
'c3': (f, 'b3'),
'd1': (f, 'c1'),
'd2': (f, 'c2'),
'd3': (f, 'c3'),
'e': (f, 'd1', 'd2', 'd3'),
'f': (f, 'e'),
'g': (f, 'f'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'a': 1,
'd1': (f, (f, (f, 'a'))),
'd2': (f, (f, (f, 'a'))),
'd3': (f, (f, (f, 'a'))),
'g': (f, (f, (f, 'd1', 'd2', 'd3'))),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a': 1,
'b1-c1-d1': (f, (f, (f, 'a'))),
'b2-c2-d2': (f, (f, (f, 'a'))),
'b3-c3-d3': (f, (f, (f, 'a'))),
'e-f-g': (f, (f, (f, 'd1', 'd2', 'd3'))),
'd1': 'b1-c1-d1',
'd2': 'b2-c2-d2',
'd3': 'b3-c3-d3',
'g': 'e-f-g',
})
d = {
'a': 1,
'b': (f, 'a'),
'c': (f, 'b'),
'd': (f, 'b', 'c'),
'e': (f, 'd'),
'f': (f, 'e'),
'g': (f, 'd', 'f'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'b': (f, 1),
'd': (f, 'b', (f, 'b')),
'g': (f, 'd', (f, (f, 'd'))),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a-b': (f, 1),
'c-d': (f, 'b', (f, 'b')),
'e-f-g': (f, 'd', (f, (f, 'd'))),
'b': 'a-b',
'd': 'c-d',
'g': 'e-f-g',
})
def test_fuse():
assert fuse({
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}) == {
'w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a': 1,
'b': 2,
}
assert fuse({
'NEW': (inc, 'y'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}) == {
'NEW': (inc, 'y'),
'w': (inc, (inc, 'y')),
'y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
}
assert fuse({
'v': (inc, 'y'),
'u': (inc, 'w'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': (inc, 'c'),
'b': (inc, 'd'),
'c': 1,
'd': 2,
}) == {
'u': (inc, (inc, (inc, 'y'))),
'v': (inc, 'y'),
'y': (inc, (add, 'a', 'b')),
'a': (inc, 1),
'b': (inc, 2),
}
assert fuse({
'a': (inc, 'x'),
'b': (inc, 'x'),
'c': (inc, 'x'),
'd': (inc, 'c'),
'x': (inc, 'y'),
'y': 0,
}) == {
'a': (inc, 'x'),
'b': (inc, 'x'),
'd': (inc, (inc, 'x')),
'x': (inc, 0),
}
assert fuse({
'a': 1,
'b': (inc, 'a'),
'c': (add, 'b', 'b')
}) == {
'b': (inc, 1),
'c': (add, 'b', 'b')
}
def test_fuse():
fuse = fuse2 # tests both `fuse` and `fuse_linear`
d = {
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
assert fuse(d, rename_keys=False) == with_deps({
'w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a': 1,
'b': 2,
})
assert fuse(d, rename_keys=True) == with_deps({
'z-y-x-w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a': 1,
'b': 2,
'w': 'z-y-x-w',
})
d = {
'NEW': (inc, 'y'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
assert fuse(d, rename_keys=False) == with_deps({
'NEW': (inc, 'y'),
'w': (inc, (inc, 'y')),
'y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
})
assert fuse(d, rename_keys=True) == with_deps({
'NEW': (inc, 'z-y'),
'x-w': (inc, (inc, 'z-y')),
'z-y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
'w': 'x-w',
'y': 'z-y',
})
d = {
'v': (inc, 'y'),
'u': (inc, 'w'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': (inc, 'c'),
'b': (inc, 'd'),
'c': 1,
'd': 2,
}
assert fuse(d, rename_keys=False) == with_deps({
'u': (inc, (inc, (inc, 'y'))),
'v': (inc, 'y'),
'y': (inc, (add, 'a', 'b')),
'a': (inc, 1),
'b': (inc, 2),
})
assert fuse(d, rename_keys=True) == with_deps({
'x-w-u': (inc, (inc, (inc, 'z-y'))),
'v': (inc, 'z-y'),
'z-y': (inc, (add, 'c-a', 'd-b')),
'c-a': (inc, 1),
'd-b': (inc, 2),
'a': 'c-a',
'b': 'd-b',
'u': 'x-w-u',
'y': 'z-y',
})
d = {
'a': (inc, 'x'),
'b': (inc, 'x'),
'c': (inc, 'x'),
'd': (inc, 'c'),
'x': (inc, 'y'),
'y': 0,
}
assert fuse(d, rename_keys=False) == with_deps({
'a': (inc, 'x'),
'b': (inc, 'x'),
'd': (inc, (inc, 'x')),
'x': (inc, 0)
})
assert fuse(d, rename_keys=True) == with_deps({
'a': (inc, 'y-x'),
'b': (inc, 'y-x'),
'c-d': (inc, (inc, 'y-x')),
'y-x': (inc, 0),
'd': 'c-d',
'x': 'y-x',
})
d = {
'a': 1,
'b': (inc, 'a'),
'c': (add, 'b', 'b'),
}
assert fuse(d, rename_keys=False) == with_deps({
'b': (inc, 1),
'c': (add, 'b', 'b'),
})
assert fuse(d, rename_keys=True) == with_deps({
'a-b': (inc, 1),
'c': (add, 'a-b', 'a-b'),
'b': 'a-b',
})
def get(dsk, keys, optimizations=[], num_workers=None,
docker='lensa/dask.mesos',
zk=os.getenv('ZOOKEEPER_HOST', '127.0.0.1:2181'),
mesos=os.getenv('MESOS_MASTER', '127.0.0.1:5050'),
**kwargs):
"""Mesos get function appropriate for Bags
Parameters
----------
dsk: dict
dask graph
keys: object or list
Desired results from graph
optimizations: list of functions
optimizations to perform on graph before execution
num_workers: int
Number of worker processes (defaults to number of cores)
docker: string
Default docker image name to run the dask in
zk: string
Zookeeper host and port the distributed Queue should connect to
mesos: string
Mesos Master hostname and port the Satyr framework should connect to
"""
pool, kazoo = _globals['pool'], _globals['kazoo']
if pool is None:
pool = Pool(name='dask-pool', master=mesos, processes=num_workers)
pool.start()
cleanup_pool = True
else:
cleanup_pool = False
if kazoo is None:
kazoo = KazooClient(hosts=zk)
kazoo.start()
cleanup_kazoo = True
else:
cleanup_kazoo = False
# Optimize Dask
dsk2, dependencies = cull(dsk, keys)
dsk3, dependencies = fuse(dsk2, keys, dependencies)
dsk4 = pipe(dsk3, *optimizations)
def apply_async(execute_task, args):
key = args[0]
func = args[1][0]
params = func.params if isinstance(func, SatyrPack) else {}
params['id'] = key
if 'docker' not in params:
params['docker'] = docker
return pool.apply_async(execute_task, args, **params)
try:
# Run
queue = Queue(kazoo, str(uuid4()))
result = get_async(apply_async, 1e4, dsk3, keys, queue=queue, **kwargs)
finally:
if cleanup_kazoo:
kazoo.stop()
if cleanup_pool:
pool.stop()
return result
def test_fuse_stressed():
def f(*args):
return args
d = {
'array-original-27b9f9d257a80fa6adae06a98faf71eb': 1,
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0): (
f,
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0),
),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0): (
f,
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
),
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 0): (
f,
'array-original-27b9f9d257a80fa6adae06a98faf71eb',
(slice(0, 10, None), slice(0, 10, None)),
),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0): ('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1): (
f,
(f,
('array-27b9f9d257a80fa6adae06a98faf71eb', 1, 1),
(f, [('cholesky-lt-dot-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0, 1, 0)]))
),
('cholesky-lt-dot-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0, 1, 0): (
f,
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
),
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 1): (
f,
'array-original-27b9f9d257a80fa6adae06a98faf71eb',
(slice(0, 10, None), slice(10, 20, None)),
),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1): (
f,
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1)
),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1): (
f,
(10, 10)
),
('array-27b9f9d257a80fa6adae06a98faf71eb', 1, 1): (
f,
'array-original-27b9f9d257a80fa6adae06a98faf71eb',
(slice(10, 20, None), slice(10, 20, None)),
),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1): (
f,
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0),
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 1),
),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0): (
f,
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 0),
),
}
keys = {
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1),
}
rv = fuse(d, keys=keys, ave_width=2, rename_keys=True)
assert rv == with_deps(rv[0])
def test_fuse_stressed():
def f(*args):
return args
d = {
'array-original-27b9f9d257a80fa6adae06a98faf71eb':
1,
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0): (
f,
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0),
),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0): (
f,
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
),
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 0): (
f,
'array-original-27b9f9d257a80fa6adae06a98faf71eb',
(slice(0, 10, None), slice(0, 10, None)),
),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0):
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1):
(f, (f, ('array-27b9f9d257a80fa6adae06a98faf71eb', 1, 1), (f, [
('cholesky-lt-dot-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0, 1, 0)
]))),
('cholesky-lt-dot-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0, 1, 0): (
f,
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
),
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 1): (
f,
'array-original-27b9f9d257a80fa6adae06a98faf71eb',
(slice(0, 10, None), slice(10, 20, None)),
),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1):
(f, ('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1)),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1): (f, (10, 10)),
('array-27b9f9d257a80fa6adae06a98faf71eb', 1, 1): (
f,
'array-original-27b9f9d257a80fa6adae06a98faf71eb',
(slice(10, 20, None), slice(10, 20, None)),
),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1): (
f,
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0),
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 1),
),
('cholesky-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0): (
f,
('array-27b9f9d257a80fa6adae06a98faf71eb', 0, 0),
),
}
keys = {
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 0),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 0, 1),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 0),
('cholesky-upper-26a6b670a8aabb7e2f8936db7ccb6a88', 1, 1),
}
rv = fuse(d, keys=keys, ave_width=2, rename_keys=True)
assert rv == with_deps(rv[0])
def test_fuse_reductions_multiple_input():
def f(*args):
return args
d = {
'a1': 1,
'a2': 2,
'b': (f, 'a1', 'a2'),
'c': (f, 'b'),
}
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'c': (f, (f, 1, 2)),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a1-a2-b-c': (f, (f, 1, 2)),
'c': 'a1-a2-b-c',
})
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'a1': 1,
'a2': 2,
'c': (f, (f, 'a1', 'a2')),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a1': 1,
'a2': 2,
'b-c': (f, (f, 'a1', 'a2')),
'c': 'b-c',
})
d = {
'a1': 1,
'a2': 2,
'b1': (f, 'a1'),
'b2': (f, 'a1', 'a2'),
'b3': (f, 'a2'),
'c': (f, 'b1', 'b2', 'b3'),
}
expected = with_deps(d)
assert fuse(d, ave_width=1, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
assert fuse(d, ave_width=1, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a1': 1,
'a2': 2,
'c': (f, (f, 'a1'), (f, 'a1', 'a2'), (f, 'a2')),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a1': 1,
'a2': 2,
'b1-b2-b3-c': (f, (f, 'a1'), (f, 'a1', 'a2'), (f, 'a2')),
'c': 'b1-b2-b3-c',
})
d = {
'a1': 1,
'a2': 2,
'b1': (f, 'a1'),
'b2': (f, 'a1', 'a2'),
'b3': (f, 'a2'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b2', 'b3'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a1': 1,
'a2': 2,
'b2': (f, 'a1', 'a2'),
'c1': (f, (f, 'a1'), 'b2'),
'c2': (f, 'b2', (f, 'a2')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a1': 1,
'a2': 2,
'b2': (f, 'a1', 'a2'),
'b1-c1': (f, (f, 'a1'), 'b2'),
'b3-c2': (f, 'b2', (f, 'a2')),
'c1': 'b1-c1',
'c2': 'b3-c2',
})
d = {
'a1': 1,
'a2': 2,
'b1': (f, 'a1'),
'b2': (f, 'a1', 'a2'),
'b3': (f, 'a2'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b2', 'b3'),
'd': (f, 'c1', 'c2'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1, rename_keys=True) == with_deps(d)
# A more aggressive heuristic could do this at `ave_width=2`. Perhaps
# we can improve this. Nevertheless, this is behaving as intended.
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a1': 1,
'a2': 2,
'b2': (f, 'a1', 'a2'),
'd': (f, (f, (f, 'a1'), 'b2'), (f, 'b2', (f, 'a2'))),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a1': 1,
'a2': 2,
'b2': (f, 'a1', 'a2'),
'b1-b3-c1-c2-d': (f, (f, (f, 'a1'), 'b2'), (f, 'b2', (f, 'a2'))),
'd': 'b1-b3-c1-c2-d',
})
def test_fuse():
fuse = fuse2 # tests both `fuse` and `fuse_linear`
d = {
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
assert fuse(d, rename_keys=False) == with_deps({
'w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a':
1,
'b':
2,
})
assert fuse(d, rename_keys=True) == with_deps({
'z-y-x-w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a':
1,
'b':
2,
'w':
'z-y-x-w',
})
d = {
'NEW': (inc, 'y'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
assert fuse(d, rename_keys=False) == with_deps({
'NEW': (inc, 'y'),
'w': (inc, (inc, 'y')),
'y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
})
assert fuse(d, rename_keys=True) == with_deps({
'NEW': (inc, 'z-y'),
'x-w': (inc, (inc, 'z-y')),
'z-y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
'w': 'x-w',
'y': 'z-y',
})
d = {
'v': (inc, 'y'),
'u': (inc, 'w'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': (inc, 'c'),
'b': (inc, 'd'),
'c': 1,
'd': 2,
}
assert fuse(d, rename_keys=False) == with_deps({
'u': (inc, (inc, (inc, 'y'))),
'v': (inc, 'y'),
'y': (inc, (add, 'a', 'b')),
'a': (inc, 1),
'b': (inc, 2),
})
assert fuse(d, rename_keys=True) == with_deps({
'x-w-u': (inc, (inc, (inc, 'z-y'))),
'v': (inc, 'z-y'),
'z-y': (inc, (add, 'c-a', 'd-b')),
'c-a': (inc, 1),
'd-b': (inc, 2),
'a':
'c-a',
'b':
'd-b',
'u':
'x-w-u',
'y':
'z-y',
})
d = {
'a': (inc, 'x'),
'b': (inc, 'x'),
'c': (inc, 'x'),
'd': (inc, 'c'),
'x': (inc, 'y'),
'y': 0,
}
assert fuse(d, rename_keys=False) == with_deps({
'a': (inc, 'x'),
'b': (inc, 'x'),
'd': (inc, (inc, 'x')),
'x': (inc, 0)
})
assert fuse(d, rename_keys=True) == with_deps({
'a': (inc, 'y-x'),
'b': (inc, 'y-x'),
'c-d': (inc, (inc, 'y-x')),
'y-x': (inc, 0),
'd': 'c-d',
'x': 'y-x',
})
d = {
'a': 1,
'b': (inc, 'a'),
'c': (add, 'b', 'b'),
}
assert fuse(d, rename_keys=False) == with_deps({
'b': (inc, 1),
'c': (add, 'b', 'b'),
})
assert fuse(d, rename_keys=True) == with_deps({
'a-b': (inc, 1),
'c': (add, 'a-b', 'a-b'),
'b': 'a-b',
})
def test_fuse():
d = {
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
dsk, dependencies = fuse(d, rename_fused_keys=False)
assert dsk == {
'w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a': 1,
'b': 2,
}
assert dependencies == {'a': set(), 'b': set(), 'w': set(['a', 'b'])}
dsk, dependencies = fuse(d, rename_fused_keys=True)
assert dsk == {
'z-y-x-w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a': 1,
'b': 2,
'w': 'z-y-x-w',
}
assert dependencies == {
'a': set(),
'b': set(),
'z-y-x-w': set(['a', 'b']),
'w': set(['z-y-x-w'])
}
d = {
'NEW': (inc, 'y'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
dsk, dependencies = fuse(d, rename_fused_keys=False)
assert dsk == {
'NEW': (inc, 'y'),
'w': (inc, (inc, 'y')),
'y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
}
assert dependencies == {
'a': set(),
'b': set(),
'y': set(['a', 'b']),
'w': set(['y']),
'NEW': set(['y'])
}
dsk, dependencies = fuse(d, rename_fused_keys=True)
assert dsk == {
'NEW': (inc, 'z-y'),
'x-w': (inc, (inc, 'z-y')),
'z-y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
'w': 'x-w',
'y': 'z-y',
}
assert dependencies == {
'a': set(),
'b': set(),
'z-y': set(['a', 'b']),
'x-w': set(['z-y']),
'NEW': set(['z-y']),
'w': set(['x-w']),
'y': set(['z-y'])
}
d = {
'v': (inc, 'y'),
'u': (inc, 'w'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': (inc, 'c'),
'b': (inc, 'd'),
'c': 1,
'd': 2,
}
dsk, dependencies = fuse(d, rename_fused_keys=False)
assert dsk == {
'u': (inc, (inc, (inc, 'y'))),
'v': (inc, 'y'),
'y': (inc, (add, 'a', 'b')),
'a': (inc, 1),
'b': (inc, 2),
}
assert dependencies == {
'a': set(),
'b': set(),
'y': set(['a', 'b']),
'v': set(['y']),
'u': set(['y'])
}
dsk, dependencies = fuse(d, rename_fused_keys=True)
assert dsk == {
'x-w-u': (inc, (inc, (inc, 'z-y'))),
'v': (inc, 'z-y'),
'z-y': (inc, (add, 'c-a', 'd-b')),
'c-a': (inc, 1),
'd-b': (inc, 2),
'a': 'c-a',
'b': 'd-b',
'u': 'x-w-u',
'y': 'z-y',
}
assert dependencies == {
'c-a': set(),
'd-b': set(),
'z-y': set(['c-a', 'd-b']),
'v': set(['z-y']),
'x-w-u': set(['z-y']),
'a': set(['c-a']),
'b': set(['d-b']),
'u': set(['x-w-u']),
'y': set(['z-y']),
}
d = {
'a': (inc, 'x'),
'b': (inc, 'x'),
'c': (inc, 'x'),
'd': (inc, 'c'),
'x': (inc, 'y'),
'y': 0,
}
dsk, dependencies = fuse(d, rename_fused_keys=False)
assert dsk == {
'a': (inc, 'x'),
'b': (inc, 'x'),
'd': (inc, (inc, 'x')),
'x': (inc, 0)
}
assert dependencies == {
'x': set(),
'd': set(['x']),
'a': set(['x']),
'b': set(['x'])
}
dsk, dependencies = fuse(d, rename_fused_keys=True)
assert dsk == {
'a': (inc, 'y-x'),
'b': (inc, 'y-x'),
'c-d': (inc, (inc, 'y-x')),
'y-x': (inc, 0),
'd': 'c-d',
'x': 'y-x',
}
assert dependencies == {
'y-x': set(),
'c-d': set(['y-x']),
'a': set(['y-x']),
'b': set(['y-x']),
'd': set(['c-d']),
'x': set(['y-x'])
}
d = {
'a': 1,
'b': (inc, 'a'),
'c': (add, 'b', 'b'),
}
dsk, dependencies = fuse(d, rename_fused_keys=False)
assert dsk == {
'b': (inc, 1),
'c': (add, 'b', 'b'),
}
assert dependencies == {'b': set(), 'c': set(['b'])}
dsk, dependencies = fuse(d, rename_fused_keys=True)
assert dsk == {
'a-b': (inc, 1),
'c': (add, 'a-b', 'a-b'),
'b': 'a-b',
}
assert dependencies == {'a-b': set(), 'c': set(['a-b']), 'b': set(['a-b'])}
def test_fuse():
assert fuse({
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}) == {
'w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a': 1,
'b': 2,
}
assert fuse({
'NEW': (inc, 'y'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}) == {
'NEW': (inc, 'y'),
'w': (inc, (inc, 'y')),
'y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2,
}
assert fuse({
'v': (inc, 'y'),
'u': (inc, 'w'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': (inc, 'c'),
'b': (inc, 'd'),
'c': 1,
'd': 2,
}) == {
'u': (inc, (inc, (inc, 'y'))),
'v': (inc, 'y'),
'y': (inc, (add, 'a', 'b')),
'a': (inc, 1),
'b': (inc, 2),
}
assert fuse({
'a': (inc, 'x'),
'b': (inc, 'x'),
'c': (inc, 'x'),
'd': (inc, 'c'),
'x': (inc, 'y'),
'y': 0,
}) == {
'a': (inc, 'x'),
'b': (inc, 'x'),
'd': (inc, (inc, 'x')),
'x': (inc, 0),
}
assert fuse({
'a': 1,
'b': (inc, 'a'),
'c': (add, 'b', 'b')
}) == {
'b': (inc, 1),
'c': (add, 'b', 'b')
}
def test_fuse():
dsk, dependencies = fuse({'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2})
assert dsk == {'w': (inc, (inc, (inc, (add, 'a', 'b')))),
'a': 1,
'b': 2}
assert dependencies == {'a': set(), 'b': set(), 'w': set(['a', 'b'])}
assert (fuse({'NEW': (inc, 'y'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2}) ==
({'NEW': (inc, 'y'),
'w': (inc, (inc, 'y')),
'y': (inc, (add, 'a', 'b')),
'a': 1,
'b': 2},
{'a': set(), 'b': set(), 'y': set(['a', 'b']),
'w': set(['y']), 'NEW': set(['y'])}))
assert (fuse({'v': (inc, 'y'),
'u': (inc, 'w'),
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': (inc, 'c'),
'b': (inc, 'd'),
'c': 1,
'd': 2}) ==
({'u': (inc, (inc, (inc, 'y'))),
'v': (inc, 'y'),
'y': (inc, (add, 'a', 'b')),
'a': (inc, 1),
'b': (inc, 2)},
{'a': set(), 'b': set(), 'y': set(['a', 'b']),
'v': set(['y']), 'u': set(['y'])}))
assert (fuse({'a': (inc, 'x'),
'b': (inc, 'x'),
'c': (inc, 'x'),
'd': (inc, 'c'),
'x': (inc, 'y'),
'y': 0}) ==
({'a': (inc, 'x'),
'b': (inc, 'x'),
'd': (inc, (inc, 'x')),
'x': (inc, 0)},
{'x': set(), 'd': set(['x']),
'a': set(['x']), 'b': set(['x'])}))
assert (fuse({'a': 1,
'b': (inc, 'a'),
'c': (add, 'b', 'b')}) ==
({'b': (inc, 1),
'c': (add, 'b', 'b')},
{'b': set(), 'c': set(['b'])}))
def test_fuse_reductions_multiple_input():
def f(*args):
return args
d = {
'a1': 1,
'a2': 2,
'b': (f, 'a1', 'a2'),
'c': (f, 'b'),
}
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'c': (f, (f, 1, 2)),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a1-a2-b-c': (f, (f, 1, 2)),
'c':
'a1-a2-b-c',
})
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'a1':
1,
'a2':
2,
'c': (f, (f, 'a1', 'a2')),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a1':
1,
'a2':
2,
'b-c': (f, (f, 'a1', 'a2')),
'c':
'b-c',
})
d = {
'a1': 1,
'a2': 2,
'b1': (f, 'a1'),
'b2': (f, 'a1', 'a2'),
'b3': (f, 'a2'),
'c': (f, 'b1', 'b2', 'b3'),
}
expected = with_deps(d)
assert fuse(d, ave_width=1, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
assert fuse(d, ave_width=1, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a1':
1,
'a2':
2,
'c': (f, (f, 'a1'), (f, 'a1', 'a2'), (f, 'a2')),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a1':
1,
'a2':
2,
'b1-b2-b3-c': (f, (f, 'a1'), (f, 'a1', 'a2'), (f, 'a2')),
'c':
'b1-b2-b3-c',
})
d = {
'a1': 1,
'a2': 2,
'b1': (f, 'a1'),
'b2': (f, 'a1', 'a2'),
'b3': (f, 'a2'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b2', 'b3'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a1':
1,
'a2':
2,
'b2': (f, 'a1', 'a2'),
'c1': (f, (f, 'a1'), 'b2'),
'c2': (f, 'b2', (f, 'a2')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a1':
1,
'a2':
2,
'b2': (f, 'a1', 'a2'),
'b1-c1': (f, (f, 'a1'), 'b2'),
'b3-c2': (f, 'b2', (f, 'a2')),
'c1':
'b1-c1',
'c2':
'b3-c2',
})
d = {
'a1': 1,
'a2': 2,
'b1': (f, 'a1'),
'b2': (f, 'a1', 'a2'),
'b3': (f, 'a2'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b2', 'b3'),
'd': (f, 'c1', 'c2'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1, rename_keys=True) == with_deps(d)
# A more aggressive heuristic could do this at `ave_width=2`. Perhaps
# we can improve this. Nevertheless, this is behaving as intended.
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a1':
1,
'a2':
2,
'b2': (f, 'a1', 'a2'),
'd': (f, (f, (f, 'a1'), 'b2'), (f, 'b2', (f, 'a2'))),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a1':
1,
'a2':
2,
'b2': (f, 'a1', 'a2'),
'b1-b3-c1-c2-d': (f, (f, (f, 'a1'), 'b2'), (f, 'b2', (f, 'a2'))),
'd':
'b1-b3-c1-c2-d',
})
def test_fuse_reductions_single_input():
def f(*args):
return args
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a', 'a'),
'c': (f, 'b1', 'b2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a':
1,
'c': (f, (f, 'a'), (f, 'a', 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-c': (f, (f, 'a'), (f, 'a', 'a')),
'c':
'b1-b2-c',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a', 'a'),
'b3': (f, 'a', 'a', 'a'),
'c': (f, 'b1', 'b2', 'b3'),
}
assert fuse(d, ave_width=2.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=2.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a':
1,
'c': (f, (f, 'a'), (f, 'a', 'a'), (f, 'a', 'a', 'a')),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-b3-c': (f, (f, 'a'), (f, 'a', 'a'), (f, 'a', 'a', 'a')),
'c':
'b1-b2-b3-c',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c': (f, 'a', 'b1', 'b2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a':
1,
'c': (f, 'a', (f, 'a'), (f, 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-c': (f, 'a', (f, 'a'), (f, 'a')),
'c':
'b1-b2-c',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c': (f, 'b1', 'b2'),
'd1': (f, 'c'),
'd2': (f, 'c'),
'e': (f, 'd1', 'd2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a':
1,
'c': (f, (f, 'a'), (f, 'a')),
'e': (f, (f, 'c'), (f, 'c')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-c': (f, (f, 'a'), (f, 'a')),
'd1-d2-e': (f, (f, 'c'), (f, 'c')),
'c':
'b1-b2-c',
'e':
'd1-d2-e',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'b4': (f, 'a'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b3', 'b4'),
'd': (f, 'c1', 'c2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
expected = with_deps({
'a': 1,
'c1': (f, (f, 'a'), (f, 'a')),
'c2': (f, (f, 'a'), (f, 'a')),
'd': (f, 'c1', 'c2'),
})
assert fuse(d, ave_width=2, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-c1': (f, (f, 'a'), (f, 'a')),
'b3-b4-c2': (f, (f, 'a'), (f, 'a')),
'd': (f, 'c1', 'c2'),
'c1': 'b1-b2-c1',
'c2': 'b3-b4-c2',
})
assert fuse(d, ave_width=2, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
assert fuse(d, ave_width=3, rename_keys=False) == with_deps({
'a':
1,
'd': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
})
assert fuse(d, ave_width=3, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-b3-b4-c1-c2-d':
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd':
'b1-b2-b3-b4-c1-c2-d',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'b4': (f, 'a'),
'b5': (f, 'a'),
'b6': (f, 'a'),
'b7': (f, 'a'),
'b8': (f, 'a'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b3', 'b4'),
'c3': (f, 'b5', 'b6'),
'c4': (f, 'b7', 'b8'),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'e': (f, 'd1', 'd2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
expected = with_deps({
'a': 1,
'c1': (f, (f, 'a'), (f, 'a')),
'c2': (f, (f, 'a'), (f, 'a')),
'c3': (f, (f, 'a'), (f, 'a')),
'c4': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'e': (f, 'd1', 'd2'),
})
assert fuse(d, ave_width=2, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-c1': (f, (f, 'a'), (f, 'a')),
'b3-b4-c2': (f, (f, 'a'), (f, 'a')),
'b5-b6-c3': (f, (f, 'a'), (f, 'a')),
'b7-b8-c4': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'e': (f, 'd1', 'd2'),
'c1': 'b1-b2-c1',
'c2': 'b3-b4-c2',
'c3': 'b5-b6-c3',
'c4': 'b7-b8-c4',
})
assert fuse(d, ave_width=2, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
expected = with_deps({
'a':
1,
'd1': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd2': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e': (f, 'd1', 'd2'),
})
assert fuse(d, ave_width=3, rename_keys=False) == expected
assert fuse(d, ave_width=4.6, rename_keys=False) == expected
expected = with_deps({
'a':
1,
'b1-b2-b3-b4-c1-c2-d1':
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b5-b6-b7-b8-c3-c4-d2':
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e': (f, 'd1', 'd2'),
'd1':
'b1-b2-b3-b4-c1-c2-d1',
'd2':
'b5-b6-b7-b8-c3-c4-d2',
})
assert fuse(d, ave_width=3, rename_keys=True) == expected
assert fuse(d, ave_width=4.6, rename_keys=True) == expected
assert fuse(d, ave_width=4.7, rename_keys=False) == with_deps({
'a':
1,
'e': (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))))
})
assert fuse(d, ave_width=4.7, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e':
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
'e':
'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'b4': (f, 'a'),
'b5': (f, 'a'),
'b6': (f, 'a'),
'b7': (f, 'a'),
'b8': (f, 'a'),
'b9': (f, 'a'),
'b10': (f, 'a'),
'b11': (f, 'a'),
'b12': (f, 'a'),
'b13': (f, 'a'),
'b14': (f, 'a'),
'b15': (f, 'a'),
'b16': (f, 'a'),
'c1': (f, 'b1', 'b2'),
'c2': (f, 'b3', 'b4'),
'c3': (f, 'b5', 'b6'),
'c4': (f, 'b7', 'b8'),
'c5': (f, 'b9', 'b10'),
'c6': (f, 'b11', 'b12'),
'c7': (f, 'b13', 'b14'),
'c8': (f, 'b15', 'b16'),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'd3': (f, 'c5', 'c6'),
'd4': (f, 'c7', 'c8'),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
}
assert fuse(d, ave_width=1.9, rename_keys=False) == with_deps(d)
assert fuse(d, ave_width=1.9, rename_keys=True) == with_deps(d)
expected = with_deps({
'a': 1,
'c1': (f, (f, 'a'), (f, 'a')),
'c2': (f, (f, 'a'), (f, 'a')),
'c3': (f, (f, 'a'), (f, 'a')),
'c4': (f, (f, 'a'), (f, 'a')),
'c5': (f, (f, 'a'), (f, 'a')),
'c6': (f, (f, 'a'), (f, 'a')),
'c7': (f, (f, 'a'), (f, 'a')),
'c8': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'd3': (f, 'c5', 'c6'),
'd4': (f, 'c7', 'c8'),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
})
assert fuse(d, ave_width=2, rename_keys=False) == expected
assert fuse(d, ave_width=2.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b1-b2-c1': (f, (f, 'a'), (f, 'a')),
'b3-b4-c2': (f, (f, 'a'), (f, 'a')),
'b5-b6-c3': (f, (f, 'a'), (f, 'a')),
'b7-b8-c4': (f, (f, 'a'), (f, 'a')),
'b10-b9-c5': (f, (f, 'a'), (f, 'a')),
'b11-b12-c6': (f, (f, 'a'), (f, 'a')),
'b13-b14-c7': (f, (f, 'a'), (f, 'a')),
'b15-b16-c8': (f, (f, 'a'), (f, 'a')),
'd1': (f, 'c1', 'c2'),
'd2': (f, 'c3', 'c4'),
'd3': (f, 'c5', 'c6'),
'd4': (f, 'c7', 'c8'),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
'c1': 'b1-b2-c1',
'c2': 'b3-b4-c2',
'c3': 'b5-b6-c3',
'c4': 'b7-b8-c4',
'c5': 'b10-b9-c5',
'c6': 'b11-b12-c6',
'c7': 'b13-b14-c7',
'c8': 'b15-b16-c8',
})
assert fuse(d, ave_width=2, rename_keys=True) == expected
assert fuse(d, ave_width=2.9, rename_keys=True) == expected
expected = with_deps({
'a':
1,
'd1': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd2': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd3': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'd4': (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
})
assert fuse(d, ave_width=3, rename_keys=False) == expected
assert fuse(d, ave_width=4.6, rename_keys=False) == expected
expected = with_deps({
'a':
1,
'b1-b2-b3-b4-c1-c2-d1':
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b5-b6-b7-b8-c3-c4-d2':
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b10-b11-b12-b9-c5-c6-d3':
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'b13-b14-b15-b16-c7-c8-d4':
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
'e1': (f, 'd1', 'd2'),
'e2': (f, 'd3', 'd4'),
'f': (f, 'e1', 'e2'),
'd1':
'b1-b2-b3-b4-c1-c2-d1',
'd2':
'b5-b6-b7-b8-c3-c4-d2',
'd3':
'b10-b11-b12-b9-c5-c6-d3',
'd4':
'b13-b14-b15-b16-c7-c8-d4',
})
assert fuse(d, ave_width=3, rename_keys=True) == expected
assert fuse(d, ave_width=4.6, rename_keys=True) == expected
expected = with_deps({
'a':
1,
'e1': (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
'e2': (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
'f': (f, 'e1', 'e2'),
})
assert fuse(d, ave_width=4.7, rename_keys=False) == expected
assert fuse(d, ave_width=7.4, rename_keys=False) == expected
expected = with_deps({
'a':
1,
'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e1':
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
'b10-b11-b12-b13-b14-b15-b16-b9-c5-c6-c7-c8-d3-d4-e2':
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
'f': (f, 'e1', 'e2'),
'e1':
'b1-b2-b3-b4-b5-b6-b7-b8-c1-c2-c3-c4-d1-d2-e1',
'e2':
'b10-b11-b12-b13-b14-b15-b16-b9-c5-c6-c7-c8-d3-d4-e2',
})
assert fuse(d, ave_width=4.7, rename_keys=True) == expected
assert fuse(d, ave_width=7.4, rename_keys=True) == expected
assert fuse(d, ave_width=7.5, rename_keys=False) == with_deps({
'a':
1,
'f': (f, (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))))),
})
assert fuse(d, ave_width=7.5, rename_keys=True) == with_deps({
'a':
1,
'b1-b10-b11-b12-b13-b14-b15-b16-b2-b3-b4-b5-b6-b7-b8-b9-c1-c2-c3-c4-c5-c6-c7-c8-d1-d2-d3-d4-e1-e2-f':
(f, (f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a')))),
(f, (f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))),
(f, (f, (f, 'a'), (f, 'a')), (f, (f, 'a'), (f, 'a'))))),
'f':
'b1-b10-b11-b12-b13-b14-b15-b16-b2-b3-b4-b5-b6-b7-b8-b9-c1-c2-c3-c4-c5-c6-c7-c8-d1-d2-d3-d4-e1-e2-f',
})
d = {
'a': 1,
'b': (f, 'a'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({'b': (f, 1)})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a-b': (f, 1),
'b': 'a-b',
})
d = {
'a': 1,
'b': (f, 'a'),
'c': (f, 'b'),
'd': (f, 'c'),
}
assert fuse(d, ave_width=1,
rename_keys=False) == with_deps({'d': (f, (f, (f, 1)))})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a-b-c-d': (f, (f, (f, 1))),
'd':
'a-b-c-d',
})
d = {
'a': 1,
'b': (f, 'a'),
'c': (f, 'a', 'b'),
'd': (f, 'a', 'c'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'a':
1,
'd': (f, 'a', (f, 'a', (f, 'a'))),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a':
1,
'b-c-d': (f, 'a', (f, 'a', (f, 'a'))),
'd':
'b-c-d',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c1': (f, 'b1'),
'd1': (f, 'c1'),
'e1': (f, 'd1'),
'f': (f, 'e1', 'b2'),
}
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'e1': (f, (f, (f, (f, 'a')))),
'f': (f, 'e1', 'b2'),
})
assert fuse(d, ave_width=1, rename_keys=False) == expected
assert fuse(d, ave_width=1.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'b1-c1-d1-e1': (f, (f, (f, (f, 'a')))),
'f': (f, 'e1', 'b2'),
'e1': 'b1-c1-d1-e1',
})
assert fuse(d, ave_width=1, rename_keys=True) == expected
assert fuse(d, ave_width=1.9, rename_keys=True) == expected
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a':
1,
'f': (f, (f, (f, (f, (f, 'a')))), (f, 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-c1-d1-e1-f': (f, (f, (f, (f, (f, 'a')))), (f, 'a')),
'f':
'b1-b2-c1-d1-e1-f',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'c1': (f, 'a', 'b1'),
'd1': (f, 'a', 'c1'),
'e1': (f, 'a', 'd1'),
'f': (f, 'a', 'e1', 'b2'),
}
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'e1': (f, 'a', (f, 'a', (f, 'a', (f, 'a')))),
'f': (f, 'a', 'e1', 'b2'),
})
assert fuse(d, ave_width=1, rename_keys=False) == expected
assert fuse(d, ave_width=1.9, rename_keys=False) == expected
expected = with_deps({
'a': 1,
'b2': (f, 'a'),
'b1-c1-d1-e1': (f, 'a', (f, 'a', (f, 'a', (f, 'a')))),
'f': (f, 'a', 'e1', 'b2'),
'e1': 'b1-c1-d1-e1',
})
assert fuse(d, ave_width=1, rename_keys=True) == expected
assert fuse(d, ave_width=1.9, rename_keys=True) == expected
assert fuse(d, ave_width=2, rename_keys=False) == with_deps({
'a':
1,
'f': (f, 'a', (f, 'a', (f, 'a', (f, 'a', (f, 'a')))), (f, 'a')),
})
assert fuse(d, ave_width=2, rename_keys=True) == with_deps({
'a':
1,
'b1-b2-c1-d1-e1-f':
(f, 'a', (f, 'a', (f, 'a', (f, 'a', (f, 'a')))), (f, 'a')),
'f':
'b1-b2-c1-d1-e1-f',
})
d = {
'a': 1,
'b1': (f, 'a'),
'b2': (f, 'a'),
'b3': (f, 'a'),
'c1': (f, 'b1'),
'c2': (f, 'b2'),
'c3': (f, 'b3'),
'd1': (f, 'c1'),
'd2': (f, 'c2'),
'd3': (f, 'c3'),
'e': (f, 'd1', 'd2', 'd3'),
'f': (f, 'e'),
'g': (f, 'f'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'a':
1,
'd1': (f, (f, (f, 'a'))),
'd2': (f, (f, (f, 'a'))),
'd3': (f, (f, (f, 'a'))),
'g': (f, (f, (f, 'd1', 'd2', 'd3'))),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a':
1,
'b1-c1-d1': (f, (f, (f, 'a'))),
'b2-c2-d2': (f, (f, (f, 'a'))),
'b3-c3-d3': (f, (f, (f, 'a'))),
'e-f-g': (f, (f, (f, 'd1', 'd2', 'd3'))),
'd1':
'b1-c1-d1',
'd2':
'b2-c2-d2',
'd3':
'b3-c3-d3',
'g':
'e-f-g',
})
d = {
'a': 1,
'b': (f, 'a'),
'c': (f, 'b'),
'd': (f, 'b', 'c'),
'e': (f, 'd'),
'f': (f, 'e'),
'g': (f, 'd', 'f'),
}
assert fuse(d, ave_width=1, rename_keys=False) == with_deps({
'b': (f, 1),
'd': (f, 'b', (f, 'b')),
'g': (f, 'd', (f, (f, 'd'))),
})
assert fuse(d, ave_width=1, rename_keys=True) == with_deps({
'a-b': (f, 1),
'c-d': (f, 'b', (f, 'b')),
'e-f-g': (f, 'd', (f, (f, 'd'))),
'b':
'a-b',
'd':
'c-d',
'g':
'e-f-g',
})
def test_fuse_keys():
d = {
'a': 1,
'b': (inc, 'a'),
'c': (inc, 'b'),
}
keys = ['b']
dsk, dependencies = fuse(d, keys, rename_fused_keys=False)
assert dsk == {
'b': (inc, 1),
'c': (inc, 'b'),
}
assert dependencies == {'b': set(), 'c': set(['b'])}
dsk, dependencies = fuse(d, keys, rename_fused_keys=True)
assert dsk == {
'a-b': (inc, 1),
'c': (inc, 'a-b'),
'b': 'a-b',
}
assert dependencies == {'a-b': set(), 'c': set(['a-b']), 'b': set(['a-b'])}
d = {
'w': (inc, 'x'),
'x': (inc, 'y'),
'y': (inc, 'z'),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
keys = ['x', 'z']
dsk, dependencies = fuse(d, keys, rename_fused_keys=False)
assert dsk == {
'w': (inc, 'x'),
'x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
}
assert dependencies == {
'a': set(),
'b': set(),
'z': set(['a', 'b']),
'x': set(['z']),
'w': set(['x'])
}
dsk, dependencies = fuse(d, keys, rename_fused_keys=True)
assert dsk == {
'w': (inc, 'y-x'),
'y-x': (inc, (inc, 'z')),
'z': (add, 'a', 'b'),
'a': 1,
'b': 2,
'x': 'y-x',
}
assert dependencies == {
'a': set(),
'b': set(),
'z': set(['a', 'b']),
'y-x': set(['z']),
'w': set(['y-x']),
'x': set(['y-x'])
}
