def test_dont_concatenate_single_chunks(shape, chunks):
x = da.ones(shape, chunks=shape)
y = x.rechunk(chunks)
dsk = dict(y.dask)
assert not any(funcname(task[0]).startswith('concat')
for task in dsk.values()
if dask.istask(task))
def test_dont_concatenate_single_chunks(shape, chunks):
x = da.ones(shape, chunks=shape)
y = x.rechunk(chunks)
dsk = dict(y.dask)
assert not any(
funcname(task[0]).startswith("concat")
for task in dsk.values()
if dask.istask(task)
)
def _maybe_complex(task):
""" Possibly contains a nested task """
return (
istask(task)
or type(task) is list
and any(map(_maybe_complex, task))
or type(task) is dict
and any(map(_maybe_complex, task.values()))
)
def _hash(self, daskkey: str) -> str:
try:
return self._hashes[daskkey]
except KeyError:
obj = self._dsk[daskkey]
if dask.istask(obj):
self._hashes[daskkey] = self._hashtask(obj)
else:
self._hashes[daskkey] = self._md5hashpickle(obj)
return self._hashes[daskkey]
def convert(task):
if type(task) is list:
return [convert(v) for v in task]
if type(task) is dict:
return valmap(convert, task)
if istask(task):
return (task[0], ) + tuple(map(convert, task[1:]))
try:
if task in dsk or task in extra_values:
return tokey(task)
except TypeError:
pass
return task
def convert(task):
if type(task) is list:
return [convert(v) for v in task]
if type(task) is dict:
return valmap(convert, task)
if istask(task):
return (task[0],) + tuple(map(convert, task[1:]))
try:
if task in dsk:
return tokey(task)
except TypeError:
pass
return task
def _hashtask(self, obj: Any) -> str:
md5 = hashlib.md5()
for o in obj:
if isinstance(o, str) and o in self._dsk:
h = self._hash(o)
elif callable(o):
h = self._get_callable_src(o)
elif isinstance(o, list) or dask.istask(o):
h = self._hashtask(o)
else:
h = self._md5hashpickle(o)
md5.update(h.encode())
return md5.hexdigest()
def convert(task):
if isinstance(task, list):
return [convert(v) for v in task]
if isinstance(task, dict):
return valmap(convert, task)
if istask(task):
return (task[0], ) + tuple(map(convert, task[1:]))
try:
if task in dsk:
return tokey(task)
except TypeError:
pass
return task
def convert(task, dsk, extra_values):
if type(task) is list:
return [convert(v, dsk, extra_values) for v in task]
if type(task) is dict:
return {k: convert(v, dsk, extra_values) for k, v in task.items()}
if istask(task):
return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
try:
if task in dsk or task in extra_values:
return tokey(task)
except TypeError:
pass
return task
def convert(task, dsk, extra_values):
if type(task) is list:
return [convert(v, dsk, extra_values) for v in task]
if type(task) is dict:
return {k: convert(v, dsk, extra_values) for k, v in task.items()}
if istask(task):
return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
try:
if task in dsk or task in extra_values:
return tokey(task)
except TypeError:
pass
return task
def delayedoptimize(
dsk: Mapping[str, Any],
keys: Optional[Union[str, Iterable[str]]] = None,
cache: Optional[Cache] = None,
) -> Mapping[str, Any]:
hashes = dsktohash(dsk)
result = {}
for key in dsk:
if cache is not None and hashes[key] in cache:
result[key] = (lambda k: cache[k], hashes[key]) # type: ignore
else:
c = dsk[key]
if cache is not None and dask.istask(c):
result[key] = (CacheAfterExecution(c[0], cache,
hashes[key]), ) + c[1:]
else:
result[key] = dsk[key]
return result
def _deps(dsk, arg):
""" Get dependencies from keys or tasks
Helper function for get_dependencies.
Examples
--------
>>> inc = lambda x: x + 1
>>> add = lambda x, y: x + y
>>> dsk = {'x': 1, 'y': 2}
>>> _deps(dsk, 'x')
['x']
>>> _deps(dsk, (add, 'x', 1))
['x']
>>> _deps(dsk, ['x', 'y'])
['x', 'y']
>>> _deps(dsk, {'name': 'x'})
['x']
>>> _deps(dsk, (add, 'x', (inc, 'y'))) # doctest: +SKIP
['x', 'y']
"""
if istask(arg):
result = []
for a in arg[1:]:
result.extend(_deps(dsk, a))
return result
if isinstance(arg, list):
return sum([_deps(dsk, a) for a in arg], [])
if isinstance(arg, dict):
return sum([_deps(dsk, v) for v in arg.values()], [])
try:
if arg not in dsk:
return []
except TypeError: # not hashable
return []
return [arg]
def _deps(dsk, arg):
""" Get dependencies from keys or tasks
Helper function for get_dependencies.
Examples
--------
>>> inc = lambda x: x + 1
>>> add = lambda x, y: x + y
>>> dsk = {'x': 1, 'y': 2}
>>> _deps(dsk, 'x')
['x']
>>> _deps(dsk, (add, 'x', 1))
['x']
>>> _deps(dsk, ['x', 'y'])
['x', 'y']
>>> _deps(dsk, {'name': 'x'})
['x']
>>> _deps(dsk, (add, 'x', (inc, 'y'))) # doctest: +SKIP
['x', 'y']
"""
if istask(arg):
result = []
for a in arg[1:]:
result.extend(_deps(dsk, a))
return result
if isinstance(arg, list):
return sum([_deps(dsk, a) for a in arg], [])
if isinstance(arg, dict):
return sum([_deps(dsk, v) for v in arg.values()], [])
try:
if arg not in dsk:
return []
except TypeError: # not hashable
return []
return [arg]
def _maybe_complex(task):
""" Possibly contains a nested task """
return (istask(task) or
type(task) is list and any(map(_maybe_complex, task)) or
type(task) is dict and any(map(_maybe_complex, task.values())))
def _maybe_complex(task):
""" Possibly contains a nested task """
return (istask(task)
or isinstance(task, list) and any(map(_maybe_complex, task)) or
isinstance(task, dict) and any(map(_maybe_complex, task.values())))
def _maybe_complex(task):
""" Possibly contains a nested task """
return (istask(task) or
isinstance(task, list) and any(map(_maybe_complex, task)) or
isinstance(task, dict) and any(map(_maybe_complex, task.values())))
