def test_dont_concatenate_single_chunks(shape, chunks): x = da.ones(shape, chunks=shape) y = x.rechunk(chunks) dsk = dict(y.dask) assert not any(funcname(task[0]).startswith('concat') for task in dsk.values() if dask.istask(task))
def test_dont_concatenate_single_chunks(shape, chunks): x = da.ones(shape, chunks=shape) y = x.rechunk(chunks) dsk = dict(y.dask) assert not any( funcname(task[0]).startswith("concat") for task in dsk.values() if dask.istask(task) )
def _maybe_complex(task): """ Possibly contains a nested task """ return ( istask(task) or type(task) is list and any(map(_maybe_complex, task)) or type(task) is dict and any(map(_maybe_complex, task.values())) )
def _hash(self, daskkey: str) -> str: try: return self._hashes[daskkey] except KeyError: obj = self._dsk[daskkey] if dask.istask(obj): self._hashes[daskkey] = self._hashtask(obj) else: self._hashes[daskkey] = self._md5hashpickle(obj) return self._hashes[daskkey]
def convert(task): if type(task) is list: return [convert(v) for v in task] if type(task) is dict: return valmap(convert, task) if istask(task): return (task[0], ) + tuple(map(convert, task[1:])) try: if task in dsk or task in extra_values: return tokey(task) except TypeError: pass return task
def convert(task): if type(task) is list: return [convert(v) for v in task] if type(task) is dict: return valmap(convert, task) if istask(task): return (task[0],) + tuple(map(convert, task[1:])) try: if task in dsk: return tokey(task) except TypeError: pass return task
def _hashtask(self, obj: Any) -> str: md5 = hashlib.md5() for o in obj: if isinstance(o, str) and o in self._dsk: h = self._hash(o) elif callable(o): h = self._get_callable_src(o) elif isinstance(o, list) or dask.istask(o): h = self._hashtask(o) else: h = self._md5hashpickle(o) md5.update(h.encode()) return md5.hexdigest()
def convert(task): if isinstance(task, list): return [convert(v) for v in task] if isinstance(task, dict): return valmap(convert, task) if istask(task): return (task[0], ) + tuple(map(convert, task[1:])) try: if task in dsk: return tokey(task) except TypeError: pass return task
def convert(task, dsk, extra_values): if type(task) is list: return [convert(v, dsk, extra_values) for v in task] if type(task) is dict: return {k: convert(v, dsk, extra_values) for k, v in task.items()} if istask(task): return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:]) try: if task in dsk or task in extra_values: return tokey(task) except TypeError: pass return task
def delayedoptimize( dsk: Mapping[str, Any], keys: Optional[Union[str, Iterable[str]]] = None, cache: Optional[Cache] = None, ) -> Mapping[str, Any]: hashes = dsktohash(dsk) result = {} for key in dsk: if cache is not None and hashes[key] in cache: result[key] = (lambda k: cache[k], hashes[key]) # type: ignore else: c = dsk[key] if cache is not None and dask.istask(c): result[key] = (CacheAfterExecution(c[0], cache, hashes[key]), ) + c[1:] else: result[key] = dsk[key] return result
def _deps(dsk, arg): """ Get dependencies from keys or tasks Helper function for get_dependencies. Examples -------- >>> inc = lambda x: x + 1 >>> add = lambda x, y: x + y >>> dsk = {'x': 1, 'y': 2} >>> _deps(dsk, 'x') ['x'] >>> _deps(dsk, (add, 'x', 1)) ['x'] >>> _deps(dsk, ['x', 'y']) ['x', 'y'] >>> _deps(dsk, {'name': 'x'}) ['x'] >>> _deps(dsk, (add, 'x', (inc, 'y'))) # doctest: +SKIP ['x', 'y'] """ if istask(arg): result = [] for a in arg[1:]: result.extend(_deps(dsk, a)) return result if isinstance(arg, list): return sum([_deps(dsk, a) for a in arg], []) if isinstance(arg, dict): return sum([_deps(dsk, v) for v in arg.values()], []) try: if arg not in dsk: return [] except TypeError: # not hashable return [] return [arg]
def _maybe_complex(task): """ Possibly contains a nested task """ return (istask(task) or type(task) is list and any(map(_maybe_complex, task)) or type(task) is dict and any(map(_maybe_complex, task.values())))
def _maybe_complex(task): """ Possibly contains a nested task """ return (istask(task) or isinstance(task, list) and any(map(_maybe_complex, task)) or isinstance(task, dict) and any(map(_maybe_complex, task.values())))