def optimize(dsk, keys, **kwargs):
flatkeys = list(flatten(keys)) if isinstance(keys, list) else [keys]
dsk, dependencies = cull(dsk, flatkeys)
dsk, dependencies = fuse(dsk, keys, dependencies=dependencies,
ave_width=_globals.get('fuse_ave_width', 1))
dsk, _ = cull(dsk, keys)
return dsk
def inlined_array(a, inline_arrays=None):
""" Flatten underlying graph """
agraph = a.__dask_graph__()
akeys = set(flatten(a.__dask_keys__()))
# Inline everything except the output keys
if inline_arrays is None:
inline_keys = set(agraph.keys()) - akeys
dsk2 = inline(agraph, keys=inline_keys, inline_constants=True)
dsk3, _ = cull(dsk2, akeys)
graph = HighLevelGraph.from_collections(a.name, dsk3, [])
return da.Array(graph, a.name, a.chunks, dtype=a.dtype)
# We're given specific arrays to inline, promote to list
if isinstance(inline_arrays, da.Array):
inline_arrays = [inline_arrays]
elif isinstance(inline_arrays, tuple):
inline_arrays = list(inline_arrays)
if not isinstance(inline_arrays, list):
raise TypeError("Invalid inline_arrays, must be "
"(None, list, tuple, dask.array.Array)")
inline_names = set(a.name for a in inline_arrays)
layers = agraph.layers.copy()
deps = {k: v.copy() for k, v in agraph.dependencies.items()}
# We want to inline layers that depend on the inlined arrays
inline_layers = set(k for k, v in deps.items()
if len(inline_names.intersection(v)) > 0)
for layer_name in inline_layers:
dsk = dict(layers[layer_name])
layer_keys = set(dsk.keys())
inline_keys = set()
for array in inline_arrays:
dsk.update(layers[array.name])
deps.pop(array.name, None)
deps[layer_name].discard(array.name)
inline_keys.update(layers[array.name].keys())
dsk2 = inline(dsk, keys=inline_keys, inline_constants=True)
layers[layer_name], _ = cull(dsk2, layer_keys)
# Remove layers containing the inlined arrays
for inline_name in inline_names:
layers.pop(inline_name)
return da.Array(HighLevelGraph(layers, deps), a.name, a.chunks, a.dtype)
def inlined_array(a, inline_arrays=None):
""" Flatten underlying graph """
agraph = a.__dask_graph__()
akeys = set(flatten(a.__dask_keys__()))
# Inline everything except the output keys
if inline_arrays is None:
inline_keys = set(agraph.keys()) - akeys
dsk2 = inline(agraph, keys=inline_keys, inline_constants=True)
dsk3, _ = cull(dsk2, akeys)
graph = HighLevelGraph.from_collections(a.name, dsk3, [])
return da.Array(graph, a.name, a.chunks, dtype=a.dtype)
# We're given specific arrays to inline, promote to list
if isinstance(inline_arrays, da.Array):
inline_arrays = [inline_arrays]
elif isinstance(inline_arrays, tuple):
inline_arrays = list(inline_arrays)
if not isinstance(inline_arrays, list):
raise TypeError("Invalid inline_arrays, must be "
"(None, list, tuple, dask.array.Array)")
layers = agraph.layers.copy()
deps = agraph.dependencies.copy()
inline_keys = set()
dsk = dict(layers[a.name])
# Inline specified arrays
for array in inline_arrays:
# Remove array from layers and dependencies
try:
dsk.update(layers.pop(array.name))
del deps[array.name]
except KeyError:
raise ValueError("%s is not a valid dependency of a"
% array.name)
# Record keys to inline
inline_keys.update(flatten(array.__dask_keys__()))
dsk2 = inline(dsk, keys=inline_keys, inline_constants=True)
dsk3, _ = cull(dsk2, akeys)
layers[a.name] = dsk3
graph = HighLevelGraph(layers, deps)
return da.Array(graph, a.name, a.chunks, a.dtype)
def cached_array(array):
"""
Return a new array that functionally has the same values as array,
but flattens the underlying graph and introduces a cache lookup
when the individual array chunks are accessed.
Useful for caching data that can fit in-memory for the duration
of the graph's execution.
"""
dsk = dict(array.__dask_graph__())
keys = set(flatten(array.__dask_keys__()))
# Inline + cull everything except the current array
inline_keys = set(dsk.keys() - keys)
dsk2 = inline(dsk, inline_keys, inline_constants=True)
dsk3, _ = cull(dsk2, keys)
# Create a cache used to store array values
cache = ArrayCache(uuid.uuid4().hex)
for k in keys:
dsk3[k] = (cache_entry, cache, Key(k), dsk3.pop(k))
graph = HighLevelGraph.from_collections(array.name, dsk3, [])
return da.Array(graph, array.name, array.chunks, array.dtype)
def optimize(dsk, keys, **kwargs):
if not isinstance(keys, (list, set)):
keys = [keys]
keys = list(core.flatten(keys))
if not isinstance(dsk, HighLevelGraph):
dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
else:
# Perform Blockwise optimizations for HLG input
dsk = optimize_dataframe_getitem(dsk, keys=keys)
dsk = optimize_blockwise(dsk, keys=keys)
dsk = fuse_roots(dsk, keys=keys)
dsk = dsk.cull(set(keys))
# Do not perform low-level fusion unless the user has
# specified True explicitly. The configuration will
# be None by default.
if not config.get("optimization.fuse.active"):
return dsk
dependencies = dsk.get_all_dependencies()
dsk = ensure_dict(dsk)
fuse_subgraphs = config.get("optimization.fuse.subgraphs")
if fuse_subgraphs is None:
fuse_subgraphs = True
dsk, _ = fuse(
dsk,
keys,
dependencies=dependencies,
fuse_subgraphs=fuse_subgraphs,
)
dsk, _ = cull(dsk, keys)
return dsk
def inline_pattern(dsk: dict, pat_ls: List[str], inline_constants: bool) -> dict:
"""
Inline tasks whose keys match certain patterns.
Parameters
----------
dsk : dict
Input dask graph.
pat_ls : List[str]
List of patterns to check.
inline_constants : bool
Whether to inline constants.
Returns
-------
dsk : dict
Dask graph with keys inlined.
See Also
-------
dask.optimization.inline
"""
keys = [k for k in dsk.keys() if check_pat(k, pat_ls)]
if keys:
dsk = inline(dsk, keys, inline_constants=inline_constants)
for k in keys:
del dsk[k]
if inline_constants:
dsk, dep = cull(dsk, set(list(flatten(keys))))
return dsk
def get(self, key=None):
"""Execute and return the result of the computation *key*.
Only *key* and its dependencies are computed.
Parameters
----------
key : str, optional
If not provided, :attr:`default_key` is used.
Raises
------
ValueError
If `key` and :attr:`default_key` are both :obj:`None`.
"""
if key is None:
if self.default_key is not None:
key = self.default_key
else:
raise ValueError('no default reporting key set')
# Cull the graph, leaving only those needed to compute *key*
dsk, deps = cull(self.graph, key)
log.debug('Cull {} -> {} keys'.format(len(self.graph), len(dsk)))
try:
return dask_get(dsk, key)
except Exception as exc:
raise ComputationError from exc
def test_SubgraphCallable():
non_hashable = [1, 2, 3]
dsk = {
'a': (apply, add, ['in1', 2]),
'b': (apply, partial_by_order, ['in2'], {
'function': func_with_kwargs,
'other': [(1, 20)],
'c': 4
}),
'c': (apply, partial_by_order, ['in2', 'in1'], {
'function': func_with_kwargs,
'other': [(1, 20)]
}),
'd': (inc, 'a'),
'e': (add, 'c', 'd'),
'f': ['a', 2, 'b', (add, 'b', (sum, non_hashable))],
'h': (add, (sum, 'f'), (sum, ['a', 'b']))
}
f = SubgraphCallable(dsk, 'h', ['in1', 'in2'], name='test')
assert f.name == 'test'
assert repr(f) == 'test'
dsk2 = dsk.copy()
dsk2.update({'in1': 1, 'in2': 2})
assert f(1, 2) == get_sync(cull(dsk2, ['h'])[0], ['h'])[0]
assert f(1, 2) == f(1, 2)
f2 = pickle.loads(pickle.dumps(f))
assert f2(1, 2) == f(1, 2)
def test_SubgraphCallable():
non_hashable = [1, 2, 3]
dsk = {'a': (apply, add, ['in1', 2]),
'b': (apply, partial_by_order, ['in2'],
{'function': func_with_kwargs, 'other': [(1, 20)], 'c': 4}),
'c': (apply, partial_by_order, ['in2', 'in1'],
{'function': func_with_kwargs, 'other': [(1, 20)]}),
'd': (inc, 'a'),
'e': (add, 'c', 'd'),
'f': ['a', 2, 'b', (add, 'b', (sum, non_hashable))],
'g': (dontcall, 'in1'),
'h': (add, (sum, 'f'), (sum, ['a', 'b']))}
f = SubgraphCallable(dsk, 'h', ['in1', 'in2'], name='test')
assert f.name == 'test'
assert repr(f) == 'test'
dsk2 = dsk.copy()
dsk2.update({'in1': 1, 'in2': 2})
assert f(1, 2) == get_sync(cull(dsk2, ['h'])[0], ['h'])[0]
assert f(1, 2) == f(1, 2)
f2 = pickle.loads(pickle.dumps(f))
assert f2(1, 2) == f(1, 2)
def test_SubgraphCallable():
non_hashable = [1, 2, 3]
dsk = {
"a": (apply, add, ["in1", 2]),
"b": (
apply,
partial_by_order,
["in2"],
{"function": func_with_kwargs, "other": [(1, 20)], "c": 4},
),
"c": (
apply,
partial_by_order,
["in2", "in1"],
{"function": func_with_kwargs, "other": [(1, 20)]},
),
"d": (inc, "a"),
"e": (add, "c", "d"),
"f": ["a", 2, "b", (add, "b", (sum, non_hashable))],
"h": (add, (sum, "f"), (sum, ["a", "b"])),
}
f = SubgraphCallable(dsk, "h", ["in1", "in2"], name="test")
assert f.name == "test"
assert repr(f) == "test"
dsk2 = dsk.copy()
dsk2.update({"in1": 1, "in2": 2})
assert f(1, 2) == get_sync(cull(dsk2, ["h"])[0], ["h"])[0]
assert f(1, 2) == f(1, 2)
f2 = pickle.loads(pickle.dumps(f))
assert f2(1, 2) == f(1, 2)
def test_shuffle_hlg_layer():
# This test checks that the `ShuffleLayer` HLG Layer
# is used (as expected) for a multi-stage shuffle.
ddf = dd.from_pandas(pd.DataFrame({"a": np.random.randint(0, 10, 100)}),
npartitions=10)
ddf_shuffled = ddf.shuffle("a", max_branch=3, shuffle="tasks")
keys = [(ddf_shuffled._name, i) for i in range(ddf_shuffled.npartitions)]
# Make sure HLG culling reduces the graph size
dsk = ddf_shuffled.__dask_graph__()
dsk_culled = dsk.cull(set(keys))
assert len(dsk_culled) < len(dsk)
assert isinstance(dsk_culled, dask.highlevelgraph.HighLevelGraph)
# Ensure we have ShuffleLayers
assert any(
isinstance(layer, dd.shuffle.ShuffleLayer)
for layer in dsk.layers.values())
# Check ShuffleLayer names
for name, layer in dsk.layers.items():
if isinstance(layer, dd.shuffle.ShuffleLayer):
assert name.startswith("shuffle-")
# Since we already culled the HLG,
# culling the dictionary should not change the graph
dsk_dict = dict(dsk_culled)
dsk_dict_culled, _ = cull(dsk_dict, keys)
assert dsk_dict_culled == dsk_dict
def test_fuse_getitem():
def load(*args):
pass
dsk = {"x": (load, "store", "part", ["a", "b"]), "y": (getitem, "x", "a")}
dsk2 = fuse_getitem(dsk, load, 3)
dsk2, dependencies = cull(dsk2, "y")
assert dsk2 == {"y": (load, "store", "part", "a")}
def test_fuse_getitem():
def load(*args):
pass
dsk = {'x': (load, 'store', 'part', ['a', 'b']),
'y': (getitem, 'x', 'a')}
dsk2 = fuse_getitem(dsk, load, 3)
dsk2, dependencies = cull(dsk2, 'y')
assert dsk2 == {'y': (load, 'store', 'part', 'a')}
def _generate_dask_graph(data, keys):
"""
Generate a dask graph from a subset of REGISTRY.
"""
tasks = cull(REGISTRY, keys)[0]
for k in unresolved(tasks, keys):
tasks[k] = data[k]
return tasks
def test_fuse_getitem():
def load(*args):
pass
dsk = {'x': (load, 'store', 'part', ['a', 'b']),
'y': (getitem, 'x', 'a')}
dsk2 = fuse_getitem(dsk, load, 3)
dsk2, dependencies = cull(dsk2, 'y')
assert dsk2 == {'y': (load, 'store', 'part', 'a')}
def test_fuse_selections():
def load(*args):
pass
dsk = {'x': (load, 'store', 'part', ['a', 'b']),
'y': (getitem, 'x', 'a')}
merge = lambda t1, t2: (load, t2[1], t2[2], t1[2])
dsk2 = fuse_selections(dsk, getitem, load, merge)
dsk2, dependencies = cull(dsk2, 'y')
assert dsk2 == {'y': (load, 'store', 'part', 'a')}
def test_inline_cull_dependencies():
d = {'a': 1,
'b': 'a',
'c': 'b',
'd': ['a', 'b', 'c'],
'e': (add, (len, 'd'), 'a')}
d2, dependencies = cull(d, ['d', 'e'])
inline(d2, {'b'}, dependencies=dependencies)
def test_inline_cull_dependencies():
d = {'a': 1,
'b': 'a',
'c': 'b',
'd': ['a', 'b', 'c'],
'e': (add, (len, 'd'), 'a')}
d2, dependencies = cull(d, ['d', 'e'])
inline(d2, {'b'}, dependencies=dependencies)
def test_fuse_selections():
def load(*args):
pass
dsk = {'x': (load, 'store', 'part', ['a', 'b']),
'y': (getitem, 'x', 'a')}
merge = lambda t1, t2: (load, t2[1], t2[2], t1[2])
dsk2 = fuse_selections(dsk, getitem, load, merge)
dsk2, dependencies = cull(dsk2, 'y')
assert dsk2 == {'y': (load, 'store', 'part', 'a')}
def test_fuse_selections():
def load(*args):
pass
dsk = {"x": (load, "store", "part", ["a", "b"]), "y": (getitem, "x", "a")}
merge = lambda t1, t2: (load, t2[1], t2[2], t1[2])
dsk2 = fuse_selections(dsk, getitem, load, merge)
dsk2, dependencies = cull(dsk2, "y")
assert dsk2 == {"y": (load, "store", "part", "a")}
def __dask_optimize__(dsk, keys, **kwargs):
"""
Notes
-----
The dask default optimizer induces too many (unnecesarry)
IO calls -- we turn this off feature off by default, and only apply a culling.
"""
from dask.optimization import cull
dsk2, dependencies = cull(dsk, keys)
return dsk2
def test_inline_cull_dependencies():
d = {
"a": 1,
"b": "a",
"c": "b",
"d": ["a", "b", "c"],
"e": (add, (len, "d"), "a")
}
d2, dependencies = cull(d, ["d", "e"])
inline(d2, {"b"}, dependencies=dependencies)
def get(graph, key, local=True):
# new function because from version 19.1 on, dask no longer culls graphs (apparently it's only done by distributed)
# so the function combines a get call with previous culling (which means to modify a graph in such a way that only the relevant bit to the key remains)
import dask
from dask.optimization import cull
cgraph = cull(graph, key)[0]
# with ProgressBar():
if local:
return dask.local.get_sync(cgraph, key)
else:
return dask.threaded.get(cgraph, key)
def test_cull():
# 'out' depends on 'x' and 'y', but not 'z'
d = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'x'), 'out': (add, 'y', 10)}
culled, dependencies = cull(d, 'out')
assert culled == {'x': 1, 'y': (inc, 'x'), 'out': (add, 'y', 10)}
assert dependencies == {'x': [], 'y': ['x'], 'out': ['y']}
assert cull(d, 'out') == cull(d, ['out'])
assert cull(d, ['out', 'z'])[0] == d
assert cull(d, [['out'], ['z']]) == cull(d, ['out', 'z'])
pytest.raises(KeyError, lambda: cull(d, 'badkey'))
def test_cull():
# 'out' depends on 'x' and 'y', but not 'z'
d = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'x'), 'out': (add, 'y', 10)}
culled, dependencies = cull(d, 'out')
assert culled == {'x': 1, 'y': (inc, 'x'), 'out': (add, 'y', 10)}
assert dependencies == {'x': [], 'y': ['x'], 'out': ['y']}
assert cull(d, 'out') == cull(d, ['out'])
assert cull(d, ['out', 'z'])[0] == d
assert cull(d, [['out'], ['z']]) == cull(d, ['out', 'z'])
pytest.raises(KeyError, lambda: cull(d, 'badkey'))
def test_cull():
# 'out' depends on 'x' and 'y', but not 'z'
d = {"x": 1, "y": (inc, "x"), "z": (inc, "x"), "out": (add, "y", 10)}
culled, dependencies = cull(d, "out")
assert culled == {"x": 1, "y": (inc, "x"), "out": (add, "y", 10)}
assert dependencies == {"x": [], "y": ["x"], "out": ["y"]}
assert cull(d, "out") == cull(d, ["out"])
assert cull(d, ["out", "z"])[0] == d
assert cull(d, [["out"], ["z"]]) == cull(d, ["out", "z"])
pytest.raises(KeyError, lambda: cull(d, "badkey"))
def __dask_optimize__(dsk, keys, **kwargs):
"""
Optimize the dask object.
.. note::
The dask default optimizer induces too many (unnecesarry)
IO calls. We turn this feature off by default, and only apply a culling.
"""
from dask.optimization import cull
dsk2, dependencies = cull(dsk, keys)
return dsk2
def __dask_optimize__(cls, dsk, keys):
dsk1, _ = optimization.cull(dsk, keys)
dsk2 = {}
coll = []
for key, val in dsk1.items():
if isinstance(key, tuple) and key[0].startswith('image'):
name, z, x, y = key
dfn, url, token, chunk = val
dsk2[key] = (operator.getitem, "load_urls", (z, x, y))
coll.append([url, token, (z, x, y)])
else:
dsk2[key] = val
dsk2['load_urls'] = (cls.__fetch__, coll)
return dsk2
def __dask_optimize__(cls, dsk, keys):
dsk1, _ = optimization.cull(dsk, keys)
dsk2 = {}
coll = []
for key, val in dsk1.items():
if isinstance(key, tuple) and key[0].startswith('image'):
name, z, x, y = key
dfn, url, token, chunk = val
dsk2[key] = (operator.getitem, "load_urls", (z, x, y))
coll.append([url, token, (z, x, y)])
else:
dsk2[key] = val
dsk2['load_urls'] = (cls.__fetch__, coll)
return dsk2
def get(self, key=None):
"""Execute and return the result of the computation *key*.
Only *key* and its dependencies are computed.
Parameters
----------
key : str, optional
If not provided, :attr:`default_key` is used.
Raises
------
ValueError
If `key` and :attr:`default_key` are both :obj:`None`.
"""
if key is None:
if self.default_key is not None:
key = self.default_key
else:
raise ValueError('no default reporting key set')
# Cull the graph, leaving only those needed to compute *key*
dsk, deps = cull(self.graph, key)
log.debug('Cull {} -> {} keys'.format(len(self.graph), len(dsk)))
try:
# Protect 'config' dict, so that dask schedulers do not try to
# interpret its contents as further tasks. Workaround for
# https://github.com/dask/dask/issues/3523
dsk['config'] = dask.core.quote(dsk['config'])
except KeyError:
pass
try:
return dask_get(dsk, key)
except Exception as exc:
# Print the exception in case ComputationError.__str__ fails;
# workaround for https://github.com/iiasa/ixmp/issues/206
print(exc)
raise ComputationError from exc
def cached_array(array, token=None):
"""
Return a new array that functionally has the same values as array,
but flattens the underlying graph and introduces a cache lookup
when the individual array chunks are accessed.
Useful for caching data that can fit in-memory for the duration
of the graph's execution.
Parameters
----------
array : :class:`dask.array.Array`
dask array to cache.
token : optional, str
A unique token for identifying the internal cache.
If None, it will be automatically generated.
"""
dsk = dict(array.__dask_graph__())
keys = set(flatten(array.__dask_keys__()))
if token is None:
token = uuid.uuid4().hex
# Inline + cull everything except the current array
inline_keys = set(dsk.keys() - keys)
dsk2 = inline(dsk, inline_keys, inline_constants=True)
dsk3, _ = cull(dsk2, keys)
# Create a cache used to store array values
cache = ArrayCache(token)
assert len(dsk3) == len(keys)
for k in keys:
dsk3[k] = (cache_entry, cache, Key(k), dsk3.pop(k))
graph = HighLevelGraph.from_collections(array.name, dsk3, [])
return da.Array(graph, array.name, array.chunks, array.dtype)
def optimize(dsk, keys, **kwargs):
dsk, _ = cull(dsk, keys)
return dsk
def cull2(dsk, keys): # type: ignore
return cull(dsk if type(dsk) is dict else dict(dsk), keys)
def from_darray(cls, darr, new=tuple.__new__, len=len):
dsk, _ = optimization.cull(darr.dask, darr.__dask_keys__())
itr = [dsk, darr.name, darr.chunks, darr.dtype, darr.shape]
return cls._make(itr)
def warp(self, dem=None, proj="EPSG:4326", **kwargs):
"""Delayed warp across an entire AOI or Image
Creates a new dask image by deferring calls to the warp_geometry on chunks
Args:
dem (ndarray): optional. A DEM for warping to specific elevation planes
proj (str): optional. An EPSG proj string to project the image data into ("EPSG:32612")
Returns:
daskarray: a warped image as deferred image array
"""
try:
img_md = self.rda.metadata["image"]
x_size = img_md["tileXSize"]
y_size = img_md["tileYSize"]
except (AttributeError, KeyError):
x_size = kwargs.get("chunk_size", 256)
y_size = kwargs.get("chunk_size", 256)
# Create an affine transform to convert between real-world and pixels
if self.proj is None:
from_proj = "EPSG:4326"
else:
from_proj = self.proj
try:
# NOTE: this only works on images that have rda rpcs metadata
center = wkt.loads(self.rda.metadata["image"]["imageBoundsWGS84"]).centroid
g = box(*(center.buffer(self.rda.metadata["rpcs"]["gsd"] / 2).bounds))
tfm = partial(pyproj.transform, pyproj.Proj(init="EPSG:4326"), pyproj.Proj(init=proj))
gsd = kwargs.get("gsd", ops.transform(tfm, g).area ** 0.5)
current_bounds = wkt.loads(self.rda.metadata["image"]["imageBoundsWGS84"]).bounds
except (AttributeError, KeyError, TypeError):
tfm = partial(pyproj.transform, pyproj.Proj(init=self.proj), pyproj.Proj(init=proj))
gsd = kwargs.get("gsd", (ops.transform(tfm, shape(self)).area / (self.shape[1] * self.shape[2])) ** 0.5 )
current_bounds = self.bounds
tfm = partial(pyproj.transform, pyproj.Proj(init=from_proj), pyproj.Proj(init=proj))
itfm = partial(pyproj.transform, pyproj.Proj(init=proj), pyproj.Proj(init=from_proj))
output_bounds = ops.transform(tfm, box(*current_bounds)).bounds
gtf = Affine.from_gdal(output_bounds[0], gsd, 0.0, output_bounds[3], 0.0, -1 * gsd)
ll = ~gtf * (output_bounds[:2])
ur = ~gtf * (output_bounds[2:])
x_chunks = int((ur[0] - ll[0]) / x_size) + 1
y_chunks = int((ll[1] - ur[1]) / y_size) + 1
num_bands = self.shape[0]
try:
dtype = RDA_TO_DTYPE[img_md["dataType"]]
except:
dtype = 'uint8'
daskmeta = {
"dask": {},
"chunks": (num_bands, y_size, x_size),
"dtype": dtype,
"name": "warp-{}".format(self.name),
"shape": (num_bands, y_chunks * y_size, x_chunks * x_size)
}
def px_to_geom(xmin, ymin):
xmax = int(xmin + x_size)
ymax = int(ymin + y_size)
bounds = list((gtf * (xmin, ymax)) + (gtf * (xmax, ymin)))
return box(*bounds)
full_bounds = box(*output_bounds)
dasks = []
if isinstance(dem, GeoDaskImage):
if dem.proj != proj:
dem = dem.warp(proj=proj, dem=dem)
dasks.append(dem.dask)
for y in xrange(y_chunks):
for x in xrange(x_chunks):
xmin = x * x_size
ymin = y * y_size
geometry = px_to_geom(xmin, ymin)
daskmeta["dask"][(daskmeta["name"], 0, y, x)] = (self._warp, geometry, gsd, dem, proj, dtype, 5)
daskmeta["dask"], _ = optimization.cull(HighLevelGraph.merge(daskmeta["dask"], *dasks), list(daskmeta["dask"].keys()))
gi = mapping(full_bounds)
gt = AffineTransform(gtf, proj)
image = GeoDaskImage(daskmeta, __geo_interface__ = gi, __geo_transform__ = gt)
return image[box(*output_bounds)]
def featurize_single_ts(ts,
features_to_use,
custom_script_path=None,
custom_functions=None,
raise_exceptions=True):
"""Compute feature values for a given single time-series. Data is
returned as dictionaries/lists of lists.
Parameters
----------
ts : TimeSeries object
Single time series to be featurized.
features_to_use : list of str
List of feature names to be generated.
custom_functions : dict, optional
Dictionary of custom feature functions to be evaluated for the given
time series, or a dictionary representing a dask graph of function
evaluations. Dictionaries of functions should have keys `feature_name`
and values functions that take arguments (t, m, e); in the case of a
dask graph, these arrays should be referenced as 't', 'm', 'e',
respectively, and any values with keys present in `features_to_use`
will be computed.
raise_exceptions : bool, optional
If True, exceptions during feature computation are raised immediately;
if False, exceptions are supressed and `np.nan` is returned for the
given feature and any dependent features. Defaults to True.
Returns
-------
dict
Dictionary with feature names as keys, lists of feature values (one per
channel) as values.
"""
# Initialize empty feature array for all channels
feature_values = np.empty((len(features_to_use), ts.n_channels))
for (t_i, m_i, e_i), i in zip(ts.channels(), range(ts.n_channels)):
feature_graph = generate_dask_graph(t_i, m_i, e_i)
feature_graph.update(ts.meta_features)
if custom_functions:
# If values in custom_functions are functions, add calls to graph
if all(hasattr(v, '__call__') for v in custom_functions.values()):
feature_graph.update({
feat: f(t_i, m_i, e_i)
for feat, f in custom_functions.items()
})
# Otherwise, custom_functions is another dask graph
else:
feature_graph.update(custom_functions)
# Do not execute in parallel; parallelization has already taken place
# at the level of time series, so we compute features for a single time
# series in serial.
if raise_exceptions:
raise_callback = reraise
else:
raise_callback = lambda e, tb: None
culled_feature_graph, _ = cull(feature_graph, features_to_use)
dask_values = dask.get(culled_feature_graph,
features_to_use,
raise_exception=raise_callback,
pack_exception=pack_exception)
feature_values[:, i] = [
x if not isinstance(x, Exception) else np.nan for x in dask_values
]
index = pd.MultiIndex.from_product((features_to_use, range(ts.n_channels)),
names=('feature', 'channel'))
return pd.Series(feature_values.ravel(), index=index)
def warp(self, dem=None, proj="EPSG:4326", **kwargs):
"""Delayed warp across an entire AOI or Image
Creates a new dask image by deferring calls to the warp_geometry on chunks
Args:
dem (ndarray): optional. A DEM for warping to specific elevation planes
proj (str): optional. An EPSG proj string to project the image data into ("EPSG:32612")
Returns:
daskarray: a warped image as deferred image array
"""
try:
img_md = self.rda.metadata["image"]
x_size = img_md["tileXSize"]
y_size = img_md["tileYSize"]
except (AttributeError, KeyError):
x_size = kwargs.get("chunk_size", 256)
y_size = kwargs.get("chunk_size", 256)
# Create an affine transform to convert between real-world and pixels
if self.proj is None:
from_proj = "EPSG:4326"
else:
from_proj = self.proj
try:
# NOTE: this only works on images that have rda rpcs metadata
center = wkt.loads(
self.rda.metadata["image"]["imageBoundsWGS84"]).centroid
g = box(*(center.buffer(self.rda.metadata["rpcs"]["gsd"] /
2).bounds))
tfm = partial(pyproj.transform, pyproj.Proj(init="EPSG:4326"),
pyproj.Proj(init=proj))
gsd = kwargs.get("gsd", ops.transform(tfm, g).area**0.5)
current_bounds = wkt.loads(
self.rda.metadata["image"]["imageBoundsWGS84"]).bounds
except (AttributeError, KeyError, TypeError):
tfm = partial(pyproj.transform, pyproj.Proj(init=self.proj),
pyproj.Proj(init=proj))
gsd = kwargs.get("gsd", (ops.transform(tfm, shape(self)).area /
(self.shape[1] * self.shape[2]))**0.5)
current_bounds = self.bounds
tfm = partial(pyproj.transform, pyproj.Proj(init=from_proj),
pyproj.Proj(init=proj))
itfm = partial(pyproj.transform, pyproj.Proj(init=proj),
pyproj.Proj(init=from_proj))
output_bounds = ops.transform(tfm, box(*current_bounds)).bounds
gtf = Affine.from_gdal(output_bounds[0], gsd, 0.0, output_bounds[3],
0.0, -1 * gsd)
ll = ~gtf * (output_bounds[:2])
ur = ~gtf * (output_bounds[2:])
x_chunks = int((ur[0] - ll[0]) / x_size) + 1
y_chunks = int((ll[1] - ur[1]) / y_size) + 1
num_bands = self.shape[0]
try:
dtype = RDA_TO_DTYPE[img_md["dataType"]]
except:
dtype = 'uint8'
daskmeta = {
"dask": {},
"chunks": (num_bands, y_size, x_size),
"dtype": dtype,
"name": "warp-{}".format(self.name),
"shape": (num_bands, y_chunks * y_size, x_chunks * x_size)
}
def px_to_geom(xmin, ymin):
xmax = int(xmin + x_size)
ymax = int(ymin + y_size)
bounds = list((gtf * (xmin, ymax)) + (gtf * (xmax, ymin)))
return box(*bounds)
full_bounds = box(*output_bounds)
dasks = []
if isinstance(dem, GeoDaskImage):
if dem.proj != proj:
dem = dem.warp(proj=proj, dem=dem)
dasks.append(dem.dask)
for y in xrange(y_chunks):
for x in xrange(x_chunks):
xmin = x * x_size
ymin = y * y_size
geometry = px_to_geom(xmin, ymin)
daskmeta["dask"][(daskmeta["name"], 0, y,
x)] = (self._warp, geometry, gsd, dem, proj,
dtype, 5)
daskmeta["dask"], _ = optimization.cull(
sharedict.merge(daskmeta["dask"], *dasks),
list(daskmeta["dask"].keys()))
gi = mapping(full_bounds)
gt = AffineTransform(gtf, proj)
image = GeoDaskImage(daskmeta,
__geo_interface__=gi,
__geo_transform__=gt)
return image[box(*output_bounds)]
def from_darray(cls, darr, new=tuple.__new__, len=len):
dsk, _ = optimization.cull(darr.dask, darr.__dask_keys__())
itr = [dsk, darr.name, darr.chunks, darr.dtype, darr.shape]
return cls._make(itr)