def _merge_vals_by_time(multi, times, kwargs):
""" Merge chunks using indices. """
# determine the unique times and assign result bands
sorted_times = Group._unique_times(times)
bands = dict((y, x) for x, y in enumerate(sorted_times))
fillvalue = get_dtype_max(kwargs["dtype"])
# initialize values array
shape = (len(sorted_times), ) + multi[0]["values"].shape[1:]
values = np.full(shape, fillvalue, dtype=kwargs["dtype"])
# populate values array
for data, time in zip(multi, times):
# source_index is the index into the source array
for source_index, datetime in enumerate(time["time"]):
source_band = data["values"][source_index]
# determine data index
index = get_index(values=source_band,
no_data_value=data["no_data_value"])
# find corresponding target band by source datetime
target_band = values[bands[datetime]]
# paste source into target provided there is data
target_band[index] = source_band[index]
# check if single band result required
start, stop = kwargs["start"], kwargs["stop"]
if stop is None and len(sorted_times) > 1:
index = Group._nearest_index(sorted_times, start)
values = values[index:index + 1]
return {"values": values, "no_data_value": fillvalue}
def process(process_kwargs, *args):
"""Combine data, filling in nodata values."""
data_list = []
no_data_values = []
for data in args:
if data is None:
continue
elif "time" in data or "meta" in data:
# return the time / meta right away. assumes there are no
# mixed requests and that time is aligned
return data
elif "values" in data and "no_data_value" in data:
data_list.append(data["values"])
no_data_values.append(data["no_data_value"])
dtype = process_kwargs["dtype"]
fillvalue = get_dtype_max(dtype)
if not data_list:
return
# initialize values array
values = np.full(data_list[0].shape, fillvalue, dtype=dtype)
# populate values array
for data, no_data_value in zip(data_list, no_data_values):
# index is where the source has data
index = get_index(data, no_data_value)
values[index] = data[index]
return {"values": values, "no_data_value": fillvalue}
def preprocess(*args):
data_list = []
no_data_values = []
for data in args:
if data is None:
continue
elif "time" in data or "meta" in data:
# return the time / meta right away. assumes there are no
# mixed requests and that time is aligned
return data
elif "values" in data and "no_data_value" in data:
data_list.append(data["values"])
no_data_values.append(data["no_data_value"])
dtype = np.result_type(*data_list)
fillvalue = get_dtype_max(dtype)
# initialize values array
values = np.full(data_list[0].shape, fillvalue, dtype=dtype)
# populate values array
multivalues = []
for data, no_data_value in zip(data_list, no_data_values):
# initialize values array
array_values = np.full(data_list[0].shape, np.nan, dtype=dtype)
# index is where the source has data
index = get_index(data, no_data_value)
array_values[index] = data[index]
multivalues.append(array_values)
return {"values": values, "no_data_value": fillvalue}, multivalues
def process(data, value):
if data is None or "values" not in data:
return data
index = get_index(values=data["values"], no_data_value=data["no_data_value"])
fillvalue = 1 if value == 0 else 0
dtype = "float32" if isinstance(value, float) else "uint8"
values = np.full_like(data["values"], fillvalue, dtype=dtype)
values[index] = value
return {"values": values, "no_data_value": fillvalue}
def process(data, value):
if data is None or "values" not in data:
return data
index = utils.get_index(values=data["values"],
no_data_value=data["no_data_value"])
fillvalue = 1 if value == 0 else 0
dtype = Mask._dtype_from_value(value)
values = np.full_like(data["values"], fillvalue, dtype=dtype)
values[index] = value
return {"values": values, "no_data_value": fillvalue}
def process(data):
if data is None or "values" not in data:
return data
index = utils.get_index(values=data["values"],
no_data_value=data["no_data_value"])
fillvalue = 255
dtype = "float32"
values = np.full_like(data["values"], fillvalue, dtype=dtype)
values[index] = random.random()
return {"values": values, "no_data_value": fillvalue}
def _merge_vals_by_bands(multi, bands, dtype):
""" Merge chunks using slices. """
# analyze band structure
starts, stops = zip(*bands)
fillvalue = get_dtype_max(dtype)
# initialize values array
shape = (max(stops), ) + multi[0]["values"].shape[1:]
values = np.full(shape, fillvalue, dtype=dtype)
# populate values array
for data, (a, b) in zip(multi, bands):
# index is where the source has data
index = get_index(values=data["values"],
no_data_value=data["no_data_value"])
values[a:b][index] = data["values"][index]
return {"values": values, "no_data_value": fillvalue}
def test_get_index(self):
index = utils.get_index(values=np.array([0, 1]), no_data_value=1)
self.assertTrue((index == np.array([True, False])).all())
def reduce_rasters(stack, statistic, no_data_value=None, dtype=None):
"""Apply a statistic (e.g. "mean") to a stack of rasters, skipping
'no data' values.
In this context, reduce means that the dimensionality of the input data
is reduced by one.
Args:
stack (list): a list of dicts containing "values" (ndarray)
and "no_data_value". If the list has zero length or if the ndarrays
do not have the same shape, a ValueError is raised.
statistic (str): the applied statistic (no data is ignored). One of:
{"last", "first", "count", "sum", "mean", "min",
"max", "argmin", "argmax", "product", "std", "var", "p<number>"}
no_data_value (number, optional): the 'no data' value in the output
array. Defaults to the no data value of the first element in the stack.
dtype (str or dtype): the datatype of the output array. Defaults to the
dtype of the first element in the stack. If the input
data cannot be cast to this dtype, a ValueError is raised.
Returns:
dict with "values" and "no_data_value"
"""
if statistic not in STATISTICS:
percentile = parse_percentile_statistic(statistic)
if percentile is None:
raise KeyError('Unknown statistic "{}"'.format(statistic))
else:
statistic = "percentile"
if len(stack) == 0:
raise ValueError("Cannot reduce a zero-length stack")
# get the output array properties (dtype, no_data_value, shape)
if dtype is None:
dtype = stack[0]["values"].dtype
if no_data_value is None:
no_data_value = stack[0]["no_data_value"]
shape = stack[0]["values"].shape
# sum, count and nans output do not contain no data: fill zeroes right away
if statistic in {"sum", "count", "nans"}:
fill_value = 0
else:
fill_value = no_data_value
# create the output array
out = np.full(shape, fill_value, dtype)
if statistic == "last":
# populate 'out' with the last value that is not 'no data'
for data in stack:
index = get_index(data["values"], data["no_data_value"])
out[index] = data["values"][index]
elif statistic == "first":
# populate 'out' with the first value that is not 'no data'
for data in stack[::-1]:
index = get_index(data["values"], data["no_data_value"])
out[index] = data["values"][index]
elif statistic == "count":
# count the number of values that are not 'no data'
for data in stack:
out += get_index(data["values"], data["no_data_value"])
else:
if statistic == "percentile":
func = partial(np.nanpercentile, q=percentile)
else:
func = STATISTICS[statistic]
# transform 'no data' into 'nan' to be able to use numpy functions
# NB: the dtype is at least float16 to accomodate NaN
stack_array = np.full((len(stack), ) + shape, np.nan,
np.result_type(dtype, np.float16))
for i, data in enumerate(stack):
index = get_index(data["values"], data["no_data_value"])
stack_array[i, index] = data["values"][index]
# protect against all-NaN slice warnings and errors
not_all_nan = ~np.all(np.isnan(stack_array), axis=0)
# perform the math
out[not_all_nan] = func(stack_array[:, not_all_nan], axis=0)
return {"values": out, "no_data_value": no_data_value}
def process(process_kwargs, *multi):
if process_kwargs["mode"] in {"meta", "time"}:
return multi[0]
if process_kwargs["mode"] == "null":
return
if process_kwargs["mode"] == "empty":
data = multi[0]
if data is None:
return
out_shape = (
len(data["time"]),
process_kwargs["height"],
process_kwargs["width"],
)
out_no_data_value = process_kwargs["fillvalue"]
out_dtype = process_kwargs["dtype"]
stack = []
elif process_kwargs["mode"] == "group":
# We have a bunch of arrays that are already shifted. Stack them.
stack = [data for data in multi if data is not None]
if len(stack) == 0:
return # instead of returning nodata (because inputs are None)
elif process_kwargs["mode"] == "warp":
# There is a single 'source' raster that we are going to shift
# multiple times into the result. The cellsize is already correct.
data = multi[0]
if data is None:
return
out_no_data_value = data["no_data_value"]
source = data["values"]
out_dtype = source.dtype
# convert the anchor to pixels (indices inside 'source')
anchor = process_kwargs["anchor"]
src_bbox = process_kwargs["src_bbox"]
size_x, size_y = process_kwargs["cellsize"]
anchor_px = (
(anchor[0] - src_bbox[0]) / size_x,
(anchor[1] - src_bbox[1]) / size_y,
)
# compute the output shape
x1, y1, x2, y2 = process_kwargs["dst_bbox"]
coordinates = process_kwargs["coordinates"]
dst_h = round((y2 - y1) / size_y)
dst_w = round((x2 - x1) / size_x)
src_d, src_h, src_w = source.shape
out_shape = (src_d, dst_h, dst_w)
# determine what indices in 'source' have data
k, j, i = np.where(get_index(source, out_no_data_value))
# place the data on each coordinate
stack = []
for x, y in coordinates:
if i.size == 0: # shortcut: no data at all to place
break
# transform coordinate into pixels (indices in 'values')
coord_px = (x - x1) / size_x, (y - y1) / size_y
di = round(coord_px[0] - anchor_px[0])
dj = round(coord_px[1] - anchor_px[1])
# because of the y-axis inversion: dj is measured from the
# other side of the array. if you draw it, you'll arrive at:
dj = dst_h - src_h - dj
if di <= -src_w or di >= dst_w or dj <= -src_h or dj >= dst_h:
# skip as it would shift completely outside
continue
elif 0 <= di <= (dst_w - src_w) and 0 <= dj <= (dst_h - src_h):
# complete place
values = np.full(out_shape, out_no_data_value, out_dtype)
values[k, j + dj, i + di] = source[k, j, i]
stack.append({"values": values, "no_data_value": out_no_data_value})
else:
# partial place
i_s = i + di
j_s = j + dj
m = (i_s >= 0) & (j_s >= 0) & (i_s < dst_w) & (j_s < dst_h)
if not m.any():
continue
values = np.full(out_shape, out_no_data_value, out_dtype)
values[k[m], j_s[m], i_s[m]] = source[k[m], j[m], i[m]]
stack.append({"values": values, "no_data_value": out_no_data_value})
# merge the values_stack
if len(stack) == 0:
return {
"values": np.full(out_shape, out_no_data_value, out_dtype),
"no_data_value": out_no_data_value,
}
else:
return reduce_rasters(stack, process_kwargs["statistic"])
