def _merge_vals_by_time(multi, times, kwargs): """ Merge chunks using indices. """ # determine the unique times and assign result bands sorted_times = Group._unique_times(times) bands = dict((y, x) for x, y in enumerate(sorted_times)) fillvalue = get_dtype_max(kwargs["dtype"]) # initialize values array shape = (len(sorted_times), ) + multi[0]["values"].shape[1:] values = np.full(shape, fillvalue, dtype=kwargs["dtype"]) # populate values array for data, time in zip(multi, times): # source_index is the index into the source array for source_index, datetime in enumerate(time["time"]): source_band = data["values"][source_index] # determine data index index = get_index(values=source_band, no_data_value=data["no_data_value"]) # find corresponding target band by source datetime target_band = values[bands[datetime]] # paste source into target provided there is data target_band[index] = source_band[index] # check if single band result required start, stop = kwargs["start"], kwargs["stop"] if stop is None and len(sorted_times) > 1: index = Group._nearest_index(sorted_times, start) values = values[index:index + 1] return {"values": values, "no_data_value": fillvalue}
def process(process_kwargs, *args): """Combine data, filling in nodata values.""" data_list = [] no_data_values = [] for data in args: if data is None: continue elif "time" in data or "meta" in data: # return the time / meta right away. assumes there are no # mixed requests and that time is aligned return data elif "values" in data and "no_data_value" in data: data_list.append(data["values"]) no_data_values.append(data["no_data_value"]) dtype = process_kwargs["dtype"] fillvalue = get_dtype_max(dtype) if not data_list: return # initialize values array values = np.full(data_list[0].shape, fillvalue, dtype=dtype) # populate values array for data, no_data_value in zip(data_list, no_data_values): # index is where the source has data index = get_index(data, no_data_value) values[index] = data[index] return {"values": values, "no_data_value": fillvalue}
def preprocess(*args): data_list = [] no_data_values = [] for data in args: if data is None: continue elif "time" in data or "meta" in data: # return the time / meta right away. assumes there are no # mixed requests and that time is aligned return data elif "values" in data and "no_data_value" in data: data_list.append(data["values"]) no_data_values.append(data["no_data_value"]) dtype = np.result_type(*data_list) fillvalue = get_dtype_max(dtype) # initialize values array values = np.full(data_list[0].shape, fillvalue, dtype=dtype) # populate values array multivalues = [] for data, no_data_value in zip(data_list, no_data_values): # initialize values array array_values = np.full(data_list[0].shape, np.nan, dtype=dtype) # index is where the source has data index = get_index(data, no_data_value) array_values[index] = data[index] multivalues.append(array_values) return {"values": values, "no_data_value": fillvalue}, multivalues
def process(data, value): if data is None or "values" not in data: return data index = get_index(values=data["values"], no_data_value=data["no_data_value"]) fillvalue = 1 if value == 0 else 0 dtype = "float32" if isinstance(value, float) else "uint8" values = np.full_like(data["values"], fillvalue, dtype=dtype) values[index] = value return {"values": values, "no_data_value": fillvalue}
def process(data, value): if data is None or "values" not in data: return data index = utils.get_index(values=data["values"], no_data_value=data["no_data_value"]) fillvalue = 1 if value == 0 else 0 dtype = Mask._dtype_from_value(value) values = np.full_like(data["values"], fillvalue, dtype=dtype) values[index] = value return {"values": values, "no_data_value": fillvalue}
def process(data): if data is None or "values" not in data: return data index = utils.get_index(values=data["values"], no_data_value=data["no_data_value"]) fillvalue = 255 dtype = "float32" values = np.full_like(data["values"], fillvalue, dtype=dtype) values[index] = random.random() return {"values": values, "no_data_value": fillvalue}
def _merge_vals_by_bands(multi, bands, dtype): """ Merge chunks using slices. """ # analyze band structure starts, stops = zip(*bands) fillvalue = get_dtype_max(dtype) # initialize values array shape = (max(stops), ) + multi[0]["values"].shape[1:] values = np.full(shape, fillvalue, dtype=dtype) # populate values array for data, (a, b) in zip(multi, bands): # index is where the source has data index = get_index(values=data["values"], no_data_value=data["no_data_value"]) values[a:b][index] = data["values"][index] return {"values": values, "no_data_value": fillvalue}
def test_get_index(self): index = utils.get_index(values=np.array([0, 1]), no_data_value=1) self.assertTrue((index == np.array([True, False])).all())
def reduce_rasters(stack, statistic, no_data_value=None, dtype=None): """Apply a statistic (e.g. "mean") to a stack of rasters, skipping 'no data' values. In this context, reduce means that the dimensionality of the input data is reduced by one. Args: stack (list): a list of dicts containing "values" (ndarray) and "no_data_value". If the list has zero length or if the ndarrays do not have the same shape, a ValueError is raised. statistic (str): the applied statistic (no data is ignored). One of: {"last", "first", "count", "sum", "mean", "min", "max", "argmin", "argmax", "product", "std", "var", "p<number>"} no_data_value (number, optional): the 'no data' value in the output array. Defaults to the no data value of the first element in the stack. dtype (str or dtype): the datatype of the output array. Defaults to the dtype of the first element in the stack. If the input data cannot be cast to this dtype, a ValueError is raised. Returns: dict with "values" and "no_data_value" """ if statistic not in STATISTICS: percentile = parse_percentile_statistic(statistic) if percentile is None: raise KeyError('Unknown statistic "{}"'.format(statistic)) else: statistic = "percentile" if len(stack) == 0: raise ValueError("Cannot reduce a zero-length stack") # get the output array properties (dtype, no_data_value, shape) if dtype is None: dtype = stack[0]["values"].dtype if no_data_value is None: no_data_value = stack[0]["no_data_value"] shape = stack[0]["values"].shape # sum, count and nans output do not contain no data: fill zeroes right away if statistic in {"sum", "count", "nans"}: fill_value = 0 else: fill_value = no_data_value # create the output array out = np.full(shape, fill_value, dtype) if statistic == "last": # populate 'out' with the last value that is not 'no data' for data in stack: index = get_index(data["values"], data["no_data_value"]) out[index] = data["values"][index] elif statistic == "first": # populate 'out' with the first value that is not 'no data' for data in stack[::-1]: index = get_index(data["values"], data["no_data_value"]) out[index] = data["values"][index] elif statistic == "count": # count the number of values that are not 'no data' for data in stack: out += get_index(data["values"], data["no_data_value"]) else: if statistic == "percentile": func = partial(np.nanpercentile, q=percentile) else: func = STATISTICS[statistic] # transform 'no data' into 'nan' to be able to use numpy functions # NB: the dtype is at least float16 to accomodate NaN stack_array = np.full((len(stack), ) + shape, np.nan, np.result_type(dtype, np.float16)) for i, data in enumerate(stack): index = get_index(data["values"], data["no_data_value"]) stack_array[i, index] = data["values"][index] # protect against all-NaN slice warnings and errors not_all_nan = ~np.all(np.isnan(stack_array), axis=0) # perform the math out[not_all_nan] = func(stack_array[:, not_all_nan], axis=0) return {"values": out, "no_data_value": no_data_value}
def process(process_kwargs, *multi): if process_kwargs["mode"] in {"meta", "time"}: return multi[0] if process_kwargs["mode"] == "null": return if process_kwargs["mode"] == "empty": data = multi[0] if data is None: return out_shape = ( len(data["time"]), process_kwargs["height"], process_kwargs["width"], ) out_no_data_value = process_kwargs["fillvalue"] out_dtype = process_kwargs["dtype"] stack = [] elif process_kwargs["mode"] == "group": # We have a bunch of arrays that are already shifted. Stack them. stack = [data for data in multi if data is not None] if len(stack) == 0: return # instead of returning nodata (because inputs are None) elif process_kwargs["mode"] == "warp": # There is a single 'source' raster that we are going to shift # multiple times into the result. The cellsize is already correct. data = multi[0] if data is None: return out_no_data_value = data["no_data_value"] source = data["values"] out_dtype = source.dtype # convert the anchor to pixels (indices inside 'source') anchor = process_kwargs["anchor"] src_bbox = process_kwargs["src_bbox"] size_x, size_y = process_kwargs["cellsize"] anchor_px = ( (anchor[0] - src_bbox[0]) / size_x, (anchor[1] - src_bbox[1]) / size_y, ) # compute the output shape x1, y1, x2, y2 = process_kwargs["dst_bbox"] coordinates = process_kwargs["coordinates"] dst_h = round((y2 - y1) / size_y) dst_w = round((x2 - x1) / size_x) src_d, src_h, src_w = source.shape out_shape = (src_d, dst_h, dst_w) # determine what indices in 'source' have data k, j, i = np.where(get_index(source, out_no_data_value)) # place the data on each coordinate stack = [] for x, y in coordinates: if i.size == 0: # shortcut: no data at all to place break # transform coordinate into pixels (indices in 'values') coord_px = (x - x1) / size_x, (y - y1) / size_y di = round(coord_px[0] - anchor_px[0]) dj = round(coord_px[1] - anchor_px[1]) # because of the y-axis inversion: dj is measured from the # other side of the array. if you draw it, you'll arrive at: dj = dst_h - src_h - dj if di <= -src_w or di >= dst_w or dj <= -src_h or dj >= dst_h: # skip as it would shift completely outside continue elif 0 <= di <= (dst_w - src_w) and 0 <= dj <= (dst_h - src_h): # complete place values = np.full(out_shape, out_no_data_value, out_dtype) values[k, j + dj, i + di] = source[k, j, i] stack.append({"values": values, "no_data_value": out_no_data_value}) else: # partial place i_s = i + di j_s = j + dj m = (i_s >= 0) & (j_s >= 0) & (i_s < dst_w) & (j_s < dst_h) if not m.any(): continue values = np.full(out_shape, out_no_data_value, out_dtype) values[k[m], j_s[m], i_s[m]] = source[k[m], j[m], i[m]] stack.append({"values": values, "no_data_value": out_no_data_value}) # merge the values_stack if len(stack) == 0: return { "values": np.full(out_shape, out_no_data_value, out_dtype), "no_data_value": out_no_data_value, } else: return reduce_rasters(stack, process_kwargs["statistic"])