def process(process_kwargs, *args):
"""Combine data, filling in nodata values."""
data_list = []
no_data_values = []
for data in args:
if data is None:
continue
elif "time" in data or "meta" in data:
# return the time / meta right away. assumes there are no
# mixed requests and that time is aligned
return data
elif "values" in data and "no_data_value" in data:
data_list.append(data["values"])
no_data_values.append(data["no_data_value"])
dtype = process_kwargs["dtype"]
fillvalue = get_dtype_max(dtype)
if not data_list:
return
# initialize values array
values = np.full(data_list[0].shape, fillvalue, dtype=dtype)
# populate values array
for data, no_data_value in zip(data_list, no_data_values):
# index is where the source has data
index = get_index(data, no_data_value)
values[index] = data[index]
return {"values": values, "no_data_value": fillvalue}
def _merge_vals_by_time(multi, times, kwargs):
""" Merge chunks using indices. """
# determine the unique times and assign result bands
sorted_times = Group._unique_times(times)
bands = dict((y, x) for x, y in enumerate(sorted_times))
fillvalue = get_dtype_max(kwargs["dtype"])
# initialize values array
shape = (len(sorted_times), ) + multi[0]["values"].shape[1:]
values = np.full(shape, fillvalue, dtype=kwargs["dtype"])
# populate values array
for data, time in zip(multi, times):
# source_index is the index into the source array
for source_index, datetime in enumerate(time["time"]):
source_band = data["values"][source_index]
# determine data index
index = get_index(values=source_band,
no_data_value=data["no_data_value"])
# find corresponding target band by source datetime
target_band = values[bands[datetime]]
# paste source into target provided there is data
target_band[index] = source_band[index]
# check if single band result required
start, stop = kwargs["start"], kwargs["stop"]
if stop is None and len(sorted_times) > 1:
index = Group._nearest_index(sorted_times, start)
values = values[index:index + 1]
return {"values": values, "no_data_value": fillvalue}
def preprocess(*args):
data_list = []
no_data_values = []
for data in args:
if data is None:
continue
elif "time" in data or "meta" in data:
# return the time / meta right away. assumes there are no
# mixed requests and that time is aligned
return data
elif "values" in data and "no_data_value" in data:
data_list.append(data["values"])
no_data_values.append(data["no_data_value"])
dtype = np.result_type(*data_list)
fillvalue = get_dtype_max(dtype)
# initialize values array
values = np.full(data_list[0].shape, fillvalue, dtype=dtype)
# populate values array
multivalues = []
for data, no_data_value in zip(data_list, no_data_values):
# initialize values array
array_values = np.full(data_list[0].shape, np.nan, dtype=dtype)
# index is where the source has data
index = get_index(data, no_data_value)
array_values[index] = data[index]
multivalues.append(array_values)
return {"values": values, "no_data_value": fillvalue}, multivalues
def process(data, bins, right):
if data is None or "values" not in data:
return data
values = data["values"]
dtype = utils.get_uint_dtype(len(bins) + 2)
fillvalue = utils.get_dtype_max(dtype)
result_values = np.digitize(values, bins, right).astype(dtype)
result_values[values == data["no_data_value"]] = fillvalue
return {"values": result_values, "no_data_value": fillvalue}
def _merge_vals_by_bands(multi, bands, dtype):
""" Merge chunks using slices. """
# analyze band structure
starts, stops = zip(*bands)
fillvalue = get_dtype_max(dtype)
# initialize values array
shape = (max(stops), ) + multi[0]["values"].shape[1:]
values = np.full(shape, fillvalue, dtype=dtype)
# populate values array
for data, (a, b) in zip(multi, bands):
# index is where the source has data
index = get_index(values=data["values"],
no_data_value=data["no_data_value"])
values[a:b][index] = data["values"][index]
return {"values": values, "no_data_value": fillvalue}
def process(process_kwargs, time_data=None, data=None):
mode = process_kwargs["mode"]
# handle empty data
if process_kwargs.get("empty"):
return None if mode == "vals" else {mode: []}
if mode == "time":
return time_data
if time_data is None or not time_data.get("time"):
return None if mode == "vals" else {mode: []}
start = process_kwargs["start"]
stop = process_kwargs["stop"]
frequency = process_kwargs["frequency"]
timezone = process_kwargs["timezone"]
closed = process_kwargs["closed"]
label = process_kwargs["label"]
times = (pd.Series(
index=time_data["time"]).tz_localize("UTC").tz_convert(timezone))
if frequency is None:
# the first (and only label) will be the statistic of all frames
indices = {None: range(len(times))}
else:
# construct a pandas Resampler object to map labels to frames
resampler = times.resample(frequency, closed=closed, label=label)
# get the frame indices belonging to each bin
indices = resampler.indices
start_ts = _dt_to_ts(start, timezone)
stop_ts = _dt_to_ts(stop, timezone)
if mode == "meta":
if data is None or "meta" not in data:
return {"meta": []}
meta = data["meta"]
result = []
for indices_in_bin in indices.values(): # [0, 1], [2, 3], ...
for length in range(1, len(indices_in_bin) + 1):
indices_for_cumulative = indices_in_bin[:length]
ts = times.index[indices_for_cumulative[-1]]
if ts < start_ts or (stop_ts is not None and ts > stop_ts):
continue
result.append([meta[i] for i in indices_for_cumulative])
return {"meta": result}
# mode == 'vals'
if data is None or "values" not in data:
return
values = data["values"]
if values.shape[0] != len(times):
raise RuntimeError(
"Shape of raster does not match number of timestamps")
statistic = process_kwargs["statistic"]
percentile = parse_percentile_statistic(statistic)
if percentile:
extensive = False
agg_func = partial(np.nanpercentile, q=percentile)
else:
extensive = Cumulative.STATISTICS[statistic]["extensive"]
agg_func = Cumulative.STATISTICS[statistic]["func"]
dtype = process_kwargs["dtype"]
fillvalue = 0 if extensive else get_dtype_max(dtype)
# cast to at least float32 so that we can fit in NaN (and make copy)
values = values.astype(np.result_type(np.float32, dtype))
# put NaN for no data
values[data["values"] == data["no_data_value"]] = np.nan
output_mask = (times.index >= start_ts) & (times.index <= stop_ts)
output_offset = np.where(output_mask)[0][0]
n_frames = output_mask.sum()
result = np.full(
shape=(n_frames, values.shape[1], values.shape[2]),
fill_value=fillvalue,
dtype=dtype,
)
for indices_in_bin in indices.values():
mask = output_mask[indices_in_bin]
data = values[indices_in_bin]
accumulated = agg_func(data, axis=0)[mask]
# keep track of NaN or inf values before casting to target dtype
no_data_mask = ~np.isfinite(accumulated)
# cast to target dtype
if dtype != accumulated.dtype:
accumulated = accumulated.astype(dtype)
# set fillvalue to NaN values
accumulated[no_data_mask] = fillvalue
indices_in_result = np.array(indices_in_bin)[mask] - output_offset
result[indices_in_result] = accumulated
return {"values": result, "no_data_value": get_dtype_max(dtype)}
def fillvalue(self):
return get_dtype_max(self.dtype)
def process(process_kwargs, time_data=None, data=None):
mode = process_kwargs["mode"]
# handle empty data
if process_kwargs.get("empty"):
return None if mode == "vals" else {mode: []}
start = process_kwargs["start"]
stop = process_kwargs["stop"]
frequency = process_kwargs["frequency"]
if frequency is None:
labels = pd.DatetimeIndex([start])
else:
labels = pd.date_range(start, stop or start, freq=frequency)
if mode == "time":
return {"time": labels.to_pydatetime().tolist()}
if time_data is None or not time_data.get("time"):
return None if mode == "vals" else {mode: []}
timezone = process_kwargs["timezone"]
closed = process_kwargs["closed"]
label = process_kwargs["label"]
times = time_data["time"]
# convert times to a pandas series
series = pd.Series(index=times).tz_localize("UTC").tz_convert(timezone)
# localize the labels so we can use it as an index
labels = labels.tz_localize("UTC").tz_convert(timezone)
if frequency is None:
# the first (and only label) will be the statistic of all frames
indices = {labels[0]: range(len(times))}
else:
# construct a pandas Resampler object to map labels to frames
resampler = series.resample(frequency, closed=closed, label=label)
# get the frame indices belonging to each bin
indices = resampler.indices
if mode == "meta":
if data is None or "meta" not in data:
return {"meta": []}
meta = data["meta"]
return {"meta": [[meta[i] for i in indices[ts]] for ts in labels]}
# mode == 'vals'
if data is None or "values" not in data:
return
values = data["values"]
if values.shape[0] != len(times):
raise RuntimeError(
"Shape of raster does not match number of timestamps")
statistic = process_kwargs["statistic"]
percentile = parse_percentile_statistic(statistic)
if percentile:
extensive = False
agg_func = partial(np.nanpercentile, q=percentile)
else:
extensive = TemporalAggregate.STATISTICS[statistic]["extensive"]
agg_func = TemporalAggregate.STATISTICS[statistic]["func"]
dtype = process_kwargs["dtype"]
fillvalue = 0 if extensive else get_dtype_max(dtype)
# cast to at least float32 so that we can fit in NaN (and make copy)
values = values.astype(np.result_type(np.float32, dtype))
# put NaN for no data
values[data["values"] == data["no_data_value"]] = np.nan
result = np.full(
shape=(len(labels), values.shape[1], values.shape[2]),
fill_value=fillvalue,
dtype=dtype,
)
for i, timestamp in enumerate(labels):
inds = indices[timestamp]
if len(inds) == 0:
continue
aggregated = agg_func(values[inds], axis=0)
# keep track of NaN or inf values before casting to target dtype
no_data_mask = ~np.isfinite(aggregated)
# cast to target dtype
if dtype != aggregated.dtype:
aggregated = aggregated.astype(dtype)
# set fillvalue to NaN values
aggregated[no_data_mask] = fillvalue
result[i] = aggregated
return {"values": result, "no_data_value": get_dtype_max(dtype)}
def fillvalue(self):
return None if self.dtype == np.bool else utils.get_dtype_max(
self.dtype)
def test_get_dtype_max(self):
self.assertIsInstance(utils.get_dtype_max("f4"), float)
self.assertIsInstance(utils.get_dtype_max("u4"), int)
def fillvalue(self):
dtype = self.dtype
if dtype == np.bool:
return
else:
return get_dtype_max(dtype)
def process(args, request):
origin, timedelta, bands, value, src_projection = args
if origin is None or timedelta is None or bands is None:
return
td_seconds = timedelta.total_seconds()
lo = origin
start = request.get("start", None)
stop = request.get("stop", None)
if start is None:
# take the latest
bands_lo = bands - 1
bands_hi = bands
elif stop is None:
# take the nearest to start
start_band = (request["start"] - lo).total_seconds() / td_seconds
bands_lo = min(max(int(round(start_band)), 0), bands - 1)
bands_hi = bands_lo + 1
else:
bands_lo = (request["start"] - lo).total_seconds() / td_seconds
bands_hi = (request["stop"] - lo).total_seconds() / td_seconds
bands_lo = max(int(math.ceil(bands_lo)), 0)
bands_hi = min(int(math.floor(bands_hi)) + 1, bands)
depth = bands_hi - bands_lo
if depth <= 0:
return
if request["mode"] == "time":
return {"time": [origin + i * timedelta for i in range(bands_lo, bands_hi)]}
if request["mode"] == "meta":
return {
"meta": [
"Testmeta for band {}".format(i) for i in range(bands_lo, bands_hi)
]
}
if request["mode"] != "vals":
raise ValueError('Invalid mode "{}"'.format(request["mode"]))
height = request.get("height", 1)
width = request.get("width", 1)
shape = (depth, height, width)
# simple mode: return a filled value with type uint8
if not hasattr(value, "shape"):
fillvalue = 255
result = np.full(shape, value, dtype=np.uint8)
return {"values": result, "no_data_value": fillvalue}
# there is an actual data array
fillvalue = get_dtype_max(value.dtype)
bbox = request.get("bbox", (0, 0, width, height))
projection = request.get("projection", "EPSG:3857")
if projection != src_projection:
extent = Extent(bbox, get_sr(projection))
bbox = extent.transformed(get_sr(src_projection)).bbox
x1, y1, x2, y2 = [int(round(x)) for x in bbox]
if x1 == x2 or y1 == y2: # point request
if x1 < 0 or x1 >= value.shape[1] or y1 < 0 or y1 >= value.shape[0]:
result = np.array([[255]], dtype=np.uint8)
else:
result = value[y1 : y1 + 1, x1 : x1 + 1]
else:
_x1 = max(x1, 0)
_y1 = max(y1, 0)
_x2 = min(x2, value.shape[1])
_y2 = min(y2, value.shape[0])
result = value[_y1:_y2, _x1:_x2]
result = np.pad(
result,
((_y1 - y1, y2 - _y2), (_x1 - x1, x2 - _x2)),
mode=str("constant"),
constant_values=fillvalue,
)
if result.shape != (height, width):
zoom = (height / result.shape[0], width / result.shape[1])
mask = ndimage.zoom((result == fillvalue).astype(np.float), zoom) > 0.5
result[result == fillvalue] = 0
result = ndimage.zoom(result, zoom)
result[mask] = fillvalue
result = np.repeat(result[np.newaxis], depth, axis=0)
# fill nan values
result[~np.isfinite(result)] = fillvalue
return {"values": result, "no_data_value": fillvalue}
