def write(self, process_tile):
"""
Write data from process tiles into PNG file(s).
Parameters
----------
process_tile : ``BufferedTile``
must be member of process ``TilePyramid``
"""
data = prepare_array(process_tile.data,
dtype="uint8",
masked=False,
nodata=0)
if self.old_band_num:
data = np.stack(
(np.zeros(process_tile.shape), np.zeros(process_tile.shape),
np.zeros(process_tile.shape), data[0]))
else:
data = np.stack((np.zeros(process_tile.shape), data[0]))
process_tile.data = prepare_array(data,
dtype="uint8",
masked=True,
nodata=255)
# Convert from process_tile to output_tiles
for tile in self.pyramid.intersecting(process_tile):
# skip if file exists and overwrite is not set
out_path = self.get_path(tile)
self.prepare_path(tile)
out_tile = BufferedTile(tile, self.pixelbuffer)
write_raster_window(in_tile=process_tile,
out_profile=self.profile(out_tile),
out_tile=out_tile,
out_path=out_path)
def _prepare_array(self, data):
data = prepare_array(
-(data - 255), dtype="uint8", masked=False, nodata=0)
if self.old_band_num:
data = np.stack((
np.zeros(data[0].shape), np.zeros(data[0].shape),
np.zeros(data[0].shape), data[0]))
else:
data = np.stack((np.zeros(data[0].shape), data[0]))
return prepare_array(data, dtype="uint8", masked=True, nodata=255)
def _prepare_array(self, data):
data = prepare_array(-(data - 255),
dtype="uint8",
masked=False,
nodata=0)[0]
zeros = np.zeros(data.shape)
if self.old_band_num:
data = np.stack([zeros, zeros, zeros, data])
else:
data = np.stack([zeros, data])
return prepare_array(data, dtype="uint8", masked=True, nodata=255)
def write(self, process_tile, data):
"""
Write data from process tiles into GeoTIFF file(s).
Parameters
----------
process_tile : ``BufferedTile``
must be member of process ``TilePyramid``
"""
data = prepare_array(data,
masked=True,
nodata=self.output_params["nodata"],
dtype=self.profile(process_tile)["dtype"])
if data.mask.all():
logger.debug("data empty, nothing to write")
else:
# Convert from process_tile to output_tiles and write
for tile in self.pyramid.intersecting(process_tile):
out_tile = BufferedTile(tile, self.pixelbuffer)
write_window = from_bounds(
*out_tile.bounds,
transform=self.rio_file.transform,
height=self.rio_file.height,
width=self.rio_file.width).round_lengths(
pixel_precision=0).round_offsets(pixel_precision=0)
if _window_in_out_file(write_window, self.rio_file):
logger.debug("write data to window: %s", write_window)
self.rio_file.write(
extract_from_array(in_raster=data,
in_affine=process_tile.affine,
out_tile=out_tile)
if process_tile != out_tile else data,
window=write_window,
)
def write(self, process_tile, data):
"""
Write data from process tiles into GeoTIFF file(s).
Parameters
----------
process_tile : ``BufferedTile``
must be member of process ``TilePyramid``
"""
if (
isinstance(data, tuple) and
len(data) == 2 and
isinstance(data[1], dict)
):
data, tags = data
else:
tags = {}
data = prepare_array(
data, masked=True, nodata=self.nodata,
dtype=self.profile(process_tile)["dtype"])
if data.mask.all():
return
# Convert from process_tile to output_tiles
for tile in self.pyramid.intersecting(process_tile):
out_path = self.get_path(tile)
self.prepare_path(tile)
out_tile = BufferedTile(tile, self.pixelbuffer)
write_raster_window(
in_tile=process_tile, in_data=data,
out_profile=self.profile(out_tile), out_tile=out_tile,
out_path=out_path, tags=tags)
def write(self, process_tile, data):
"""
Write data from process tiles into GeoTIFF file(s).
Parameters
----------
process_tile : ``BufferedTile``
must be member of process ``TilePyramid``
"""
if (isinstance(data, tuple) and len(data) == 2
and isinstance(data[1], dict)):
data, tags = data
else:
tags = {}
data = prepare_array(data,
masked=True,
nodata=self.nodata,
dtype=self.profile(process_tile)["dtype"])
if data.mask.all():
logger.debug((process_tile.id, "empty data"))
return
bucket = boto3.resource('s3').Bucket(self.bucket)
for tile in self.pyramid.intersecting(process_tile):
logger.debug((tile.id, "prepare to upload", self.get_path(tile)))
out_tile = BufferedTile(tile, self.pixelbuffer)
with RasterWindowMemoryFile(in_tile=process_tile,
in_data=data,
out_profile=self.profile(out_tile),
out_tile=out_tile,
tags=tags) as memfile:
logger.debug(
(tile.id, "upload tile", self.get_bucket_key(tile)))
bucket.put_object(Key=self.get_bucket_key(tile), Body=memfile)
def write(self, process_tile):
"""
Write data from process tiles into GeoTIFF file(s).
Parameters
----------
process_tile : ``BufferedTile``
must be member of process ``TilePyramid``
"""
process_tile.data = prepare_array(
process_tile.data,
masked=True,
nodata=self.nodata,
dtype=self.profile(process_tile)["dtype"])
if process_tile.data.mask.all():
return
# Convert from process_tile to output_tiles
for tile in self.pyramid.intersecting(process_tile):
out_path = self.get_path(tile)
self.prepare_path(tile)
out_tile = BufferedTile(tile, self.pixelbuffer)
write_raster_window(in_tile=process_tile,
out_profile=self.profile(out_tile),
out_tile=out_tile,
out_path=out_path)
def test_prepare_array_errors():
"""Convert ndarray data into a proper array."""
# input is iterable
data = [None]
try:
prepare_array(data)
raise Exception()
except ValueError:
pass
# input is not array
data = 5
try:
prepare_array(data)
raise Exception()
except ValueError:
pass
def test_prepare_array_iterables():
"""Convert iterable data into a proper array."""
# input is iterable
# iterable contains arrays
data = [np.zeros((1, 1))]
# output ndarray
output = prepare_array(data, masked=False)
assert isinstance(output, np.ndarray)
assert not isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# output masked array
output = prepare_array(data)
assert isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# iterable contains masked arrays
data = [ma.empty((1, 1))]
output = prepare_array(data, masked=False)
assert isinstance(output, np.ndarray)
assert not isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# output masked array
output = prepare_array(data)
assert isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# iterable contains masked arrays with full mask
data = [ma.masked_array(data=np.ones((1, 1)), mask=np.ones((1, 1)))]
output = prepare_array(data, masked=False)
assert isinstance(output, np.ndarray)
assert not isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# output masked array
output = prepare_array(data)
assert isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
def _extract(self, in_tile=None, in_data=None, out_tile=None):
"""Extract data from tile."""
if self.config.output.METADATA["data_type"] == "raster":
return raster.extract_from_array(in_raster=raster.prepare_array(
in_data,
nodata=self.config.output.nodata,
dtype=self.config.output.output_params["dtype"]),
in_affine=in_tile.affine,
out_tile=out_tile)
elif self.config.output.METADATA["data_type"] == "vector":
return [
feature for feature in in_data
if shape(feature["geometry"]).intersects(out_tile.bbox)
]
def _extract(self, process_tile, tile):
try:
process_tile = self.process_tile_cache[process_tile.id]
except Exception:
pass
if self.config.output.METADATA["data_type"] == "raster":
process_tile.data = raster.prepare_array(
process_tile.data,
nodata=self.config.output.nodata,
dtype=self.config.output.output_params["dtype"])
tile.data = raster.extract_from_tile(process_tile, tile)
elif self.config.output.METADATA["data_type"] == "vector":
tile.data = vector.extract_from_tile(process_tile, tile)
return tile
def test_prepare_array_ndarrays():
"""Convert ndarray data into a proper array."""
# input is np.ndarray
data = np.zeros((1, 1, 1))
# output ndarray
output = prepare_array(data, masked=False)
assert isinstance(output, np.ndarray)
assert not isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# output masked array
output = prepare_array(data)
assert isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# input is 2D np.ndarray
data = np.zeros((1, 1))
# output ndarray
output = prepare_array(data, masked=False)
assert isinstance(output, np.ndarray)
assert not isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# output masked array
output = prepare_array(data)
assert isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
def test_prepare_array_maskedarrays():
"""Convert masked array data into a proper array."""
# input is ma.masked_array
data = ma.empty((1, 1, 1))
# output ndarray
output = prepare_array(data, masked=False)
assert isinstance(output, np.ndarray)
assert not isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# output masked array
output = prepare_array(data)
assert isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# input is ma.masked_array with full mask
data = ma.masked_array(data=np.ones((1, 1, 1)), mask=np.ones((1, 1, 1)))
# output ndarray
output = prepare_array(data, masked=False)
assert isinstance(output, np.ndarray)
assert not isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
# output masked array
output = prepare_array(data)
assert isinstance(output, ma.masked_array)
assert output.shape == (1, 1, 1)
def for_web(self, data):
"""
Convert data to web output (raster only).
Parameters
----------
data : array
Returns
-------
web data : array
"""
data = prepare_array(
data, masked=True, nodata=self.nodata,
dtype=self.profile()["dtype"])
return memory_file(data, self.profile()), "image/tiff"
def write(self, process_tile, data):
"""
Write data from process tiles into GeoTIFF file(s).
Parameters
----------
process_tile : ``BufferedTile``
must be member of process ``TilePyramid``
data : ``np.ndarray``
"""
if (
isinstance(data, tuple) and
len(data) == 2 and
isinstance(data[1], dict)
):
data, tags = data
else:
tags = {}
data = prepare_array(
data,
masked=True,
nodata=self.output_params["nodata"],
dtype=self.profile(process_tile)["dtype"]
)
if data.mask.all():
logger.debug("data empty, nothing to write")
else:
# in case of S3 output, create an boto3 resource
bucket_resource = get_boto3_bucket(self._bucket) if self._bucket else None
# Convert from process_tile to output_tiles and write
for tile in self.pyramid.intersecting(process_tile):
out_path = self.get_path(tile)
self.prepare_path(tile)
out_tile = BufferedTile(tile, self.pixelbuffer)
write_raster_window(
in_tile=process_tile,
in_data=data,
out_profile=self.profile(out_tile),
out_tile=out_tile,
out_path=out_path,
tags=tags,
bucket_resource=bucket_resource
)
def _prepare_array_for_png(self, data):
data = prepare_array(data, dtype=np.uint8)
# Create 3D NumPy array with alpha channel.
if len(data) == 1:
rgba = np.stack((data[0], data[0], data[0],
np.where(data[0].data == self.nodata, 0,
255).astype("uint8")))
elif len(data) == 2:
rgba = np.stack((data[0], data[0], data[0], data[1]))
elif len(data) == 3:
rgba = np.stack((data[0], data[1], data[2],
np.where(data[0].data == self.nodata, 0,
255).astype("uint8")))
elif len(data) == 4:
rgba = np.array(data).astype("uint8")
else:
raise TypeError("invalid number of bands: %s" % len(data))
return rgba
def for_web(self, data):
"""
Convert data to web output.
Parameters
----------
data : array
Returns
-------
web data : array
"""
data = prepare_array(data, masked=False, dtype="uint8")[0]
zeros = np.zeros(data.shape)
out_rgb = (zeros, zeros, zeros, data)
reshaped = np.stack(out_rgb).transpose(1, 2, 0).astype("uint8")
empty_image = Image.fromarray(reshaped, mode='RGBA')
out_img = io.BytesIO()
empty_image.save(out_img, 'PNG')
out_img.seek(0)
return send_file(out_img, mimetype='image/png')
def write(self, process_tile):
"""
Write data from process tiles into NumPy array dumps.
Parameters
----------
process_tile : ``BufferedTile``
must be member of process ``TilePyramid``
"""
if not os.path.exists(self.path):
os.makedirs(self.path)
process_tile.data = prepare_array(
process_tile.data, masked=True, nodata=self.nodata,
dtype=self.output_params["dtype"]
)
# Convert from process_tile to output_tiles
for tile in self.pyramid.intersecting(process_tile):
out_tile = BufferedTile(tile, self.pixelbuffer)
out_data = extract_from_tile(process_tile, out_tile)
self.prepare_path(tile)
out_data.dump(self.get_path(out_tile))
def extract_subset(self, input_data_tiles=None, out_tile=None):
"""
Extract subset from multiple tiles.
input_data_tiles : list of (``Tile``, process data) tuples
out_tile : ``Tile``
Returns
-------
NumPy array or list of features.
"""
if self.METADATA["data_type"] == "raster":
mosaic = create_mosaic(input_data_tiles)
return extract_from_array(in_raster=prepare_array(
mosaic.data,
nodata=self.output_params["nodata"],
dtype=self.output_params["dtype"]),
in_affine=mosaic.affine,
out_tile=out_tile)
elif self.METADATA["data_type"] == "vector":
return [
feature for feature in list(
chain.from_iterable(
[features for _, features in input_data_tiles]))
if shape(feature["geometry"]).intersects(out_tile.bbox)
]