def copy_to_workspace(source_uri, dest_uri):
"""
Translates an image from a URI to a compressed, tiled GeoTIFF version in the workspace
"""
creation_options = {
"driver": "GTiff",
"tiled": True,
"compress": "lzw",
"predictor": 2,
"sparse_ok": True,
"blockxsize": 512,
"blockysize": 512
}
with rasterio.open(source_uri, "r") as src:
meta = src.meta.copy()
meta.update(creation_options)
tmp_path = "/vsimem/" + get_filename(dest_uri)
with rasterio.open(tmp_path, "w", **meta) as tmp:
tmp.write(src.read())
contents = bytearray(virtual_file_to_buffer(tmp_path))
write_bytes_to_target(dest_uri, contents)
def _encode_as_png(data, profile, dst_transform):
"""
Uses rasterio's virtual file system to encode a (3, 512, 512)
array as a png-encoded bytearray.
Parameters
-----------
data: ndarray
(3 x 512 x 512) uint8 RGB array
profile: dictionary
dictionary of kwargs for png writing
affine: Affine
affine transform for output tile
Returns
--------
contents: bytearray
png-encoded bytearray of the provided input data
"""
profile["affine"] = dst_transform
with rasterio.open("/vsimem/tileimg", "w", **profile) as dst:
dst.write(data)
contents = bytearray(virtual_file_to_buffer("/vsimem/tileimg"))
return contents
def save_hillshade(tile, data, meta):
s3 = boto3.resource("s3")
meta.update(
driver="GTiff",
dtype=rasterio.uint8,
compress="deflate",
predictor=1,
nodata=None,
tiled=True,
sparse_ok=True,
blockxsize=DST_BLOCK_SIZE,
blockysize=DST_BLOCK_SIZE,
)
with rasterio.open(TMP_PATH, "w", **meta) as tmp:
tmp.write(data, 1)
key = "3857/{}/{}/{}.tif".format(tile.z, tile.x, tile.y)
s3.Object(
S3_BUCKET,
key,
).put(
Body=bytes(bytearray(virtual_file_to_buffer(TMP_PATH))),
ACL="public-read",
ContentType="image/tiff",
CacheControl="public, max-age=2592000",
StorageClass="REDUCED_REDUNDANCY",
)
return "http://{}.s3.amazonaws.com/{}".format(S3_BUCKET, key)
def _encode_as_png(data, profile, dst_transform):
"""
Uses rasterio's virtual file system to encode a (3, 512, 512)
array as a png-encoded bytearray.
Parameters
-----------
data: ndarray
(3 x 512 x 512) uint8 RGB array
profile: dictionary
dictionary of kwargs for png writing
affine: Affine
affine transform for output tile
Returns
--------
contents: bytearray
png-encoded bytearray of the provided input data
"""
profile['affine'] = dst_transform
with rasterio.open('/vsimem/tileimg', 'w', **profile) as dst:
dst.write(data)
contents = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
return contents
def _write((tile, data)):
if not contains_data((tile, data)):
return
print("Writing", tile)
# Get the bounds of the tile.
ulx, uly = mercantile.xy(
*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(
*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
# TODO constantize
tmp_path = "/vsimem/tile"
# create GeoTIFF
meta = creation_options.copy()
meta["count"] = 1
meta["nodata"] = data.fill_value
meta["dtype"] = data.dtype
meta["width"] = CHUNK_SIZE
meta["height"] = CHUNK_SIZE
meta["transform"] = from_bounds(ulx, lry, lrx, uly, CHUNK_SIZE, CHUNK_SIZE)
with rasterio.drivers():
with rasterio.open(tmp_path, "w", **meta) as tmp:
tmp.write(data, 1)
# write out
output_uri = urlparse(out_dir)
contents = bytearray(virtual_file_to_buffer(tmp_path))
if output_uri.scheme == "s3":
# TODO use mapPartitions so that the client only needs to be
# instantiated once per partition
client = boto3.client("s3")
bucket = output_uri.netloc
# TODO strip out trailing slashes on the path if necessary
key = "%s/%d/%d/%d.tif" % (output_uri.path[1:], tile.z, tile.x, tile.y)
response = client.put_object(
ACL="public-read",
Body=bytes(contents),
Bucket=bucket,
# CacheControl="TODO",
ContentType="image/tiff",
Key=key
)
else:
output_path = os.path.join(out_dir, "%d/%d/%d.tif" % (tile.z, tile.x, tile.y))
mkdir_p(os.path.dirname(output_path))
f = open(output_path, "w")
f.write(contents)
f.close()
def process_chunk_task(task):
"""
Chunks the image into tile_dim x tile_dim tiles,
and saves them to the target folder (s3 or local)
Returns the extent of the output raster.
"""
creation_options = {
"driver": "GTiff",
"crs": "EPSG:3857",
"tiled": True,
"compress": "deflate",
"predictor": 2, # 3 for floats, 2 otherwise
"sparse_ok": True
}
with rasterio.open(task.source_uri, "r") as src:
meta = src.meta.copy()
meta.update(creation_options)
meta.update(task.target_meta)
cols = meta["width"]
rows = meta["height"]
tmp_path = "/vsimem/" + get_filename(task.target)
with rasterio.open(tmp_path, "w", **meta) as tmp:
# Reproject the src dataset into image tile.
warped = []
for bidx in src.indexes:
source = rasterio.band(src, bidx)
warped.append(numpy.zeros((cols, rows), dtype=meta['dtype']))
warp.reproject(
source=source,
src_nodata=0,
destination=warped[bidx - 1],
dst_transform=meta["transform"],
dst_crs=meta["crs"],
# resampling=RESAMPLING.bilinear
)
# check for chunks containing only zero values
if not any(map(lambda b: b.any(), warped)):
return False
# write out our warped data to the vsimem raster
for bidx in src.indexes:
tmp.write_band(bidx, warped[bidx - 1])
contents = bytearray(virtual_file_to_buffer(tmp_path))
write_bytes_to_target(task.target, contents)
return True
def process_chunk_task(task):
"""
Chunks the image into tile_dim x tile_dim tiles,
and saves them to the target folder (s3 or local)
Returns the extent of the output raster.
"""
creation_options = {
"driver": "GTiff",
"crs": "EPSG:3857",
"tiled": True,
"compress": "deflate",
"predictor": 2, # 3 for floats, 2 otherwise
"sparse_ok": True
}
with rasterio.open(task.source_uri, "r") as src:
meta = src.meta.copy()
meta.update(creation_options)
meta.update(task.target_meta)
cols = meta["width"]
rows = meta["height"]
tmp_path = "/vsimem/" + get_filename(task.target)
with rasterio.open(tmp_path, "w", **meta) as tmp:
# Reproject the src dataset into image tile.
warped = []
for bidx in src.indexes:
source = rasterio.band(src, bidx)
warped.append(numpy.zeros((cols, rows), dtype=meta['dtype']))
warp.reproject(
source=source,
src_nodata=0,
destination=warped[bidx - 1],
dst_transform=meta["transform"],
dst_crs=meta["crs"],
# resampling=RESAMPLING.bilinear
)
# check for chunks containing only zero values
if not any(map(lambda b: b.any(), warped)):
return False
# write out our warped data to the vsimem raster
for bidx in src.indexes:
tmp.write_band(bidx, warped[bidx - 1])
contents = bytearray(virtual_file_to_buffer(tmp_path))
write_bytes_to_target(task.target, contents)
return True
def process_tile(tile):
"""Process a single MBTiles tile
Parameters
----------
tile : mercantile.Tile
Returns:
tile : mercantile.Tile
The input tile.
bytes : bytearray
Image bytes corresponding to the tile.
"""
global base_kwds, resampling, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(
*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(
*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
src_nodata = kwds.pop('src_nodata', None)
dst_nodata = kwds.pop('dst_nodata', None)
with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
reproject(rasterio.band(src, src.indexes),
rasterio.band(tmp, tmp.indexes),
src_nodata=src_nodata,
dst_nodata=dst_nodata,
num_threads=1,
resampling=resampling)
data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
# Workaround for https://bugs.python.org/issue23349.
if sys.version_info[0] == 2 and sys.version_info[2] < 10:
# Check for backported bug fix before re-ordering
if kwds['driver'] == 'PNG' and data[0:8] == png_header:
# Properly constructed PNG, no need to re-order bytes
pass
elif kwds['driver'] == 'JPEG' and data[0:4] == jpeg_header:
# Properly constructed JPEG, no need to re-order bytes
pass
else:
data[:] = data[-1:] + data[:-1]
return tile, data
def process_tile(tile):
"""Process a single MBTiles tile."""
global base_kwds, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds["transform"] = from_bounds(ulx, lry, lrx, uly, 256, 256)
with rasterio.open("/vsimem/tileimg", "w", **kwds) as tmp:
# Reproject the src dataset into image tile.
for bidx in tmp.indexes:
reproject(rasterio.band(src, bidx), rasterio.band(tmp, bidx))
# Get contents of the virtual file.
contents = bytearray(virtual_file_to_buffer("/vsimem/tileimg"))
return tile, contents
def process_tile(tile):
"""Process a single MBTiles tile."""
global base_kwds, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
# Reproject the src dataset into image tile.
for bidx in tmp.indexes:
reproject(rasterio.band(src, bidx), rasterio.band(tmp, bidx))
# Get contents of the virtual file and repair it.
contents = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
return tile, contents[-1:] + contents[:-1]
def copy_image(copy_job):
(image_uri, dest_uri) = copy_job
image_uri = vsi_curlify(image_uri)
creation_options = {
"driver": "GTiff",
"tiled": True,
"compress": "lzw",
"predictor": 2,
"sparse_ok": True,
"blockxsize": 512,
"blockysize": 512
}
with rasterio.open(image_uri, 'r') as src:
meta = src.meta.copy()
meta.update(creation_options)
tmp_path = "/vsimem/" + get_filename(dest_uri)
with rasterio.open(tmp_path, 'w', **meta) as tmp:
tmp.write(src.read())
contents = bytearray(virtual_file_to_buffer(tmp_path))
write_bytes_to_target(dest_uri, contents)
def process_tile(tile):
"""Process a single MBTiles tile
Parameters
----------
tile : mercantile.Tile
Returns:
tile : mercantile.Tile
The input tile.
bytes : bytearray
Image bytes corresponding to the tile.
"""
global base_kwds, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
src_nodata = kwds.pop('src_nodata', None)
dst_nodata = kwds.pop('dst_nodata', None)
with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
reproject(rasterio.band(src, src.indexes),
rasterio.band(tmp, tmp.indexes),
src_nodata=src_nodata,
dst_nodata=dst_nodata,
num_threads=1)
data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
# Workaround for https://bugs.python.org/issue23349.
if sys.version_info[0] == 2 and sys.version_info[2] < 10:
data[:] = data[-1:] + data[:-1]
return tile, data
def process_tile(tile):
"""Process a single MBTiles tile
Parameters
----------
tile : mercantile.Tile
Returns
-------
tile : mercantile.Tile
The input tile.
bytes : bytearray
Image bytes corresponding to the tile.
"""
global base_kwds, resampling, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
# kwds['transform'] = transform_from_bounds(ulx, lry, lrx, uly, 256, 256)
kwds['transform'] = transform_from_bounds(ulx, lry, lrx, uly, 512, 512)
src_nodata = kwds.pop('src_nodata', None)
dst_nodata = kwds.pop('dst_nodata', None)
with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
# determine window of source raster corresponding to the tile
# image, with small buffer at edges
try:
west, south, east, north = transform_bounds(
TILES_CRS, src.crs, ulx, lry, lrx, uly)
tile_window = window_from_bounds(west,
south,
east,
north,
transform=src.transform)
adjusted_tile_window = Window(tile_window.col_off - 1,
tile_window.row_off - 1,
tile_window.width + 2,
tile_window.height + 2)
tile_window = adjusted_tile_window.round_offsets().round_shape()
# if no data in window, skip processing the tile
if not src.read_masks(1, window=tile_window).any():
return tile, None
except ValueError:
log.info(
"Tile %r will not be skipped, even if empty. This is harmless.",
tile)
reproject(rasterio.band(src, src.indexes),
rasterio.band(tmp, tmp.indexes),
src_nodata=src_nodata,
dst_nodata=dst_nodata,
num_threads=1,
resampling=resampling)
data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
return tile, data