Esempio n. 1
0
def copy_to_workspace(source_uri, dest_uri):
    """
    Translates an image from a URI to a compressed, tiled GeoTIFF version in the workspace
    """

    creation_options = {
        "driver": "GTiff",
        "tiled": True,
        "compress": "lzw",
        "predictor":   2,
        "sparse_ok": True,
        "blockxsize": 512, 
        "blockysize": 512
    }

    with rasterio.open(source_uri, "r") as src:
        meta = src.meta.copy()
        meta.update(creation_options)

        tmp_path = "/vsimem/" + get_filename(dest_uri)

        with rasterio.open(tmp_path, "w", **meta) as tmp:
            tmp.write(src.read())

    contents = bytearray(virtual_file_to_buffer(tmp_path))

    write_bytes_to_target(dest_uri, contents)
Esempio n. 2
0
def _encode_as_png(data, profile, dst_transform):
    """
    Uses rasterio's virtual file system to encode a (3, 512, 512)
    array as a png-encoded bytearray.

    Parameters
    -----------
    data: ndarray
        (3 x 512 x 512) uint8 RGB array
    profile: dictionary
        dictionary of kwargs for png writing
    affine: Affine
        affine transform for output tile

    Returns
    --------
    contents: bytearray
        png-encoded bytearray of the provided input data
    """
    profile["affine"] = dst_transform

    with rasterio.open("/vsimem/tileimg", "w", **profile) as dst:
        dst.write(data)

    contents = bytearray(virtual_file_to_buffer("/vsimem/tileimg"))

    return contents
Esempio n. 3
0
def save_hillshade(tile, data, meta):
    s3 = boto3.resource("s3")
    meta.update(
        driver="GTiff",
        dtype=rasterio.uint8,
        compress="deflate",
        predictor=1,
        nodata=None,
        tiled=True,
        sparse_ok=True,
        blockxsize=DST_BLOCK_SIZE,
        blockysize=DST_BLOCK_SIZE,
    )

    with rasterio.open(TMP_PATH, "w", **meta) as tmp:
        tmp.write(data, 1)

    key = "3857/{}/{}/{}.tif".format(tile.z, tile.x, tile.y)

    s3.Object(
        S3_BUCKET,
        key,
    ).put(
        Body=bytes(bytearray(virtual_file_to_buffer(TMP_PATH))),
        ACL="public-read",
        ContentType="image/tiff",
        CacheControl="public, max-age=2592000",
        StorageClass="REDUCED_REDUNDANCY",
    )

    return "http://{}.s3.amazonaws.com/{}".format(S3_BUCKET, key)
Esempio n. 4
0
def _encode_as_png(data, profile, dst_transform):
    """
    Uses rasterio's virtual file system to encode a (3, 512, 512)
    array as a png-encoded bytearray.

    Parameters
    -----------
    data: ndarray
        (3 x 512 x 512) uint8 RGB array
    profile: dictionary
        dictionary of kwargs for png writing
    affine: Affine
        affine transform for output tile

    Returns
    --------
    contents: bytearray
        png-encoded bytearray of the provided input data
    """
    profile['affine'] = dst_transform

    with rasterio.open('/vsimem/tileimg', 'w', **profile) as dst:
        dst.write(data)

    contents = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))

    return contents
Esempio n. 5
0
    def _write((tile, data)):
        if not contains_data((tile, data)):
            return

        print("Writing", tile)

        # Get the bounds of the tile.
        ulx, uly = mercantile.xy(
            *mercantile.ul(tile.x, tile.y, tile.z))
        lrx, lry = mercantile.xy(
            *mercantile.ul(tile.x + 1, tile.y + 1, tile.z))

        # TODO constantize
        tmp_path = "/vsimem/tile"

        # create GeoTIFF
        meta = creation_options.copy()
        meta["count"] = 1
        meta["nodata"] = data.fill_value
        meta["dtype"] = data.dtype
        meta["width"] = CHUNK_SIZE
        meta["height"] = CHUNK_SIZE
        meta["transform"] = from_bounds(ulx, lry, lrx, uly, CHUNK_SIZE, CHUNK_SIZE)

        with rasterio.drivers():
            with rasterio.open(tmp_path, "w", **meta) as tmp:
                tmp.write(data, 1)

        # write out
        output_uri = urlparse(out_dir)
        contents = bytearray(virtual_file_to_buffer(tmp_path))

        if output_uri.scheme == "s3":
            # TODO use mapPartitions so that the client only needs to be
            # instantiated once per partition
            client = boto3.client("s3")

            bucket = output_uri.netloc
            # TODO strip out trailing slashes on the path if necessary
            key = "%s/%d/%d/%d.tif" % (output_uri.path[1:], tile.z, tile.x, tile.y)

            response = client.put_object(
                ACL="public-read",
                Body=bytes(contents),
                Bucket=bucket,
                # CacheControl="TODO",
                ContentType="image/tiff",
                Key=key
            )
        else:
            output_path = os.path.join(out_dir, "%d/%d/%d.tif" % (tile.z, tile.x, tile.y))
            mkdir_p(os.path.dirname(output_path))

            f = open(output_path, "w")
            f.write(contents)
            f.close()
Esempio n. 6
0
def process_chunk_task(task):
    """
    Chunks the image into tile_dim x tile_dim tiles,
    and saves them to the target folder (s3 or local)

    Returns the extent of the output raster.
    """

    creation_options = {
        "driver": "GTiff",
        "crs": "EPSG:3857",
        "tiled": True,
        "compress": "deflate",
        "predictor":   2, # 3 for floats, 2 otherwise
        "sparse_ok": True
    }

    with rasterio.open(task.source_uri, "r") as src:
        meta = src.meta.copy()
        meta.update(creation_options)
        meta.update(task.target_meta)
        
        cols = meta["width"]
        rows = meta["height"]

        tmp_path = "/vsimem/" + get_filename(task.target)

        with rasterio.open(tmp_path, "w", **meta) as tmp:
            # Reproject the src dataset into image tile.
            warped = []
            for bidx in src.indexes:
                source = rasterio.band(src, bidx)
                warped.append(numpy.zeros((cols, rows), dtype=meta['dtype']))

                warp.reproject(
                    source=source,
                    src_nodata=0,
                    destination=warped[bidx - 1],
                    dst_transform=meta["transform"],
                    dst_crs=meta["crs"],
                    # resampling=RESAMPLING.bilinear
                )

            # check for chunks containing only zero values
            if not any(map(lambda b: b.any(), warped)):
                return False

            # write out our warped data to the vsimem raster
            for bidx in src.indexes:
                tmp.write_band(bidx, warped[bidx - 1])

    contents = bytearray(virtual_file_to_buffer(tmp_path))

    write_bytes_to_target(task.target, contents)
    return True
Esempio n. 7
0
def process_chunk_task(task):
    """
    Chunks the image into tile_dim x tile_dim tiles,
    and saves them to the target folder (s3 or local)

    Returns the extent of the output raster.
    """

    creation_options = {
        "driver": "GTiff",
        "crs": "EPSG:3857",
        "tiled": True,
        "compress": "deflate",
        "predictor": 2,  # 3 for floats, 2 otherwise
        "sparse_ok": True
    }

    with rasterio.open(task.source_uri, "r") as src:
        meta = src.meta.copy()
        meta.update(creation_options)
        meta.update(task.target_meta)

        cols = meta["width"]
        rows = meta["height"]

        tmp_path = "/vsimem/" + get_filename(task.target)

        with rasterio.open(tmp_path, "w", **meta) as tmp:
            # Reproject the src dataset into image tile.
            warped = []
            for bidx in src.indexes:
                source = rasterio.band(src, bidx)
                warped.append(numpy.zeros((cols, rows), dtype=meta['dtype']))

                warp.reproject(
                    source=source,
                    src_nodata=0,
                    destination=warped[bidx - 1],
                    dst_transform=meta["transform"],
                    dst_crs=meta["crs"],
                    # resampling=RESAMPLING.bilinear
                )

            # check for chunks containing only zero values
            if not any(map(lambda b: b.any(), warped)):
                return False

            # write out our warped data to the vsimem raster
            for bidx in src.indexes:
                tmp.write_band(bidx, warped[bidx - 1])

    contents = bytearray(virtual_file_to_buffer(tmp_path))

    write_bytes_to_target(task.target, contents)
    return True
Esempio n. 8
0
def process_tile(tile):
    """Process a single MBTiles tile

    Parameters
    ----------
    tile : mercantile.Tile

    Returns:
    tile : mercantile.Tile
        The input tile.
    bytes : bytearray
        Image bytes corresponding to the tile.
    """
    global base_kwds, resampling, src
    # Get the bounds of the tile.
    ulx, uly = mercantile.xy(
        *mercantile.ul(tile.x, tile.y, tile.z))
    lrx, lry = mercantile.xy(
        *mercantile.ul(tile.x + 1, tile.y + 1, tile.z))

    kwds = base_kwds.copy()
    kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
    src_nodata = kwds.pop('src_nodata', None)
    dst_nodata = kwds.pop('dst_nodata', None)

    with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
        reproject(rasterio.band(src, src.indexes),
                  rasterio.band(tmp, tmp.indexes),
                  src_nodata=src_nodata,
                  dst_nodata=dst_nodata,
                  num_threads=1,
                  resampling=resampling)

    data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))

    # Workaround for https://bugs.python.org/issue23349.
    if sys.version_info[0] == 2 and sys.version_info[2] < 10:
        # Check for backported bug fix before re-ordering
	if kwds['driver'] == 'PNG' and data[0:8] == png_header:
            # Properly constructed PNG, no need to re-order bytes
            pass
	elif kwds['driver'] == 'JPEG' and data[0:4] == jpeg_header:
            # Properly constructed JPEG, no need to re-order bytes
            pass
	else:
            data[:] = data[-1:] + data[:-1]

    return tile, data
Esempio n. 9
0
def process_tile(tile):
    """Process a single MBTiles tile."""
    global base_kwds, src
    # Get the bounds of the tile.
    ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
    lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))

    kwds = base_kwds.copy()
    kwds["transform"] = from_bounds(ulx, lry, lrx, uly, 256, 256)

    with rasterio.open("/vsimem/tileimg", "w", **kwds) as tmp:
        # Reproject the src dataset into image tile.
        for bidx in tmp.indexes:
            reproject(rasterio.band(src, bidx), rasterio.band(tmp, bidx))

    # Get contents of the virtual file.
    contents = bytearray(virtual_file_to_buffer("/vsimem/tileimg"))
    return tile, contents
Esempio n. 10
0
def process_tile(tile):
    """Process a single MBTiles tile."""
    global base_kwds, src
    # Get the bounds of the tile.
    ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
    lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))

    kwds = base_kwds.copy()
    kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)

    with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
        # Reproject the src dataset into image tile.
        for bidx in tmp.indexes:
            reproject(rasterio.band(src, bidx), rasterio.band(tmp, bidx))

    # Get contents of the virtual file and repair it.
    contents = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
    return tile, contents[-1:] + contents[:-1]
def copy_image(copy_job):
    (image_uri, dest_uri) = copy_job
    image_uri = vsi_curlify(image_uri)
    creation_options = {
        "driver": "GTiff",
        "tiled": True,
        "compress": "lzw",
        "predictor":   2,
        "sparse_ok": True,
        "blockxsize": 512,
        "blockysize": 512
    }
    with rasterio.open(image_uri, 'r') as src:
        meta = src.meta.copy()
        meta.update(creation_options)
        tmp_path = "/vsimem/" + get_filename(dest_uri)
        with rasterio.open(tmp_path, 'w', **meta) as tmp:
            tmp.write(src.read())

    contents = bytearray(virtual_file_to_buffer(tmp_path))
    write_bytes_to_target(dest_uri, contents)
Esempio n. 12
0
def process_tile(tile):
    """Process a single MBTiles tile

    Parameters
    ----------
    tile : mercantile.Tile

    Returns:
    tile : mercantile.Tile
        The input tile.
    bytes : bytearray
        Image bytes corresponding to the tile.
    """
    global base_kwds, src
    # Get the bounds of the tile.
    ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
    lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))

    kwds = base_kwds.copy()
    kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
    src_nodata = kwds.pop('src_nodata', None)
    dst_nodata = kwds.pop('dst_nodata', None)

    with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
        reproject(rasterio.band(src, src.indexes),
                  rasterio.band(tmp, tmp.indexes),
                  src_nodata=src_nodata,
                  dst_nodata=dst_nodata,
                  num_threads=1)

    data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))

    # Workaround for https://bugs.python.org/issue23349.
    if sys.version_info[0] == 2 and sys.version_info[2] < 10:
        data[:] = data[-1:] + data[:-1]

    return tile, data
Esempio n. 13
0
def process_tile(tile):
    """Process a single MBTiles tile

    Parameters
    ----------
    tile : mercantile.Tile

    Returns
    -------

    tile : mercantile.Tile
        The input tile.
    bytes : bytearray
        Image bytes corresponding to the tile.

    """
    global base_kwds, resampling, src

    # Get the bounds of the tile.
    ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
    lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))

    kwds = base_kwds.copy()
    # kwds['transform'] = transform_from_bounds(ulx, lry, lrx, uly, 256, 256)
    kwds['transform'] = transform_from_bounds(ulx, lry, lrx, uly, 512, 512)
    src_nodata = kwds.pop('src_nodata', None)
    dst_nodata = kwds.pop('dst_nodata', None)

    with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:

        # determine window of source raster corresponding to the tile
        # image, with small buffer at edges
        try:
            west, south, east, north = transform_bounds(
                TILES_CRS, src.crs, ulx, lry, lrx, uly)
            tile_window = window_from_bounds(west,
                                             south,
                                             east,
                                             north,
                                             transform=src.transform)
            adjusted_tile_window = Window(tile_window.col_off - 1,
                                          tile_window.row_off - 1,
                                          tile_window.width + 2,
                                          tile_window.height + 2)
            tile_window = adjusted_tile_window.round_offsets().round_shape()

            # if no data in window, skip processing the tile
            if not src.read_masks(1, window=tile_window).any():
                return tile, None

        except ValueError:
            log.info(
                "Tile %r will not be skipped, even if empty. This is harmless.",
                tile)

        reproject(rasterio.band(src, src.indexes),
                  rasterio.band(tmp, tmp.indexes),
                  src_nodata=src_nodata,
                  dst_nodata=dst_nodata,
                  num_threads=1,
                  resampling=resampling)

    data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
    return tile, data