def __init__(self, url, zoom=18, bounds=None):
self.zoom_level = zoom
self._name = "image-{}".format(str(uuid.uuid4()))
self._url = url
_first_tile = mercantile.Tile(z=self.zoom_level, x=0, y=0)
_last_tile = mercantile.Tile(z=self.zoom_level, x=180, y=-85.05)
g = box(*mercantile.xy_bounds(_first_tile)).union(
box(*mercantile.xy_bounds(_last_tile)))
self._full_bounds = g.bounds
# TODO: populate rest of fields automatically
self._tile_size = 256
self._nbands = 3
self._dtype = "uint8"
self.bounds = self._expand_bounds(bounds)
def test_tile_read_internal_nodata():
"""Read masked area."""
# non-boundless tile covering the nodata part
mercator_tile = mercantile.Tile(x=876431, y=1603670, z=22)
bounds = mercantile.xy_bounds(mercator_tile)
arr, mask = utils.tile_read(S3_NODATA_PATH, bounds, 256, indexes=(1, 2, 3))
assert arr.shape == (3, 256, 256)
assert not mask.all()
def test_tile_read_alpha():
"""Read masked area."""
# non-boundless tile covering the alpha masked part
mercator_tile = mercantile.Tile(x=876432, y=1603670, z=22)
bounds = mercantile.xy_bounds(mercator_tile)
arr, mask = utils.tile_read(S3_ALPHA_PATH, bounds, 256, indexes=(1, 2, 3))
assert arr.shape == (3, 256, 256)
assert not mask.all()
def test_getitem(self):
dataset = SlippyMapTiles(TestSlippyMapTiles.images)
image, tile = dataset[0]
assert tile == mercantile.Tile(69105, 105093, 18)
# Inspired by: https://github.com/python-pillow/Pillow/blob/master/Tests/test_image.py#L37-L38
self.assertEqual(repr(image)[:45], "<PIL.JpegImagePlugin.JpegImageFile image mode")
self.assertEqual(image.size, (512, 512))
def increment(z, y, x):
#print("Incrementing", (x,y,z))
while z > 1:
key = (z, y, x)
Heatmap.count_dict[key] += 1
tile = mercantile.Tile(x, y, z)
tile = mercantile.parent(tile)
x, y, z = tile.x, tile.y, tile.z
def tile_from_slippy_map(root, x, y, z):
"""Retrieve a single tile from a slippy map dir."""
path = glob.glob(os.path.join(os.path.expanduser(root), z, x, y + ".*"))
if not path:
return None
return mercantile.Tile(x, y, z), path[0]
def get_assets(self, x: int, y: int, z: int) -> List[str]:
"""Find assets."""
mercator_tile = mercantile.Tile(x=x, y=y, z=z)
quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom)
return list(
itertools.chain.from_iterable([
self._fetch_dynamodb(qk).get("assets", []) for qk in quadkeys
]))
def expression(self,
x: int,
y: int,
z: int,
expr: str,
tilesize: int = 256,
**kwargs: Any) -> Tuple[numpy.ndarray, numpy.ndarray]:
"""
Read Tile from a COG and apply simple band math.
Attributes
----------
x : int
X tile map index.
y : int
y tile map index
z : int
Z tile map index
expr : str
Band math expression.
tilesize : int, optional
Output tile size. Default is 256
kwargs : Any, optional
Additional options to forward to rio_tiler.utils._tile_read
Returns
-------
tile : numpy.ndarray
Tile data.
mask : numpy.ndarray
Mask data.
"""
if not self.tile_exists(x, y, z):
raise TileOutsideBounds(
f"Tile {z}/{x}/{y} is outside image bounds")
tile_bounds = mercantile.xy_bounds(mercantile.Tile(x=x, y=y, z=z))
bands_names = tuple(set(re.findall(r"b(?P<bands>[0-9A]{1,2})", expr)))
expr_bands = ["b{}".format(b) for b in bands_names]
indexes = tuple(map(int, bands_names))
tile, mask = _tile_read(self.src_dst,
tile_bounds,
tilesize,
indexes=indexes,
**kwargs)
tile = dict(zip(expr_bands, tile))
rgb = expr.split(",")
return (
numpy.array([
numpy.nan_to_num(
numexpr.evaluate(bloc.strip(), local_dict=tile))
for bloc in rgb
]),
mask,
)
def main(args):
dataset = load_config(args.dataset)
classes = dataset["common"]["classes"]
colors = dataset["common"]["colors"]
assert len(classes) == len(colors), "classes and colors coincide"
assert len(colors) == 2, "only binary models supported right now"
bg = colors[0]
fg = colors[1]
os.makedirs(args.out, exist_ok=True)
# We can only rasterize all tiles at a single zoom.
assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles))
with open(args.features) as f:
fc = json.load(f)
# Find all tiles the features cover and make a map object for quick lookup.
feature_map = collections.defaultdict(list)
for i, feature in enumerate(
tqdm(fc["features"], ascii=True, unit="feature")):
if feature["geometry"]["type"] != "Polygon":
continue
try:
for tile in burntiles.burn([feature], zoom=args.zoom):
feature_map[mercantile.Tile(*tile)].append(feature)
except ValueError as e:
print("Warning: invalid feature {}, skipping".format(i),
file=sys.stderr)
continue
# Burn features to tiles and write to a slippy map directory.
for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True,
unit="tile"):
if tile in feature_map:
out = burn(tile, feature_map[tile], args.size)
else:
out = np.zeros(shape=(args.size, args.size), dtype=np.uint8)
out_dir = os.path.join(args.out, str(tile.z), str(tile.x))
os.makedirs(out_dir, exist_ok=True)
out_path = os.path.join(out_dir, "{}.png".format(tile.y))
if os.path.exists(out_path):
prev = np.array(Image.open(out_path))
out = np.maximum(out, prev)
out = Image.fromarray(out, mode="P")
palette = make_palette(bg, fg)
out.putpalette(palette)
out.save(out_path, optimize=True)
async def test_read_not_in_bounds(create_cog_reader, infile):
tile = mercantile.Tile(x=0, y=0, z=25)
bounds = mercantile.xy_bounds(tile)
async with create_cog_reader(infile) as cog:
if cog.epsg != 3857:
bounds = transform_bounds("EPSG:3857", f"EPSG:{cog.epsg}", *bounds)
with pytest.raises(TileNotFoundError):
await cog.read(bounds=bounds, shape=(256, 256))
def tile(sceneid, tile_x, tile_y, tile_z, bands=("04", "03", "02"), tilesize=256):
"""
Create mercator tile from Sentinel-2 data.
Attributes
----------
sceneid : str
Sentinel-2 sceneid.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
bands : tuple, str, optional (default: ('04', '03', '02'))
Bands index for the RGB combination.
tilesize : int, optional (default: 256)
Output image size.
Returns
-------
data : numpy ndarray
mask: numpy array
"""
if not isinstance(bands, tuple):
bands = tuple((bands,))
for band in bands:
if band not in SENTINEL_BANDS:
raise InvalidBandName("{} is not a valid Sentinel band name".format(band))
scene_params = _sentinel_parse_scene_id(sceneid)
sentinel_address = "{}/{}".format(SENTINEL_BUCKET, scene_params["key"])
sentinel_preview = "{}/preview.jp2".format(sentinel_address)
with rasterio.open(sentinel_preview) as src:
wgs_bounds = transform_bounds(
*[src.crs, "epsg:4326"] + list(src.bounds), densify_pts=21
)
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds(
"Tile {}/{}/{} is outside image bounds".format(tile_z, tile_x, tile_y)
)
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
addresses = ["{}/B{}.jp2".format(sentinel_address, band) for band in bands]
_tiler = partial(utils.tile_read, bounds=tile_bounds, tilesize=tilesize, nodata=0)
with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
data, masks = zip(*list(executor.map(_tiler, addresses)))
mask = np.all(masks, axis=0).astype(np.uint8) * 255
return np.concatenate(data), mask
def tile_image_neighbour(tile, dx, dy, tiles, bands):
"""Retrieves neighbour tile image if exists."""
try:
path = tiles[mercantile.Tile(x=int(tile.x) + dx,
y=int(tile.y) + dy,
z=int(tile.z))]
except KeyError:
return None
return tile_image_from_file(path, bands)
def test_tile_read_wrong_nodata():
"""Return empty mask on wrong nodata."""
# non-boundless tile covering the nodata part
mercator_tile = mercantile.Tile(x=438217, y=801835, z=21)
bounds = mercantile.xy_bounds(mercator_tile)
arr, mask = utils.tile_read(
S3_NODATA_PATH, bounds, 256, indexes=(1, 2, 3), nodata=1000
)
assert arr.shape == (3, 256, 256)
assert mask.all()
# Mask boundless values
mercator_tile = mercantile.Tile(x=109554, y=200458, z=19)
bounds = mercantile.xy_bounds(mercator_tile)
arr, mask = utils.tile_read(
S3_NODATA_PATH, bounds, 256, indexes=(1, 2, 3), nodata=1000
)
assert arr.shape == (3, 256, 256)
assert not mask.all()
def tile(address,
tile_x,
tile_y,
tile_z,
rgb=None,
tilesize=256,
nodata=None,
alpha=None):
"""Create mercator tile from any images.
Attributes
----------
address : str
file url.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
rgb : tuple, int, optional (default: (1, 2, 3))
Bands index for the RGB combination.
tilesize : int, optional (default: 256)
Output image size.
nodata: int or float, optional (defaults: None)
alpha: int, optional (defaults: None)
Force alphaband if not present in the dataset metadata
Returns
-------
data : numpy ndarray
mask: numpy array
"""
with rasterio.open(address) as src:
wgs_bounds = transform_bounds(*[src.crs, 'epsg:4326'] +
list(src.bounds),
densify_pts=21)
nodata = nodata if nodata is not None else src.nodata
if not rgb:
rgb = src.indexes
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds('Tile {}/{}/{} is outside image bounds'.format(
tile_z, tile_x, tile_y))
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
return utils.tile_band_worker(address,
tile_bounds,
tilesize,
indexes=rgb,
nodata=nodata,
alpha=alpha)
def partial_update(dsn: str, starting: str = '000') -> None:
final_status: str = 'SUCCESS'
sql_queries: list = []
# get paths of sql files
# r=root, d=directories, f = files
for r, d, f in walk(partial_update_path):
for file in f:
if file.endswith('.sql') and file >= starting:
sql_queries.append(join(r, file))
sql_queries = [x for x in sorted(sql_queries)]
with pg.connect(dsn, **keepalive_kwargs) as conn:
cur = conn.cursor()
cur.execute('SELECT in_progress FROM process_locks WHERE process_name in (%s, %s)',
('prg_full_update', 'prg_partial_update'))
update_in_progress = [x[0] for x in cur.fetchall()]
if not any(update_in_progress):
print(datetime.now(timezone.utc).astimezone().isoformat(), '- starting partial update process.')
cur.execute('UPDATE process_locks SET (in_progress, start_time, end_time) = (true, \'now\', null) ' +
'WHERE process_name = %s',
('prg_partial_update',))
conn.commit()
cur.execute('SELECT * FROM expired_tiles WHERE processed = false FOR UPDATE SKIP LOCKED;')
for i, row in enumerate(cur.fetchall()):
x, y, z = row[2], row[3], row[1]
tile = m.Tile(x, y, z)
bbox = to_merc(m.bounds(tile))
bbox = {'xmin': bbox['west'], 'ymin': bbox['south'], 'xmax': bbox['east'], 'ymax': bbox['north']}
try:
execute_scripts_from_files(conn=conn, vacuum='never', paths=sql_queries,
query_parameters=bbox, commit_mode='off')
except:
print(datetime.now(timezone.utc).astimezone().isoformat(), '- failure in partial update process.')
final_status = 'FAIL'
conn.rollback()
cur.execute(
'UPDATE expired_tiles SET processed = false ' +
'WHERE file_name = %s and z = %s and x = %s and y = %s;',
(row[0], row[1], row[2], row[3])
)
conn.commit()
break
cur.execute(
'UPDATE expired_tiles SET processed = true WHERE file_name = %s and z = %s and x = %s and y = %s;',
(row[0], row[1], row[2], row[3])
)
if i % 10 == 0:
conn.commit()
cur.execute(
'UPDATE process_locks SET (in_progress, end_time, last_status) = (false, \'now\', %s) WHERE process_name = %s',
(final_status, 'prg_partial_update',))
conn.commit()
print(datetime.now(timezone.utc).astimezone().isoformat(), '- finished partial update process.')
else:
print(datetime.now(timezone.utc).astimezone().isoformat(),
'- update in progress skipping partial update.')
def test_tile_read_extmask():
"""Read masked area."""
# non-boundless tile covering the masked part
mercator_tile = mercantile.Tile(x=876431, y=1603669, z=22)
bounds = mercantile.xy_bounds(mercator_tile)
with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN="TRUE"):
with rasterio.open(S3_EXTMASK_PATH) as src_dst:
arr, mask = reader.part(src_dst, bounds, 256, 256)
assert arr.shape == (3, 256, 256)
assert mask.shape == (256, 256)
assert not mask.all()
# boundless tile covering the masked part
mercator_tile = mercantile.Tile(x=876431, y=1603668, z=22)
bounds = mercantile.xy_bounds(mercator_tile)
with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR"):
with rasterio.open(S3_MASK_PATH) as src_dst:
arr, mask = reader.part(src_dst, bounds, 256, 256)
assert arr.shape == (3, 256, 256)
assert not mask.all()
def all_tiles(self):
tiles = self.conn.execute(
'select zoom_level, tile_column, tile_row from tiles')
for tile in tiles:
z = tile[0]
x = tile[1]
y = tile[2]
if self.scheme == 'tms':
y = mbutil.flip_y(z, y)
yield mercantile.Tile(x, y, z)
def create_tiles_xy(self, x, y, reg, border=1):
result = []
tile = mercantile.Tile(x, y, self.zoom)
for tr in self.rasters:
if tr.register == reg:
a = self.create_single_tile(tile, tr.file, postfix=self._postfix_file(tr.file), interp=tr.interp,
border=border)
if a is not None:
result.append(a)
return result
def tile_image_adjacent(tile, dx, dy, tiles):
"""Retrieves an adjacent tile image if exists from a tile store, or None."""
try:
path = tiles[mercantile.Tile(x=int(tile.x) + dx,
y=int(tile.y) + dy,
z=int(tile.z))]
except KeyError:
return None
return tile_image(path)
def geotiff_options(
tile_x,
tile_y,
tile_z,
tilesize: int = 256,
dst_crs: CRS = CRS.from_epsg(3857)) -> Dict:
"""GeoTIFF options."""
bounds = mercantile.xy_bounds(mercantile.Tile(x=tile_x, y=tile_y,
z=tile_z))
dst_transform = from_bounds(*bounds, tilesize, tilesize)
return dict(crs=dst_crs, transform=dst_transform)
def test_find_quadkeys():
"""Get correct quadkeys"""
tile = mercantile.Tile(150, 182, 9)
assert utils.find_quadkeys(tile, 8) == ["03023033"]
assert utils.find_quadkeys(tile, 9) == ["030230330"]
assert utils.find_quadkeys(tile, 10) == [
"0302303300",
"0302303301",
"0302303303",
"0302303302",
]
def __init__(self, center, bbox, zoom, basemap, opacity, size=None):
self._configure_event_loop()
point = center['geometry']['coordinates']
if size is None:
size = IMAGE_SIZE
self.num_tiles = [
math.ceil(size[x] / TILE_SIZE[x]) + 1 for x in (0, 1)
]
center_tile = mercantile.tile(point[0], point[1], zoom)
mercator = Proj(init='epsg:3857')
wgs84 = Proj(init='epsg:4326')
center_tile_bbox = BBox(mercantile.bounds(*center_tile),
projection=wgs84).project(mercator,
edge_points=0)
center_to_image = world_to_image(center_tile_bbox, TILE_SIZE)
center_to_world = image_to_world(center_tile_bbox, TILE_SIZE)
center_point_px = center_to_image(*mercantile.xy(*point))
self.ul_tile = mercantile.tile(*transform(
mercator, wgs84,
*center_to_world(center_point_px[0] -
math.ceil(IMAGE_SIZE[0] / 2), center_point_px[1] -
math.ceil(IMAGE_SIZE[1] / 2)), zoom))
lr_tile = mercantile.Tile(x=min(2**zoom,
self.ul_tile.x + self.num_tiles[0]),
y=min(2**zoom,
self.ul_tile.y + self.num_tiles[1]),
z=zoom)
ul = mercantile.xy(*mercantile.ul(*self.ul_tile))
lr = mercantile.xy(*mercantile.ul(*lr_tile))
self.image_bbox = BBox((ul[0], lr[1], lr[0], ul[1]))
self.image_size = (TILE_SIZE[0] * self.num_tiles[0],
TILE_SIZE[1] * self.num_tiles[1])
self.to_image = world_to_image(self.image_bbox, self.image_size)
self.to_world = image_to_world(self.image_bbox, self.image_size)
self.point_px = [
round(x) for x in self.to_image(*mercantile.xy(*point))
]
self.target_size = size
self.point = point
self.zoom = zoom
self.basemap = basemap
self._basemap_image = None
self.opacity = opacity
def get_assets(self, x: int, y: int, z: int) -> List[str]:
"""Find assets."""
mercator_tile = mercantile.Tile(x=x, y=y, z=z)
quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom)
return list(
dict.fromkeys(
itertools.chain.from_iterable(
[self.mosaic_def.tiles.get(qk, []) for qk in quadkeys]
)
)
)
def tiles_from_csv(path):
"""Retrieve tiles from a line-delimited csv file."""
with open(os.path.expanduser(path)) as fp:
reader = csv.reader(fp)
for row in reader:
if not row:
continue
yield mercantile.Tile(*map(int, row))
def read_tile(self, z, x, y):
"""Read raster tile data and mask."""
mercator_tile = mercantile.Tile(x=x, y=y, z=z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
return tile_read(
self.path,
tile_bounds,
self.tiles_size,
indexes=self.indexes,
nodata=self.nodata,
)
def __init__(self, access_token=os.environ.get("DG_MAPS_API_TOKEN"),
url="https://api.mapbox.com/v4/digitalglobe.nal0g75k/{z}/{x}/{y}.png",
zoom=22, bounds=None):
self.zoom_level = zoom
self._token = access_token
self._name = "image-{}".format(str(uuid.uuid4()))
self._url_template = url + "?access_token={token}"
_first_tile = mercantile.Tile(z=self.zoom_level, x=0, y=0)
_last_tile = mercantile.Tile(z=self.zoom_level, x=180, y=-85.05)
g = box(*mercantile.xy_bounds(_first_tile)).union(box(*mercantile.xy_bounds(_last_tile)))
self._full_bounds = g.bounds
# TODO: populate rest of fields automatically
self._tile_size = 256
self._nbands = 3
self._dtype = "uint8"
self.bounds = self._expand_bounds(bounds)
self._chunks = tuple([self._nbands] + [self._tile_size, self._tile_size])
def download_tiles(minzoom, maxzoom, bbox, url, path, tile_cover=False, skip_existing=False):
if not os.path.exists(path):
os.makedirs(path)
if tile_cover:
ul = mercantile.tile(bbox[0], bbox[3], minzoom)
lr = mercantile.tile(bbox[2], bbox[1], minzoom)
ul_bounds = mercantile.bounds(ul.x, ul.y, ul.z)
lr_bounds = mercantile.bounds(lr.x, lr.y, lr.z)
bbox = (ul_bounds.west, lr_bounds.south, lr_bounds.east, ul_bounds.north)
for zoom in range(minzoom, maxzoom + 1):
tile_url_z = url.replace("{z}", str(zoom))
z_dir = os.path.join(path, str(zoom))
if not os.path.exists(z_dir):
os.makedirs(z_dir)
ul = mercantile.tile(bbox[0], bbox[3], zoom)
lr = mercantile.tile(bbox[2], bbox[1], zoom)
if ul.x < 0:
ul = mercantile.Tile(x=0, y=ul.y, z=ul.z)
if ul.y < 0:
ul = mercantile.Tile(x=ul.x, y=0, z=ul.z)
max_tile = pow(2, zoom) - 1
if lr.x > max_tile:
lr = mercantile.Tile(x=max_tile, y=lr.y, z=lr.z)
if lr.y > max_tile:
lr = mercantile.Tile(x=lr.x, y=max_tile, z=lr.z)
logging.info("Downloading tiles for zoom %d x:%d-%d y:%d-%d " % (zoom, ul.x, lr.x, ul.y, lr.y))
for x in range(ul.x, lr.x + 1):
x_dir = os.path.join(z_dir, str(x))
if not os.path.exists(x_dir):
os.makedirs(x_dir)
tile_url_x = tile_url_z.replace("{x}", str(x))
for y in range(ul.y, lr.y + 1):
tile_url_y = tile_url_x.replace("{y}", str(y))
file_path = os.path.join(x_dir, str(y) + ".png")
try:
download_tile(tile_url_y, file_path, skip_existing=skip_existing)
except Exception as e:
logging.debug(e)
logging.error("Failed to download tile: " + tile_url_y)
def in_tile(self, request, zoom_level, x, y):
"""Returns all entries within the given tile
For more info about tiles and slippy maps, see https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames
"""
tile = mercantile.Tile(int(x), int(y), int(zoom_level))
bbox = mercantile.bounds(tile)
polygon = geos.Polygon.from_bbox(bbox)
dumpsters = Dumpster.objects.filter(location__within=polygon)
serializer = DumpsterSerializer(dumpsters, many=True)
return Response(serializer.data)
def tile(bucket, key, tile_x, tile_y, tile_z, rgb=(1, 2, 3), tilesize=256,
prefix='s3:/'):
"""Create mercator tile from AWS hosted images and encodes it in base64.
Attributes
----------
bucket : str
AWS bucket's name.
key : str
AWS file's key.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
tileformat : str
Image format to return (Accepted: "jpg" or "png")
rgb : tuple, int, optional (default: (1, 2, 3))
Bands index for the RGB combination.
tilesize : int, optional (default: 256)
Output image size.
Returns
-------
out : numpy ndarray
"""
source_address = '{}/{}/{}'.format(prefix, bucket, key)
with rasterio.open(source_address) as src:
wgs_bounds = transform_bounds(
*[src.crs, 'epsg:4326'] + list(src.bounds), densify_pts=21)
nodata = src.nodata
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds(
'Tile {}/{}/{} is outside image bounds'.format(
tile_z, tile_x, tile_y))
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
w, s, e, n = tile_bounds
out = utils.tile_band_worker(source_address,
tile_bounds,
tilesize,
indexes=rgb,
nodata=nodata)
return out
def tile_from_xyz(root, x, y, z):
"""Retrieve a single tile from a slippy map dir."""
path = glob.glob(
os.path.join(os.path.expanduser(root), str(z), str(x),
str(y) + ".*"))
if not path:
return None
assert len(path) == 1, "ambiguous tile path"
return mercantile.Tile(x, y, z), path[0]
