def __init__(self, url, zoom=18, bounds=None): self.zoom_level = zoom self._name = "image-{}".format(str(uuid.uuid4())) self._url = url _first_tile = mercantile.Tile(z=self.zoom_level, x=0, y=0) _last_tile = mercantile.Tile(z=self.zoom_level, x=180, y=-85.05) g = box(*mercantile.xy_bounds(_first_tile)).union( box(*mercantile.xy_bounds(_last_tile))) self._full_bounds = g.bounds # TODO: populate rest of fields automatically self._tile_size = 256 self._nbands = 3 self._dtype = "uint8" self.bounds = self._expand_bounds(bounds)
def test_tile_read_internal_nodata(): """Read masked area.""" # non-boundless tile covering the nodata part mercator_tile = mercantile.Tile(x=876431, y=1603670, z=22) bounds = mercantile.xy_bounds(mercator_tile) arr, mask = utils.tile_read(S3_NODATA_PATH, bounds, 256, indexes=(1, 2, 3)) assert arr.shape == (3, 256, 256) assert not mask.all()
def test_tile_read_alpha(): """Read masked area.""" # non-boundless tile covering the alpha masked part mercator_tile = mercantile.Tile(x=876432, y=1603670, z=22) bounds = mercantile.xy_bounds(mercator_tile) arr, mask = utils.tile_read(S3_ALPHA_PATH, bounds, 256, indexes=(1, 2, 3)) assert arr.shape == (3, 256, 256) assert not mask.all()
def test_getitem(self): dataset = SlippyMapTiles(TestSlippyMapTiles.images) image, tile = dataset[0] assert tile == mercantile.Tile(69105, 105093, 18) # Inspired by: https://github.com/python-pillow/Pillow/blob/master/Tests/test_image.py#L37-L38 self.assertEqual(repr(image)[:45], "<PIL.JpegImagePlugin.JpegImageFile image mode") self.assertEqual(image.size, (512, 512))
def increment(z, y, x): #print("Incrementing", (x,y,z)) while z > 1: key = (z, y, x) Heatmap.count_dict[key] += 1 tile = mercantile.Tile(x, y, z) tile = mercantile.parent(tile) x, y, z = tile.x, tile.y, tile.z
def tile_from_slippy_map(root, x, y, z): """Retrieve a single tile from a slippy map dir.""" path = glob.glob(os.path.join(os.path.expanduser(root), z, x, y + ".*")) if not path: return None return mercantile.Tile(x, y, z), path[0]
def get_assets(self, x: int, y: int, z: int) -> List[str]: """Find assets.""" mercator_tile = mercantile.Tile(x=x, y=y, z=z) quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom) return list( itertools.chain.from_iterable([ self._fetch_dynamodb(qk).get("assets", []) for qk in quadkeys ]))
def expression(self, x: int, y: int, z: int, expr: str, tilesize: int = 256, **kwargs: Any) -> Tuple[numpy.ndarray, numpy.ndarray]: """ Read Tile from a COG and apply simple band math. Attributes ---------- x : int X tile map index. y : int y tile map index z : int Z tile map index expr : str Band math expression. tilesize : int, optional Output tile size. Default is 256 kwargs : Any, optional Additional options to forward to rio_tiler.utils._tile_read Returns ------- tile : numpy.ndarray Tile data. mask : numpy.ndarray Mask data. """ if not self.tile_exists(x, y, z): raise TileOutsideBounds( f"Tile {z}/{x}/{y} is outside image bounds") tile_bounds = mercantile.xy_bounds(mercantile.Tile(x=x, y=y, z=z)) bands_names = tuple(set(re.findall(r"b(?P<bands>[0-9A]{1,2})", expr))) expr_bands = ["b{}".format(b) for b in bands_names] indexes = tuple(map(int, bands_names)) tile, mask = _tile_read(self.src_dst, tile_bounds, tilesize, indexes=indexes, **kwargs) tile = dict(zip(expr_bands, tile)) rgb = expr.split(",") return ( numpy.array([ numpy.nan_to_num( numexpr.evaluate(bloc.strip(), local_dict=tile)) for bloc in rgb ]), mask, )
def main(args): dataset = load_config(args.dataset) classes = dataset["common"]["classes"] colors = dataset["common"]["colors"] assert len(classes) == len(colors), "classes and colors coincide" assert len(colors) == 2, "only binary models supported right now" bg = colors[0] fg = colors[1] os.makedirs(args.out, exist_ok=True) # We can only rasterize all tiles at a single zoom. assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles)) with open(args.features) as f: fc = json.load(f) # Find all tiles the features cover and make a map object for quick lookup. feature_map = collections.defaultdict(list) for i, feature in enumerate( tqdm(fc["features"], ascii=True, unit="feature")): if feature["geometry"]["type"] != "Polygon": continue try: for tile in burntiles.burn([feature], zoom=args.zoom): feature_map[mercantile.Tile(*tile)].append(feature) except ValueError as e: print("Warning: invalid feature {}, skipping".format(i), file=sys.stderr) continue # Burn features to tiles and write to a slippy map directory. for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit="tile"): if tile in feature_map: out = burn(tile, feature_map[tile], args.size) else: out = np.zeros(shape=(args.size, args.size), dtype=np.uint8) out_dir = os.path.join(args.out, str(tile.z), str(tile.x)) os.makedirs(out_dir, exist_ok=True) out_path = os.path.join(out_dir, "{}.png".format(tile.y)) if os.path.exists(out_path): prev = np.array(Image.open(out_path)) out = np.maximum(out, prev) out = Image.fromarray(out, mode="P") palette = make_palette(bg, fg) out.putpalette(palette) out.save(out_path, optimize=True)
async def test_read_not_in_bounds(create_cog_reader, infile): tile = mercantile.Tile(x=0, y=0, z=25) bounds = mercantile.xy_bounds(tile) async with create_cog_reader(infile) as cog: if cog.epsg != 3857: bounds = transform_bounds("EPSG:3857", f"EPSG:{cog.epsg}", *bounds) with pytest.raises(TileNotFoundError): await cog.read(bounds=bounds, shape=(256, 256))
def tile(sceneid, tile_x, tile_y, tile_z, bands=("04", "03", "02"), tilesize=256): """ Create mercator tile from Sentinel-2 data. Attributes ---------- sceneid : str Sentinel-2 sceneid. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. bands : tuple, str, optional (default: ('04', '03', '02')) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. Returns ------- data : numpy ndarray mask: numpy array """ if not isinstance(bands, tuple): bands = tuple((bands,)) for band in bands: if band not in SENTINEL_BANDS: raise InvalidBandName("{} is not a valid Sentinel band name".format(band)) scene_params = _sentinel_parse_scene_id(sceneid) sentinel_address = "{}/{}".format(SENTINEL_BUCKET, scene_params["key"]) sentinel_preview = "{}/preview.jp2".format(sentinel_address) with rasterio.open(sentinel_preview) as src: wgs_bounds = transform_bounds( *[src.crs, "epsg:4326"] + list(src.bounds), densify_pts=21 ) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds( "Tile {}/{}/{} is outside image bounds".format(tile_z, tile_x, tile_y) ) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) addresses = ["{}/B{}.jp2".format(sentinel_address, band) for band in bands] _tiler = partial(utils.tile_read, bounds=tile_bounds, tilesize=tilesize, nodata=0) with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor: data, masks = zip(*list(executor.map(_tiler, addresses))) mask = np.all(masks, axis=0).astype(np.uint8) * 255 return np.concatenate(data), mask
def tile_image_neighbour(tile, dx, dy, tiles, bands): """Retrieves neighbour tile image if exists.""" try: path = tiles[mercantile.Tile(x=int(tile.x) + dx, y=int(tile.y) + dy, z=int(tile.z))] except KeyError: return None return tile_image_from_file(path, bands)
def test_tile_read_wrong_nodata(): """Return empty mask on wrong nodata.""" # non-boundless tile covering the nodata part mercator_tile = mercantile.Tile(x=438217, y=801835, z=21) bounds = mercantile.xy_bounds(mercator_tile) arr, mask = utils.tile_read( S3_NODATA_PATH, bounds, 256, indexes=(1, 2, 3), nodata=1000 ) assert arr.shape == (3, 256, 256) assert mask.all() # Mask boundless values mercator_tile = mercantile.Tile(x=109554, y=200458, z=19) bounds = mercantile.xy_bounds(mercator_tile) arr, mask = utils.tile_read( S3_NODATA_PATH, bounds, 256, indexes=(1, 2, 3), nodata=1000 ) assert arr.shape == (3, 256, 256) assert not mask.all()
def tile(address, tile_x, tile_y, tile_z, rgb=None, tilesize=256, nodata=None, alpha=None): """Create mercator tile from any images. Attributes ---------- address : str file url. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. rgb : tuple, int, optional (default: (1, 2, 3)) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. nodata: int or float, optional (defaults: None) alpha: int, optional (defaults: None) Force alphaband if not present in the dataset metadata Returns ------- data : numpy ndarray mask: numpy array """ with rasterio.open(address) as src: wgs_bounds = transform_bounds(*[src.crs, 'epsg:4326'] + list(src.bounds), densify_pts=21) nodata = nodata if nodata is not None else src.nodata if not rgb: rgb = src.indexes if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds('Tile {}/{}/{} is outside image bounds'.format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) return utils.tile_band_worker(address, tile_bounds, tilesize, indexes=rgb, nodata=nodata, alpha=alpha)
def partial_update(dsn: str, starting: str = '000') -> None: final_status: str = 'SUCCESS' sql_queries: list = [] # get paths of sql files # r=root, d=directories, f = files for r, d, f in walk(partial_update_path): for file in f: if file.endswith('.sql') and file >= starting: sql_queries.append(join(r, file)) sql_queries = [x for x in sorted(sql_queries)] with pg.connect(dsn, **keepalive_kwargs) as conn: cur = conn.cursor() cur.execute('SELECT in_progress FROM process_locks WHERE process_name in (%s, %s)', ('prg_full_update', 'prg_partial_update')) update_in_progress = [x[0] for x in cur.fetchall()] if not any(update_in_progress): print(datetime.now(timezone.utc).astimezone().isoformat(), '- starting partial update process.') cur.execute('UPDATE process_locks SET (in_progress, start_time, end_time) = (true, \'now\', null) ' + 'WHERE process_name = %s', ('prg_partial_update',)) conn.commit() cur.execute('SELECT * FROM expired_tiles WHERE processed = false FOR UPDATE SKIP LOCKED;') for i, row in enumerate(cur.fetchall()): x, y, z = row[2], row[3], row[1] tile = m.Tile(x, y, z) bbox = to_merc(m.bounds(tile)) bbox = {'xmin': bbox['west'], 'ymin': bbox['south'], 'xmax': bbox['east'], 'ymax': bbox['north']} try: execute_scripts_from_files(conn=conn, vacuum='never', paths=sql_queries, query_parameters=bbox, commit_mode='off') except: print(datetime.now(timezone.utc).astimezone().isoformat(), '- failure in partial update process.') final_status = 'FAIL' conn.rollback() cur.execute( 'UPDATE expired_tiles SET processed = false ' + 'WHERE file_name = %s and z = %s and x = %s and y = %s;', (row[0], row[1], row[2], row[3]) ) conn.commit() break cur.execute( 'UPDATE expired_tiles SET processed = true WHERE file_name = %s and z = %s and x = %s and y = %s;', (row[0], row[1], row[2], row[3]) ) if i % 10 == 0: conn.commit() cur.execute( 'UPDATE process_locks SET (in_progress, end_time, last_status) = (false, \'now\', %s) WHERE process_name = %s', (final_status, 'prg_partial_update',)) conn.commit() print(datetime.now(timezone.utc).astimezone().isoformat(), '- finished partial update process.') else: print(datetime.now(timezone.utc).astimezone().isoformat(), '- update in progress skipping partial update.')
def test_tile_read_extmask(): """Read masked area.""" # non-boundless tile covering the masked part mercator_tile = mercantile.Tile(x=876431, y=1603669, z=22) bounds = mercantile.xy_bounds(mercator_tile) with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN="TRUE"): with rasterio.open(S3_EXTMASK_PATH) as src_dst: arr, mask = reader.part(src_dst, bounds, 256, 256) assert arr.shape == (3, 256, 256) assert mask.shape == (256, 256) assert not mask.all() # boundless tile covering the masked part mercator_tile = mercantile.Tile(x=876431, y=1603668, z=22) bounds = mercantile.xy_bounds(mercator_tile) with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR"): with rasterio.open(S3_MASK_PATH) as src_dst: arr, mask = reader.part(src_dst, bounds, 256, 256) assert arr.shape == (3, 256, 256) assert not mask.all()
def all_tiles(self): tiles = self.conn.execute( 'select zoom_level, tile_column, tile_row from tiles') for tile in tiles: z = tile[0] x = tile[1] y = tile[2] if self.scheme == 'tms': y = mbutil.flip_y(z, y) yield mercantile.Tile(x, y, z)
def create_tiles_xy(self, x, y, reg, border=1): result = [] tile = mercantile.Tile(x, y, self.zoom) for tr in self.rasters: if tr.register == reg: a = self.create_single_tile(tile, tr.file, postfix=self._postfix_file(tr.file), interp=tr.interp, border=border) if a is not None: result.append(a) return result
def tile_image_adjacent(tile, dx, dy, tiles): """Retrieves an adjacent tile image if exists from a tile store, or None.""" try: path = tiles[mercantile.Tile(x=int(tile.x) + dx, y=int(tile.y) + dy, z=int(tile.z))] except KeyError: return None return tile_image(path)
def geotiff_options( tile_x, tile_y, tile_z, tilesize: int = 256, dst_crs: CRS = CRS.from_epsg(3857)) -> Dict: """GeoTIFF options.""" bounds = mercantile.xy_bounds(mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)) dst_transform = from_bounds(*bounds, tilesize, tilesize) return dict(crs=dst_crs, transform=dst_transform)
def test_find_quadkeys(): """Get correct quadkeys""" tile = mercantile.Tile(150, 182, 9) assert utils.find_quadkeys(tile, 8) == ["03023033"] assert utils.find_quadkeys(tile, 9) == ["030230330"] assert utils.find_quadkeys(tile, 10) == [ "0302303300", "0302303301", "0302303303", "0302303302", ]
def __init__(self, center, bbox, zoom, basemap, opacity, size=None): self._configure_event_loop() point = center['geometry']['coordinates'] if size is None: size = IMAGE_SIZE self.num_tiles = [ math.ceil(size[x] / TILE_SIZE[x]) + 1 for x in (0, 1) ] center_tile = mercantile.tile(point[0], point[1], zoom) mercator = Proj(init='epsg:3857') wgs84 = Proj(init='epsg:4326') center_tile_bbox = BBox(mercantile.bounds(*center_tile), projection=wgs84).project(mercator, edge_points=0) center_to_image = world_to_image(center_tile_bbox, TILE_SIZE) center_to_world = image_to_world(center_tile_bbox, TILE_SIZE) center_point_px = center_to_image(*mercantile.xy(*point)) self.ul_tile = mercantile.tile(*transform( mercator, wgs84, *center_to_world(center_point_px[0] - math.ceil(IMAGE_SIZE[0] / 2), center_point_px[1] - math.ceil(IMAGE_SIZE[1] / 2)), zoom)) lr_tile = mercantile.Tile(x=min(2**zoom, self.ul_tile.x + self.num_tiles[0]), y=min(2**zoom, self.ul_tile.y + self.num_tiles[1]), z=zoom) ul = mercantile.xy(*mercantile.ul(*self.ul_tile)) lr = mercantile.xy(*mercantile.ul(*lr_tile)) self.image_bbox = BBox((ul[0], lr[1], lr[0], ul[1])) self.image_size = (TILE_SIZE[0] * self.num_tiles[0], TILE_SIZE[1] * self.num_tiles[1]) self.to_image = world_to_image(self.image_bbox, self.image_size) self.to_world = image_to_world(self.image_bbox, self.image_size) self.point_px = [ round(x) for x in self.to_image(*mercantile.xy(*point)) ] self.target_size = size self.point = point self.zoom = zoom self.basemap = basemap self._basemap_image = None self.opacity = opacity
def get_assets(self, x: int, y: int, z: int) -> List[str]: """Find assets.""" mercator_tile = mercantile.Tile(x=x, y=y, z=z) quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom) return list( dict.fromkeys( itertools.chain.from_iterable( [self.mosaic_def.tiles.get(qk, []) for qk in quadkeys] ) ) )
def tiles_from_csv(path): """Retrieve tiles from a line-delimited csv file.""" with open(os.path.expanduser(path)) as fp: reader = csv.reader(fp) for row in reader: if not row: continue yield mercantile.Tile(*map(int, row))
def read_tile(self, z, x, y): """Read raster tile data and mask.""" mercator_tile = mercantile.Tile(x=x, y=y, z=z) tile_bounds = mercantile.xy_bounds(mercator_tile) return tile_read( self.path, tile_bounds, self.tiles_size, indexes=self.indexes, nodata=self.nodata, )
def __init__(self, access_token=os.environ.get("DG_MAPS_API_TOKEN"), url="https://api.mapbox.com/v4/digitalglobe.nal0g75k/{z}/{x}/{y}.png", zoom=22, bounds=None): self.zoom_level = zoom self._token = access_token self._name = "image-{}".format(str(uuid.uuid4())) self._url_template = url + "?access_token={token}" _first_tile = mercantile.Tile(z=self.zoom_level, x=0, y=0) _last_tile = mercantile.Tile(z=self.zoom_level, x=180, y=-85.05) g = box(*mercantile.xy_bounds(_first_tile)).union(box(*mercantile.xy_bounds(_last_tile))) self._full_bounds = g.bounds # TODO: populate rest of fields automatically self._tile_size = 256 self._nbands = 3 self._dtype = "uint8" self.bounds = self._expand_bounds(bounds) self._chunks = tuple([self._nbands] + [self._tile_size, self._tile_size])
def download_tiles(minzoom, maxzoom, bbox, url, path, tile_cover=False, skip_existing=False): if not os.path.exists(path): os.makedirs(path) if tile_cover: ul = mercantile.tile(bbox[0], bbox[3], minzoom) lr = mercantile.tile(bbox[2], bbox[1], minzoom) ul_bounds = mercantile.bounds(ul.x, ul.y, ul.z) lr_bounds = mercantile.bounds(lr.x, lr.y, lr.z) bbox = (ul_bounds.west, lr_bounds.south, lr_bounds.east, ul_bounds.north) for zoom in range(minzoom, maxzoom + 1): tile_url_z = url.replace("{z}", str(zoom)) z_dir = os.path.join(path, str(zoom)) if not os.path.exists(z_dir): os.makedirs(z_dir) ul = mercantile.tile(bbox[0], bbox[3], zoom) lr = mercantile.tile(bbox[2], bbox[1], zoom) if ul.x < 0: ul = mercantile.Tile(x=0, y=ul.y, z=ul.z) if ul.y < 0: ul = mercantile.Tile(x=ul.x, y=0, z=ul.z) max_tile = pow(2, zoom) - 1 if lr.x > max_tile: lr = mercantile.Tile(x=max_tile, y=lr.y, z=lr.z) if lr.y > max_tile: lr = mercantile.Tile(x=lr.x, y=max_tile, z=lr.z) logging.info("Downloading tiles for zoom %d x:%d-%d y:%d-%d " % (zoom, ul.x, lr.x, ul.y, lr.y)) for x in range(ul.x, lr.x + 1): x_dir = os.path.join(z_dir, str(x)) if not os.path.exists(x_dir): os.makedirs(x_dir) tile_url_x = tile_url_z.replace("{x}", str(x)) for y in range(ul.y, lr.y + 1): tile_url_y = tile_url_x.replace("{y}", str(y)) file_path = os.path.join(x_dir, str(y) + ".png") try: download_tile(tile_url_y, file_path, skip_existing=skip_existing) except Exception as e: logging.debug(e) logging.error("Failed to download tile: " + tile_url_y)
def in_tile(self, request, zoom_level, x, y): """Returns all entries within the given tile For more info about tiles and slippy maps, see https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames """ tile = mercantile.Tile(int(x), int(y), int(zoom_level)) bbox = mercantile.bounds(tile) polygon = geos.Polygon.from_bbox(bbox) dumpsters = Dumpster.objects.filter(location__within=polygon) serializer = DumpsterSerializer(dumpsters, many=True) return Response(serializer.data)
def tile(bucket, key, tile_x, tile_y, tile_z, rgb=(1, 2, 3), tilesize=256, prefix='s3:/'): """Create mercator tile from AWS hosted images and encodes it in base64. Attributes ---------- bucket : str AWS bucket's name. key : str AWS file's key. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. tileformat : str Image format to return (Accepted: "jpg" or "png") rgb : tuple, int, optional (default: (1, 2, 3)) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. Returns ------- out : numpy ndarray """ source_address = '{}/{}/{}'.format(prefix, bucket, key) with rasterio.open(source_address) as src: wgs_bounds = transform_bounds( *[src.crs, 'epsg:4326'] + list(src.bounds), densify_pts=21) nodata = src.nodata if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds( 'Tile {}/{}/{} is outside image bounds'.format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) w, s, e, n = tile_bounds out = utils.tile_band_worker(source_address, tile_bounds, tilesize, indexes=rgb, nodata=nodata) return out
def tile_from_xyz(root, x, y, z): """Retrieve a single tile from a slippy map dir.""" path = glob.glob( os.path.join(os.path.expanduser(root), str(z), str(x), str(y) + ".*")) if not path: return None assert len(path) == 1, "ambiguous tile path" return mercantile.Tile(x, y, z), path[0]