def worker(idx): """ """ nb_features = len(features) while True: zoom = randint(min_zoom, max_zoom) feature = features[randint(0, nb_features - 1)] tiles = burn([feature], zoom) nb_mctiles = len(tiles) tile = tiles[randint(0, nb_mctiles - 1)] tile_x = tile[0] tile_y = tile[1] tile_z = tile[2] url = f'https://api.mapbox.com/v4/{mapid}/{tile_z}/{tile_x}/{tile_y}@2x.jpg?access_token={access_token}' response = requests.get(url) if response.status_code != 200: continue header = response.headers size = header.get('Content-Length') if int(size) < 10000: continue new_im.paste(Image.open(BytesIO(response.content)), (idx['c'], idx['r'])) return True
def geojson_parse_polygon(zoom, srid, feature_map, polygon): for i, ring in enumerate( polygon["coordinates"] ): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[ x, y ] for point in ring for x, y in zip([point[0]], [point[1]])] if srid != 4326: polygon = transform_geom(CRS.from_epsg(srid), CRS.from_epsg(4326), polygon) try: for tile in burntiles.burn([{ "type": "feature", "geometry": polygon }], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({ "type": "feature", "geometry": polygon }) except: pass return feature_map
def geojson_parse_polygon(zoom, srid, feature_map, polygon): if srid != 4326: polygon = [ xy for xy in geojson_reproject( { "type": "feature", "geometry": polygon }, srid, 4326) ][0] # GeoJSON coordinates could be N dimensionals for i, ring in enumerate(polygon["coordinates"]): polygon["coordinates"][i] = [[ x, y ] for point in ring for x, y in zip([point[0]], [point[1]])] if polygon["coordinates"]: for tile in burntiles.burn([{ "type": "feature", "geometry": polygon }], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({ "type": "feature", "geometry": polygon }) return feature_map
def geojson_parse_polygon(zoom, srid, feature_map, polygon): if isinstance(polygon["coordinates"], list): # https://github.com/Toblerity/Shapely/issues/245 for i, ring in enumerate( polygon["coordinates"] ): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[ x, y ] for point in ring for x, y in zip([point[0]], [point[1]])] if srid != 4326: try: polygon = transform_geom(CRS.from_epsg(srid), CRS.from_epsg(4326), polygon) except: # negative buffer could lead to empty/invalid geom return feature_map try: for tile in burntiles.burn([{ "type": "feature", "geometry": polygon }], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({ "type": "feature", "geometry": polygon }) except: pass return feature_map
def main(args): if not args.zoom and args.type in ["geojson", "bbox"]: sys.exit("Zoom parameter is required") cover = [] if args.type == "geojson": with open(args.input) as f: features = json.load(f) for feature in tqdm(features["features"], ascii=True, unit="feature"): cover.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist())) cover = list(set(cover)) # tiles can overlap for multiple features; unique tile ids elif args.type == "bbox": west, south, east, north = map(float, args.input.split(",")) cover = tiles(west, south, east, north, args.zoom) elif args.type == "dir": cover = [tile for tile, _ in tiles_from_slippy_map(args.input)] if os.path.dirname(args.out) and not os.path.isdir(os.path.dirname(args.out)): os.makedirs(os.path.dirname(args.out), exist_ok=True) with open(args.out, "w") as fp: csv.writer(fp).writerows(cover)
def get_list_of_mercator_tiles(path): bounds = get_bounds_from_raster(path) with rasterio.open(path) as src: pixel_size = src.res[0] zoom_level = get_zoom_level_for_pixel_size(pixel_size) tile_set = burntiles.burn(bounds, zoom_level) return tile_set
def main(args): dataset = load_config(args.dataset) classes = dataset["common"]["classes"] colors = dataset["common"]["colors"] assert len(classes) == len(colors), "classes and colors coincide" assert len(colors) == 2, "only binary models supported right now" bg = colors[0] fg = colors[1] os.makedirs(args.out, exist_ok=True) # We can only rasterize all tiles at a single zoom. assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles)) with open(args.features) as f: fc = json.load(f) # Find all tiles the features cover and make a map object for quick lookup. feature_map = collections.defaultdict(list) for i, feature in enumerate( tqdm(fc["features"], ascii=True, unit="feature")): if feature["geometry"]["type"] != "Polygon": continue try: for tile in burntiles.burn([feature], zoom=args.zoom): feature_map[mercantile.Tile(*tile)].append(feature) except ValueError as e: print("Warning: invalid feature {}, skipping".format(i), file=sys.stderr) continue # Burn features to tiles and write to a slippy map directory. for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit="tile"): if tile in feature_map: out = burn(tile, feature_map[tile], args.size) else: out = np.zeros(shape=(args.size, args.size), dtype=np.uint8) out_dir = os.path.join(args.out, str(tile.z), str(tile.x)) os.makedirs(out_dir, exist_ok=True) out_path = os.path.join(out_dir, "{}.png".format(tile.y)) if os.path.exists(out_path): prev = np.array(Image.open(out_path)) out = np.maximum(out, prev) out = Image.fromarray(out, mode="P") palette = make_palette(bg, fg) out.putpalette(palette) out.save(out_path, optimize=True)
def burn(features, sequence, zoom): """ Burn a stream of GeoJSONs into a output stream of the tiles they intersect for a given zoom. """ features = [f for f in super_utils.filter_polygons(features)] tiles = burntiles.burn(features, zoom) for t in tiles: click.echo(t.tolist())
def tile_cover(geometry, z): import geopandas as gpd from supermercado import burntiles, super_utils from pygeotile.tile import Tile geo = gpd.GeoSeries([geometry]).__geo_interface__['features'][0] geo = [f for f in super_utils.filter_polygons([geo])] return [ Tile.from_google(*geo).quad_tree for geo in [f for f in burntiles.burn(geo, z)] ]
def main(args): with open(args.features) as f: features = json.load(f) tiles = [] for feature in tqdm(features['features'], ascii=True, unit='feature'): tiles.extend(burntiles.burn([feature], args.zoom)) with open(args.out, 'w') as fp: writer = csv.writer(fp) writer.writerows(tiles)
def main(args): dataset = load_config(args.dataset) classes = dataset['common']['classes'] colors = dataset['common']['colors'] assert len(classes) == len(colors), 'classes and colors coincide' assert len(colors) == 2, 'only binary models supported right now' bg = colors[0] fg = colors[1] os.makedirs(args.out, exist_ok=True) # We can only rasterize all tiles at a single zoom. assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles)) with open(args.features) as f: fc = json.load(f) # Find all tiles the features cover and make a map object for quick lookup. feature_map = collections.defaultdict(list) for i, feature in enumerate( tqdm(fc['features'], ascii=True, unit='feature')): if feature['geometry']['type'] != 'Polygon': continue try: for tile in burntiles.burn([feature], zoom=args.zoom): feature_map[mercantile.Tile(*tile)].append(feature) except ValueError as e: print('Warning: invalid feature {}, skipping'.format(i), file=sys.stderr) continue # Burn features to tiles and write to a slippy map directory. for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit='tile'): if tile in feature_map: out = burn(tile, feature_map[tile], args.size) else: out = Image.fromarray(np.zeros(shape=(args.size, args.size)).astype(int), mode='P') palette = make_palette(bg, fg) out.putpalette(palette) out_path = os.path.join(args.out, str(tile.z), str(tile.x)) os.makedirs(out_path, exist_ok=True) out.save(os.path.join(out_path, '{}.png'.format(tile.y)), optimize=True)
def geojson_parse_polygon(zoom, feature_map, polygon, i): try: for i, ring in enumerate(polygon["coordinates"]): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[x, y] for point in ring for x, y in zip([point[0]], [point[1]])] for tile in burntiles.burn([{"type": "feature", "geometry": polygon}], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({"type": "feature", "geometry": polygon}) except ValueError: log.log("Warning: invalid feature {}, skipping".format(i)) return feature_map
def main(args): with open(args.features) as f: features = json.load(f) tiles = [] for feature in tqdm(features['features'], ascii=True, unit='feature'): tiles.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist())) # tiles can overlap for multiple features; unique tile ids tiles = list(set(tiles)) with open(args.out, 'w') as fp: writer = csv.writer(fp) writer.writerows(tiles)
def main(args): with open(args.features) as f: features = json.load(f) tiles = [] for feature in tqdm(features['features'], ascii=True, unit='feature'): tiles.extend(burntiles.burn([feature], args.zoom)) # tiles can overlap for multiple features; unique tile ids tiles = list(set(tiles)) with open(args.out, 'w') as fp: writer = csv.writer(fp) writer.writerows(tiles)
def main(args): with open(args.features) as f: features = json.load(f) tiles = [] for feature in tqdm(features["features"], ascii=True, unit="feature"): tiles.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist())) # tiles can overlap for multiple features; unique tile ids tiles = list(set(tiles)) with open(args.out, "w") as fp: writer = csv.writer(fp) writer.writerows(tiles)
def tiles_cover(self): """generates tiles covering GeoJSON file. """ features = json.load(open(self.gis_path)) tiles = [] for feature in features["features"]: tiles.extend( map(tuple, burntiles.burn([feature], self.zoom).tolist())) tiles = list(set(tiles)) with open(self.tiles_cover_path, "w") as fp: writer = csv.writer(fp) writer.writerows(tiles)
def main(args): dataset = load_config(args.dataset) classes = dataset['common']['classes'] colors = dataset['common']['colors'] assert len(classes) == len(colors), 'classes and colors coincide' assert len(colors) == 2, 'only binary models supported right now' bg = colors[0] fg = colors[1] os.makedirs(args.out, exist_ok=True) # We can only rasterize all tiles at a single zoom. assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles)) with open(args.features) as f: fc = json.load(f) # Find all tiles the features cover and make a map object for quick lookup. feature_map = collections.defaultdict(list) for i, feature in enumerate(tqdm(fc['features'], ascii=True, unit='feature')): if feature['geometry']['type'] != 'Polygon': continue try: for tile in burntiles.burn([feature], zoom=args.zoom): feature_map[mercantile.Tile(*tile)].append(feature) except ValueError as e: print('Warning: invalid feature {}, skipping'.format(i), file=sys.stderr) continue # Burn features to tiles and write to a slippy map directory. for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit='tile'): if tile in feature_map: out = burn(tile, feature_map[tile], args.size) else: out = Image.fromarray(np.zeros(shape=(args.size, args.size)).astype(int), mode='P') palette = make_palette(bg, fg) out.putpalette(palette) out_path = os.path.join(args.out, str(tile.z), str(tile.x)) os.makedirs(out_path, exist_ok=True) out.save(os.path.join(out_path, '{}.png'.format(tile.y)), optimize=True)
def map_masks(self, colors): os.makedirs(self.masks_path, exist_ok=True) assert all(tile.z == self.zoom for tile in tiles_from_csv(self.tiles_cover_path)) with open(self.gis_path) as f: fc = json.load(f) # Find all tiles the features cover and make a map object for quick lookup. feature_map = collections.defaultdict(list) for i, feature in enumerate(fc["features"]): if feature["geometry"]["type"] != "Polygon": continue try: for tile in burntiles.burn([feature], zoom=self.zoom): feature_map[mercantile.Tile(*tile)].append(feature) except ValueError: # as e: print("Warning: invalid feature {}, skipping".format(i), file=sys.stderr) continue # Burn features to tiles and write to a png image. for tile in list(tiles_from_csv(self.tiles_cover_path)): if tile in feature_map: out = burn(tile, feature_map[tile], self.size) else: out = np.zeros(shape=(self.size, self.size), dtype=np.uint8) x, y, z = map(str, [tile.x, tile.y, tile.z]) out_path = os.path.join( self.masks_path, "{}_{}_{}.{}".format(z, x, y, self.mask_format)) bg = colors[0] fg = colors[1] out = Image.fromarray(out, mode="P") palette = make_palette(bg, fg) out.putpalette(palette) out.save(out_path, optimize=True)
def vector_tile_to_geojson(uri, zoom, map_extent): """Get GeoJSON features that overlap with an extent from a vector tile endpoint.""" log.info('Downloading and converting vector tiles to GeoJSON...') # Figure out which tiles cover the extent. extent_polys = [{ 'type': 'Feature', 'properties': {}, 'geometry': { 'type': 'Polygon', 'coordinates': [map_extent.geojson_coordinates()] } }] xyzs = burn(extent_polys, zoom) # Retrieve tile features. features = [] for xyz in xyzs: x, y, z = xyz # If this isn't a zxy schema, this is a no-op. tile_uri = uri.format(x=x, y=y, z=z) tile_features = get_tile_features(tile_uri, z, x, y) features.extend(tile_features) # Crop features to extent extent_geom = map_extent.to_shapely() cropped_features = [] for f in features: geom = shape(f['geometry']) if f['geometry']['type'].lower() in ['polygon', 'multipolygon']: geom = geom.buffer(0) geom = geom.intersection(extent_geom) if not geom.is_empty: f = dict(f) f['geometry'] = mapping(geom) cropped_features.append(f) return {'type': 'FeatureCollection', 'features': cropped_features}
def main(args): dataset = load_config(args.dataset) classes = dataset["common"]["classes"] colors = dataset["common"]["colors"] assert len(classes) == len(colors), "classes and colors coincide" assert len(colors) == 2, "only binary models supported right now" bg = colors[0] fg = colors[1] os.makedirs(args.out, exist_ok=True) # We can only rasterize all tiles at a single zoom. assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles)) with open(args.features) as f: fc = json.load(f) # Find all tiles the features cover and make a map object for quick lookup. feature_map = collections.defaultdict(list) for i, feature in enumerate( tqdm(fc["features"], ascii=True, unit="feature")): if feature["geometry"]["type"] != "Polygon": continue try: for tile in burntiles.burn([feature], zoom=args.zoom): feature_map[mercantile.Tile(*tile)].append(feature) except ValueError as e: print("Warning: invalid feature {}, skipping".format(i), file=sys.stderr) continue single_burning(args, feature_map, bg, fg)
def main(args): if ( int(args.bbox is not None) + int(args.geojson is not None) + int(args.dir is not None) + int(args.xyz is not None) + int(args.raster is not None) + int(args.cover is not None) != 1 ): sys.exit("ERROR: One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson.") if args.bbox: try: w, s, e, n, crs = args.bbox.split(",") w, s, e, n = map(float, (w, s, e, n)) except: try: crs = None w, s, e, n = map(float, args.bbox.split(",")) except: sys.exit("ERROR: invalid bbox parameter.") if args.splits: try: splits = [int(split) for split in args.splits.split("/")] assert len(splits) == len(args.out) assert sum(splits) == 100 except: sys.exit("ERROR: Invalid split value or incoherent with provided out paths.") if not args.zoom and (args.geojson or args.bbox or args.raster): sys.exit("ERROR: Zoom parameter is required.") args.out = [os.path.expanduser(out) for out in args.out] cover = [] if args.raster: print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom)) with rasterio_open(os.path.expanduser(args.raster)) as r: try: w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds) except: sys.exit("ERROR: unable to deal with raster projection") cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.geojson: print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom)) with open(os.path.expanduser(args.geojson)) as f: features = json.load(f) try: for feature in tqdm(features["features"], ascii=True, unit="feature"): cover.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist())) except: sys.exit("ERROR: invalid or unsupported GeoJSON.") cover = list(set(cover)) # tiles can overlap for multiple features; unique tile ids if args.bbox: print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom)) if crs: try: w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n) except: sys.exit("ERROR: unable to deal with raster projection") cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.cover: print("RoboSat.pink - cover from {}".format(args.cover)) cover = [tile for tile in tiles_from_csv(args.cover)] if args.dir: print("RoboSat.pink - cover from {}".format(args.dir)) cover = [tile for tile in tiles_from_dir(args.dir, xyz=False)] if args.xyz: print("RoboSat.pink - cover from {}".format(args.xyz)) cover = [tile for tile in tiles_from_dir(args.xyz, xyz=True)] _cover = [] for tile in tqdm(cover, ascii=True, unit="tile"): if args.zoom and tile.z != args.zoom: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) for t in tiles(w, s, n, e, args.zoom): unique = True for _t in _cover: if _t == t: unique = False if unique: _cover.append(t) else: _cover.append(tile) cover = _cover if args.splits: shuffle(cover) # in-place splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)] s = 0 covers = [] for e in splits: covers.append(cover[s : s + e - 1]) s += e else: covers = [cover] for i, cover in enumerate(covers): if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])): os.makedirs(os.path.dirname(args.out[i]), exist_ok=True) with open(args.out[i], "w") as fp: csv.writer(fp).writerows(cover)
def create_mosaic( dataset_list: Tuple, minzoom: int = None, maxzoom: int = None, max_threads: int = 20, minimum_tile_cover: float = None, tile_cover_sort: bool = False, version: str = "0.0.2", quiet: bool = True, ) -> Dict: """ Create mosaic definition content. Attributes ---------- dataset_list : tuple or list, required Dataset urls. minzoom: int, optional Force mosaic min-zoom. maxzoom: int, optional Force mosaic max-zoom. minimum_tile_cover: float, optional (default: 0) Filter files with low tile intersection coverage. tile_cover_sort: bool, optional (default: None) Sort intersecting files by coverage. max_threads : int Max threads to use (default: 20). version: str, optional mosaicJSON definition version quiet: bool, optional (default: True) Mask processing steps. Returns ------- mosaic_definition : dict Mosaic definition. """ if version not in ["0.0.1", "0.0.2"]: raise Exception(f"Invalid mosaicJSON's version: {version}") if not quiet: click.echo("Get files footprint", err=True) results = get_footprints(dataset_list, max_threads=max_threads, quiet=quiet) if minzoom is None: minzoom = list(set([feat["properties"]["minzoom"] for feat in results])) if len(minzoom) > 1: warnings.warn("Multiple MinZoom, Assets different minzoom values", UserWarning) minzoom = max(minzoom) if maxzoom is None: maxzoom = list(set([feat["properties"]["maxzoom"] for feat in results])) if len(maxzoom) > 1: warnings.warn( "Multiple MaxZoom, Assets have multiple resolution values", UserWarning) maxzoom = max(maxzoom) quadkey_zoom = minzoom datatype = list(set([feat["properties"]["datatype"] for feat in results])) if len(datatype) > 1: raise Exception("Dataset should have the same data type") if not quiet: click.echo(f"Get quadkey list for zoom: {quadkey_zoom}", err=True) tiles = burntiles.burn(results, quadkey_zoom) tiles = ["{2}-{0}-{1}".format(*tile.tolist()) for tile in tiles] bounds = burntiles.find_extrema(results) mosaic_definition = dict( mosaicjson=version, minzoom=minzoom, maxzoom=maxzoom, bounds=bounds, center=[(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, minzoom], tiles={}, version="1.0.0", ) if version == "0.0.2": mosaic_definition.update(dict(quadkey_zoom=quadkey_zoom)) if not quiet: click.echo(f"Feed Quadkey index", err=True) dataset_geoms = polygons( [feat["geometry"]["coordinates"][0] for feat in results]) dataset = [{ "path": f["properties"]["path"], "geometry": geom } for (f, geom) in zip(results, dataset_geoms)] for parent in tiles: z, x, y = list(map(int, parent.split("-"))) parent = mercantile.Tile(x=x, y=y, z=z) quad = mercantile.quadkey(*parent) tile_geometry = polygons( mercantile.feature(parent)["geometry"]["coordinates"][0]) fdataset = [ dataset[idx] for idx in numpy.nonzero( intersects(tile_geometry, dataset_geoms))[0] ] if minimum_tile_cover is not None or tile_cover_sort: fdataset = _filter_and_sort( tile_geometry, fdataset, minimum_cover=minimum_tile_cover, sort_cover=tile_cover_sort, ) if len(fdataset): mosaic_definition["tiles"][quad] = [f["path"] for f in fdataset] return mosaic_definition
def update_mosaic( dataset_list: Tuple, mosaic_def: dict, max_threads: int = 20, minimum_tile_cover: float = None, ) -> Dict: """ Create mosaic definition content. Attributes ---------- dataset_list : tuple or list, required Dataset urls. mosaic_def : dict Mosaic definition to update. max_threads : int Max threads to use (default: 20). minimum_tile_cover: float, optional (default: 0) Filter files with low tile intersection coverage. Returns ------- mosaic_definition : dict Updated mosaic definition. """ version = mosaic_def.get("version") if version: version = list(map(int, version.split("."))) version[-1] += 1 version = ".".join(map(str, version)) else: version = "1.0.0" mosaic_def["version"] = version results = get_footprints(dataset_list, max_threads=max_threads) min_zoom = mosaic_def["minzoom"] quadkey_zoom = mosaic_def.get("quadkey_zoom", min_zoom) dataset_geoms = polygons( [feat["geometry"]["coordinates"][0] for feat in results]) for idx, r in enumerate(results): tiles = burntiles.burn([r], quadkey_zoom) tiles = ["{2}-{0}-{1}".format(*tile.tolist()) for tile in tiles] dataset = [{ "path": r["properties"]["path"], "geometry": dataset_geoms[idx] }] for parent in tiles: z, x, y = list(map(int, parent.split("-"))) parent = mercantile.Tile(x=x, y=y, z=z) quad = mercantile.quadkey(*parent) tile_geometry = polygons( mercantile.feature(parent)["geometry"]["coordinates"][0]) fdataset = dataset if minimum_tile_cover is not None: fdataset = _filter_and_sort(tile_geometry, fdataset, minimum_cover=minimum_tile_cover) if len(fdataset): dst_quad = mosaic_def["tiles"].get(quad, []) for f in fdataset: dst_quad.append(f["path"]) mosaic_def["tiles"][quad] = dst_quad tiles = [ mercantile.quadkey_to_tile(qk) for qk in mosaic_def["tiles"].keys() ] bounds = tiles_to_bounds(tiles) mosaic_def["bounds"] = bounds mosaic_def["center"] = [ (bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, mosaic_def["minzoom"], ] return mosaic_def
def tile_image(imageFile, output_dir, zoom, cover=None, indexes = None, quant = None, aws_profile = None, skip_blanks = True, max_nodata_pct = 0.0): """ Produce either A) all tiles covering <image> at <zoom> or B) all tiles in <cover> if <cover> is not None at <zoom> and place OSM directory structure in <imageFile>/Z/X/Y.png format inside output_dir. If quant, divide all bands by Quant first. Can write to s3:// destinations with aws_profile. """ from shapely.geometry import box from json import loads from supermercado import burntiles def __load_cover_tiles(coverfile): coverTiles = pd.read_csv(coverfile) if len(coverTiles.columns) != 3: raise Exception("cover file needs to have 3 columns (z, x, y)") return [Tile(z, x, y) for _, (z, x, y) in list(coverTiles.iterrows())] f = None if (imageFile.startswith("s3://")): with rio.Env(profile_name = aws_profile): f = rio.open(imageFile) else: f = rio.open(imageFile) # check crs: if int(f.crs.to_dict()['init'].split(":")[1]) != 4326: print(f"invalid crs ({f.crs.to_dict()['init']}), reprojecting raster....") f.close() mf = rio.io.MemoryFile() reproject_raster(imageFile, 4326, mf) mf.seek(0) f = mf.open() print(f"reproject successful {f.crs.to_dict()}") bbox = box(f.bounds.left, f.bounds.bottom, f.bounds.right, f.bounds.top) bbox = loads(gpd.GeoSeries(bbox).to_json())['features'] # need geojson dict tiles = [Tile(z, x, y) for z, x, y in burntiles.burn(bbox, zoom)] covertiles = set() if cover is not None: covertiles = set(__load_cover_tiles(cover)) tiles = set(tiles).intersection(covertiles) __TILER = partial(_write_tile, image = f, output_dir = output_dir, bands = indexes, quant = quant, aws_profile = aws_profile, skip_blanks = skip_blanks, nodata_val = f.nodata, max_nodata_pct = max_nodata_pct) with futures.ThreadPoolExecutor() as executor: responses = list(executor.map(__TILER, tiles)) tiles, status = zip(*responses) print("#tiles: {} | written: {}\tfailed:{}".format(len(tiles), sum(status), len(tiles) - sum(status))) return(responses)
def _create_mosaic( cls, features: Sequence[Dict], minzoom: int, maxzoom: int, quadkey_zoom: Optional[int] = None, accessor: Callable[[Dict], str] = default_accessor, asset_filter: Callable = default_filter, version: str = "0.0.2", quiet: bool = True, **kwargs, ): """ Create mosaic definition content. Attributes ---------- features : List, required List of GeoJSON features. minzoom: int, required Force mosaic min-zoom. maxzoom: int, required Force mosaic max-zoom. quadkey_zoom: int, optional Force mosaic quadkey zoom. accessor: callable, required Function called on each feature to get its identifier (default is feature["properties"]["path"]). asset_filter: callable, required Function to filter features. version: str, optional mosaicJSON definition version (default: 0.0.2). quiet: bool, optional (default: True) Mask processing steps. kwargs: any Options forwarded to `asset_filter` Returns ------- mosaic_definition : MosaicJSON Mosaic definition. """ quadkey_zoom = quadkey_zoom or minzoom if not quiet: click.echo(f"Get quadkey list for zoom: {quadkey_zoom}", err=True) # Find dataset geometries dataset_geoms = polygons( [feat["geometry"]["coordinates"][0] for feat in features]) bounds = list(total_bounds(dataset_geoms)) tiles = burntiles.burn(features, quadkey_zoom) tiles = [mercantile.Tile(*tile) for tile in tiles] mosaic_definition: Dict[str, Any] = dict( mosaicjson=version, minzoom=minzoom, maxzoom=maxzoom, quadkey_zoom=quadkey_zoom, bounds=bounds, center=((bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, minzoom), tiles={}, version="1.0.0", ) if not quiet: click.echo(f"Feed Quadkey index", err=True) # Create tree and find assets that overlap each tile tree = STRtree(dataset_geoms) for tile in tiles: quadkey = mercantile.quadkey(tile) tile_geom = polygons( mercantile.feature(tile)["geometry"]["coordinates"][0]) # Find intersections from rtree intersections_idx = sorted( tree.query(tile_geom, predicate="intersects")) if len(intersections_idx) == 0: continue intersect_dataset, intersect_geoms = zip( *[(features[idx], dataset_geoms[idx]) for idx in intersections_idx]) dataset = asset_filter(tile, intersect_dataset, intersect_geoms, **kwargs) if dataset: mosaic_definition["tiles"][quadkey] = [ accessor(f) for f in dataset ] return cls(**mosaic_definition)
def stac_to_mosaicJSON( query: Dict, minzoom: int = 7, maxzoom: int = 12, optimized_selection: bool = True, maximum_items_per_tile: int = 20, stac_collection_limit: int = 500, seasons: Tuple = ["spring", "summer", "autumn", "winter"], stac_url: str = os.environ.get("SATAPI_URL", "https://sat-api.developmentseed.org"), ) -> Dict: """ Create a mosaicJSON from a stac request. Attributes ---------- query : str sat-api query. minzoom : int, optional, (default: 7) Mosaic Min Zoom. maxzoom : int, optional (default: 12) Mosaic Max Zoom. optimized_selection : bool, optional (default: true) Limit one Path-Row scene per quadkey. maximum_items_per_tile : int, optional (default: 20) Limit number of scene per quadkey. Use 0 to use all items. stac_collection_limit : int, optional (default: None) Limits the number of items returned by sat-api stac_url : str, optional (default: from ENV) Returns ------- out : dict MosaicJSON definition. """ if stac_collection_limit: query.update(limit=stac_collection_limit) logger.debug(json.dumps(query)) def fetch_sat_api(query): headers = { "Content-Type": "application/json", "Accept-Encoding": "gzip", "Accept": "application/geo+json", } url = f"{stac_url}/stac/search" data = requests.post(url, headers=headers, json=query).json() error = data.get("message", "") if error: raise Exception(f"SAT-API failed and returned: {error}") meta = data.get("meta", {}) if not meta.get("found"): return [] logger.debug(json.dumps(meta)) features = data["features"] if data["links"]: curr_page = int(meta["page"]) query["page"] = curr_page + 1 query["limit"] = meta["limit"] features = list(itertools.chain(features, fetch_sat_api(query))) return features features = fetch_sat_api(query) if not features: raise Exception(f"No asset found for query '{json.dumps(query)}'") logger.debug(f"Found: {len(features)} scenes") features = list( filter( lambda x: _get_season(x["properties"]["datetime"], max(x["bbox"][1], x["bbox"][3])) in seasons, features, )) if optimized_selection: dataset = [] prs = [] for item in features: pr = item["properties"]["eo:column"] + "-" + item["properties"][ "eo:row"] if pr not in prs: prs.append(pr) dataset.append(item) else: dataset = features if query.get("bbox"): bounds = query["bbox"] else: bounds = burntiles.find_extrema(dataset) for i in range(len(dataset)): dataset[i]["geometry"] = shape(dataset[i]["geometry"]) tiles = burntiles.burn([bbox_to_geojson(bounds)], minzoom) tiles = list(set(["{2}-{0}-{1}".format(*tile.tolist()) for tile in tiles])) logger.debug(f"Number tiles: {len(tiles)}") mosaic_definition = dict( mosaicjson="0.0.1", minzoom=minzoom, maxzoom=maxzoom, bounds=bounds, center=[(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, minzoom], tiles={}, ) for tile in tiles: z, x, y = list(map(int, tile.split("-"))) tile = mercantile.Tile(x=x, y=y, z=z) quadkey = mercantile.quadkey(*tile) geometry = box(*mercantile.bounds(tile)) intersect_dataset = list( filter(lambda x: geometry.intersects(x["geometry"]), dataset)) if len(intersect_dataset): # We limit the item per quadkey to 20 if maximum_items_per_tile: intersect_dataset = intersect_dataset[0:maximum_items_per_tile] mosaic_definition["tiles"][quadkey] = [ scene["properties"]["landsat:product_id"] for scene in intersect_dataset ] return mosaic_definition
def _create_mosaic( cls, features: Sequence[Dict], minzoom: int, maxzoom: int, quadkey_zoom: Optional[int] = None, accessor: Callable[[Dict], str] = default_accessor, asset_filter: Callable = default_filter, version: str = "0.0.2", quiet: bool = True, **kwargs, ): """Create mosaic definition content. Attributes: features (list): List of GeoJSON features. minzoom (int): Force mosaic min-zoom. maxzoom (int): Force mosaic max-zoom. quadkey_zoom (int): Force mosaic quadkey zoom (optional). accessor (callable): Function called on each feature to get its identifier (default is feature["properties"]["path"]). asset_filter (callable): Function to filter features. version (str): mosaicJSON definition version (default: 0.0.2). quiet (bool): Mask processing steps (default is True). kwargs (any): Options forwarded to `asset_filter` Returns: mosaic_definition (MosaicJSON): Mosaic definition. Examples: >>> MosaicJSON._create_mosaic([], 12, 14) """ quadkey_zoom = quadkey_zoom or minzoom if not quiet: click.echo(f"Get quadkey list for zoom: {quadkey_zoom}", err=True) # If Pygeos throws an error, fall back to non-vectorized operation # Ref: https://github.com/developmentseed/cogeo-mosaic/issues/81 try: dataset_geoms = polygons( [feat["geometry"]["coordinates"][0] for feat in features] ) except TypeError: dataset_geoms = [ polygons(feat["geometry"]["coordinates"][0]) for feat in features ] bounds = tuple(total_bounds(dataset_geoms)) tiles = burntiles.burn(features, quadkey_zoom) tiles = [mercantile.Tile(*tile) for tile in tiles] mosaic_definition: Dict[str, Any] = dict( mosaicjson=version, minzoom=minzoom, maxzoom=maxzoom, quadkey_zoom=quadkey_zoom, bounds=bounds, center=((bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, minzoom), tiles={}, version="1.0.0", ) if not quiet: click.echo("Feed Quadkey index", err=True) # Create tree and find assets that overlap each tile tree = STRtree(dataset_geoms) fout = os.devnull if quiet else sys.stderr with click.progressbar( # type: ignore tiles, file=fout, show_percent=True, label="Iterate over quadkeys" ) as bar: for tile in bar: quadkey = mercantile.quadkey(tile) tile_geom = polygons( mercantile.feature(tile)["geometry"]["coordinates"][0] ) # Find intersections from rtree intersections_idx = sorted( tree.query(tile_geom, predicate="intersects") ) if len(intersections_idx) == 0: continue intersect_dataset, intersect_geoms = zip( *[(features[idx], dataset_geoms[idx]) for idx in intersections_idx] ) dataset = asset_filter( tile, intersect_dataset, intersect_geoms, **kwargs ) if dataset: mosaic_definition["tiles"][quadkey] = [accessor(f) for f in dataset] return cls(**mosaic_definition)