def test_read_tiles(self): filename = "tests/fixtures/tiles.csv" tiles = [tile for tile in tiles_from_csv(filename)] tiles.sort() self.assertEqual(len(tiles), 3) self.assertEqual(tiles[1], mercantile.Tile(69623, 104945, 18))
def main(args): assert args.out or args.delete, "out parameter is required" args.out = os.path.expanduser(args.out) print("RoboSat.pink - subset {} with cover {}, on CPU".format( args.dir, args.cover), file=sys.stderr, flush=True) ext = set() tiles = set(tiles_from_csv(os.path.expanduser(args.cover))) for tile in tqdm(tiles, ascii=True, unit="tiles"): if isinstance(tile, mercantile.Tile): src_tile = tile_from_xyz(args.dir, tile.x, tile.y, tile.z) if not src_tile: print("WARNING: skipping tile {}".format(tile), file=sys.stderr, flush=True) continue _, src = src_tile dst_dir = os.path.join(args.out, str(tile.z), str(tile.x)) else: src = tile dst_dir = os.path.join(args.out, os.path.dirname(tile)) assert os.path.isfile(src) dst = os.path.join(dst_dir, os.path.basename(src)) ext.add(os.path.splitext(src)[1][1:]) if not os.path.isdir(dst_dir): os.makedirs(dst_dir, exist_ok=True) if args.delete: os.remove(src) assert not os.path.lexists(src) elif args.copy: shutil.copyfile(src, dst) assert os.path.exists(dst) else: if os.path.islink(dst): os.remove(dst) os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) assert os.path.islink(dst) if tiles and not args.no_web_ui and not args.delete: assert len(ext) == 1, "ERROR: Mixed extensions, can't generate Web UI" template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiles, tiles, list(ext)[0], template)
def main(args): if not args.out and not args.delete: sys.exit("ERROR: out parameter is required") args.out = os.path.expanduser(args.out) extension = "" print("RoboSat.pink - subset {} with cover {}".format( args.dir, args.cover)) tiles = set(tiles_from_csv(os.path.expanduser(args.cover))) for tile in tqdm(tiles, desc="Subset", unit="tiles", ascii=True): paths = glob( os.path.join(os.path.expanduser(args.dir), str(tile.z), str(tile.x), "{}.*".format(tile.y))) if len(paths) != 1: print("Warning: {} skipped.".format(tile)) continue src = paths[0] try: if not os.path.isdir( os.path.join(args.out, str(tile.z), str(tile.x))): os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True) extension = os.path.splitext(src)[1][1:] dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}.{}".format(tile.y, extension)) if args.move: assert os.path.isfile(src) shutil.move(src, dst) elif args.delete: assert os.path.isfile(src) os.remove(src) else: shutil.copyfile(src, dst) except: sys.exit("Error: Unable to process tile: {}".format(str(tile))) if not args.no_web_ui and not args.delete: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" web_ui(args.out, base_url, tiles, tiles, extension, template)
def main(args): if not args.out and args.mode in ["copy", "move"]: sys.exit("Zoom parameter is required") tiles = set(tiles_from_csv(args.cover)) extension = "" for tile in tqdm(tiles, desc="Subset", unit="tiles", ascii=True): paths = glob( os.path.join(args.dir, str(tile.z), str(tile.x), "{}.*".format(tile.y))) if len(paths) != 1: print("Warning: {} skipped.".format(tile)) continue src = paths[0] try: if not os.path.isdir( os.path.join(args.out, str(tile.z), str(tile.x))): os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True) extension = os.path.splitext(src)[1][1:] dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}.{}".format(tile.y, extension)) if args.mode == "move": assert os.path.isfile(src) shutil.move(src, dst) if args.mode == "copy": shutil.copyfile(src, dst) if args.mode == "delete": assert os.path.isfile(src) os.remove(src) except: sys.exit("Error: Unable to process {}".format(tile)) if args.web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" web_ui(args.out, base_url, tiles, tiles, extension, template)
def main(args): assert args.out or args.delete, "out parameter is required" args.out = os.path.expanduser(args.out) if not args.workers: args.workers = 4 if os.cpu_count() >= 4 else os.cpu_count() print("RoboSat.pink - subset {} with cover {}, on CPU, with {} workers".format(args.dir, args.cover, args.workers)) tiles = set(tiles_from_csv(os.path.expanduser(args.cover))) progress = tqdm(total=len(tiles), ascii=True, unit="tiles") with futures.ThreadPoolExecutor(args.workers) as executor: def worker(tile): if isinstance(tile, mercantile.Tile): _, src = tile_from_xyz(args.dir, tile.x, tile.y, tile.z) dst_dir = os.path.join(args.out, str(tile.z), str(tile.x)) else: src = tile dst_dir = os.path.join(args.out, os.path.dirname(tile)) assert os.path.isfile(src) dst = os.path.join(dst_dir, os.path.basename(src)) if not os.path.isdir(dst_dir): os.makedirs(dst_dir, exist_ok=True) if args.move: shutil.move(src, dst) elif args.delete: os.remove(src) else: shutil.copyfile(src, dst) return os.path.splitext(src)[1][1:] # ext for extension in executor.map(worker, tiles): progress.update() if not args.no_web_ui and not args.delete: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" web_ui(args.out, base_url, tiles, tiles, extension, template)
def main(args): tiles = list(tiles_from_csv(args.cover)) os.makedirs(os.path.expanduser(args.out), exist_ok=True) if not args.workers: args.workers = max(1, math.floor(os.cpu_count() * 0.5)) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) log.log( "RoboSat.pink - download with {} workers, at max {} req/s, from: {}". format(args.workers, args.rate, args.url)) already_dl = 0 dl = 0 with requests.Session() as session: progress = tqdm(total=len(tiles), ascii=True, unit="image") with futures.ThreadPoolExecutor(args.workers) as executor: def worker(tile): tick = time.monotonic() progress.update() try: x, y, z = map(str, [tile.x, tile.y, tile.z]) os.makedirs(os.path.join(args.out, z, x), exist_ok=True) except: return tile, None, False path = os.path.join(args.out, z, x, "{}.{}".format(y, args.format)) if os.path.isfile(path): # already downloaded return tile, None, True if args.type == "XYZ": url = args.url.format(x=tile.x, y=tile.y, z=tile.z) elif args.type == "TMS": y = (2**tile.z) - tile.y - 1 url = args.url.format(x=tile.x, y=y, z=tile.z) elif args.type == "WMS": xmin, ymin, xmax, ymax = xy_bounds(tile) url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax) res = tile_image_from_url(session, url, args.timeout) if res is None: # let's retry once res = tile_image_from_url(session, url, args.timeout) if res is None: return tile, url, False try: tile_image_to_file(args.out, tile, res) except OSError: return tile, url, False tock = time.monotonic() time_for_req = tock - tick time_per_worker = args.workers / args.rate if time_for_req < time_per_worker: time.sleep(time_per_worker - time_for_req) return tile, url, True for tile, url, ok in executor.map(worker, tiles): if url and ok: dl += 1 elif not url and ok: already_dl += 1 else: log.log("Warning:\n {} failed, skipping.\n {}\n".format( tile, url)) if already_dl: log.log( "Notice: {} tiles were already downloaded previously, and so skipped now." .format(already_dl)) if already_dl + dl == len(tiles): log.log("Notice: Coverage is fully downloaded.") if not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" web_ui(args.out, base_url, tiles, tiles, args.format, template)
def main(args): assert not (args.sql and args.geojson), "You can only use at once --pg OR --geojson." assert not (args.pg and not args.sql ), "With PostgreSQL --pg, --sql must also be provided" assert len(args.ts.split( ",")) == 2, "--ts expect width,height value (e.g 512,512)" config = load_config(args.config) check_classes(config) palette = make_palette([classe["color"] for classe in config["classes"]], complementary=True) index = [ config["classes"].index(classe) for classe in config["classes"] if classe["title"] == args.type ] assert index, "Requested type is not contains in your config file classes." burn_value = int(math.pow(2, index[0] - 1)) # 8bits One Hot Encoding assert 0 <= burn_value <= 128 args.out = os.path.expanduser(args.out) os.makedirs(args.out, exist_ok=True) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) if args.geojson: tiles = [ tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] assert tiles, "Empty cover" zoom = tiles[0].z assert not [tile for tile in tiles if tile.z != zoom ], "Unsupported zoom mixed cover. Use PostGIS instead" feature_map = collections.defaultdict(list) log.log("RoboSat.pink - rasterize - Compute spatial index") for geojson_file in args.geojson: with open(os.path.expanduser(geojson_file)) as geojson: feature_collection = json.load(geojson) srid = geojson_srid(feature_collection) feature_map = collections.defaultdict(list) for i, feature in enumerate( tqdm(feature_collection["features"], ascii=True, unit="feature")): feature_map = geojson_parse_feature( zoom, srid, feature_map, feature) features = args.geojson if args.pg: conn = psycopg2.connect(args.pg) db = conn.cursor() assert "limit" not in args.sql.lower(), "LIMIT is not supported" assert "TILE_GEOM" in args.sql, "TILE_GEOM filter not found in your SQL" sql = re.sub(r"ST_Intersects( )*\((.*)?TILE_GEOM(.*)?\)", "1=1", args.sql, re.I) assert sql and sql != args.sql db.execute( """SELECT ST_Srid("1") AS srid FROM ({} LIMIT 1) AS t("1")""". format(sql)) srid = db.fetchone()[0] assert srid and int(srid) > 0, "Unable to retrieve geometry SRID." features = args.sql log.log( "RoboSat.pink - rasterize - rasterizing {} from {} on cover {}".format( args.type, features, args.cover)) with open(os.path.join(os.path.expanduser(args.out), "instances_" + args.type.lower() + ".cover"), mode="w") as cover: for tile in tqdm(list(tiles_from_csv(os.path.expanduser(args.cover))), ascii=True, unit="tile"): geojson = None if args.pg: w, s, e, n = tile_bbox(tile) tile_geom = "ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), {})".format( w, s, e, n, srid) query = """ WITH sql AS ({}), geom AS (SELECT "1" AS geom FROM sql AS t("1")), json AS (SELECT '{{"type": "Feature", "geometry": ' || ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6) || '}}' AS features FROM geom) SELECT '{{"type": "FeatureCollection", "features": [' || Array_To_String(array_agg(features), ',') || ']}}' FROM json """.format(args.sql.replace("TILE_GEOM", tile_geom)) db.execute(query) row = db.fetchone() try: geojson = json.loads( row[0])["features"] if row and row[0] else None except Exception: log.log("Warning: Invalid geometries, skipping {}".format( tile)) conn = psycopg2.connect(args.pg) db = conn.cursor() if args.geojson: geojson = feature_map[tile] if tile in feature_map else None if geojson: num = len(geojson) out = geojson_tile_burn(tile, geojson, 4326, list(map(int, args.ts.split(","))), burn_value) if not geojson or out is None: num = 0 out = np.zeros(shape=list(map(int, args.ts.split(","))), dtype=np.uint8) tile_label_to_file(args.out, tile, palette, out, append=args.append) cover.write("{},{},{} {}{}".format(tile.x, tile.y, tile.z, num, os.linesep)) if not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." tiles = [tile for tile in tiles_from_csv(args.cover)] web_ui(args.out, base_url, tiles, tiles, "png", template)
def main(args): if (args.geojson and args.postgis) or (not args.geojson and not args.postgis): sys.exit( "ERROR: Input features to rasterize must be either GeoJSON or PostGIS" ) if args.postgis and not args.pg_dsn: sys.exit( "ERROR: With PostGIS input features, --pg_dsn must be provided") config = load_config(args.config) check_classes(config) palette = make_palette(*[classe["color"] for classe in config["classes"]], complementary=True) burn_value = next(config["classes"].index(classe) for classe in config["classes"] if classe["title"] == args.type) if "burn_value" not in locals(): sys.exit( "ERROR: asked type to rasterize is not contains in your config file classes." ) args.out = os.path.expanduser(args.out) os.makedirs(args.out, exist_ok=True) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) def geojson_parse_polygon(zoom, srid, feature_map, polygon, i): try: if srid != 4326: polygon = [ xy for xy in geojson_reproject( { "type": "feature", "geometry": polygon }, srid, 4326) ][0] for i, ring in enumerate( polygon["coordinates"] ): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[ x, y ] for point in ring for x, y in zip([point[0]], [point[1]])] if polygon["coordinates"]: for tile in burntiles.burn([{ "type": "feature", "geometry": polygon }], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({ "type": "feature", "geometry": polygon }) except ValueError: log.log("Warning: invalid feature {}, skipping".format(i)) return feature_map def geojson_parse_geometry(zoom, srid, feature_map, geometry, i): if geometry["type"] == "Polygon": feature_map = geojson_parse_polygon(zoom, srid, feature_map, geometry, i) elif geometry["type"] == "MultiPolygon": for polygon in geometry["coordinates"]: feature_map = geojson_parse_polygon(zoom, srid, feature_map, { "type": "Polygon", "coordinates": polygon }, i) else: log.log( "Notice: {} is a non surfacic geometry type, skipping feature {}" .format(geometry["type"], i)) return feature_map if args.geojson: try: tiles = [ tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] zoom = tiles[0].z assert not [tile for tile in tiles if tile.z != zoom] except: sys.exit("ERROR: Inconsistent cover {}".format(args.cover)) feature_map = collections.defaultdict(list) log.log("RoboSat.pink - rasterize - Compute spatial index") for geojson_file in args.geojson: with open(os.path.expanduser(geojson_file)) as geojson: feature_collection = json.load(geojson) try: crs_mapping = {"CRS84": "4326", "900913": "3857"} srid = feature_collection["crs"]["properties"][ "name"].split(":")[-1] srid = int(srid) if srid not in crs_mapping else int( crs_mapping[srid]) except: srid = int(4326) for i, feature in enumerate( tqdm(feature_collection["features"], ascii=True, unit="feature")): if feature["geometry"]["type"] == "GeometryCollection": for geometry in feature["geometry"]["geometries"]: feature_map = geojson_parse_geometry( zoom, srid, feature_map, geometry, i) else: feature_map = geojson_parse_geometry( zoom, srid, feature_map, feature["geometry"], i) features = args.geojson if args.postgis: pg_conn = psycopg2.connect(args.pg_dsn) pg = pg_conn.cursor() pg.execute( "SELECT ST_Srid(geom) AS srid FROM ({} LIMIT 1) AS sub".format( args.postgis)) try: srid = pg.fetchone()[0] except Exception: sys.exit("Unable to retrieve geometry SRID.") features = args.postgis log.log( "RoboSat.pink - rasterize - rasterizing {} from {} on cover {}".format( args.type, features, args.cover)) with open(os.path.join(os.path.expanduser(args.out), "instances.cover"), mode="w") as cover: for tile in tqdm(list(tiles_from_csv(os.path.expanduser(args.cover))), ascii=True, unit="tile"): if args.postgis: s, w, e, n = mercantile.bounds(tile) query = """ WITH a AS ({}), b AS (SELECT ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), {}) AS geom) SELECT '{{ "type": "FeatureCollection", "features": [{{"type": "Feature", "geometry": ' || ST_AsGeoJSON(ST_Transform(ST_Intersection(a.geom, b.geom), 4326), 6) || '}}]}}' FROM a, b WHERE ST_Intersects(a.geom, b.geom) """.format(args.postgis, s, w, e, n, srid) try: pg.execute(query) row = pg.fetchone() geojson = json.loads(row[0])["features"] if row else None except Exception: log.log("Warning: Invalid geometries, skipping {}".format( tile)) pg_conn = psycopg2.connect(args.pg_dsn) pg = pg_conn.cursor() if args.geojson: geojson = feature_map[tile] if tile in feature_map else None if geojson: num = len(geojson) out = geojson_tile_burn(tile, geojson, 4326, args.ts, burn_value) if not geojson or out is None: num = 0 out = np.zeros(shape=(args.ts, args.ts), dtype=np.uint8) tile_label_to_file(args.out, tile, palette, out) cover.write("{},{},{} {}{}".format(tile.x, tile.y, tile.z, num, os.linesep)) if not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" tiles = [tile for tile in tiles_from_csv(args.cover)] web_ui(args.out, base_url, tiles, tiles, "png", template)
def main(args): assert not (args.extent and args.splits ), "--splits and --extent are mutually exclusive options." assert not (args.extent and len(args.out) > 1), "--extent option imply a single output." assert not (args.sql and not args.pg), "--sql option imply --pg" assert ( int(args.bbox is not None) + int(args.geojson is not None) + int(args.sql is not None) + int(args.dir is not None) + int(args.raster is not None) + int(args.cover is not None) == 1 ), "One, and only one, input type must be provided, among: --dir, --bbox, --cover, --raster, --geojson or --sql" if args.bbox: try: w, s, e, n, crs = args.bbox.split(",") w, s, e, n = map(float, (w, s, e, n)) except: crs = None w, s, e, n = map(float, args.bbox.split(",")) assert isinstance(w, float) and isinstance( s, float), "Invalid bbox parameter." if args.splits: splits = [int(split) for split in args.splits.split("/")] assert len(splits) == len(args.out) and 0 < sum( splits) <= 100, "Invalid split value or incoherent with out paths." assert not (not args.zoom and (args.geojson or args.bbox or args.raster)), "Zoom parameter is required." args.out = [os.path.expanduser(out) for out in args.out] cover = [] if args.raster: print("RoboSat.pink - cover from {} at zoom {}".format( args.raster, args.zoom), file=sys.stderr, flush=True) with rasterio_open(os.path.expanduser(args.raster)) as r: w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds) assert isinstance(w, float) and isinstance( s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.geojson: print("RoboSat.pink - cover from {} at zoom {}".format( args.geojson, args.zoom), file=sys.stderr, flush=True) with open(os.path.expanduser(args.geojson)) as f: feature_collection = json.load(f) srid = geojson_srid(feature_collection) feature_map = collections.defaultdict(list) for feature in tqdm(feature_collection["features"], ascii=True, unit="feature"): feature_map = geojson_parse_feature(args.zoom, srid, feature_map, feature) cover = feature_map.keys() if args.sql: print("RoboSat.pink - cover from {} {} at zoom {}".format( args.sql, args.pg, args.zoom), file=sys.stderr, flush=True) conn = psycopg2.connect(args.pg) assert conn, "Unable to connect to PostgreSQL database." db = conn.cursor() query = """ WITH sql AS ({}), geom AS (SELECT "1" AS geom FROM sql AS t("1")) SELECT '{{"type": "Feature", "geometry": ' || ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6) || '}}' AS features FROM geom """.format(args.sql) db.execute(query) assert db.rowcount is not None and db.rowcount != -1, "SQL Query return no result." feature_map = collections.defaultdict(list) for feature in tqdm( db.fetchall(), ascii=True, unit="feature" ): # FIXME: fetchall will not always fit in memory... feature_map = geojson_parse_feature(args.zoom, 4326, feature_map, json.loads(feature[0])) cover = feature_map.keys() if args.bbox: print("RoboSat.pink - cover from {} at zoom {}".format( args.bbox, args.zoom), file=sys.stderr, flush=True) if crs: w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n) assert isinstance(w, float) and isinstance( s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.cover: print("RoboSat.pink - cover from {}".format(args.cover), file=sys.stderr, flush=True) cover = [tile for tile in tiles_from_csv(args.cover)] if args.dir: print("RoboSat.pink - cover from {}".format(args.dir), file=sys.stderr, flush=True) cover = [ tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz)) ] _cover = [] extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0) for tile in tqdm(cover, ascii=True, unit="tile"): if args.zoom and tile.z != args.zoom: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) for t in tiles(w, s, n, e, args.zoom): unique = True for _t in _cover: if _t == t: unique = False if unique: _cover.append(t) else: if args.extent: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) _cover.append(tile) if args.extent: extent_w, extent_s, extent_n, extent_e = (min(extent_w, w), min(extent_s, s), max(extent_n, n), max(extent_e, e)) cover = _cover if args.splits: shuffle(cover) # in-place cover_splits = [ math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1) ] if len(splits) > 1 and sum(map( int, splits)) == 100 and len(cover) > sum(map(int, splits)): cover_splits[0] = len(cover) - sum(map( int, cover_splits[1:])) # no tile waste s = 0 covers = [] for e in cover_splits: covers.append(cover[s:s + e]) s += e else: covers = [cover] if args.extent: if args.out and os.path.dirname(args.out[0]) and not os.path.isdir( os.path.dirname(args.out[0])): os.makedirs(os.path.dirname(args.out[0]), exist_ok=True) extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s, extent_n, extent_e) if args.out: with open(args.out[0], "w") as fp: fp.write(extent) else: print(extent) else: for i, cover in enumerate(covers): if os.path.dirname(args.out[i]) and not os.path.isdir( os.path.dirname(args.out[i])): os.makedirs(os.path.dirname(args.out[i]), exist_ok=True) with open(args.out[i], "w") as fp: csv.writer(fp).writerows(cover)
def main(args): if ( int(args.bbox is not None) + int(args.geojson is not None) + int(args.dir is not None) + int(args.xyz is not None) + int(args.raster is not None) + int(args.cover is not None) != 1 ): sys.exit("ERROR: One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson.") if args.bbox: try: w, s, e, n, crs = args.bbox.split(",") w, s, e, n = map(float, (w, s, e, n)) except: try: crs = None w, s, e, n = map(float, args.bbox.split(",")) except: sys.exit("ERROR: invalid bbox parameter.") if args.splits: try: splits = [int(split) for split in args.splits.split("/")] assert len(splits) == len(args.out) assert sum(splits) == 100 except: sys.exit("ERROR: Invalid split value or incoherent with provided out paths.") if not args.zoom and (args.geojson or args.bbox or args.raster): sys.exit("ERROR: Zoom parameter is required.") args.out = [os.path.expanduser(out) for out in args.out] cover = [] if args.raster: print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom)) with rasterio_open(os.path.expanduser(args.raster)) as r: try: w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds) except: sys.exit("ERROR: unable to deal with raster projection") cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.geojson: print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom)) with open(os.path.expanduser(args.geojson)) as f: features = json.load(f) try: for feature in tqdm(features["features"], ascii=True, unit="feature"): cover.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist())) except: sys.exit("ERROR: invalid or unsupported GeoJSON.") cover = list(set(cover)) # tiles can overlap for multiple features; unique tile ids if args.bbox: print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom)) if crs: try: w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n) except: sys.exit("ERROR: unable to deal with raster projection") cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.cover: print("RoboSat.pink - cover from {}".format(args.cover)) cover = [tile for tile in tiles_from_csv(args.cover)] if args.dir: print("RoboSat.pink - cover from {}".format(args.dir)) cover = [tile for tile in tiles_from_dir(args.dir, xyz=False)] if args.xyz: print("RoboSat.pink - cover from {}".format(args.xyz)) cover = [tile for tile in tiles_from_dir(args.xyz, xyz=True)] _cover = [] for tile in tqdm(cover, ascii=True, unit="tile"): if args.zoom and tile.z != args.zoom: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) for t in tiles(w, s, n, e, args.zoom): unique = True for _t in _cover: if _t == t: unique = False if unique: _cover.append(t) else: _cover.append(tile) cover = _cover if args.splits: shuffle(cover) # in-place splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)] s = 0 covers = [] for e in splits: covers.append(cover[s : s + e - 1]) s += e else: covers = [cover] for i, cover in enumerate(covers): if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])): os.makedirs(os.path.dirname(args.out[i]), exist_ok=True) with open(args.out[i], "w") as fp: csv.writer(fp).writerows(cover)
def main(args): if args.pg: if not args.sql: sys.exit("ERROR: With PostgreSQL db, --sql must be provided") if (args.sql and args.geojson) or (args.sql and not args.pg): sys.exit( "ERROR: You can use either --pg or --geojson inputs, but only one at once." ) config = load_config(args.config) check_classes(config) palette = make_palette(*[classe["color"] for classe in config["classes"]], complementary=True) burn_value = next(config["classes"].index(classe) for classe in config["classes"] if classe["title"] == args.type) if "burn_value" not in locals(): sys.exit( "ERROR: asked type to rasterize is not contains in your config file classes." ) args.out = os.path.expanduser(args.out) os.makedirs(args.out, exist_ok=True) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) def geojson_parse_polygon(zoom, srid, feature_map, polygon, i): try: if srid != 4326: polygon = [ xy for xy in geojson_reproject( { "type": "feature", "geometry": polygon }, srid, 4326) ][0] for i, ring in enumerate( polygon["coordinates"] ): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[ x, y ] for point in ring for x, y in zip([point[0]], [point[1]])] if polygon["coordinates"]: for tile in burntiles.burn([{ "type": "feature", "geometry": polygon }], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({ "type": "feature", "geometry": polygon }) except ValueError: log.log("Warning: invalid feature {}, skipping".format(i)) return feature_map def geojson_parse_geometry(zoom, srid, feature_map, geometry, i): if geometry["type"] == "Polygon": feature_map = geojson_parse_polygon(zoom, srid, feature_map, geometry, i) elif geometry["type"] == "MultiPolygon": for polygon in geometry["coordinates"]: feature_map = geojson_parse_polygon(zoom, srid, feature_map, { "type": "Polygon", "coordinates": polygon }, i) else: log.log( "Notice: {} is a non surfacic geometry type, skipping feature {}" .format(geometry["type"], i)) return feature_map if args.geojson: tiles = [ tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] assert tiles, "Empty cover" zoom = tiles[0].z assert not [tile for tile in tiles if tile.z != zoom ], "Unsupported zoom mixed cover. Use PostGIS instead" feature_map = collections.defaultdict(list) log.log("RoboSat.pink - rasterize - Compute spatial index") for geojson_file in args.geojson: with open(os.path.expanduser(geojson_file)) as geojson: feature_collection = json.load(geojson) try: crs_mapping = {"CRS84": "4326", "900913": "3857"} srid = feature_collection["crs"]["properties"][ "name"].split(":")[-1] srid = int(srid) if srid not in crs_mapping else int( crs_mapping[srid]) except: srid = int(4326) for i, feature in enumerate( tqdm(feature_collection["features"], ascii=True, unit="feature")): if feature["geometry"]["type"] == "GeometryCollection": for geometry in feature["geometry"]["geometries"]: feature_map = geojson_parse_geometry( zoom, srid, feature_map, geometry, i) else: feature_map = geojson_parse_geometry( zoom, srid, feature_map, feature["geometry"], i) features = args.geojson if args.pg: conn = psycopg2.connect(args.pg) db = conn.cursor() assert "limit" not in args.sql.lower(), "LIMIT is not supported" db.execute( "SELECT ST_Srid(geom) AS srid FROM ({} LIMIT 1) AS sub".format( args.sql)) srid = db.fetchone()[0] assert srid, "Unable to retrieve geometry SRID." if "where" not in args.sql.lower( ): # TODO: Find a more reliable way to handle feature filtering args.sql += " WHERE ST_Intersects(tile.geom, geom)" else: args.sql += " AND ST_Intersects(tile.geom, geom)" features = args.sql log.log( "RoboSat.pink - rasterize - rasterizing {} from {} on cover {}".format( args.type, features, args.cover)) with open(os.path.join(os.path.expanduser(args.out), "instances.cover"), mode="w") as cover: for tile in tqdm(list(tiles_from_csv(os.path.expanduser(args.cover))), ascii=True, unit="tile"): geojson = None if args.pg: w, s, e, n = tile_bbox(tile) query = """ WITH tile AS (SELECT ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), {}) AS geom), geom AS (SELECT ST_Intersection(tile.geom, sql.geom) AS geom FROM tile CROSS JOIN LATERAL ({}) sql), json AS (SELECT '{{"type": "Feature", "geometry": ' || ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6) || '}}' AS features FROM geom) SELECT '{{"type": "FeatureCollection", "features": [' || Array_To_String(array_agg(features), ',') || ']}}' FROM json """.format(w, s, e, n, srid, args.sql) db.execute(query) row = db.fetchone() try: geojson = json.loads( row[0])["features"] if row and row[0] else None except Exception: log.log("Warning: Invalid geometries, skipping {}".format( tile)) conn = psycopg2.connect(args.pg) db = conn.cursor() if args.geojson: geojson = feature_map[tile] if tile in feature_map else None if geojson: num = len(geojson) out = geojson_tile_burn(tile, geojson, 4326, args.ts, burn_value) if not geojson or out is None: num = 0 out = np.zeros(shape=(args.ts, args.ts), dtype=np.uint8) tile_label_to_file(args.out, tile, palette, out) cover.write("{},{},{} {}{}".format(tile.x, tile.y, tile.z, num, os.linesep)) if not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" tiles = [tile for tile in tiles_from_csv(args.cover)] web_ui(args.out, base_url, tiles, tiles, "png", template)
def main(args): config = load_config(args.config) check_channels(config) check_classes(config) palette = make_palette([classe["color"] for classe in config["classes"]]) if not args.bs: try: args.bs = config["model"]["bs"] except: pass assert args.bs, "For rsp predict, model/bs must be set either in config file, or pass trought parameter --bs" args.workers = args.bs if not args.workers else args.workers cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] if args.cover else None log = Logs(os.path.join(args.out, "log")) if torch.cuda.is_available(): log.log("RoboSat.pink - predict on {} GPUs, with {} workers".format( torch.cuda.device_count(), args.workers)) log.log("(Torch:{} Cuda:{} CudNN:{})".format( torch.__version__, torch.version.cuda, torch.backends.cudnn.version())) device = torch.device("cuda") torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True else: log.log("RoboSat.pink - predict on CPU, with {} workers".format( args.workers)) log.log("") log.log("============================================================") log.log("WARNING: Are you -really- sure about not predicting on GPU ?") log.log("============================================================") log.log("") device = torch.device("cpu") chkpt = torch.load(args.checkpoint, map_location=device) nn_module = load_module("robosat_pink.nn.{}".format(chkpt["nn"].lower())) nn = getattr(nn_module, chkpt["nn"])(chkpt["shape_in"], chkpt["shape_out"], chkpt["encoder"].lower()).to(device) nn = torch.nn.DataParallel(nn) nn.load_state_dict(chkpt["state_dict"]) nn.eval() log.log("Model {} - UUID: {}".format(chkpt["nn"], chkpt["uuid"])) with torch.no_grad( ): # don't track tensors with autograd during prediction tiled = [] if args.passes in ["first", "both"]: log.log("== Predict First Pass ==") tiled = predict(config, cover, args, palette, chkpt, nn, device, "predict") if args.passes in ["second", "both"]: log.log("== Predict Second Pass ==") predict(config, cover, args, palette, chkpt, nn, device, "predict_translate") if not args.no_web_ui and tiled: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiled, tiled, "png", template)
def main(args): if not args.workers: args.workers = min(os.cpu_count(), len(args.rasters)) if args.label: config = load_config(args.config) check_classes(config) colors = [classe["color"] for classe in config["classes"]] palette = make_palette(colors) assert len(args.ts.split( ",")) == 2, "--ts expect width,height value (e.g 512,512)" width, height = list(map(int, args.ts.split(","))) cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] if args.cover else None splits_path = os.path.join(os.path.expanduser(args.out), ".splits") tiles_map = {} print("RoboSat.pink - tile on CPU, with {} workers".format(args.workers), file=sys.stderr, flush=True) bands = -1 for path in args.rasters: raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) if bands != -1: assert bands == len( raster.indexes), "Coverage must be bands consistent" bands = len(raster.indexes) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiles = list(set(tiles) & set(cover)) if cover else tiles for tile in tiles: tile_key = (str(tile.x), str(tile.y), str(tile.z)) if tile_key not in tiles_map.keys(): tiles_map[tile_key] = [] tiles_map[tile_key].append(path) if args.label: ext = "png" bands = 1 if not args.label: if bands == 1: ext = "png" if bands == 3: ext = "webp" if bands > 3: ext = "tiff" tiles = [] progress = tqdm(total=len(tiles_map), ascii=True, unit="tile") # Begin to tile plain tiles with futures.ThreadPoolExecutor(args.workers) as executor: def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, width, height), width=width, height=height, ) data = warp_vrt.read(out_shape=(len(raster.indexes), width, height), window=warp_vrt.window(w, s, e, n)) if data.dtype == "uint16": # GeoTiff could be 16 bits data = np.uint8(data / 256) elif data.dtype == "uint32": # or 32 bits data = np.uint8(data / (256 * 256)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if (not args.label and len(tiles_map[tile_key]) == 1 and is_nodata( image, args.nodata, args.nodata_threshold, args.keep_borders)): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled for tiled in executor.map(worker, args.rasters): if tiled is not None: tiles.extend(tiled) # Aggregate remaining tiles splits with futures.ThreadPoolExecutor(args.workers) as executor: def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((width, height, bands), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) split = split.reshape((width, height, 1)) # H,W -> H,W,C assert image.shape == split.shape image[np.where(image == 0)] += split[np.where(image == 0)] if not args.label and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: tile_image_to_file(args.out, tile, image) if args.label: tile_label_to_file(args.out, tile, palette, image) progress.update() return tile for tiled in executor.map(worker, tiles_map.keys()): if tiled is not None: tiles.append(tiled) if splits_path and os.path.isdir(splits_path): shutil.rmtree(splits_path) # Delete suffixes dir if any if tiles and not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args): if (args.geojson and args.postgis) or (not args.geojson and not args.postgis): sys.exit( "ERROR: Input features to rasterize must be either GeoJSON or PostGIS" ) if args.postgis and not args.pg_dsn: sys.exit( "ERROR: With PostGIS input features, --pg_dsn must be provided") config = load_config(args.config) check_classes(config) palette = make_palette(*[classe["color"] for classe in config["classes"]], complementary=True) burn_value = 1 args.out = os.path.expanduser(args.out) os.makedirs(args.out, exist_ok=True) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) def geojson_parse_polygon(zoom, srid, feature_map, polygon, i): try: if srid != 4326: polygon = [ xy for xy in geojson_reproject( { "type": "feature", "geometry": polygon }, srid, 4326) ][0] for i, ring in enumerate( polygon["coordinates"] ): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[ x, y ] for point in ring for x, y in zip([point[0]], [point[1]])] if polygon["coordinates"]: for tile in burntiles.burn([{ "type": "feature", "geometry": polygon }], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({ "type": "feature", "geometry": polygon }) except ValueError: log.log("Warning: invalid feature {}, skipping".format(i)) return feature_map def geojson_parse_geometry(zoom, srid, feature_map, geometry, i): if geometry["type"] == "Polygon": feature_map = geojson_parse_polygon(zoom, srid, feature_map, geometry, i) elif geometry["type"] == "MultiPolygon": for polygon in geometry["coordinates"]: feature_map = geojson_parse_polygon(zoom, srid, feature_map, { "type": "Polygon", "coordinates": polygon }, i) else: log.log( "Notice: {} is a non surfacic geometry type, skipping feature {}" .format(geometry["type"], i)) return feature_map if args.geojson: try: tiles = [ tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] zoom = tiles[0].z assert not [tile for tile in tiles if tile.z != zoom] except: sys.exit("ERROR: Inconsistent cover {}".format(args.cover)) feature_map = collections.defaultdict(list) log.log("RoboSat.pink - rasterize - Compute spatial index") for geojson_file in args.geojson: with open(os.path.expanduser(geojson_file)) as geojson: try: feature_collection = json.load(geojson) except: sys.exit("ERROR: {} is not a valid JSON file.".format( geojson_file)) try: crs_mapping = {"CRS84": "4326", "900913": "3857"} srid = feature_collection["crs"]["properties"][ "name"].split(":")[-1] srid = int(srid) if srid not in crs_mapping else int( crs_mapping[srid]) except: srid = int(4326) for i, feature in enumerate( tqdm(feature_collection["features"], ascii=True, unit="feature")): try: if feature["geometry"]["type"] == "GeometryCollection": for geometry in feature["geometry"]["geometries"]: feature_map = geojson_parse_geometry( zoom, srid, feature_map, geometry, i) else: feature_map = geojson_parse_geometry( zoom, srid, feature_map, feature["geometry"], i) except: sys.exit( "ERROR: Unable to parse {} file. Seems not a valid GEOJSON file." .format(geojson_file)) log.log( "RoboSat.pink - rasterize - rasterizing tiles from {} on cover {}". format(args.geojson, args.cover)) with open(os.path.join(os.path.expanduser(args.out), "instances.cover"), mode="w") as cover: for tile in tqdm(list( tiles_from_csv(os.path.expanduser(args.cover))), ascii=True, unit="tile"): try: if tile in feature_map: cover.write("{},{},{} {}{}".format( tile.x, tile.y, tile.z, len(feature_map[tile]), os.linesep)) out = geojson_tile_burn(tile, feature_map[tile], 4326, args.ts, burn_value) else: cover.write("{},{},{} {}{}".format( tile.x, tile.y, tile.z, 0, os.linesep)) out = np.zeros(shape=(args.ts, args.ts), dtype=np.uint8) tile_label_to_file(args.out, tile, palette, out) except: log.log("Warning: Unable to rasterize tile. Skipping {}". format(str(tile))) if args.postgis: try: pg_conn = psycopg2.connect(args.pg_dsn) pg = pg_conn.cursor() except Exception: sys.exit("Unable to connect PostgreSQL: {}".format(args.pg_dsn)) log.log( "RoboSat.pink - rasterize - rasterizing tiles from PostGIS on cover {}" .format(args.cover)) log.log(" SQL {}".format(args.postgis)) try: pg.execute( "SELECT ST_Srid(geom) AS srid FROM ({} LIMIT 1) AS sub".format( args.postgis)) srid = pg.fetchone()[0] except Exception: sys.exit("Unable to retrieve geometry SRID.") for tile in tqdm(list(tiles_from_csv(args.cover)), ascii=True, unit="tile"): s, w, e, n = mercantile.bounds(tile) raster = np.zeros((args.ts, args.ts)) query = """ WITH bbox AS (SELECT ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), {} ) AS bbox), bbox_merc AS (SELECT ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), 3857) AS bbox), rast_a AS (SELECT ST_AddBand( ST_SetSRID( ST_MakeEmptyRaster({}, {}, ST_Xmin(bbox), ST_Ymax(bbox), (ST_YMax(bbox) - ST_YMin(bbox)) / {}), 3857), '8BUI'::text, 0) AS rast FROM bbox_merc), features AS (SELECT ST_Union(ST_Transform(ST_Force2D(geom), 3857)) AS geom FROM ({}) AS sub, bbox WHERE ST_Intersects(geom, bbox)), rast_b AS (SELECT ST_AsRaster(geom, rast, '8BUI', {}) AS rast FROM features, rast_a WHERE NOT ST_IsEmpty(geom)) SELECT ST_AsBinary(ST_MapAlgebra(rast_a.rast, rast_b.rast, '{}', NULL, 'FIRST')) AS wkb FROM rast_a, rast_b """.format(s, w, e, n, srid, s, w, e, n, args.ts, args.ts, args.ts, args.postgis, burn_value, burn_value) try: pg.execute(query) row = pg.fetchone() if row: raster = np.squeeze(wkb_to_numpy(io.BytesIO(row[0])), axis=2) except Exception: log.log( "Warning: Invalid geometries, skipping {}".format(tile)) pg_conn = psycopg2.connect(args.pg_dsn) pg = pg_conn.cursor() try: tile_label_to_file(args.out, tile, palette, raster) except: log.log( "Warning: Unable to rasterize tile. Skipping {}".format( str(tile))) if not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" tiles = [tile for tile in tiles_from_csv(args.cover)] web_ui(args.out, base_url, tiles, tiles, "png", template)
def main(args): assert not (args.extent and args.splits), "--splits and --extent are mutually exclusive options." assert not (args.extent and len(args.out) > 1), "--extent option imply a single output." assert ( int(args.bbox is not None) + int(args.geojson is not None) + int(args.dir is not None) + int(args.raster is not None) + int(args.cover is not None) == 1 ), "One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson." if args.bbox: try: w, s, e, n, crs = args.bbox.split(",") w, s, e, n = map(float, (w, s, e, n)) except: crs = None w, s, e, n = map(float, args.bbox.split(",")) assert isinstance(w, float) and isinstance(s, float), "Invalid bbox parameter." if args.splits: splits = [int(split) for split in args.splits.split("/")] assert len(splits) == len(args.out) and 0 < sum(splits) <= 100, "Invalid split value or incoherent with out paths." assert not (not args.zoom and (args.geojson or args.bbox or args.raster)), "Zoom parameter is required." args.out = [os.path.expanduser(out) for out in args.out] cover = [] if args.raster: print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom), file=sys.stderr, flush=True) with rasterio_open(os.path.expanduser(args.raster)) as r: w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds) assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.geojson: print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom), file=sys.stderr, flush=True) with open(os.path.expanduser(args.geojson)) as f: feature_collection = json.load(f) srid = geojson_srid(feature_collection) feature_map = collections.defaultdict(list) for i, feature in enumerate(tqdm(feature_collection["features"], ascii=True, unit="feature")): feature_map = geojson_parse_feature(args.zoom, srid, feature_map, feature) cover = feature_map.keys() if args.bbox: print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom), file=sys.stderr, flush=True) if crs: w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n) assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.cover: print("RoboSat.pink - cover from {}".format(args.cover), file=sys.stderr, flush=True) cover = [tile for tile in tiles_from_csv(args.cover)] if args.dir: print("RoboSat.pink - cover from {}".format(args.dir), file=sys.stderr, flush=True) cover = [tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz))] _cover = [] extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0) for tile in tqdm(cover, ascii=True, unit="tile"): if args.zoom and tile.z != args.zoom: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) for t in tiles(w, s, n, e, args.zoom): unique = True for _t in _cover: if _t == t: unique = False if unique: _cover.append(t) else: if args.extent: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) _cover.append(tile) if args.extent: extent_w, extent_s, extent_n, extent_e = (min(extent_w, w), min(extent_s, s), max(extent_n, n), max(extent_e, e)) cover = _cover if args.splits: shuffle(cover) # in-place cover_splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)] if len(splits) > 1 and sum(map(int, splits)) == 100 and len(cover) > sum(map(int, splits)): cover_splits[0] = len(cover) - sum(map(int, cover_splits[1:])) # no tile waste s = 0 covers = [] for e in cover_splits: covers.append(cover[s : s + e]) s += e else: covers = [cover] if args.extent: if args.out and os.path.dirname(args.out[0]) and not os.path.isdir(os.path.dirname(args.out[0])): os.makedirs(os.path.dirname(args.out[0]), exist_ok=True) extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s, extent_n, extent_e) if args.out: with open(args.out[0], "w") as fp: fp.write(extent) else: print(extent) else: for i, cover in enumerate(covers): if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])): os.makedirs(os.path.dirname(args.out[i]), exist_ok=True) with open(args.out[i], "w") as fp: csv.writer(fp).writerows(cover)
def main(args): if (args.geojson and args.postgis) or (not args.geojson and not args.postgis): sys.exit("Input features to rasterize must be either GeoJSON or PostGIS") config = load_config(args.config) tile_size = args.tile_size if args.tile_size else config["model"]["tile_size"] colors = [classe["color"] for classe in config["classes"]] burn_value = 1 os.makedirs(args.out, exist_ok=True) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) def geojson_parse_polygon(zoom, feature_map, polygon, i): try: for i, ring in enumerate(polygon["coordinates"]): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[x, y] for point in ring for x, y in zip([point[0]], [point[1]])] for tile in burntiles.burn([{"type": "feature", "geometry": polygon}], zoom=zoom): feature_map[mercantile.Tile(*tile)].append({"type": "feature", "geometry": polygon}) except ValueError: log.log("Warning: invalid feature {}, skipping".format(i)) return feature_map def geojson_parse_geometry(zoom, feature_map, geometry, i): if geometry["type"] == "Polygon": feature_map = geojson_parse_polygon(zoom, feature_map, geometry, i) elif geometry["type"] == "MultiPolygon": for polygon in geometry["coordinates"]: feature_map = geojson_parse_polygon(zoom, feature_map, {"type": "Polygon", "coordinates": polygon}, i) else: log.log("Notice: {} is a non surfacic geometry type, skipping feature {}".format(geometry["type"], i)) return feature_map if args.geojson: tiles = [tile for tile in tiles_from_csv(args.cover)] zoom = tiles[0].z if [tile for tile in tiles if tile.z != zoom]: sys.exit("With GeoJson input, all tiles z values have to be the same, in the csv cover file.") feature_map = collections.defaultdict(list) # Compute a spatial index like for geojson_file in args.geojson: with open(geojson_file) as geojson: feature_collection = json.load(geojson) for i, feature in enumerate(tqdm(feature_collection["features"], ascii=True, unit="feature")): if feature["geometry"]["type"] == "GeometryCollection": for geometry in feature["geometry"]["geometries"]: feature_map = geojson_parse_geometry(zoom, feature_map, geometry, i) else: feature_map = geojson_parse_geometry(zoom, feature_map, feature["geometry"], i) # Rasterize tiles for tile in tqdm(list(tiles_from_csv(args.cover)), ascii=True, unit="tile"): if tile in feature_map: out = geojson_tile_burn(tile, feature_map[tile], tile_size, burn_value) else: out = np.zeros(shape=(tile_size, tile_size), dtype=np.uint8) write_tile(args.out, tile, colors, out) if args.postgis: try: pg_conn = psycopg2.connect(config["dataset"]["pg_dsn"]) pg = pg_conn.cursor() except Exception: sys.exit("Unable to connect PostgreSQL: {}".format(config["dataset"]["pg_dsn"])) try: pg.execute("SELECT ST_Srid(geom) AS srid FROM ({} LIMIT 1) AS sub".format(args.postgis)) srid = pg.fetchone()[0] except Exception: sys.exit("Unable to retrieve geometry SRID.") for tile in tqdm(list(tiles_from_csv(args.cover)), ascii=True, unit="tile"): s, w, e, n = mercantile.bounds(tile) raster = np.zeros((tile_size, tile_size)) query = """ WITH bbox AS (SELECT ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), {} ) AS bbox), bbox_merc AS (SELECT ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), 3857) AS bbox), rast_a AS (SELECT ST_AddBand( ST_SetSRID( ST_MakeEmptyRaster({}, {}, ST_Xmin(bbox), ST_Ymax(bbox), (ST_YMax(bbox) - ST_YMin(bbox)) / {}), 3857), '8BUI'::text, 0) AS rast FROM bbox_merc), features AS (SELECT ST_Union(ST_Transform(ST_Force2D(geom), 3857)) AS geom FROM ({}) AS sub, bbox WHERE ST_Intersects(geom, bbox)), rast_b AS (SELECT ST_AsRaster(geom, rast, '8BUI', {}) AS rast FROM features, rast_a WHERE NOT ST_IsEmpty(geom)) SELECT ST_AsBinary(ST_MapAlgebra(rast_a.rast, rast_b.rast, '{}', NULL, 'FIRST')) AS wkb FROM rast_a, rast_b """.format( s, w, e, n, srid, s, w, e, n, tile_size, tile_size, tile_size, args.postgis, burn_value, burn_value ) try: pg.execute(query) row = pg.fetchone() if row: raster = np.squeeze(wkb_to_numpy(io.BytesIO(row[0])), axis=2) except Exception: log.log("Warning: Invalid geometries, skipping {}".format(tile)) pg_conn = psycopg2.connect(config["dataset"]["pg_dsn"]) pg = pg_conn.cursor() write_tile(args.out, tile, colors, raster) if args.web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" tiles = [tile for tile in tiles_from_csv(args.cover)] web_ui(args.out, base_url, tiles, tiles, "png", template)
def main(args): config = load_config(args.config) check_channels(config) check_classes(config) palette = make_palette([classe["color"] for classe in config["classes"]]) args.workers = torch.cuda.device_count() * 2 if torch.device( "cuda") and not args.workers else args.workers cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] if args.cover else None log = Logs(os.path.join(args.out, "log")) if torch.cuda.is_available(): log.log("RoboSat.pink - predict on {} GPUs, with {} workers".format( torch.cuda.device_count(), args.workers)) log.log("(Torch:{} Cuda:{} CudNN:{})".format( torch.__version__, torch.version.cuda, torch.backends.cudnn.version())) device = torch.device("cuda") torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True else: log.log("RoboSat.pink - predict on CPU, with {} workers".format( args.workers)) log.log("") log.log("============================================================") log.log("WARNING: Are you -really- sure about not predicting on GPU ?") log.log("============================================================") log.log("") device = torch.device("cpu") chkpt = torch.load(args.checkpoint, map_location=device) model_module = load_module("robosat_pink.models.{}".format( chkpt["nn"].lower())) nn = getattr(model_module, chkpt["nn"])(chkpt["shape_in"], chkpt["shape_out"]).to(device) nn = torch.nn.DataParallel(nn) nn.load_state_dict(chkpt["state_dict"]) nn.eval() log.log("Model {} - UUID: {}".format(chkpt["nn"], chkpt["uuid"])) mode = "predict" if not args.translate else "predict_translate" loader_module = load_module("robosat_pink.loaders.{}".format( chkpt["loader"].lower())) loader_predict = getattr(loader_module, chkpt["loader"])(config, chkpt["shape_in"][1:3], args.dataset, cover, mode=mode) loader = DataLoader(loader_predict, batch_size=args.bs, num_workers=args.workers) assert len(loader), "Empty predict dataset directory. Check your path." tiled = [] with torch.no_grad( ): # don't track tensors with autograd during prediction for images, tiles in tqdm(loader, desc="Eval", unit="batch", ascii=True): images = images.to(device) outputs = nn(images) probs = torch.nn.functional.softmax(outputs, dim=1).data.cpu().numpy() for tile, prob in zip(tiles, probs): x, y, z = list(map(int, tile)) mask = np.around(prob[1:, :, :]).astype(np.uint8).squeeze() if args.translate: tile_translate_to_file(args.out, mercantile.Tile(x, y, z), palette, mask) else: tile_label_to_file(args.out, mercantile.Tile(x, y, z), palette, mask) tiled.append(mercantile.Tile(x, y, z)) if not args.no_web_ui and not args.translate: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiled, tiled, "png", template)
def main(args): tiles = list(tiles_from_csv(args.tiles)) already_dl = 0 dl = 0 with requests.Session() as session: num_workers = args.rate os.makedirs(os.path.join(args.out), exist_ok=True) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) log.log("Begin download from {}".format(args.url)) progress = tqdm(total=len(tiles), ascii=True, unit="image") with futures.ThreadPoolExecutor(num_workers) as executor: def worker(tile): tick = time.monotonic() x, y, z = map(str, [tile.x, tile.y, tile.z]) os.makedirs(os.path.join(args.out, z, x), exist_ok=True) path = os.path.join(args.out, z, x, "{}.{}".format(y, args.ext)) if os.path.isfile(path): progress.update() return tile, None, True if args.type == "XYZ": url = args.url.format(x=tile.x, y=tile.y, z=tile.z) elif args.type == "TMS": tile.y = (2**tile.z) - tile.y - 1 url = args.url.format(x=tile.x, y=tile.y, z=tile.z) elif args.type == "WMS": xmin, ymin, xmax, ymax = xy_bounds(tile) url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax) res = tile_image_from_url(session, url, args.timeout) if not res: return tile, url, False try: cv2.imwrite( path, cv2.imdecode(np.fromstring(res.read(), np.uint8), cv2.IMREAD_COLOR)) progress.update() except OSError: return tile, url, False tock = time.monotonic() time_for_req = tock - tick time_per_worker = num_workers / args.rate if time_for_req < time_per_worker: time.sleep(time_per_worker - time_for_req) return tile, url, True for tile, url, ok in executor.map(worker, tiles): if url and ok: dl += 1 elif not url and ok: already_dl += 1 else: log.log("Warning:\n {} failed, skipping.\n {}\n".format( tile, url)) if already_dl: log.log( "Notice:\n {} tiles were already downloaded previously, and so skipped now." .format(already_dl)) if already_dl + dl == len(tiles): log.log(" Coverage is fully downloaded.") if args.web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" web_ui(args.out, base_url, tiles, tiles, args.ext, template)
def main(args): if not args.workers: args.workers = min(os.cpu_count(), len(args.rasters)) if args.label: config = load_config(args.config) check_classes(config) colors = [classe["color"] for classe in config["classes"]] palette = make_palette(*colors) cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] if args.cover else None splits_path = os.path.join(os.path.expanduser(args.out), ".splits") tiles_map = {} print("RoboSat.pink - tile on CPU, with {} workers".format(args.workers)) bands = -1 for path in args.rasters: raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) if bands != -1: assert bands == len( raster.indexes), "Coverage must be bands consistent" bands = len(raster.indexes) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiles = list(set(tiles) & set(cover)) if cover else tiles for tile in tiles: tile_key = (str(tile.x), str(tile.y), str(tile.z)) if tile_key not in tiles_map.keys(): tiles_map[tile_key] = [] tiles_map[tile_key].append(path) if args.label: ext = "png" bands = 1 if not args.label: if bands == 1: ext = "png" if bands == 3: ext = "webp" if bands > 3: ext = "tiff" tiles = [] progress = tqdm(total=len(tiles_map), ascii=True, unit="tile") # Begin to tile plain tiles with futures.ThreadPoolExecutor(args.workers) as executor: def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) transform, _, _ = calculate_default_transform( raster.crs, "EPSG:3857", raster.width, raster.height, w, s, e, n) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) # inspired by rio-tiler, cf: https://github.com/mapbox/rio-tiler/pull/45 warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, args.ts, args.ts), width=math.ceil((e - w) / transform.a), height=math.ceil((s - n) / transform.e), ) data = warp_vrt.read(out_shape=(len(raster.indexes), args.ts, args.ts), window=warp_vrt.window(w, s, e, n)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if not args.label and len( tiles_map[tile_key]) == 1 and is_nodata( image, threshold=args.nodata_threshold): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: ret = tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: ret = tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) assert ret, "Unable to write tile {} from raster {}.".format( str(tile), raster) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled for tiled in executor.map(worker, args.rasters): if tiled is not None: tiles.extend(tiled) # Aggregate remaining tiles splits with futures.ThreadPoolExecutor(args.workers) as executor: def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((args.ts, args.ts, bands), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) split = split.reshape( (args.ts, args.ts, 1)) # H,W -> H,W,C assert image.shape == split.shape image[:, :, :] += split[:, :, :] if not args.label and is_nodata(image, threshold=args.nodata_threshold): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: ret = tile_image_to_file(args.out, tile, image) if args.label: ret = tile_label_to_file(args.out, tile, palette, image) assert ret, "Unable to write tile {} from raster {}.".format( str(tile_key)) progress.update() return tile for tiled in executor.map(worker, tiles_map.keys()): if tiled is not None: tiles.append(tiled) if splits_path and os.path.isdir(splits_path): shutil.rmtree(splits_path) # Delete suffixes dir if any if tiles and not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args): config = load_config(args.config) tile_size = args.tile_size if args.tile_size else config["model"]["tile_size"] colors = [classe["color"] for classe in config["classes"]] os.makedirs(args.out, exist_ok=True) # We can only rasterize all tiles at a single zoom. assert all(tile.z == args.zoom for tile in tiles_from_csv(args.cover)) # Find all tiles the features cover and make a map object for quick lookup. feature_map = collections.defaultdict(list) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) def parse_polygon(feature_map, polygon, i): try: for i, ring in enumerate(polygon["coordinates"]): # GeoJSON coordinates could be N dimensionals polygon["coordinates"][i] = [[x, y] for point in ring for x, y in zip([point[0]], [point[1]])] for tile in burntiles.burn([{"type": "feature", "geometry": polygon}], zoom=args.zoom): feature_map[mercantile.Tile(*tile)].append({"type": "feature", "geometry": polygon}) except ValueError: log.log("Warning: invalid feature {}, skipping".format(i)) return feature_map def parse_geometry(feature_map, geometry, i): if geometry["type"] == "Polygon": feature_map = parse_polygon(feature_map, geometry, i) elif geometry["type"] == "MultiPolygon": for polygon in geometry["coordinates"]: feature_map = parse_polygon(feature_map, {"type": "Polygon", "coordinates": polygon}, i) else: log.log("Notice: {} is a non surfacic geometry type, skipping feature {}".format(geometry["type"], i)) return feature_map for feature in args.features: with open(feature) as f: fc = json.load(f) for i, feature in enumerate(tqdm(fc["features"], ascii=True, unit="feature")): if feature["geometry"]["type"] == "GeometryCollection": for geometry in feature["geometry"]["geometries"]: feature_map = parse_geometry(feature_map, geometry, i) else: feature_map = parse_geometry(feature_map, feature["geometry"], i) # Burn features to tiles and write to a slippy map directory. for tile in tqdm(list(tiles_from_csv(args.cover)), ascii=True, unit="tile"): if tile in feature_map: out = burn(tile, feature_map[tile], tile_size) else: out = np.zeros(shape=(tile_size, tile_size), dtype=np.uint8) out_dir = os.path.join(args.out, str(tile.z), str(tile.x)) os.makedirs(out_dir, exist_ok=True) out_path = os.path.join(out_dir, "{}.png".format(tile.y)) if os.path.exists(out_path): prev = np.array(Image.open(out_path)) out = np.maximum(out, prev) out = Image.fromarray(out, mode="P") out_path = os.path.join(args.out, str(tile.z), str(tile.x)) os.makedirs(out_path, exist_ok=True) out.putpalette(complementary_palette(make_palette(colors[0], colors[1]))) out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True) if args.web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "./" tiles = [tile for tile in tiles_from_csv(args.cover)] web_ui(args.out, base_url, tiles, tiles, "png", template)