示例#1
0
def main(args):

    assert not (args.extent and args.splits), "--splits and --extent are mutually exclusive options."
    assert not (args.extent and len(args.out) > 1), "--extent option imply a single output."
    assert (
        int(args.bbox is not None)
        + int(args.geojson is not None)
        + int(args.dir is not None)
        + int(args.raster is not None)
        + int(args.cover is not None)
        == 1
    ), "One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson."

    if args.bbox:
        try:
            w, s, e, n, crs = args.bbox.split(",")
            w, s, e, n = map(float, (w, s, e, n))
        except:
            crs = None
            w, s, e, n = map(float, args.bbox.split(","))
        assert isinstance(w, float) and isinstance(s, float), "Invalid bbox parameter."

    if args.splits:
        splits = [int(split) for split in args.splits.split("/")]
        assert len(splits) == len(args.out) and 0 < sum(splits) <= 100, "Invalid split value or incoherent with out paths."

    assert not (not args.zoom and (args.geojson or args.bbox or args.raster)), "Zoom parameter is required."

    args.out = [os.path.expanduser(out) for out in args.out]

    cover = []

    if args.raster:
        print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom), file=sys.stderr, flush=True)
        with rasterio_open(os.path.expanduser(args.raster)) as r:
            w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds)
            assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection"

            cover = [tile for tile in tiles(w, s, e, n, args.zoom)]

    if args.geojson:
        print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom), file=sys.stderr, flush=True)
        with open(os.path.expanduser(args.geojson)) as f:
            feature_collection = json.load(f)
            srid = geojson_srid(feature_collection)
            feature_map = collections.defaultdict(list)

            for i, feature in enumerate(tqdm(feature_collection["features"], ascii=True, unit="feature")):
                feature_map = geojson_parse_feature(args.zoom, srid, feature_map, feature)

        cover = feature_map.keys()

    if args.bbox:
        print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom), file=sys.stderr, flush=True)
        if crs:
            w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n)
            assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection"

        cover = [tile for tile in tiles(w, s, e, n, args.zoom)]

    if args.cover:
        print("RoboSat.pink - cover from {}".format(args.cover), file=sys.stderr, flush=True)
        cover = [tile for tile in tiles_from_csv(args.cover)]

    if args.dir:
        print("RoboSat.pink - cover from {}".format(args.dir), file=sys.stderr, flush=True)
        cover = [tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz))]

    _cover = []
    extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0)
    for tile in tqdm(cover, ascii=True, unit="tile"):
        if args.zoom and tile.z != args.zoom:
            w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile))
            for t in tiles(w, s, n, e, args.zoom):
                unique = True
                for _t in _cover:
                    if _t == t:
                        unique = False
                if unique:
                    _cover.append(t)
        else:
            if args.extent:
                w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile))
            _cover.append(tile)

        if args.extent:
            extent_w, extent_s, extent_n, extent_e = (min(extent_w, w), min(extent_s, s), max(extent_n, n), max(extent_e, e))

    cover = _cover

    if args.splits:
        shuffle(cover)  # in-place
        cover_splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)]
        if len(splits) > 1 and sum(map(int, splits)) == 100 and len(cover) > sum(map(int, splits)):
            cover_splits[0] = len(cover) - sum(map(int, cover_splits[1:]))  # no tile waste
        s = 0
        covers = []
        for e in cover_splits:
            covers.append(cover[s : s + e])
            s += e
    else:
        covers = [cover]

    if args.extent:
        if args.out and os.path.dirname(args.out[0]) and not os.path.isdir(os.path.dirname(args.out[0])):
            os.makedirs(os.path.dirname(args.out[0]), exist_ok=True)

        extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s, extent_n, extent_e)

        if args.out:
            with open(args.out[0], "w") as fp:
                fp.write(extent)
        else:
            print(extent)
    else:
        for i, cover in enumerate(covers):

            if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])):
                os.makedirs(os.path.dirname(args.out[i]), exist_ok=True)

            with open(args.out[i], "w") as fp:
                csv.writer(fp).writerows(cover)
示例#2
0
def main(args):

    assert not (args.extent and args.splits
                ), "--splits and --extent are mutually exclusive options."
    assert not (args.extent and
                len(args.out) > 1), "--extent option imply a single output."
    assert not (args.sql and not args.pg), "--sql option imply --pg"
    assert (
        int(args.bbox is not None) + int(args.geojson is not None) +
        int(args.sql is not None) + int(args.dir is not None) +
        int(args.raster is not None) + int(args.cover is not None) == 1
    ), "One, and only one, input type must be provided, among: --dir, --bbox, --cover, --raster, --geojson or --sql"

    if args.bbox:
        try:
            w, s, e, n, crs = args.bbox.split(",")
            w, s, e, n = map(float, (w, s, e, n))
        except:
            crs = None
            w, s, e, n = map(float, args.bbox.split(","))
        assert isinstance(w, float) and isinstance(
            s, float), "Invalid bbox parameter."

    if args.splits:
        splits = [int(split) for split in args.splits.split("/")]
        assert len(splits) == len(args.out) and 0 < sum(
            splits) <= 100, "Invalid split value or incoherent with out paths."

    assert not (not args.zoom and
                (args.geojson or args.bbox
                 or args.raster)), "Zoom parameter is required."

    args.out = [os.path.expanduser(out) for out in args.out]

    cover = []

    if args.raster:
        print("RoboSat.pink - cover from {} at zoom {}".format(
            args.raster, args.zoom),
              file=sys.stderr,
              flush=True)
        with rasterio_open(os.path.expanduser(args.raster)) as r:
            w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds)
            assert isinstance(w, float) and isinstance(
                s, float), "Unable to deal with raster projection"

            cover = [tile for tile in tiles(w, s, e, n, args.zoom)]

    if args.geojson:
        print("RoboSat.pink - cover from {} at zoom {}".format(
            args.geojson, args.zoom),
              file=sys.stderr,
              flush=True)
        with open(os.path.expanduser(args.geojson)) as f:
            feature_collection = json.load(f)
            srid = geojson_srid(feature_collection)
            feature_map = collections.defaultdict(list)

            for feature in tqdm(feature_collection["features"],
                                ascii=True,
                                unit="feature"):
                feature_map = geojson_parse_feature(args.zoom, srid,
                                                    feature_map, feature)

        cover = feature_map.keys()

    if args.sql:
        print("RoboSat.pink - cover from {} {} at zoom {}".format(
            args.sql, args.pg, args.zoom),
              file=sys.stderr,
              flush=True)
        conn = psycopg2.connect(args.pg)
        assert conn, "Unable to connect to PostgreSQL database."
        db = conn.cursor()

        query = """
            WITH
              sql  AS ({}),
              geom AS (SELECT "1" AS geom FROM sql AS t("1"))
              SELECT '{{"type": "Feature", "geometry": '
                     || ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6)
                     || '}}' AS features
              FROM geom
            """.format(args.sql)

        db.execute(query)
        assert db.rowcount is not None and db.rowcount != -1, "SQL Query return no result."

        feature_map = collections.defaultdict(list)

        for feature in tqdm(
                db.fetchall(), ascii=True, unit="feature"
        ):  # FIXME: fetchall will not always fit in memory...
            feature_map = geojson_parse_feature(args.zoom, 4326, feature_map,
                                                json.loads(feature[0]))

        cover = feature_map.keys()

    if args.bbox:
        print("RoboSat.pink - cover from {} at zoom {}".format(
            args.bbox, args.zoom),
              file=sys.stderr,
              flush=True)
        if crs:
            w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n)
            assert isinstance(w, float) and isinstance(
                s, float), "Unable to deal with raster projection"

        cover = [tile for tile in tiles(w, s, e, n, args.zoom)]

    if args.cover:
        print("RoboSat.pink - cover from {}".format(args.cover),
              file=sys.stderr,
              flush=True)
        cover = [tile for tile in tiles_from_csv(args.cover)]

    if args.dir:
        print("RoboSat.pink - cover from {}".format(args.dir),
              file=sys.stderr,
              flush=True)
        cover = [
            tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz))
        ]

    _cover = []
    extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0)
    for tile in tqdm(cover, ascii=True, unit="tile"):
        if args.zoom and tile.z != args.zoom:
            w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
                                          *xy_bounds(tile))
            for t in tiles(w, s, n, e, args.zoom):
                unique = True
                for _t in _cover:
                    if _t == t:
                        unique = False
                if unique:
                    _cover.append(t)
        else:
            if args.extent:
                w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
                                              *xy_bounds(tile))
            _cover.append(tile)

        if args.extent:
            extent_w, extent_s, extent_n, extent_e = (min(extent_w,
                                                          w), min(extent_s, s),
                                                      max(extent_n,
                                                          n), max(extent_e, e))

    cover = _cover

    if args.splits:
        shuffle(cover)  # in-place
        cover_splits = [
            math.floor(len(cover) * split / 100)
            for i, split in enumerate(splits, 1)
        ]
        if len(splits) > 1 and sum(map(
                int, splits)) == 100 and len(cover) > sum(map(int, splits)):
            cover_splits[0] = len(cover) - sum(map(
                int, cover_splits[1:]))  # no tile waste
        s = 0
        covers = []
        for e in cover_splits:
            covers.append(cover[s:s + e])
            s += e
    else:
        covers = [cover]

    if args.extent:
        if args.out and os.path.dirname(args.out[0]) and not os.path.isdir(
                os.path.dirname(args.out[0])):
            os.makedirs(os.path.dirname(args.out[0]), exist_ok=True)

        extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s,
                                                      extent_n, extent_e)

        if args.out:
            with open(args.out[0], "w") as fp:
                fp.write(extent)
        else:
            print(extent)
    else:
        for i, cover in enumerate(covers):

            if os.path.dirname(args.out[i]) and not os.path.isdir(
                    os.path.dirname(args.out[i])):
                os.makedirs(os.path.dirname(args.out[i]), exist_ok=True)

            with open(args.out[i], "w") as fp:
                csv.writer(fp).writerows(cover)
示例#3
0
def main(args):

    assert not (args.sql and
                args.geojson), "You can only use at once --pg OR --geojson."
    assert not (args.pg and not args.sql
                ), "With PostgreSQL --pg, --sql must also be provided"
    assert len(args.ts.split(
        ",")) == 2, "--ts expect width,height value (e.g 512,512)"

    config = load_config(args.config)
    check_classes(config)

    palette = make_palette([classe["color"] for classe in config["classes"]],
                           complementary=True)
    index = [
        config["classes"].index(classe) for classe in config["classes"]
        if classe["title"] == args.type
    ]
    assert index, "Requested type is not contains in your config file classes."
    burn_value = int(math.pow(2, index[0] - 1))  # 8bits One Hot Encoding
    assert 0 <= burn_value <= 128

    args.out = os.path.expanduser(args.out)
    os.makedirs(args.out, exist_ok=True)
    log = Logs(os.path.join(args.out, "log"), out=sys.stderr)

    if args.geojson:

        tiles = [
            tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
        ]
        assert tiles, "Empty cover"

        zoom = tiles[0].z
        assert not [tile for tile in tiles if tile.z != zoom
                    ], "Unsupported zoom mixed cover. Use PostGIS instead"

        feature_map = collections.defaultdict(list)

        log.log("RoboSat.pink - rasterize - Compute spatial index")
        for geojson_file in args.geojson:

            with open(os.path.expanduser(geojson_file)) as geojson:
                feature_collection = json.load(geojson)
                srid = geojson_srid(feature_collection)

                feature_map = collections.defaultdict(list)

                for i, feature in enumerate(
                        tqdm(feature_collection["features"],
                             ascii=True,
                             unit="feature")):
                    feature_map = geojson_parse_feature(
                        zoom, srid, feature_map, feature)

        features = args.geojson

    if args.pg:

        conn = psycopg2.connect(args.pg)
        db = conn.cursor()

        assert "limit" not in args.sql.lower(), "LIMIT is not supported"
        assert "TILE_GEOM" in args.sql, "TILE_GEOM filter not found in your SQL"
        sql = re.sub(r"ST_Intersects( )*\((.*)?TILE_GEOM(.*)?\)", "1=1",
                     args.sql, re.I)
        assert sql and sql != args.sql

        db.execute(
            """SELECT ST_Srid("1") AS srid FROM ({} LIMIT 1) AS t("1")""".
            format(sql))
        srid = db.fetchone()[0]
        assert srid and int(srid) > 0, "Unable to retrieve geometry SRID."

        features = args.sql

    log.log(
        "RoboSat.pink - rasterize - rasterizing {} from {} on cover {}".format(
            args.type, features, args.cover))
    with open(os.path.join(os.path.expanduser(args.out),
                           "instances_" + args.type.lower() + ".cover"),
              mode="w") as cover:

        for tile in tqdm(list(tiles_from_csv(os.path.expanduser(args.cover))),
                         ascii=True,
                         unit="tile"):

            geojson = None

            if args.pg:

                w, s, e, n = tile_bbox(tile)
                tile_geom = "ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), {})".format(
                    w, s, e, n, srid)

                query = """
                WITH
                  sql  AS ({}),
                  geom AS (SELECT "1" AS geom FROM sql AS t("1")),
                  json AS (SELECT '{{"type": "Feature", "geometry": '
                         || ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6)
                         || '}}' AS features
                        FROM geom)
                SELECT '{{"type": "FeatureCollection", "features": [' || Array_To_String(array_agg(features), ',') || ']}}'
                FROM json
                """.format(args.sql.replace("TILE_GEOM", tile_geom))

                db.execute(query)
                row = db.fetchone()
                try:
                    geojson = json.loads(
                        row[0])["features"] if row and row[0] else None
                except Exception:
                    log.log("Warning: Invalid geometries, skipping {}".format(
                        tile))
                    conn = psycopg2.connect(args.pg)
                    db = conn.cursor()

            if args.geojson:
                geojson = feature_map[tile] if tile in feature_map else None

            if geojson:
                num = len(geojson)
                out = geojson_tile_burn(tile, geojson, 4326,
                                        list(map(int, args.ts.split(","))),
                                        burn_value)

            if not geojson or out is None:
                num = 0
                out = np.zeros(shape=list(map(int, args.ts.split(","))),
                               dtype=np.uint8)

            tile_label_to_file(args.out,
                               tile,
                               palette,
                               out,
                               append=args.append)
            cover.write("{},{},{}  {}{}".format(tile.x, tile.y, tile.z, num,
                                                os.linesep))

    if not args.no_web_ui:
        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
        base_url = args.web_ui_base_url if args.web_ui_base_url else "."
        tiles = [tile for tile in tiles_from_csv(args.cover)]
        web_ui(args.out, base_url, tiles, tiles, "png", template)