def main(args):
config = load_config(args.config)
check_classes(config)
index = [i for i in (list(range(len(config["classes"])))) if config["classes"][i]["title"] == args.type]
assert index, "Requested type {} not found among classes title in the config file.".format(args.type)
masks = list(tiles_from_dir(args.masks, xyz_path=True))
assert len(masks), "empty masks directory: {}".format(args.masks)
print("abd vectorize {} from {}".format(args.type, args.masks), file=sys.stderr, flush=True)
if os.path.dirname(os.path.expanduser(args.out)):
os.makedirs(os.path.dirname(os.path.expanduser(args.out)), exist_ok=True)
out = open(args.out, "w", encoding="utf-8")
assert out, "Unable to write in output file"
out.write('{"type":"FeatureCollection","features":[')
first = True
for tile, path in tqdm(masks, ascii=True, unit="mask"):
mask = (np.array(Image.open(path).convert("P"), dtype=np.uint8) == index).astype(np.uint8)
try:
C, W, H = mask.shape
except:
W, H = mask.shape
transform = rasterio.transform.from_bounds((*mercantile.bounds(tile.x, tile.y, tile.z)), W, H)
for shape, value in rasterio.features.shapes(mask, transform=transform, mask=mask):
geom = '"geometry":{{"type": "Polygon", "coordinates":{}}}'.format(json.dumps(shape["coordinates"]))
out.write('{}{{"type":"Feature",{}}}'.format("" if first else ",", geom))
first = False
out.write("]}")
def __init__(self, root, num_classes, cover=None):
super().__init__()
self.num_classes = num_classes
self.tiles = [
path for tile, path in tiles_from_dir(
os.path.join(root, "labels"), cover=cover, xyz_path=True)
]
assert len(self.tiles), "Empty Dataset"
def __init__(self, config, ts, root, cover=None, tiles_weights=None, mode=None, metatiles=False, keep_borders=False):
super().__init__()
self.mode = mode
self.config = config
self.tiles_weights = tiles_weights
self.metatiles = metatiles
self.da = True if "da" in self.config["train"].keys() and self.config["train"]["da"]["p"] > 0.0 else False
assert mode in ["train", "eval", "predict"]
path = os.path.join(root, config["channels"][0]["name"])
self.tiles_paths = [(tile, path) for tile, path in tiles_from_dir(path, cover=cover, xyz_path=True)]
if metatiles:
self.metatiles_paths = self.tiles_paths
if not keep_borders:
self.tiles_paths = [
(tile, path) for tile, path in self.metatiles_paths if tile_is_neighboured(tile, self.metatiles_paths)
]
self.cover = {tile for tile, path in self.tiles_paths}
assert len(self.tiles_paths), "Empty Dataset"
self.tiles = {}
num_channels = 0
for channel in config["channels"]:
path = os.path.join(root, channel["name"])
self.tiles[channel["name"]] = [
(tile, path) for tile, path in tiles_from_dir(path, cover=self.cover, xyz_path=True)
]
num_channels += len(channel["bands"])
self.shape_in = (num_channels,) + tuple(ts) # C,W,H
self.shape_out = (len(config["classes"]),) + tuple(ts) # C,W,H
if self.mode in ["train", "eval"]:
path = os.path.join(root, "labels")
self.tiles["labels"] = [(tile, path) for tile, path in tiles_from_dir(path, cover=self.cover, xyz_path=True)]
for channel in config["channels"]: # Order images and labels accordingly
self.tiles[channel["name"]].sort(key=lambda tile: tile[0])
self.tiles["labels"].sort(key=lambda tile: tile[0])
assert len(self.tiles), "Empty Dataset"
def main(args):
if not args.masks or not args.labels:
assert args.mode != "list", "Parameters masks and labels are mandatories in list mode."
assert not (
args.min or args.max
), "Both --masks and --labels mandatory, for metric filtering."
if args.min or args.max:
config = load_config(args.config)
args.out = os.path.expanduser(args.out)
cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
args_minmax = set()
args.min = {(m[0], m[1]): m[2] for m in args.min} if args.min else dict()
args.max = {(m[0], m[1]): m[2] for m in args.max} if args.max else dict()
args_minmax.update(args.min.keys())
args_minmax.update(args.max.keys())
minmax = dict()
for mm in args_minmax:
mm_min = float(args.min[mm]) if mm in args.min else 0.0
mm_max = float(args.max[mm]) if mm in args.max else 1.0
assert mm_min < mm_max, "--min must be lower than --max, on {}".format(
mm)
minmax[mm] = {
"min":
mm_min,
"max":
mm_max,
"class_id": [
c for c, classe in enumerate(config["classes"])
if classe["title"] == mm[0]
][0],
"module":
load_module("abd_model.metrics." + mm[1]),
}
if not args.workers:
args.workers = os.cpu_count()
print("abd compare {} on CPU, with {} workers".format(
args.mode, args.workers),
file=sys.stderr,
flush=True)
if args.images:
tiles = [tile for tile in tiles_from_dir(args.images[0], cover=cover)]
assert len(tiles), "Empty images dir: {}".format(args.images[0])
for image in args.images[1:]:
assert sorted(tiles) == sorted([
tile for tile in tiles_from_dir(image, cover=cover)
]), "Unconsistent images dirs"
if args.labels and args.masks:
tiles_masks = [
tile for tile in tiles_from_dir(args.masks, cover=cover)
]
tiles_labels = [
tile for tile in tiles_from_dir(args.labels, cover=cover)
]
if args.images:
assert sorted(tiles) == sorted(tiles_masks) == sorted(
tiles_labels), "Unconsistent images/label/mask directories"
else:
assert len(tiles_masks), "Empty masks dir: {}".format(args.masks)
assert len(tiles_labels), "Empty labels dir: {}".format(
args.labels)
assert sorted(tiles_masks) == sorted(
tiles_labels), "Label and Mask directories are not consistent"
tiles = tiles_masks
tiles_list = []
tiles_compare = []
progress = tqdm(total=len(tiles), ascii=True, unit="tile")
log = False if args.mode == "list" else Logs(os.path.join(args.out, "log"))
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile):
x, y, z = list(map(str, tile))
if args.masks and args.labels:
label = np.array(
Image.open(
os.path.join(args.labels, z, x, "{}.png".format(y))))
mask = np.array(
Image.open(
os.path.join(args.masks, z, x, "{}.png".format(y))))
assert label.shape == mask.shape, "Inconsistent tiles (size or dimensions)"
metrics = dict()
for mm in minmax:
try:
metrics[mm] = getattr(minmax[mm]["module"], "get")(
torch.as_tensor(label, device="cpu"),
torch.as_tensor(mask, device="cpu"),
minmax[mm]["class_id"],
)
except:
progress.update()
return False, tile
if not (minmax[mm]["min"] <= metrics[mm] <=
minmax[mm]["max"]):
progress.update()
return True, tile
tiles_compare.append(tile)
if args.mode == "side":
for i, root in enumerate(args.images):
img = tile_image_from_file(tile_from_xyz(root, x, y, z)[1],
force_rgb=True)
if i == 0:
side = np.zeros(
(img.shape[0], img.shape[1] * len(args.images), 3))
side = np.swapaxes(side, 0,
1) if args.vertical else side
image_shape = img.shape
else:
assert image_shape[0:2] == img.shape[
0:2], "Unconsistent image size to compare"
if args.vertical:
side[i * image_shape[0]:(i + 1) *
image_shape[0], :, :] = img
else:
side[:, i * image_shape[0]:(i + 1) *
image_shape[0], :] = img
tile_image_to_file(args.out, tile, np.uint8(side))
elif args.mode == "stack":
for i, root in enumerate(args.images):
tile_image = tile_image_from_file(tile_from_xyz(
root, x, y, z)[1],
force_rgb=True)
if i == 0:
image_shape = tile_image.shape[0:2]
stack = tile_image / len(args.images)
else:
assert image_shape == tile_image.shape[
0:2], "Unconsistent image size to compare"
stack = stack + (tile_image / len(args.images))
tile_image_to_file(args.out, tile, np.uint8(stack))
elif args.mode == "list":
tiles_list.append([tile, metrics])
progress.update()
return True, tile
for ok, tile in executor.map(worker, tiles):
if not ok and log:
log.log("Warning: skipping. {}".format(str(tile)))
if args.mode == "list":
with open(args.out, mode="w") as out:
if args.geojson:
out.write('{"type":"FeatureCollection","features":[')
first = True
for tile_list in tiles_list:
tile, metrics = tile_list
x, y, z = list(map(str, tile))
if args.geojson:
prop = '"properties":{{"x":{},"y":{},"z":{}'.format(
x, y, z)
for metric in metrics:
prop += ',"{}":{:.3f}'.format(metric, metrics[metric])
geom = '"geometry":{}'.format(
json.dumps(feature(tile, precision=6)["geometry"]))
out.write('{}{{"type":"Feature",{},{}}}}}'.format(
"," if not first else "", geom, prop))
first = False
if not args.geojson:
out.write("{},{},{}".format(x, y, z))
for metric in metrics:
out.write("\t{:.3f}".format(metrics[metric]))
out.write(os.linesep)
if args.geojson:
out.write("]}")
out.close()
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
if args.mode == "side" and not args.no_web_ui:
template = "compare.html" if not args.web_ui_template else args.web_ui_template
web_ui(args.out,
base_url,
tiles,
tiles_compare,
args.format,
template,
union_tiles=False)
if args.mode == "stack" and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
tiles = [tile for tile in tiles_from_dir(args.images[0])]
web_ui(args.out, base_url, tiles, tiles_compare, args.format, template)
def main(args):
assert not (args.type == "extent" and args.splits
), "--splits and --type extent are mutually exclusive options"
assert not (args.type == "extent" and args.out and len(args.out) > 1
), "--type extent option imply a single --out path"
assert not (args.type != "extent" and
not args.out), "--out mandatory [except with --type extent]"
assert not (args.union
and args.type != "geojson"), "--union imply --type geojson"
assert not (args.sql and not args.pg), "--sql option imply --pg"
assert (
int(args.bbox is not None) + int(args.geojson is not None) +
int(args.sql is not None) + int(args.dir is not None) +
int(args.raster is not None) + int(args.cover is not None) == 1
), "One, and only one, input type must be provided, among: --dir, --bbox, --cover, --raster, --geojson or --sql"
if args.splits:
splits = [int(split) for split in args.splits.split("/")]
assert len(splits) == len(args.out) and 0 < sum(
splits) <= 100, "Invalid split value or incoherent with out paths."
assert args.zoom or (args.dir or args.cover), "Zoom parameter is required."
args.out = [os.path.expanduser(out)
for out in args.out] if args.out else None
cover = []
if args.raster:
print("abd cover from {} at zoom {}".format(args.raster, args.zoom),
file=sys.stderr,
flush=True)
cover = set()
for raster_file in args.raster:
with rasterio_open(os.path.expanduser(raster_file)) as r:
try:
w, s, e, n = transform_bounds(r.crs, "EPSG:4326",
*r.bounds)
except:
print("WARNING: projection error, SKIPPING: {}".format(
raster_file),
file=sys.stderr,
flush=True)
continue
cover.update([tile for tile in tiles(w, s, e, n, args.zoom)])
cover = list(cover)
if args.geojson:
print("abd cover from {} at zoom {}".format(args.geojson, args.zoom),
file=sys.stderr,
flush=True)
feature_map = collections.defaultdict(list)
for geojson_file in args.geojson:
with open(os.path.expanduser(geojson_file)) as f:
feature_collection = json.load(f)
srid = geojson_srid(feature_collection)
for feature in tqdm(feature_collection["features"],
ascii=True,
unit="feature"):
feature_map = geojson_parse_feature(
args.zoom, srid, feature_map, feature)
cover = feature_map.keys()
if args.sql:
print("abd cover from {} {} at zoom {}".format(args.sql, args.pg,
args.zoom),
file=sys.stderr,
flush=True)
conn = psycopg2.connect(args.pg)
assert conn, "Unable to connect to PostgreSQL database."
db = conn.cursor()
query = """
WITH
sql AS ({}),
geom AS (SELECT "1" AS geom FROM sql AS t("1"))
SELECT '{{"type": "Feature", "geometry": '
|| ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6)
|| '}}' AS features
FROM geom
""".format(args.sql)
db.execute(query)
assert db.rowcount is not None and db.rowcount != -1, "SQL Query return no result."
feature_map = collections.defaultdict(list)
for feature in tqdm(
db.fetchall(), ascii=True, unit="feature"
): # FIXME: fetchall will not always fit in memory...
feature_map = geojson_parse_feature(args.zoom, 4326, feature_map,
json.loads(feature[0]))
cover = feature_map.keys()
if args.bbox:
try:
w, s, e, n, crs = args.bbox.split(",")
w, s, e, n = map(float, (w, s, e, n))
except:
crs = None
w, s, e, n = map(float, args.bbox.split(","))
assert isinstance(w, float) and isinstance(
s, float) and w < e and s < n, "Invalid bbox parameter."
print("abd cover from {} at zoom {}".format(args.bbox, args.zoom),
file=sys.stderr,
flush=True)
if crs:
w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n)
assert isinstance(w, float) and isinstance(
s, float), "Unable to deal with raster projection"
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.cover:
print("abd cover from {}".format(args.cover),
file=sys.stderr,
flush=True)
cover = [
tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
]
if args.dir:
print("abd cover from {}".format(args.dir),
file=sys.stderr,
flush=True)
cover = [
tile for tile in tiles_from_dir(os.path.expanduser(args.dir),
xyz=not (args.no_xyz))
]
assert len(cover), "Empty tiles inputs"
_cover = []
extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0)
for tile in tqdm(cover, ascii=True, unit="tile"):
if args.zoom and tile.z != args.zoom:
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
*xy_bounds(tile))
for t in tiles(w, s, n, e, args.zoom):
unique = True
for _t in _cover:
if _t == t:
unique = False
if unique:
_cover.append(t)
else:
if args.type == "extent":
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
*xy_bounds(tile))
_cover.append(tile)
if args.type == "extent":
extent_w, extent_s, extent_n, extent_e = (min(extent_w,
w), min(extent_s, s),
max(extent_n,
n), max(extent_e, e))
cover = _cover
if args.splits:
shuffle(cover) # in-place
cover_splits = [
math.floor(len(cover) * split / 100)
for i, split in enumerate(splits, 1)
]
if len(splits) > 1 and sum(map(
int, splits)) == 100 and len(cover) > sum(map(int, splits)):
cover_splits[0] = len(cover) - sum(map(
int, cover_splits[1:])) # no tile waste
s = 0
covers = []
for e in cover_splits:
covers.append(cover[s:s + e])
s += e
else:
covers = [cover]
if args.type == "extent":
extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s,
extent_n, extent_e)
if args.out:
if os.path.dirname(args.out[0]) and not os.path.isdir(
os.path.dirname(args.out[0])):
os.makedirs(os.path.dirname(args.out[0]), exist_ok=True)
with open(args.out[0], "w") as fp:
fp.write(extent)
else:
print(extent)
else:
for i, cover in enumerate(covers):
if os.path.dirname(args.out[i]) and not os.path.isdir(
os.path.dirname(args.out[i])):
os.makedirs(os.path.dirname(args.out[i]), exist_ok=True)
with open(args.out[i], "w") as fp:
if args.type == "geojson":
fp.write(tiles_to_geojson(cover, union=args.union))
else:
csv.writer(fp).writerows(cover)