def main(args):
assert os.path.isdir(os.path.expanduser(
args.dataset)), "--dataset path is not a directory"
args.cover = [
tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
config = load_config(args.config)
if not args.workers:
args.workers = os.cpu_count()
print("abd dataset {} on CPU, with {} workers".format(
args.mode, args.workers),
file=sys.stderr,
flush=True)
if args.mode == "check":
check_classes(config)
check_channels(config)
# TODO check dataset
if args.mode == "weights":
check_classes(config)
weights = compute_classes_weights(args.dataset, config["classes"],
args.cover, args.workers)
print(",".join(map(str, weights)))
def main(args):
config = load_config(args.config)
check_channels(config)
check_classes(config)
assert torch.cuda.is_available(
), "No GPU support found. Check CUDA and NVidia Driver install."
assert torch.distributed.is_nccl_available(
), "No NCCL support found. Check your PyTorch install."
world_size = torch.cuda.device_count()
args.bs = args.bs if args.bs is not None else math.floor(os.cpu_count() /
world_size)
args.workers = args.workers if args.workers is not None else args.bs
palette, transparency = make_palette(
[classe["color"] for classe in config["classes"]])
args.cover = [
tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
args.out = os.path.expanduser(args.out)
log = Logs(os.path.join(args.out, "log"))
chkpt = torch.load(args.checkpoint, map_location=torch.device("cpu"))
log.log("abd predict on {} GPUs, with {} workers/GPU and {} tiles/batch".
format(world_size, args.workers, args.bs))
log.log("Model {} - UUID: {}".format(chkpt["nn"], chkpt["uuid"]))
log.log("---")
loader = load_module("abd_model.loaders.{}".format(
chkpt["loader"].lower()))
lock_file = os.path.abspath(os.path.join(args.out, str(uuid.uuid1())))
dataset = getattr(loader, chkpt["loader"])(
config,
chkpt["shape_in"][1:3],
args.dataset,
args.cover,
mode="predict",
metatiles=args.metatiles,
keep_borders=args.keep_borders,
)
mp.spawn(gpu_worker,
nprocs=world_size,
args=(world_size, lock_file, args, config, dataset, palette,
transparency))
if os.path.exists(lock_file):
os.remove(lock_file)
if not args.no_web_ui and dataset.cover:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
web_ui(args.out, base_url, dataset.cover, dataset.cover, "png",
template)
def main(args):
assert args.out or args.delete, "out parameter is required"
args.out = os.path.expanduser(args.out)
print("abd subset {} with cover {}, on CPU".format(args.dir, args.cover), file=sys.stderr, flush=True)
ext = set()
tiles = set(tiles_from_csv(os.path.expanduser(args.cover)))
assert len(tiles), "Empty Cover: {}".format(args.cover)
for tile in tqdm(tiles, ascii=True, unit="tiles"):
if isinstance(tile, mercantile.Tile):
src_tile = tile_from_xyz(args.dir, tile.x, tile.y, tile.z)
if not src_tile:
if not args.quiet:
print("WARNING: skipping tile {}".format(tile), file=sys.stderr, flush=True)
continue
_, src = src_tile
dst_dir = os.path.join(args.out, str(tile.z), str(tile.x))
else:
src = tile
dst_dir = os.path.join(args.out, os.path.dirname(tile))
assert os.path.isfile(src)
dst = os.path.join(dst_dir, os.path.basename(src))
ext.add(os.path.splitext(src)[1][1:])
if not os.path.isdir(dst_dir):
os.makedirs(dst_dir, exist_ok=True)
if args.delete:
os.remove(src)
assert not os.path.lexists(src)
elif args.copy:
shutil.copyfile(src, dst)
assert os.path.exists(dst)
else:
if os.path.islink(dst):
os.remove(dst)
os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
assert os.path.islink(dst)
if tiles and not args.no_web_ui and not args.delete:
assert len(ext) == 1, "ERROR: Mixed extensions, can't generate Web UI"
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
web_ui(args.out, base_url, tiles, tiles, list(ext)[0], template)
def main(args):
if not args.masks or not args.labels:
assert args.mode != "list", "Parameters masks and labels are mandatories in list mode."
assert not (
args.min or args.max
), "Both --masks and --labels mandatory, for metric filtering."
if args.min or args.max:
config = load_config(args.config)
args.out = os.path.expanduser(args.out)
cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
args_minmax = set()
args.min = {(m[0], m[1]): m[2] for m in args.min} if args.min else dict()
args.max = {(m[0], m[1]): m[2] for m in args.max} if args.max else dict()
args_minmax.update(args.min.keys())
args_minmax.update(args.max.keys())
minmax = dict()
for mm in args_minmax:
mm_min = float(args.min[mm]) if mm in args.min else 0.0
mm_max = float(args.max[mm]) if mm in args.max else 1.0
assert mm_min < mm_max, "--min must be lower than --max, on {}".format(
mm)
minmax[mm] = {
"min":
mm_min,
"max":
mm_max,
"class_id": [
c for c, classe in enumerate(config["classes"])
if classe["title"] == mm[0]
][0],
"module":
load_module("abd_model.metrics." + mm[1]),
}
if not args.workers:
args.workers = os.cpu_count()
print("abd compare {} on CPU, with {} workers".format(
args.mode, args.workers),
file=sys.stderr,
flush=True)
if args.images:
tiles = [tile for tile in tiles_from_dir(args.images[0], cover=cover)]
assert len(tiles), "Empty images dir: {}".format(args.images[0])
for image in args.images[1:]:
assert sorted(tiles) == sorted([
tile for tile in tiles_from_dir(image, cover=cover)
]), "Unconsistent images dirs"
if args.labels and args.masks:
tiles_masks = [
tile for tile in tiles_from_dir(args.masks, cover=cover)
]
tiles_labels = [
tile for tile in tiles_from_dir(args.labels, cover=cover)
]
if args.images:
assert sorted(tiles) == sorted(tiles_masks) == sorted(
tiles_labels), "Unconsistent images/label/mask directories"
else:
assert len(tiles_masks), "Empty masks dir: {}".format(args.masks)
assert len(tiles_labels), "Empty labels dir: {}".format(
args.labels)
assert sorted(tiles_masks) == sorted(
tiles_labels), "Label and Mask directories are not consistent"
tiles = tiles_masks
tiles_list = []
tiles_compare = []
progress = tqdm(total=len(tiles), ascii=True, unit="tile")
log = False if args.mode == "list" else Logs(os.path.join(args.out, "log"))
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile):
x, y, z = list(map(str, tile))
if args.masks and args.labels:
label = np.array(
Image.open(
os.path.join(args.labels, z, x, "{}.png".format(y))))
mask = np.array(
Image.open(
os.path.join(args.masks, z, x, "{}.png".format(y))))
assert label.shape == mask.shape, "Inconsistent tiles (size or dimensions)"
metrics = dict()
for mm in minmax:
try:
metrics[mm] = getattr(minmax[mm]["module"], "get")(
torch.as_tensor(label, device="cpu"),
torch.as_tensor(mask, device="cpu"),
minmax[mm]["class_id"],
)
except:
progress.update()
return False, tile
if not (minmax[mm]["min"] <= metrics[mm] <=
minmax[mm]["max"]):
progress.update()
return True, tile
tiles_compare.append(tile)
if args.mode == "side":
for i, root in enumerate(args.images):
img = tile_image_from_file(tile_from_xyz(root, x, y, z)[1],
force_rgb=True)
if i == 0:
side = np.zeros(
(img.shape[0], img.shape[1] * len(args.images), 3))
side = np.swapaxes(side, 0,
1) if args.vertical else side
image_shape = img.shape
else:
assert image_shape[0:2] == img.shape[
0:2], "Unconsistent image size to compare"
if args.vertical:
side[i * image_shape[0]:(i + 1) *
image_shape[0], :, :] = img
else:
side[:, i * image_shape[0]:(i + 1) *
image_shape[0], :] = img
tile_image_to_file(args.out, tile, np.uint8(side))
elif args.mode == "stack":
for i, root in enumerate(args.images):
tile_image = tile_image_from_file(tile_from_xyz(
root, x, y, z)[1],
force_rgb=True)
if i == 0:
image_shape = tile_image.shape[0:2]
stack = tile_image / len(args.images)
else:
assert image_shape == tile_image.shape[
0:2], "Unconsistent image size to compare"
stack = stack + (tile_image / len(args.images))
tile_image_to_file(args.out, tile, np.uint8(stack))
elif args.mode == "list":
tiles_list.append([tile, metrics])
progress.update()
return True, tile
for ok, tile in executor.map(worker, tiles):
if not ok and log:
log.log("Warning: skipping. {}".format(str(tile)))
if args.mode == "list":
with open(args.out, mode="w") as out:
if args.geojson:
out.write('{"type":"FeatureCollection","features":[')
first = True
for tile_list in tiles_list:
tile, metrics = tile_list
x, y, z = list(map(str, tile))
if args.geojson:
prop = '"properties":{{"x":{},"y":{},"z":{}'.format(
x, y, z)
for metric in metrics:
prop += ',"{}":{:.3f}'.format(metric, metrics[metric])
geom = '"geometry":{}'.format(
json.dumps(feature(tile, precision=6)["geometry"]))
out.write('{}{{"type":"Feature",{},{}}}}}'.format(
"," if not first else "", geom, prop))
first = False
if not args.geojson:
out.write("{},{},{}".format(x, y, z))
for metric in metrics:
out.write("\t{:.3f}".format(metrics[metric]))
out.write(os.linesep)
if args.geojson:
out.write("]}")
out.close()
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
if args.mode == "side" and not args.no_web_ui:
template = "compare.html" if not args.web_ui_template else args.web_ui_template
web_ui(args.out,
base_url,
tiles,
tiles_compare,
args.format,
template,
union_tiles=False)
if args.mode == "stack" and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
tiles = [tile for tile in tiles_from_dir(args.images[0])]
web_ui(args.out, base_url, tiles, tiles_compare, args.format, template)
def main(args):
assert not (
args.label and args.format
), "Format option not supported for label, output must be kept as png"
try:
args.bands = list(map(int,
args.bands.split(","))) if args.bands else None
except:
raise ValueError("invalid --args.bands value")
if not args.workers:
args.workers = min(os.cpu_count(), len(args.rasters))
if args.label:
config = load_config(args.config)
check_classes(config)
colors = [classe["color"] for classe in config["classes"]]
palette = make_palette(colors)
assert len(args.ts.split(
",")) == 2, "--ts expect width,height value (e.g 512,512)"
width, height = list(map(int, args.ts.split(",")))
cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
splits_path = os.path.join(os.path.expanduser(args.out), ".splits")
args.out = os.path.expanduser(args.out)
if os.path.dirname(os.path.expanduser(args.out)):
os.makedirs(args.out, exist_ok=True)
log = Logs(os.path.join(args.out, "log"), out=sys.stderr)
raster = rasterio_open(os.path.expanduser(args.rasters[0]))
args.bands = args.bands if args.bands else raster.indexes
raster.close()
print(
"abd tile {} rasters on bands {}, on CPU with {} workers".format(
len(args.rasters), args.bands, args.workers),
file=sys.stderr,
flush=True,
)
skip = []
tiles_map = {}
total = 0
for path in args.rasters:
raster = rasterio_open(os.path.expanduser(path))
assert set(args.bands).issubset(set(
raster.indexes)), "Missing bands in raster {}".format(path)
try:
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326",
*raster.bounds)
except:
log.log(
"WARNING: missing or invalid raster projection, SKIPPING: {}".
format(path))
skip.append(path)
continue
tiles = [
mercantile.Tile(x=x, y=y, z=z)
for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)
]
tiles = list(set(tiles) & set(cover)) if cover else tiles
total += len(tiles)
for tile in tiles:
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if tile_key not in tiles_map.keys():
tiles_map[tile_key] = []
tiles_map[tile_key].append(path)
raster.close()
assert total, "Nothing left to tile"
if len(args.bands) == 1 or args.label:
ext = "png" if args.format is None else args.format
if len(args.bands) == 3:
ext = "webp" if args.format is None else args.format
if len(args.bands) > 3:
ext = "tiff" if args.format is None else args.format
tiles = []
progress = tqdm(desc="Coverage tiling",
total=total,
ascii=True,
unit="tile")
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(path):
if path in skip:
return None
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326",
*raster.bounds)
tiles = [
mercantile.Tile(x=x, y=y, z=z)
for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)
]
tiled = []
for tile in tiles:
if cover and tile not in cover:
continue
w, s, e, n = mercantile.xy_bounds(tile)
warp_vrt = WarpedVRT(
raster,
crs="epsg:3857",
resampling=Resampling.bilinear,
add_alpha=False,
transform=from_bounds(w, s, e, n, width, height),
width=width,
height=height,
)
data = warp_vrt.read(out_shape=(len(args.bands), width,
height),
indexes=args.bands,
window=warp_vrt.window(w, s, e, n))
if data.dtype == "uint16": # GeoTiff could be 16 bits
data = np.uint8(data / 256)
elif data.dtype == "uint32": # or 32 bits
data = np.uint8(data / (256 * 256))
image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if (not args.label
and len(tiles_map[tile_key]) == 1 and is_nodata(
image, args.nodata, args.nodata_threshold,
args.keep_borders)):
progress.update()
continue
if len(tiles_map[tile_key]) > 1:
out = os.path.join(splits_path,
str(tiles_map[tile_key].index(path)))
else:
out = args.out
x, y, z = map(int, tile)
if not args.label:
tile_image_to_file(out,
mercantile.Tile(x=x, y=y, z=z),
image,
ext=ext)
if args.label:
tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z),
palette, args.nodata, image)
if len(tiles_map[tile_key]) == 1:
tiled.append(mercantile.Tile(x=x, y=y, z=z))
progress.update()
raster.close()
return tiled
for tiled in executor.map(worker, args.rasters):
if tiled is not None:
tiles.extend(tiled)
total = sum(
[1 for tile_key in tiles_map.keys() if len(tiles_map[tile_key]) > 1])
progress = tqdm(desc="Aggregate splits",
total=total,
ascii=True,
unit="tile")
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile_key):
if len(tiles_map[tile_key]) == 1:
return
image = np.zeros((width, height, len(args.bands)), np.uint8)
x, y, z = map(int, tile_key)
for i in range(len(tiles_map[tile_key])):
root = os.path.join(splits_path, str(i))
_, path = tile_from_xyz(root, x, y, z)
if not args.label:
split = tile_image_from_file(path)
if args.label:
split = tile_label_from_file(path)
if len(split.shape) == 2:
split = split.reshape((width, height, 1)) # H,W -> H,W,C
assert image.shape == split.shape, "{}, {}".format(
image.shape, split.shape)
image[np.where(image == 0)] += split[np.where(image == 0)]
if not args.label and is_nodata(image, args.nodata,
args.nodata_threshold,
args.keep_borders):
progress.update()
return
tile = mercantile.Tile(x=x, y=y, z=z)
if not args.label:
tile_image_to_file(args.out, tile, image)
if args.label:
tile_label_to_file(args.out, tile, palette, image)
progress.update()
return tile
for tiled in executor.map(worker, tiles_map.keys()):
if tiled is not None:
tiles.append(tiled)
if splits_path and os.path.isdir(splits_path):
shutil.rmtree(splits_path) # Delete suffixes dir if any
if tiles and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args):
config = load_config(args.config)
args.cover = [
tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
if args.classes_weights:
try:
args.classes_weights = list(
map(float, args.classes_weights.split(",")))
except:
assert args.classes_weights == "auto", "invalid --classes_weights value"
args.classes_weights = compute_classes_weights(
args.dataset, config["classes"], args.cover, os.cpu_count())
else:
args.classes_weights = [
classe["weight"] for classe in config["classes"]
]
args.tiles_weights = ([(tile, weight) for tile, weight in tiles_from_csv(
os.path.expanduser(args.tiles_weights), extra_columns=True)]
if args.tiles_weights else None)
args.bs = args.bs if args.bs else config["train"]["bs"]
check_classes(config)
check_channels(config)
check_model(config)
assert torch.cuda.is_available(
), "No GPU support found. Check CUDA and NVidia Driver install."
assert torch.distributed.is_nccl_available(
), "No NCCL support found. Check your PyTorch install."
world_size = 1 # Hard Coded since eval MultiGPUs not yet implemented
args.workers = min(args.bs if not args.workers else args.workers,
math.floor(os.cpu_count() / world_size))
print("abd eval on 1 GPU, with {} workers, and {} tiles/batch".format(
args.workers, args.bs))
loader = load_module("abd_model.loaders.{}".format(
config["model"]["loader"].lower()))
assert os.path.isdir(os.path.expanduser(
args.dataset)), "--dataset path is not a directory"
dataset = getattr(loader,
config["model"]["loader"])(config, config["model"]["ts"],
args.dataset, args.cover,
args.tiles_weights, "eval")
assert len(dataset), "Empty or Invalid --dataset content"
shape_in = dataset.shape_in
shape_out = dataset.shape_out
print("DataSet Eval: {}".format(args.dataset))
print("\n--- Input tensor")
num_channel = 1 # 1-based numerotation
for channel in config["channels"]:
for band in channel["bands"]:
print("Channel {}:\t\t {} - (band:{})".format(
num_channel, channel["name"], band))
num_channel += 1
print("\n--- Output Classes ---")
for c, classe in enumerate(config["classes"]):
print("Class {}:\t\t {} ({:.2f})".format(c, classe["title"],
args.classes_weights[c]))
print("\n--- Model ---")
for hp in config["model"]:
print("{}{}".format(hp.ljust(25, " "), config["model"][hp]))
lock_file = os.path.abspath(os.path.join("/tmp", str(uuid.uuid1())))
mp.spawn(gpu_worker,
nprocs=world_size,
args=(world_size, lock_file, dataset, shape_in, shape_out, args,
config))
if os.path.exists(lock_file):
os.remove(lock_file)
def main(args):
assert args.cover or args.granules or args.scenes, "Either --cover OR --granules OR --scenes is mandatory"
assert not (args.download and not args.out), "--download implies out parameter"
assert args.limit, "What about increasing --limit value ?"
config = load_config(args.config)
if args.cover:
args.pg = args.pg if args.pg else config["auth"]["pg"]
assert args.pg, "PostgreSQL connection settting is mandatory with --cover"
args.granules = tiles_to_granules(tiles_from_csv(os.path.expanduser(args.cover)), args.pg)
if args.out:
args.out = os.path.expanduser(args.out)
os.makedirs(args.out, exist_ok=True)
log = Logs(os.path.join(args.out, "log"), out=sys.stderr)
else:
log = Logs(None, out=sys.stderr)
log.log("abd sat on granules: {}".format(" ".join(args.granules)))
scenes = search_scenes(args, log)
if args.download:
log.log("")
log.log("=============================================================================")
log.log("Downloading selected scenes")
log.log("=============================================================================")
report = []
login, password = dict([auth.split("=") for auth in config["auth"]["theia"].split(" ")]).values()
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(scene):
scene_dir = os.path.join(args.out, scene["dir"][:42]) # 42 related to Theia MD issue, dirty workaround
if not os.path.isabs(scene_dir):
scene_dir = "./" + scene_dir
if glob.glob(scene_dir + "*"):
scene["dir"] = glob.glob(scene_dir + "*")[0]
return scene, None, True # Already Downloaded
token = get_token(login, password)
url = THEIA_URL + "/resto2/collections/SENTINEL2/{}/download/?issuerId=theia".format(scene["uuid"])
resp = requests.get(url, headers={"Authorization": "Bearer {}".format(token)}, stream=True)
if resp is None:
return scene, None, False # Auth issue
zip_path = os.path.join(args.out, scene["uuid"] + ".zip")
with open(zip_path, "wb") as fp:
progress = tqdm(unit="B", desc=scene["uuid"], total=int(resp.headers["Content-Length"]))
for chunk in resp.iter_content(chunk_size=16384):
progress.update(16384)
fp.write(chunk)
return scene, zip_path, True
return scene, None, False # Write issue
for scene, zip_path, ok in executor.map(worker, scenes):
if zip_path and md5(zip_path) == scene["checksum"]:
scene["dir"] = os.path.dirname(ZipFile(zip_path).namelist()[0])
ZipFile(zip_path).extractall(args.out)
os.remove(zip_path)
report.append("Scene {} available in {}".format(scene["uuid"], scene["dir"]))
elif ok:
report.append("SKIPPING downloading {}, as already in {}".format(scene["uuid"], scene["dir"]))
else:
report.append("ERROR: Unable to retrieve Scene {}".format(scene["uuid"]))
log.log("")
log.log("=============================================================================")
for line in report:
log.log(line)
log.log("=============================================================================")
def main(args):
assert not (args.geojson is not None and args.pg
is not None), "You have to choose between --pg or --geojson"
assert len(args.ts.split(
",")) == 2, "--ts expect width,height value (e.g 512,512)"
config = load_config(args.config)
check_classes(config)
args.workers = min(os.cpu_count(),
args.workers) if args.workers else os.cpu_count()
args.pg = config["auth"][
"pg"] if not args.pg and "pg" in config["auth"].keys() else args.pg
assert not (args.sql and not args.pg
), "With --sql option, --pg dsn setting must also be provided"
palette, transparency = make_palette(
[classe["color"] for classe in config["classes"]], complementary=True)
index = [
config["classes"].index(classe) for classe in config["classes"]
if classe["title"] == args.type
]
assert index, "Requested type is not contains in your config file classes."
burn_value = index[0]
assert 0 < burn_value <= 255
if args.sql:
assert "limit" not in args.sql.lower(), "LIMIT is not supported"
assert "TILE_GEOM" in args.sql, "TILE_GEOM filter not found in your SQL"
sql = re.sub(r"ST_Intersects( )*\((.*)?TILE_GEOM(.*)?\)", "1=1",
args.sql, re.I)
assert sql and sql != args.sql, "Incorrect TILE_GEOM filter in your SQL"
if os.path.dirname(os.path.expanduser(args.out)):
os.makedirs(os.path.expanduser(args.out), exist_ok=True)
args.out = os.path.expanduser(args.out)
log = Logs(os.path.join(args.out, "log"), out=sys.stderr)
tiles = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))]
assert len(tiles), "Empty Cover: {}".format(args.cover)
if args.geojson:
zoom = tiles[0].z
assert not [tile for tile in tiles if tile.z != zoom
], "Unsupported zoom mixed cover. Use PostGIS instead"
workers = min(args.workers, len(args.geojson))
log.log("abd rasterize - Compute spatial index with {} workers".format(
workers))
progress = None
log_from = args.geojson
if len(args.geojson) > 42: # Arbitrary ∩ Funny
progress = tqdm(total=len(args.geojson), ascii=True, unit="file")
log_from = "{} geojson files".format(len(args.geojson))
feature_map = collections.defaultdict(list)
with futures.ProcessPoolExecutor(workers) as executor:
for fm in executor.map(
partial(worker_spatial_index, zoom, args.buffer,
True if progress is None else False),
args.geojson):
for k, v in fm.items():
try:
feature_map[k] += v
except KeyError:
feature_map[k] = v
if progress:
progress.update()
if progress:
progress.close()
if args.sql:
conn = psycopg2.connect(args.pg)
db = conn.cursor()
db.execute(
"""SELECT ST_Srid("1") AS srid FROM ({} LIMIT 1) AS t("1")""".
format(sql))
srid = db.fetchone()[0]
assert srid and int(srid) > 0, "Unable to retrieve geometry SRID."
log_from = args.sql
if not len(feature_map):
log.log("-----------------------------------------------")
log.log("NOTICE: no feature to rasterize, seems peculiar")
log.log("-----------------------------------------------")
log.log("abd rasterize - rasterizing {} from {} on cover {}".format(
args.type, log_from, args.cover))
with open(os.path.join(os.path.expanduser(args.out),
args.type.lower() + "_cover.csv"),
mode="w") as cover:
for tile in tqdm(tiles, ascii=True, unit="tile"):
geojson = None
if args.sql:
w, s, e, n = tile_bbox(tile)
tile_geom = "ST_Transform(ST_MakeEnvelope({},{},{},{}, 4326), {})".format(
w, s, e, n, srid)
query = """
WITH
sql AS ({}),
geom AS (SELECT "1" AS geom FROM sql AS t("1")),
json AS (SELECT '{{"type": "Feature", "geometry": '
|| ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6)
|| '}}' AS features
FROM geom)
SELECT '{{"type": "FeatureCollection", "features": [' || Array_To_String(array_agg(features), ',') || ']}}'
FROM json
""".format(args.sql.replace("TILE_GEOM", tile_geom))
db.execute(query)
row = db.fetchone()
try:
geojson = json.loads(
row[0])["features"] if row and row[0] else None
except Exception:
log.log("Warning: Invalid geometries, skipping {}".format(
tile))
conn = psycopg2.connect(args.pg)
db = conn.cursor()
if args.geojson:
geojson = feature_map[tile] if tile in feature_map else None
if geojson:
num = len(geojson)
out = geojson_tile_burn(tile, geojson, 4326,
list(map(int, args.ts.split(","))),
burn_value)
if not geojson or out is None:
num = 0
out = np.zeros(shape=list(map(int, args.ts.split(","))),
dtype=np.uint8)
tile_label_to_file(args.out,
tile,
palette,
transparency,
out,
append=args.append)
cover.write("{},{},{} {}{}".format(tile.x, tile.y, tile.z, num,
os.linesep))
if not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
tiles = [tile for tile in tiles_from_csv(args.cover)]
web_ui(args.out, base_url, tiles, tiles, "png", template)
def main(args):
assert not (args.type == "extent" and args.splits
), "--splits and --type extent are mutually exclusive options"
assert not (args.type == "extent" and args.out and len(args.out) > 1
), "--type extent option imply a single --out path"
assert not (args.type != "extent" and
not args.out), "--out mandatory [except with --type extent]"
assert not (args.union
and args.type != "geojson"), "--union imply --type geojson"
assert not (args.sql and not args.pg), "--sql option imply --pg"
assert (
int(args.bbox is not None) + int(args.geojson is not None) +
int(args.sql is not None) + int(args.dir is not None) +
int(args.raster is not None) + int(args.cover is not None) == 1
), "One, and only one, input type must be provided, among: --dir, --bbox, --cover, --raster, --geojson or --sql"
if args.splits:
splits = [int(split) for split in args.splits.split("/")]
assert len(splits) == len(args.out) and 0 < sum(
splits) <= 100, "Invalid split value or incoherent with out paths."
assert args.zoom or (args.dir or args.cover), "Zoom parameter is required."
args.out = [os.path.expanduser(out)
for out in args.out] if args.out else None
cover = []
if args.raster:
print("abd cover from {} at zoom {}".format(args.raster, args.zoom),
file=sys.stderr,
flush=True)
cover = set()
for raster_file in args.raster:
with rasterio_open(os.path.expanduser(raster_file)) as r:
try:
w, s, e, n = transform_bounds(r.crs, "EPSG:4326",
*r.bounds)
except:
print("WARNING: projection error, SKIPPING: {}".format(
raster_file),
file=sys.stderr,
flush=True)
continue
cover.update([tile for tile in tiles(w, s, e, n, args.zoom)])
cover = list(cover)
if args.geojson:
print("abd cover from {} at zoom {}".format(args.geojson, args.zoom),
file=sys.stderr,
flush=True)
feature_map = collections.defaultdict(list)
for geojson_file in args.geojson:
with open(os.path.expanduser(geojson_file)) as f:
feature_collection = json.load(f)
srid = geojson_srid(feature_collection)
for feature in tqdm(feature_collection["features"],
ascii=True,
unit="feature"):
feature_map = geojson_parse_feature(
args.zoom, srid, feature_map, feature)
cover = feature_map.keys()
if args.sql:
print("abd cover from {} {} at zoom {}".format(args.sql, args.pg,
args.zoom),
file=sys.stderr,
flush=True)
conn = psycopg2.connect(args.pg)
assert conn, "Unable to connect to PostgreSQL database."
db = conn.cursor()
query = """
WITH
sql AS ({}),
geom AS (SELECT "1" AS geom FROM sql AS t("1"))
SELECT '{{"type": "Feature", "geometry": '
|| ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6)
|| '}}' AS features
FROM geom
""".format(args.sql)
db.execute(query)
assert db.rowcount is not None and db.rowcount != -1, "SQL Query return no result."
feature_map = collections.defaultdict(list)
for feature in tqdm(
db.fetchall(), ascii=True, unit="feature"
): # FIXME: fetchall will not always fit in memory...
feature_map = geojson_parse_feature(args.zoom, 4326, feature_map,
json.loads(feature[0]))
cover = feature_map.keys()
if args.bbox:
try:
w, s, e, n, crs = args.bbox.split(",")
w, s, e, n = map(float, (w, s, e, n))
except:
crs = None
w, s, e, n = map(float, args.bbox.split(","))
assert isinstance(w, float) and isinstance(
s, float) and w < e and s < n, "Invalid bbox parameter."
print("abd cover from {} at zoom {}".format(args.bbox, args.zoom),
file=sys.stderr,
flush=True)
if crs:
w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n)
assert isinstance(w, float) and isinstance(
s, float), "Unable to deal with raster projection"
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.cover:
print("abd cover from {}".format(args.cover),
file=sys.stderr,
flush=True)
cover = [
tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
]
if args.dir:
print("abd cover from {}".format(args.dir),
file=sys.stderr,
flush=True)
cover = [
tile for tile in tiles_from_dir(os.path.expanduser(args.dir),
xyz=not (args.no_xyz))
]
assert len(cover), "Empty tiles inputs"
_cover = []
extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0)
for tile in tqdm(cover, ascii=True, unit="tile"):
if args.zoom and tile.z != args.zoom:
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
*xy_bounds(tile))
for t in tiles(w, s, n, e, args.zoom):
unique = True
for _t in _cover:
if _t == t:
unique = False
if unique:
_cover.append(t)
else:
if args.type == "extent":
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
*xy_bounds(tile))
_cover.append(tile)
if args.type == "extent":
extent_w, extent_s, extent_n, extent_e = (min(extent_w,
w), min(extent_s, s),
max(extent_n,
n), max(extent_e, e))
cover = _cover
if args.splits:
shuffle(cover) # in-place
cover_splits = [
math.floor(len(cover) * split / 100)
for i, split in enumerate(splits, 1)
]
if len(splits) > 1 and sum(map(
int, splits)) == 100 and len(cover) > sum(map(int, splits)):
cover_splits[0] = len(cover) - sum(map(
int, cover_splits[1:])) # no tile waste
s = 0
covers = []
for e in cover_splits:
covers.append(cover[s:s + e])
s += e
else:
covers = [cover]
if args.type == "extent":
extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s,
extent_n, extent_e)
if args.out:
if os.path.dirname(args.out[0]) and not os.path.isdir(
os.path.dirname(args.out[0])):
os.makedirs(os.path.dirname(args.out[0]), exist_ok=True)
with open(args.out[0], "w") as fp:
fp.write(extent)
else:
print(extent)
else:
for i, cover in enumerate(covers):
if os.path.dirname(args.out[i]) and not os.path.isdir(
os.path.dirname(args.out[i])):
os.makedirs(os.path.dirname(args.out[i]), exist_ok=True)
with open(args.out[i], "w") as fp:
if args.type == "geojson":
fp.write(tiles_to_geojson(cover, union=args.union))
else:
csv.writer(fp).writerows(cover)
def main(args):
tiles = list(tiles_from_csv(args.cover))
assert len(tiles), "Empty cover: {}".format(args.cover)
args.workers = min(os.cpu_count(),
args.rate) if not args.workers else args.workers
if os.path.dirname(os.path.expanduser(args.out)):
os.makedirs(os.path.expanduser(args.out), exist_ok=True)
log = Logs(os.path.join(args.out, "log"), out=sys.stderr)
log.log("abd download with {} workers, at max {} req/s, from: {}".format(
args.workers, args.rate, args.url))
already_dl = 0
dl = 0
with requests.Session() as session:
progress = tqdm(total=len(tiles), ascii=True, unit="image")
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile):
tick = time.monotonic()
progress.update()
try:
x, y, z = map(str, [tile.x, tile.y, tile.z])
os.makedirs(os.path.join(args.out, z, x), exist_ok=True)
except:
return tile, None, False
path = os.path.join(args.out, z, x,
"{}.{}".format(y, args.format))
if os.path.isfile(path): # already downloaded
return tile, None, True
if args.type == "XYZ":
url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
elif args.type == "WMS":
xmin, ymin, xmax, ymax = xy_bounds(tile)
url = args.url.format(xmin=xmin,
ymin=ymin,
xmax=xmax,
ymax=ymax)
res = tile_image_from_url(session, url, args.timeout)
if res is None: # let's retry once
res = tile_image_from_url(session, url, args.timeout)
if res is None:
return tile, url, False
try:
tile_image_to_file(args.out, tile, res)
except OSError:
return tile, url, False
tock = time.monotonic()
time_for_req = tock - tick
time_per_worker = args.workers / args.rate
if time_for_req < time_per_worker:
time.sleep(time_per_worker - time_for_req)
return tile, url, True
for tile, url, ok in executor.map(worker, tiles):
if url and ok:
dl += 1
elif not url and ok:
already_dl += 1
else:
log.log("Warning:\n {} failed, skipping.\n {}\n".format(
tile, url))
if already_dl:
log.log(
"Notice: {} tiles were already downloaded previously, and so skipped now."
.format(already_dl))
if already_dl + dl == len(tiles):
log.log("Notice: Coverage is fully downloaded.")
if not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
web_ui(args.out, base_url, tiles, tiles, args.format, template)