def worker(tile): tick = time.monotonic() progress.update() try: x, y, z = map(str, [tile.x, tile.y, tile.z]) os.makedirs(os.path.join(args.out, z, x), exist_ok=True) except: return tile, None, False path = os.path.join(args.out, z, x, "{}.{}".format(y, args.format)) if os.path.isfile(path): # already downloaded return tile, None, True if args.type == "XYZ": url = args.url.format(x=tile.x, y=tile.y, z=tile.z) elif args.type == "WMS": xmin, ymin, xmax, ymax = xy_bounds(tile) url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax) res = tile_image_from_url(session, url, args.timeout) if res is None: # let's retry once res = tile_image_from_url(session, url, args.timeout) if res is None: return tile, url, False try: tile_image_to_file(args.out, tile, res) except OSError: return tile, url, False tock = time.monotonic() time_for_req = tock - tick time_per_worker = args.workers / args.rate if time_for_req < time_per_worker: time.sleep(time_per_worker - time_for_req) return tile, url, True
def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((width, height, len(args.bands)), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) if len(split.shape) == 2: split = split.reshape((width, height, 1)) # H,W -> H,W,C assert image.shape == split.shape, "{}, {}".format( image.shape, split.shape) image[np.where(image == 0)] += split[np.where(image == 0)] if not args.label and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: tile_image_to_file(args.out, tile, image) if args.label: tile_label_to_file(args.out, tile, palette, image) progress.update() return tile
def worker(path): if path in skip: return None raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, width, height), width=width, height=height, ) data = warp_vrt.read( out_shape=(len(args.bands), width, height), indexes=args.bands, window=warp_vrt.window(w, s, e, n) ) if data.dtype == "uint16": # GeoTiff could be 16 bits data = np.uint8(data / 256) elif data.dtype == "uint32": # or 32 bits data = np.uint8(data / (256 * 256)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if ( not args.label and len(tiles_map[tile_key]) == 1 and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders) ): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image, ext=ext) if args.label: tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, args.nodata, image) if len(tiles_map[tile_key]) == 1: tiled.append(mercantile.Tile(x=x, y=y, z=z)) progress.update() raster.close() return tiled
def worker(tile): x, y, z = list(map(str, tile)) if args.masks and args.labels: label = np.array(Image.open(os.path.join(args.labels, z, x, "{}.png".format(y)))) mask = np.array(Image.open(os.path.join(args.masks, z, x, "{}.png".format(y)))) assert label.shape == mask.shape, "Inconsistent tiles (size or dimensions)" metrics = dict() for mm in minmax: try: metrics[mm] = getattr(minmax[mm]["module"], "get")( torch.as_tensor(label, device="cpu"), torch.as_tensor(mask, device="cpu"), minmax[mm]["class_id"], ) except: progress.update() return False, tile if not (minmax[mm]["min"] <= metrics[mm] <= minmax[mm]["max"]): progress.update() return True, tile tiles_compare.append(tile) if args.mode == "side": for i, root in enumerate(args.images): img = tile_image_from_file(tile_from_xyz(root, x, y, z)[1], force_rgb=True) if i == 0: side = np.zeros((img.shape[0], img.shape[1] * len(args.images), 3)) side = np.swapaxes(side, 0, 1) if args.vertical else side image_shape = img.shape else: assert image_shape[0:2] == img.shape[0:2], "Unconsistent image size to compare" if args.vertical: side[i * image_shape[0] : (i + 1) * image_shape[0], :, :] = img else: side[:, i * image_shape[0] : (i + 1) * image_shape[0], :] = img tile_image_to_file(args.out, tile, np.uint8(side)) elif args.mode == "stack": for i, root in enumerate(args.images): tile_image = tile_image_from_file(tile_from_xyz(root, x, y, z)[1], force_rgb=True) if i == 0: image_shape = tile_image.shape[0:2] stack = tile_image / len(args.images) else: assert image_shape == tile_image.shape[0:2], "Unconsistent image size to compare" stack = stack + (tile_image / len(args.images)) tile_image_to_file(args.out, tile, np.uint8(stack)) elif args.mode == "list": tiles_list.append([tile, metrics]) progress.update() return True, tile