def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((width, height, bands), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) split = split.reshape((width, height, 1)) # H,W -> H,W,C assert image.shape == split.shape image[np.where(image == 0)] += split[np.where(image == 0)] if not args.label and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: tile_image_to_file(args.out, tile, image) if args.label: tile_label_to_file(args.out, tile, palette, image) progress.update() return tile
def worker(tile): x, y, z = list(map(str, tile)) if args.masks and args.labels: label = np.array(Image.open(os.path.join(args.labels, z, x, "{}.png".format(y)))) mask = np.array(Image.open(os.path.join(args.masks, z, x, "{}.png".format(y)))) assert label.shape == mask.shape, "Inconsistent tiles (size or dimensions)" try: dist, fg_ratio, qod = compare(torch.as_tensor(label, device="cpu"), torch.as_tensor(mask, device="cpu")) except: progress.update() return False, tile if not args.minimum_fg <= fg_ratio <= args.maximum_fg or not args.minimum_qod <= qod <= args.maximum_qod: progress.update() return True, tile tiles_compare.append(tile) if args.mode == "side": for i, root in enumerate(args.images): img = tile_image_from_file(tile_from_xyz(root, x, y, z)[1]) if i == 0: side = np.zeros((img.shape[0], img.shape[1] * len(args.images), 3)) side = np.swapaxes(side, 0, 1) if args.vertical else side image_shape = img.shape else: assert image_shape[0:2] == img.shape[0:2], "Unconsistent image size to compare" if args.vertical: side[i * image_shape[0] : (i + 1) * image_shape[0], :, :] = img else: side[:, i * image_shape[0] : (i + 1) * image_shape[0], :] = img tile_image_to_file(args.out, tile, np.uint8(side)) elif args.mode == "stack": for i, root in enumerate(args.images): tile_image = tile_image_from_file(tile_from_xyz(root, x, y, z)[1]) if i == 0: image_shape = tile_image.shape[0:2] stack = tile_image / len(args.images) else: assert image_shape == tile_image.shape[0:2], "Unconsistent image size to compare" stack = stack + (tile_image / len(args.images)) tile_image_to_file(args.out, tile, np.uint8(stack)) elif args.mode == "list": tiles_list.append([tile, fg_ratio, qod]) progress.update() return True, tile
def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, width, height), width=width, height=height, ) data = warp_vrt.read(out_shape=(len(raster.indexes), width, height), window=warp_vrt.window(w, s, e, n)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if ( not args.label and len(tiles_map[tile_key]) == 1 and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders) ): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled
def worker(tile): tick = time.monotonic() progress.update() x, y, z = map(str, [tile.x, tile.y, tile.z]) try: os.makedirs(os.path.join(args.out, z, x), exist_ok=True) except: return tile, None, False path = os.path.join(args.out, z, x, "{}.{}".format(y, args.format)) if os.path.isfile(path): # already downloaded return tile, None, True if args.type == "XYZ": url = args.url.format(x=tile.x, y=tile.y, z=tile.z) elif args.type == "TMS": tile.y = (2**tile.z) - tile.y - 1 url = args.url.format(x=tile.x, y=tile.y, z=tile.z) elif args.type == "WMS": xmin, ymin, xmax, ymax = xy_bounds(tile) url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax) res = tile_image_from_url(session, url, args.timeout) if res is None: # let's retry once res = tile_image_from_url(session, url, args.timeout) if res is None: return tile, url, False try: tile_image_to_file(args.out, tile, res) # cv2.imwrite(path, cv2.imdecode(np.fromstring(res.read(), np.uint8), cv2.IMREAD_COLOR)) except OSError: return tile, url, False tock = time.monotonic() time_for_req = tock - tick time_per_worker = args.workers / args.rate if time_for_req < time_per_worker: time.sleep(time_per_worker - time_for_req) return tile, url, True
def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, w, s, e, n) tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)] tiled = [] for tile in tiles: try: w, s, e, n = mercantile.xy_bounds(tile) # inspired by rio-tiler, cf: https://github.com/mapbox/rio-tiler/pull/45 warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, args.ts, args.ts), width=math.ceil((e - w) / transform.a), height=math.ceil((s - n) / transform.e), ) data = warp_vrt.read( out_shape=(len(raster.indexes), args.ts, args.ts), window=warp_vrt.window(w, s, e, n) ) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C except: sys.exit("Error: Unable to tile {} from raster {}.".format(str(tile), raster)) tile_key = (str(tile.x), str(tile.y), str(tile.z)) if not args.label and len(tiles_map[tile_key]) == 1 and is_border(image): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: ret = tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: ret = tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) if not ret: sys.exit("Error: Unable to write tile {} from raster {}.".format(str(tile), raster)) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled
def worker(tile_key, nodata): if len(tiles_map[tile_key]) == 1: return image = np.zeros((args.ts, args.ts, bands), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) split = split.reshape( (args.ts, args.ts, 1)) # H,W -> H,W,C assert image.shape == split.shape image[:, :, :] += split[:, :, :] if not args.label and skip_nodata(image, nodata["border"], nodata["value"], nodata["threshold"]): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: ret = tile_image_to_file(args.out, tile, image) if args.label: ret = tile_label_to_file(args.out, tile, palette, image) if not ret: sys.exit("Error: Unable to write tile {}.".format( str(tile_key))) progress.update() return tile
def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((args.ts, args.ts, bands), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) split = split.reshape( (args.ts, args.ts, 1)) # H,W -> H,W,C assert image.shape == split.shape image[np.where(image == 0)] += split[np.where(image == 0)] if not args.label and is_nodata(image, threshold=args.nodata_threshold): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: ret = tile_image_to_file(args.out, tile, image) if args.label: ret = tile_label_to_file(args.out, tile, palette, image) assert ret, "Unable to write tile {} from raster {}.".format( str(tile_key)) progress.update() return tile
def worker(tile): x, y, z = list(map(str, tile)) if args.masks and args.labels: try: dist, fg_ratio, qod = compare(args.masks, args.labels, tile) except: progress.update() return False, tile if not args.minimum_fg <= fg_ratio <= args.maximum_fg or not args.minimum_qod <= qod <= args.maximum_qod: progress.update() return True, tile tiles_compare.append(tile) if args.mode == "side": for i, root in enumerate(args.images): img = tile_image_from_file( tile_from_slippy_map(root, x, y, z)[1]) if i == 0: side = np.zeros( (img.shape[0], img.shape[1] * len(args.images), 3)) side = np.swapaxes(side, 0, 1) if args.vertical else side image_shape = img.shape else: assert image_shape[0:2] == img.shape[ 0:2], "Unconsistent image size to compare" if args.vertical: side[i * image_shape[0]:(i + 1) * image_shape[0], :, :] = img else: side[:, i * image_shape[0]:(i + 1) * image_shape[0], :] = img tile_image_to_file(args.out, tile, np.uint8(side)) elif args.mode == "stack": for i, root in enumerate(args.images): tile_image = tile_image_from_file( tile_from_slippy_map(root, x, y, z)[1]) if i == 0: image_shape = tile_image.shape[0:2] stack = tile_image / len(args.images) else: assert image_shape == tile_image.shape[ 0:2], "Unconsistent image size to compare" stack = stack + (tile_image / len(args.images)) tile_image_to_file(args.out, tile, np.uint8(stack)) elif args.mode == "list": tiles_list.append([tile, fg_ratio, qod]) progress.update() return True, tile