def rescale_tile(tile, mask, rescale=None): if rescale: try: rescale_arr = list(map(float, rescale.split(","))) except ValueError: raise exceptions.ValidationError("Invalid rescale value") rescale_arr = list(_chunks(rescale_arr, 2)) if len(rescale_arr) != tile.shape[0]: rescale_arr = ((rescale_arr[0]), ) * tile.shape[0] for bdx in range(tile.shape[0]): if mask is not None: tile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]), 0, ) else: tile[bdx] = linear_rescale(tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]) tile = tile.astype(np.uint8) return tile, mask
def _postprocess( tile: numpy.ndarray, mask: numpy.ndarray, rescale: str = None, color_formula: str = None, ) -> Tuple[numpy.ndarray, numpy.ndarray]: """Post-process tile data.""" if rescale: rescale_arr = list(map(float, rescale.split(","))) rescale_arr = list(_chunks(rescale_arr, 2)) if len(rescale_arr) != tile.shape[0]: rescale_arr = ((rescale_arr[0]),) * tile.shape[0] for bdx in range(tile.shape[0]): tile[bdx] = numpy.where( mask, linear_rescale( tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255] ), 0, ) tile = tile.astype(numpy.uint8) if color_formula: # make sure one last time we don't have # negative value before applying color formula tile[tile < 0] = 0 for ops in parse_operations(color_formula): tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8) return tile, mask
def cli( sources, cogeo_profile, creation_options, options, prefix, topic, resolution, layers, ): """Create pdal-watchbot.""" def _create_message(source): message = { "src_path": source, "dst_prefix": prefix, "resolution": resolution, "output": layers, "profile_name": cogeo_profile, "profile_options": creation_options, "options": options, } return message messages = [_create_message(source) for source in sources] parts = _chunks(messages, 50) _send_message = partial(sns_worker, topic=topic) with futures.ThreadPoolExecutor(max_workers=50) as executor: executor.map(_send_message, parts)
def cli( sources, cogeo_profile, creation_options, options, config, allow_remote_read, copy_valid_cog, prefix, topic ): """ Create cogeo-watchbot-light jobs. Example: aws s3 ls s3://spacenet-dataset/spacenet/SN5_roads/test_public/AOI_7_Moscow/PS-RGB/ --recursive | awk '{print " https://spacenet-dataset.s3.amazonaws.com/"$NF}' > list.txt cat list.txt | python -m create_jobs - \ -p webp \ --co blockxsize=256 \ --co blockysize=256 \ --op overview_level=6 \ --op overview_resampling=bilinear \ --prefix cogs/spacenet \ --topic arn:aws:sns:us-east-1:{account}:cogeo-watchbot-light-production-WatchbotTopic """ if "indexes" in options.keys(): options["indexes"] = list(map(int, options["indexes"].split(","))) def _create_message(source): message = { "src_path": source, "dst_prefix": prefix, "profile_name": cogeo_profile, "profile_options": creation_options, "config": config, "options": options, } if allow_remote_read: message.update(dict(allow_remote_read=True)) if copy_valid_cog: message.update(dict(copy_valid_cog=True)) return message messages = [_create_message(source) for source in sources] parts = _chunks(messages, 50) _send_message = partial(sns_worker, topic=topic) with futures.ThreadPoolExecutor(max_workers=50) as executor: executor.map(_send_message, parts)
def test_chunck(): """Should split a list in multiple chunks.""" chuncks = list(utils._chunks(list(range(10)), 3)) assert len(chuncks) == 4
def mosaic_tiler( assets: Sequence[str], tile_x: int, tile_y: int, tile_z: int, tiler: Callable, pixel_selection: Optional[MosaicMethodBase] = None, chunk_size: Optional[int] = None, threads: int = MAX_THREADS, **kwargs, ) -> Tuple[numpy.ndarray, numpy.ndarray]: """ Create mercator tile from multiple observations. Attributes ---------- assets: list or tuple List of tiler compatible asset. tile_x: int Mercator tile X index. tile_y: int Mercator tile Y index. tile_z: int Mercator tile ZOOM level. tiler: callable tiler function. The function MUST take asset, x, y, z, **kwargs as arguments, and MUST return a tuple with tile data and mask e.g: def tiler(asset: str, x: int, y: int, z: int, **kwargs) -> Tuple[numpy.ndarray, numpy.ndarray]: with COGReader(asset) as cog: return cog.tile(x, y, z, **kwargs) pixel_selection: MosaicMethod, optional Instance of MosaicMethodBase class. default: "rio_tiler_mosaic.methods.defaults.FirstMethod". chunk_size: int, optional Control the number of asset to process per loop (default = threads). threads: int, optional Number of threads to use. If <=1, runs single threaded without an event loop. By default reads from the MAX_THREADS environment variable, and if not found defaults to multiprocessing.cpu_count() * 5. kwargs: dict, optional tiler specific options. Returns ------- tile, mask : tuple of ndarray Return tile and mask data. """ if pixel_selection is None: pixel_selection = FirstMethod() if not isinstance(pixel_selection, MosaicMethodBase): raise Exception("Mosaic filling algorithm should be an instance of" "'rio_tiler_mosaic.methods.base.MosaicMethodBase'") if not chunk_size: chunk_size = threads or len(assets) tasks: TaskType for chunks in _chunks(assets, chunk_size): if threads: with futures.ThreadPoolExecutor(max_workers=threads) as executor: tasks = [ executor.submit(tiler, asset, tile_x, tile_y, tile_z, **kwargs) for asset in chunks ] else: tasks = (tiler(asset, tile_x, tile_y, tile_z, **kwargs) for asset in chunks) for t, m in _filter_tasks(tasks): t = numpy.ma.array(t) t.mask = m == 0 pixel_selection.feed(t) if pixel_selection.is_done: return pixel_selection.data return pixel_selection.data
def mosaic_tiler(assets, tile_x, tile_y, tile_z, tiler, pixel_selection=None, chunk_size=None, **kwargs): """ Create mercator tile from multiple observations. Attributes ---------- assets : list, tuple List of rio-tiler compatible sceneid or url tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. tiler: function Rio-tiler's tiler function (e.g rio_tiler.landsat8.tile) pixel_selection: MosaicMethod, optional Instance of MosaicMethodBase class. default: "rio_tiler_mosaic.methods.defaults.FirstMethod". chunk_size: int, optional Control the number of asset to process per loop (default = MAX_THREADS). kwargs: dict, optional Rio-tiler tiler module specific options. Returns ------- tile, mask : tuple of ndarray Return tile and mask data. """ if pixel_selection is None: pixel_selection = FirstMethod() if not isinstance(pixel_selection, MosaicMethodBase): raise Exception("Mosaic filling algorithm should be an instance of" "'rio_tiler_mosaic.methods.base.MosaicMethodBase'") _tiler = partial(tiler, tile_x=tile_x, tile_y=tile_y, tile_z=tile_z, **kwargs) max_threads = int( os.environ.get("MAX_THREADS", multiprocessing.cpu_count() * 5)) if not chunk_size: chunk_size = max_threads for chunks in _chunks(assets, chunk_size): with futures.ThreadPoolExecutor(max_workers=max_threads) as executor: future_tasks = [executor.submit(_tiler, asset) for asset in chunks] for t, m in _filter_futures(future_tasks): t = numpy.ma.array(t) t.mask = m == 0 pixel_selection.feed(t) if pixel_selection.is_done: return pixel_selection.data return pixel_selection.data