def rescale_tile(tile, mask, rescale=None):
if rescale:
try:
rescale_arr = list(map(float, rescale.split(",")))
except ValueError:
raise exceptions.ValidationError("Invalid rescale value")
rescale_arr = list(_chunks(rescale_arr, 2))
if len(rescale_arr) != tile.shape[0]:
rescale_arr = ((rescale_arr[0]), ) * tile.shape[0]
for bdx in range(tile.shape[0]):
if mask is not None:
tile[bdx] = np.where(
mask,
linear_rescale(tile[bdx],
in_range=rescale_arr[bdx],
out_range=[0, 255]),
0,
)
else:
tile[bdx] = linear_rescale(tile[bdx],
in_range=rescale_arr[bdx],
out_range=[0, 255])
tile = tile.astype(np.uint8)
return tile, mask
def _postprocess(
tile: numpy.ndarray,
mask: numpy.ndarray,
rescale: str = None,
color_formula: str = None,
) -> Tuple[numpy.ndarray, numpy.ndarray]:
"""Post-process tile data."""
if rescale:
rescale_arr = list(map(float, rescale.split(",")))
rescale_arr = list(_chunks(rescale_arr, 2))
if len(rescale_arr) != tile.shape[0]:
rescale_arr = ((rescale_arr[0]),) * tile.shape[0]
for bdx in range(tile.shape[0]):
tile[bdx] = numpy.where(
mask,
linear_rescale(
tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]
),
0,
)
tile = tile.astype(numpy.uint8)
if color_formula:
# make sure one last time we don't have
# negative value before applying color formula
tile[tile < 0] = 0
for ops in parse_operations(color_formula):
tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8)
return tile, mask
def cli(
sources,
cogeo_profile,
creation_options,
options,
prefix,
topic,
resolution,
layers,
):
"""Create pdal-watchbot."""
def _create_message(source):
message = {
"src_path": source,
"dst_prefix": prefix,
"resolution": resolution,
"output": layers,
"profile_name": cogeo_profile,
"profile_options": creation_options,
"options": options,
}
return message
messages = [_create_message(source) for source in sources]
parts = _chunks(messages, 50)
_send_message = partial(sns_worker, topic=topic)
with futures.ThreadPoolExecutor(max_workers=50) as executor:
executor.map(_send_message, parts)
def cli(
sources,
cogeo_profile,
creation_options,
options,
config,
allow_remote_read,
copy_valid_cog,
prefix,
topic
):
"""
Create cogeo-watchbot-light jobs.
Example:
aws s3 ls s3://spacenet-dataset/spacenet/SN5_roads/test_public/AOI_7_Moscow/PS-RGB/ --recursive | awk '{print " https://spacenet-dataset.s3.amazonaws.com/"$NF}' > list.txt
cat list.txt | python -m create_jobs - \
-p webp \
--co blockxsize=256 \
--co blockysize=256 \
--op overview_level=6 \
--op overview_resampling=bilinear \
--prefix cogs/spacenet \
--topic arn:aws:sns:us-east-1:{account}:cogeo-watchbot-light-production-WatchbotTopic
"""
if "indexes" in options.keys():
options["indexes"] = list(map(int, options["indexes"].split(",")))
def _create_message(source):
message = {
"src_path": source,
"dst_prefix": prefix,
"profile_name": cogeo_profile,
"profile_options": creation_options,
"config": config,
"options": options,
}
if allow_remote_read:
message.update(dict(allow_remote_read=True))
if copy_valid_cog:
message.update(dict(copy_valid_cog=True))
return message
messages = [_create_message(source) for source in sources]
parts = _chunks(messages, 50)
_send_message = partial(sns_worker, topic=topic)
with futures.ThreadPoolExecutor(max_workers=50) as executor:
executor.map(_send_message, parts)
def test_chunck():
"""Should split a list in multiple chunks."""
chuncks = list(utils._chunks(list(range(10)), 3))
assert len(chuncks) == 4
def mosaic_tiler(
assets: Sequence[str],
tile_x: int,
tile_y: int,
tile_z: int,
tiler: Callable,
pixel_selection: Optional[MosaicMethodBase] = None,
chunk_size: Optional[int] = None,
threads: int = MAX_THREADS,
**kwargs,
) -> Tuple[numpy.ndarray, numpy.ndarray]:
"""
Create mercator tile from multiple observations.
Attributes
----------
assets: list or tuple
List of tiler compatible asset.
tile_x: int
Mercator tile X index.
tile_y: int
Mercator tile Y index.
tile_z: int
Mercator tile ZOOM level.
tiler: callable
tiler function. The function MUST take asset, x, y, z, **kwargs as arguments,
and MUST return a tuple with tile data and mask
e.g:
def tiler(asset: str, x: int, y: int, z: int, **kwargs) -> Tuple[numpy.ndarray, numpy.ndarray]:
with COGReader(asset) as cog:
return cog.tile(x, y, z, **kwargs)
pixel_selection: MosaicMethod, optional
Instance of MosaicMethodBase class.
default: "rio_tiler_mosaic.methods.defaults.FirstMethod".
chunk_size: int, optional
Control the number of asset to process per loop (default = threads).
threads: int, optional
Number of threads to use. If <=1, runs single threaded without an event
loop. By default reads from the MAX_THREADS environment variable, and if
not found defaults to multiprocessing.cpu_count() * 5.
kwargs: dict, optional
tiler specific options.
Returns
-------
tile, mask : tuple of ndarray
Return tile and mask data.
"""
if pixel_selection is None:
pixel_selection = FirstMethod()
if not isinstance(pixel_selection, MosaicMethodBase):
raise Exception("Mosaic filling algorithm should be an instance of"
"'rio_tiler_mosaic.methods.base.MosaicMethodBase'")
if not chunk_size:
chunk_size = threads or len(assets)
tasks: TaskType
for chunks in _chunks(assets, chunk_size):
if threads:
with futures.ThreadPoolExecutor(max_workers=threads) as executor:
tasks = [
executor.submit(tiler, asset, tile_x, tile_y, tile_z,
**kwargs) for asset in chunks
]
else:
tasks = (tiler(asset, tile_x, tile_y, tile_z, **kwargs)
for asset in chunks)
for t, m in _filter_tasks(tasks):
t = numpy.ma.array(t)
t.mask = m == 0
pixel_selection.feed(t)
if pixel_selection.is_done:
return pixel_selection.data
return pixel_selection.data
def mosaic_tiler(assets,
tile_x,
tile_y,
tile_z,
tiler,
pixel_selection=None,
chunk_size=None,
**kwargs):
"""
Create mercator tile from multiple observations.
Attributes
----------
assets : list, tuple
List of rio-tiler compatible sceneid or url
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
tiler: function
Rio-tiler's tiler function (e.g rio_tiler.landsat8.tile)
pixel_selection: MosaicMethod, optional
Instance of MosaicMethodBase class.
default: "rio_tiler_mosaic.methods.defaults.FirstMethod".
chunk_size: int, optional
Control the number of asset to process per loop (default = MAX_THREADS).
kwargs: dict, optional
Rio-tiler tiler module specific options.
Returns
-------
tile, mask : tuple of ndarray
Return tile and mask data.
"""
if pixel_selection is None:
pixel_selection = FirstMethod()
if not isinstance(pixel_selection, MosaicMethodBase):
raise Exception("Mosaic filling algorithm should be an instance of"
"'rio_tiler_mosaic.methods.base.MosaicMethodBase'")
_tiler = partial(tiler,
tile_x=tile_x,
tile_y=tile_y,
tile_z=tile_z,
**kwargs)
max_threads = int(
os.environ.get("MAX_THREADS",
multiprocessing.cpu_count() * 5))
if not chunk_size:
chunk_size = max_threads
for chunks in _chunks(assets, chunk_size):
with futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
future_tasks = [executor.submit(_tiler, asset) for asset in chunks]
for t, m in _filter_futures(future_tasks):
t = numpy.ma.array(t)
t.mask = m == 0
pixel_selection.feed(t)
if pixel_selection.is_done:
return pixel_selection.data
return pixel_selection.data
