def _execute(self):
# If baselevel is active and zoom is outside of baselevel,
# interpolate from other zoom levels.
if self.config_baselevels:
if self.tile.zoom < min(self.config_baselevels["zooms"]):
return self._interpolate_from_baselevel("lower")
elif self.tile.zoom > max(self.config_baselevels["zooms"]):
return self._interpolate_from_baselevel("higher")
# Otherwise, execute from process file.
process_func = get_process_func(process_path=self.process_path,
config_dir=self.config_dir)
try:
with Timer() as t:
# Actually run process.
process_data = process_func(
MapcheteProcess(tile=self.tile,
params=self.process_func_params,
input=self.input,
output_params=self.output_params),
**self.process_func_params)
except MapcheteNodataTile:
raise
except Exception as e:
# Log process time
logger.exception(
(self.tile.id, "exception in user process", e, str(t)))
new = MapcheteProcessException(format_exc())
new.old = e
raise new
return process_data
def _write(process_info=None, output_data=None, output_writer=None):
if process_info.processed:
try:
output_data = output_writer.streamline_output(output_data)
except MapcheteNodataTile:
output_data = None
if output_data is None:
message = "output empty, nothing written"
logger.debug((process_info.tile.id, message))
return ProcessInfo(tile=process_info.tile,
processed=process_info.processed,
process_msg=process_info.process_msg,
written=False,
write_msg=message)
else:
with Timer() as t:
output_writer.write(process_tile=process_info.tile,
data=output_data)
message = "output written in %s" % t
logger.debug((process_info.tile.id, message))
return ProcessInfo(tile=process_info.tile,
processed=process_info.processed,
process_msg=process_info.process_msg,
written=True,
write_msg=message)
else:
return process_info
def _execute(tile_process=None):
logger.debug((tile_process.tile.id,
"running on %s" % multiprocessing.current_process().name))
# skip execution if overwrite is disabled and tile exists
if tile_process.skip:
logger.debug((tile_process.tile.id, "tile exists, skipping"))
return None, ProcessInfo(tile=tile_process.tile,
processed=False,
process_msg="output already exists",
written=False,
write_msg="nothing written")
# execute on process tile
else:
with Timer() as t:
try:
output = tile_process.execute()
except MapcheteNodataTile:
output = "empty"
processor_message = "processed in %s" % t
logger.debug((tile_process.tile.id, processor_message))
return output, ProcessInfo(tile=tile_process.tile,
processed=True,
process_msg=processor_message,
written=None,
write_msg=None)
def _interpolate_from_baselevel(self, baselevel=None):
# This is a special tile derived from a pyramid which has the pixelbuffer setting
# from the output pyramid but metatiling from the process pyramid. This is due to
# performance reasons as for the usual case overview tiles do not need the
# process pyramid pixelbuffers.
tile = self.config_baselevels["tile_pyramid"].tile(*self.tile.id)
# get output_tiles that intersect with process tile
output_tiles = (list(
self.output_reader.pyramid.tiles_from_bounds(
tile.bounds, tile.zoom)) if tile.pixelbuffer >
self.output_reader.pyramid.pixelbuffer else
self.output_reader.pyramid.intersecting(tile))
with Timer() as t:
# resample from parent tile
if baselevel == "higher":
parent_tile = self.tile.get_parent()
process_data = raster.resample_from_array(
self.output_reader.read(parent_tile),
in_affine=parent_tile.affine,
out_tile=self.tile,
resampling=self.config_baselevels["higher"],
nodata=self.output_reader.output_params["nodata"])
# resample from children tiles
elif baselevel == "lower":
if self.output_reader.pyramid.pixelbuffer:
lower_tiles = set([
y for y in chain(*[
self.output_reader.pyramid.tiles_from_bounds(
x.bounds, x.zoom + 1) for x in output_tiles
])
])
else:
lower_tiles = [
y for y in chain(
*[x.get_children() for x in output_tiles])
]
mosaic = raster.create_mosaic(
[(lower_tile, self.output_reader.read(lower_tile))
for lower_tile in lower_tiles],
nodata=self.output_reader.output_params["nodata"])
process_data = raster.resample_from_array(
in_raster=mosaic.data,
in_affine=mosaic.affine,
out_tile=self.tile,
resampling=self.config_baselevels["lower"],
nodata=self.output_reader.output_params["nodata"])
logger.debug((self.tile.id, "generated from baselevel", str(t)))
return process_data
def write(self, process_tile, data):
"""
Write data into output format.
Parameters
----------
process_tile : BufferedTile or tile index tuple
process tile
data : NumPy array or features
data to be written
"""
process_tile = validate_tile(process_tile, self.config.process_pyramid)
if self.config.mode not in ["continue", "overwrite"]:
raise ValueError("cannot write output in current process mode")
if self.config.mode == "continue" and (
self.config.output.tiles_exist(process_tile)
):
message = "output exists, not overwritten"
logger.debug((process_tile.id, message))
return ProcessInfo(
tile=process_tile,
processed=False,
process_msg=None,
written=False,
write_msg=message
)
elif data is None:
message = "output empty, nothing written"
logger.debug((process_tile.id, message))
return ProcessInfo(
tile=process_tile,
processed=False,
process_msg=None,
written=False,
write_msg=message
)
else:
with Timer() as t:
self.config.output.write(process_tile=process_tile, data=data)
message = "output written in %s" % t
logger.debug((process_tile.id, message))
return ProcessInfo(
tile=process_tile,
processed=False,
process_msg=None,
written=True,
write_msg=message
)
def _run_multi(func=None,
zoom_levels=None,
process=None,
multi=None,
multiprocessing_start_method=None,
multiprocessing_module=None,
write_in_parent_process=False,
fkwargs=None,
skip_output_check=False):
total_tiles = process.count_tiles(min(zoom_levels), max(zoom_levels))
workers = min([multi, total_tiles])
num_processed = 0
logger.debug("run process on %s tiles using %s workers", total_tiles,
workers)
# here we store the parents of processed tiles so we can update overviews
# also in "continue" mode in case there were updates at the baselevel
overview_parents = set()
with Timer() as t, Executor(
max_workers=workers,
start_method=multiprocessing_start_method,
multiprocessing_module=multiprocessing_module) as executor:
for i, zoom in enumerate(zoom_levels):
if skip_output_check:
# don't check outputs and simply proceed
todo = process.get_process_tiles(zoom)
else:
# check which process output already exists and which process tiles need
# to be added to todo list
todo = set()
for process_info in _filter_skipable(
process=process,
tiles=process.get_process_tiles(zoom),
todo=todo,
target_set=(overview_parents if
process.config.baselevels and i else None),
):
num_processed += 1
logger.info("tile %s/%s finished: %s, %s, %s",
num_processed, total_tiles, process_info.tile,
process_info.process_msg,
process_info.write_msg)
yield process_info
# process all remaining tiles using todo list from before
for task in executor.as_completed(
func=func,
iterable=(TileProcess(
tile=tile,
config=process.config,
skip=(process.mode == "continue" and
process.config.output_reader.tiles_exist(tile))
if skip_output_check else False) for tile in todo),
fkwargs=fkwargs):
# trigger output write for driver which require parent process for writing
if write_in_parent_process:
output_data, process_info = task.result()
process_info = _write(
process_info=process_info,
output_data=output_data,
output_writer=process.config.output,
)
# output already has been written, so just use task process info
else:
process_info = task.result()
# in case of building overviews from baselevels, remember which parent
# tile needs to be updated later on
if (not skip_output_check and process.config.baselevels
and process_info.processed
and process_info.tile.zoom > 0):
overview_parents.add(process_info.tile.get_parent())
num_processed += 1
logger.info("tile %s/%s finished: %s, %s, %s", num_processed,
total_tiles, process_info.tile,
process_info.process_msg, process_info.write_msg)
yield process_info
logger.debug("%s tile(s) iterated in %s", str(num_processed), t)