def load_process_save_chunk(output_files: OutputDriver,
chunk: Tuple[slice, slice, slice], task: StatsTask,
timer: MultiTimer):
try:
with timer.time('loading_data'):
geom = geometry_for_task(task)
data = load_data(chunk, task.sources, geom=geom)
last_idx = len(task.output_products) - 1
for idx, (prod_name, stat) in enumerate(task.output_products.items()):
_LOG.debug(
"Computing %s in tile %s %s; %s", prod_name, task.spatial_id,
"({})".format(", ".join(prettier_slice(c) for c in chunk)),
timer)
measurements = stat.data_measurements
with timer.time(prod_name):
result = stat.compute(data)
if idx == last_idx: # make sure input data is released early
del data
# restore nodata values back
result = cast_back(result, measurements)
# For each of the data variables, shove this chunk into the output results
with timer.time('writing_data'):
output_files.write_chunk(prod_name, chunk, result)
except EmptyChunkException:
_LOG.debug(
'Error: No data returned while loading %s for %s. May have all been masked',
chunk, task)
def load_process_save_chunk_iteratively(output_files: OutputDriver,
chunk: Tuple[slice, slice, slice],
task: StatsTask,
timer: MultiTimer):
procs = [(stat.make_iterative_proc(), name, stat) for name, stat in task.output_products.items()]
def update(ds):
for proc, name, _ in procs:
with timer.time(name):
proc(ds)
def save(name, ds):
for var_name, var in ds.data_vars.items():
output_files.write_data(name, var_name, chunk, var.values)
for ds in load_data_lazy(chunk, task.sources, timer=timer):
update(ds)
with timer.time('writing_data'):
for proc, name, stat in procs:
save(name, cast_back(proc(), stat.data_measurements))
def load_process_save_chunk(output_files: OutputDriver,
chunk: Tuple[slice, slice, slice],
task: StatsTask, timer: MultiTimer):
try:
with timer.time('loading_data'):
data = load_data(chunk, task.sources)
# mask as per geometry now
if task.geom_feat:
geom = Geometry(task.geom_feat, CRS(task.crs_txt))
data = data.where(geometry_mask([geom], data.geobox, invert=True))
# pylint: disable=protected-access
if output_files._driver_name == 'None':
output_files.get_source(chunk, data)
last_idx = len(task.output_products) - 1
for idx, (prod_name, stat) in enumerate(task.output_products.items()):
_LOG.debug("Computing %s in tile %s %s; %s",
prod_name, task.tile_index,
"({})".format(", ".join(prettier_slice(c) for c in chunk)),
timer)
measurements = stat.data_measurements
with timer.time(prod_name):
result = stat.compute(data)
if idx == last_idx: # make sure input data is released early
del data
# restore nodata values back
result = cast_back(result, measurements)
# For each of the data variables, shove this chunk into the output results
with timer.time('writing_data'):
output_files.write_chunk(prod_name, chunk, result)
except EmptyChunkException:
_LOG.debug('Error: No data returned while loading %s for %s. May have all been masked',
chunk, task)
def load_process_save_chunk(output_files: OutputDriver,
chunk: Tuple[slice, slice, slice], task: StatsTask,
timer: MultiTimer):
try:
with timer.time('loading_data'):
data = load_data(chunk, task.sources)
# mask as per geometry now
if task.geom_feat:
geom = Geometry(task.geom_feat, CRS(task.crs_txt))
data = data.where(
geometry_mask([geom], data.geobox, invert=True))
last_idx = len(task.output_products) - 1
for idx, (prod_name, stat) in enumerate(task.output_products.items()):
_LOG.info("Computing %s in tile %s %s. Current timing: %s",
prod_name, task.tile_index, chunk, timer)
measurements = stat.data_measurements
with timer.time(prod_name):
result = stat.compute(data)
if idx == last_idx: # make sure input data is released early
del data
# restore nodata values back
result = cast_back(result, measurements)
# For each of the data variables, shove this chunk into the output results
with timer.time('writing_data'):
for var_name, var in result.data_vars.items(
): # TODO: Move this loop into output_files
output_files.write_data(prod_name, var_name, chunk,
var.values)
except EmptyChunkException:
_LOG.debug(
'Error: No data returned while loading %s for %s. May have all been masked',
chunk, task)