def __init__(self, grid_configs=[],
overwrite_existing=False, keep_intermediate=False, exit_on_error=True, **kwargs):
self.grid_manager = GridManager(*grid_configs)
self.overwrite_existing = overwrite_existing
self.keep_intermediate = keep_intermediate
self.exit_on_error = exit_on_error
self.methods = {
"ewa": self._remap_scene_ewa,
"nearest": self._remap_scene_nearest,
"sensor": self._remap_scene_sensor,
}
self.ll2cr_cache = {}
def _get_legacy_and_yaml_areas(
grid_configs: list[str,
...]) -> tuple[GridManager, dict[str, AreaDefinition]]:
if "grids.conf" in grid_configs:
logger.debug(
"Replacing 'grids.conf' with builtin YAML grid configuration file."
)
grid_configs[grid_configs.index("grids.conf")] = GRIDS_YAML_FILEPATH
if not grid_configs:
grid_configs = [GRIDS_YAML_FILEPATH]
p2g_grid_configs = [x for x in grid_configs if x.endswith(".conf")]
pyresample_area_configs = [
x for x in grid_configs if not x.endswith(".conf")
]
if p2g_grid_configs:
grid_manager = GridManager(*p2g_grid_configs)
else:
grid_manager = {}
if pyresample_area_configs:
yaml_areas = parse_area_file(pyresample_area_configs)
yaml_areas = {x.area_id: x for x in yaml_areas}
else:
yaml_areas = {}
return grid_manager, yaml_areas
def __init__(self, grid_configs=[],
overwrite_existing=False, keep_intermediate=False, exit_on_error=True, **kwargs):
self.grid_manager = GridManager(*grid_configs)
self.overwrite_existing = overwrite_existing
self.keep_intermediate = keep_intermediate
self.exit_on_error = exit_on_error
self.methods = {
"ewa": self._remap_scene_ewa,
"nearest": self._remap_scene_nearest,
}
self.ll2cr_cache = {}
def _get_legacy_and_custom_areas(grid_configs):
p2g_grid_configs = [x for x in grid_configs if x.endswith('.conf')]
pyresample_area_configs = [x for x in grid_configs if not x.endswith('.conf')]
if not grid_configs or p2g_grid_configs:
# if we were given p2g grid configs or we weren't given any to choose from
from polar2grid.grids import GridManager
grid_manager = GridManager(*p2g_grid_configs)
else:
grid_manager = {}
if pyresample_area_configs:
from pyresample.utils import parse_area_file
custom_areas = parse_area_file(pyresample_area_configs)
custom_areas = {x.area_id: x for x in custom_areas}
else:
custom_areas = {}
return grid_manager, custom_areas
class Remapper(object):
def __init__(self,
grid_configs=[],
overwrite_existing=False,
keep_intermediate=False,
exit_on_error=True,
**kwargs):
self.grid_manager = GridManager(*grid_configs)
self.overwrite_existing = overwrite_existing
self.keep_intermediate = keep_intermediate
self.exit_on_error = exit_on_error
self.methods = {
"ewa": self._remap_scene_ewa,
"nearest": self._remap_scene_nearest,
"sensor": self._remap_scene_sensor,
}
self.ll2cr_cache = {}
def highest_resolution_swath_definition(self, swath_scene_or_product):
if isinstance(swath_scene_or_product, SwathScene):
swath_defs = [
product_def["swath_definition"]
for product_def in swath_scene_or_product.values()
]
# choose the highest resolution swath definition navigation (the one with the most columns)
swath_def = max(swath_defs, key=lambda d: d["swath_columns"])
else:
# given a single product
swath_def = swath_scene_or_product["swath_definition"]
return swath_def
def remap_scene(self, swath_scene, grid_name, **kwargs):
method = kwargs.pop("remap_method", "ewa")
LOG.debug("Remap scene being run with method '%s'", method)
if method not in self.methods:
LOG.error("Unknown remapping method '%s'", method)
raise ValueError("Unknown remapping method '%s'" % (method, ))
if method == "sensor":
if grid_name != "sensor":
raise ValueError(
"'sensor' resampling only supports the 'sensor' grid")
else:
# grid def isn't used by 'sensor' resampling
grid_def = None
else:
grid_def = self.grid_manager.get_grid_definition(grid_name)
func = self.methods[method]
# FUTURE: Make this a keyword and add the logic to support it
if kwargs.get("share_dynamic_grids", True) and method != "sensor":
# Let's run ll2cr to fill in any parameters we need to and decide if the data fits in the grid
best_swath_def = self.highest_resolution_swath_definition(
swath_scene)
LOG.debug(
"Running ll2cr on the highest resolution swath to determine if it fits"
)
try:
self.run_ll2cr(best_swath_def,
grid_def,
swath_usage=kwargs.get("swath_usage",
SWATH_USAGE))
grid_str = str(grid_def).replace("\n", "\n\t")
LOG.info("Grid information:\n\t%s", grid_str)
except StandardError:
LOG.error("Remapping error")
raise
return func(swath_scene, grid_def, **kwargs)
def run_ll2cr(self,
swath_definition,
grid_definition,
swath_usage=SWATH_USAGE):
geo_id = swath_definition["swath_name"]
grid_name = grid_definition["grid_name"]
if (geo_id, grid_name) in self.ll2cr_cache:
return self.ll2cr_cache[(geo_id, grid_name)]
LOG.debug("Swath '%s' -> Grid '%s'", geo_id, grid_name)
rows_fn = "ll2cr_rows_%s_%s.dat" % (grid_name, geo_id)
cols_fn = "ll2cr_cols_%s_%s.dat" % (grid_name, geo_id)
# lon_arr = swath_definition.get_longitude_array()
# lat_arr = swath_definition.get_latitude_array()
if os.path.isfile(rows_fn):
if not self.overwrite_existing:
LOG.error("Intermediate remapping file already exists: %s" %
(rows_fn, ))
raise RuntimeError(
"Intermediate remapping file already exists: %s" %
(rows_fn, ))
else:
LOG.warning(
"Intermediate remapping file already exists, will overwrite: %s",
rows_fn)
if os.path.isfile(cols_fn):
if not self.overwrite_existing:
LOG.error("Intermediate remapping file already exists: %s" %
(cols_fn, ))
raise RuntimeError(
"Intermediate remapping file already exists: %s" %
(cols_fn, ))
else:
LOG.warning(
"Intermediate remapping file already exists, will overwrite: %s",
cols_fn)
try:
rows_arr = swath_definition.copy_latitude_array(filename=rows_fn,
read_only=False)
cols_arr = swath_definition.copy_longitude_array(filename=cols_fn,
read_only=False)
points_in_grid, _, _ = ll2cr.ll2cr(
cols_arr,
rows_arr,
grid_definition,
fill_in=swath_definition["fill_value"])
grid_str = str(grid_definition).replace("\n", "\n\t")
LOG.debug("Grid information:\n\t%s", grid_str)
except StandardError:
LOG.error(
"Unexpected error encountered during ll2cr gridding for %s -> %s",
geo_id, grid_name)
LOG.debug("ll2cr error exception: ", exc_info=True)
self._safe_remove(rows_fn, cols_fn)
raise
# if 5% of the grid will have data in it then it fits
fraction_in = points_in_grid / float(rows_arr.size)
swath_used = fraction_in > swath_usage
if not swath_used:
self._safe_remove(rows_fn, cols_fn)
LOG.error(
"Data does not fit in grid %s because it only %f%% of the swath is used"
% (grid_name, fraction_in * 100))
raise RuntimeError("Data does not fit in grid %s" % (grid_name, ))
else:
LOG.debug("Data fits in grid %s and uses %f%% of the swath",
grid_name, fraction_in * 100)
self.ll2cr_cache[(geo_id, grid_name)] = (cols_fn, rows_fn)
return cols_fn, rows_fn
def _add_prefix(self, prefix, *filepaths):
return [
os.path.join(os.path.dirname(x), prefix + os.path.basename(x))
for x in filepaths
]
def _safe_remove(self, *filepaths):
if not self.keep_intermediate:
for fp in filepaths:
if os.path.isfile(fp):
try:
LOG.debug("Removing intermediate file '%s'...", fp)
os.remove(fp)
except OSError:
LOG.warning(
"Could not remove intermediate files that aren't needed anymore."
)
LOG.debug("Intermediate output file remove exception:",
exc_info=True)
def _clear_ll2cr_cache(self):
# Remove ll2cr files now that we are done with them
for cols_fn, rows_fn in self.ll2cr_cache.values():
self._safe_remove(rows_fn, cols_fn)
self.ll2cr_cache = {}
def _remap_scene_ewa(self,
swath_scene,
grid_def,
share_dynamic_grids=True,
**kwargs):
# TODO: Make methods more flexible than just a function call
gridded_scene = GriddedScene()
grid_name = grid_def["grid_name"]
# Group products together that shared the same geolocation
product_groups = defaultdict(list)
for product_name, swath_product in swath_scene.items():
swath_def = swath_product["swath_definition"]
is_cat = swath_product.get('flag_meanings') is not None
geo_id = swath_def["swath_name"]
product_groups[(is_cat, geo_id)].append(product_name)
# keep a copy of the original grid definition
# if a shared grid definition isn't used then
# we start from the original
orig_grid_def = grid_def
for (is_cat, geo_id), product_names in product_groups.items():
try:
LOG.debug(
"Running ll2cr on the geolocation data for the following products:\n\t%s",
"\n\t".join(sorted(product_names)))
swath_def = swath_scene[product_names[0]]["swath_definition"]
if not share_dynamic_grids:
grid_def = orig_grid_def.copy()
cols_fn, rows_fn = self.run_ll2cr(swath_def,
grid_def,
swath_usage=kwargs.get(
"swath_usage",
SWATH_USAGE))
except StandardError:
LOG.error("Remapping error")
if self.exit_on_error:
raise
continue
# Run fornav for all of the products at once
LOG.debug("Running fornav for the following products:\n\t%s",
"\n\t".join(sorted(product_names)))
# XXX: May have to do something smarter if there are float products and integer products together (is_category property on SwathProduct?)
product_filepaths = list(
swath_scene.get_data_filepaths(product_names))
fornav_filepaths = self._add_prefix("grid_%s_" % (grid_name, ),
*product_filepaths)
for fp in fornav_filepaths:
if os.path.isfile(fp):
if not self.overwrite_existing:
LOG.error(
"Intermediate remapping file already exists: %s" %
(fp, ))
raise RuntimeError(
"Intermediate remapping file already exists: %s" %
(fp, ))
else:
LOG.warning(
"Intermediate remapping file already exists, will overwrite: %s",
fp)
rows_per_scan = swath_def.get("rows_per_scan", 0)
if rows_per_scan < 2:
LOG.warning(
"Data has less than 2 rows per scan, this is not optimal for the EWA resampling algorithm. All rows will be used as one scan"
)
rows_per_scan = swath_def['swath_rows']
edge_res = swath_def.get("limb_resolution", None)
fornav_D = kwargs.get("fornav_D", None)
if fornav_D is None:
if edge_res is not None:
if grid_def.is_latlong:
fornav_D = (edge_res / 2) / grid_def.cell_width_meters
else:
fornav_D = (edge_res / 2) / grid_def["cell_width"]
LOG.debug("Fornav 'D' option dynamically set to %f",
fornav_D)
else:
fornav_D = 10.0
mwm = kwargs.get('maximum_weight_mode', False)
if is_cat and not mwm:
LOG.debug(
"Turning on maximum weight mode in EWA resampling for category products"
)
mwm = True
try:
# fornav.ms2gt_fornav(
# len(product_filepaths),
# swath_def["swath_columns"],
# swath_def["swath_rows"]/rows_per_scan,
# rows_per_scan,
# cols_fn,
# rows_fn,
# product_filepaths,
# grid_def["width"],
# grid_def["height"],
# fornav_filepaths,
# swath_data_type_1="f4",
# swath_fill_1=swath_scene.get_fill_value(product_names),
# grid_fill_1=numpy.nan,
# weight_delta_max=fornav_D,
# weight_distance_max=kwargs.get("fornav_d", None),
# maximum_weight_mode=kwargs.get("maximum_weight_mode", None),
# start_scan=(0, 0),
# )
cols_array = numpy.memmap(cols_fn,
dtype=numpy.float32,
mode='r',
shape=(swath_def["swath_rows"],
swath_def["swath_columns"]))
rows_array = numpy.memmap(rows_fn,
dtype=numpy.float32,
mode='r',
shape=(swath_def["swath_rows"],
swath_def["swath_columns"]))
# Assumed that all share the same fill value and data type
input_dtype = [
swath_scene[pn]["data_type"] for pn in product_names
]
input_fill = [
swath_scene[pn]["fill_value"] for pn in product_names
]
LOG.debug("Running fornav with D={} and d={}".format(
fornav_D, kwargs.get('fornav_d', 1.0)))
valid_list = fornav.fornav(cols_array,
rows_array,
rows_per_scan,
product_filepaths,
input_dtype=input_dtype,
input_fill=input_fill,
output_arrays=fornav_filepaths,
grid_cols=grid_def["width"],
grid_rows=grid_def["height"],
weight_delta_max=fornav_D,
weight_distance_max=kwargs.get(
"fornav_d", 1.0),
maximum_weight_mode=mwm,
use_group_size=True)
except StandardError:
LOG.debug("Remapping exception: ", exc_info=True)
LOG.error("Remapping error")
self._safe_remove(*fornav_filepaths)
if self.exit_on_error:
self._clear_ll2cr_cache()
raise
continue
# Give the gridded product ownership of the remapped data
for product_name, fornav_fp, valid_points in zip(
product_names, fornav_filepaths, valid_list):
swath_product = swath_scene[product_name]
gridded_product = GriddedProduct()
gridded_product.from_swath_product(swath_product)
gridded_product["grid_definition"] = grid_def
gridded_product["fill_value"] = numpy.nan
gridded_product["grid_data"] = fornav_fp
grid_coverage = kwargs.get("grid_coverage", GRID_COVERAGE)
grid_covered_ratio = valid_points / float(
grid_def["width"] * grid_def["height"])
grid_covered = grid_covered_ratio > grid_coverage
if not grid_covered:
msg = "EWA resampling only found %f%% of the grid covered (need %f%%) for %s" % (
grid_covered_ratio * 100, grid_coverage * 100,
product_name)
LOG.warning(msg)
continue
LOG.debug(
"EWA resampling found %f%% of the grid covered for %s" %
(grid_covered_ratio * 100, product_name))
gridded_scene[product_name] = gridded_product
self._clear_ll2cr_cache()
if not gridded_scene:
self._safe_remove(*fornav_filepaths)
raise RuntimeError(
"EWA resampling could not remap any of the data to grid '%s'" %
(grid_name, ))
return gridded_scene
def _remap_scene_nearest(self,
swath_scene,
grid_def,
share_dynamic_grids=True,
share_remap_mask=True,
**kwargs):
# TODO: Make methods more flexible than just a function call
gridded_scene = GriddedScene()
grid_name = grid_def["grid_name"]
# Group products together that shared the same geolocation
product_groups = defaultdict(list)
for product_name, swath_product in swath_scene.items():
swath_def = swath_product["swath_definition"]
geo_id = swath_def["swath_name"]
product_groups[geo_id].append(product_name)
grid_coverage = kwargs.get("grid_coverage", GRID_COVERAGE)
orig_grid_def = grid_def
for geo_id, product_names in product_groups.items():
pp_names = "\n\t".join(product_names)
LOG.debug(
"Running ll2cr on the geolocation data for the following products:\n\t%s",
pp_names)
LOG.debug("Swath name: %s", geo_id)
# TODO: Move into it's own function if this gets complicated
# TODO: Add some multiprocessing
try:
swath_def = swath_scene[product_names[0]]["swath_definition"]
if not share_dynamic_grids:
grid_def = orig_grid_def.copy()
cols_fn, rows_fn = self.run_ll2cr(swath_def, grid_def)
except StandardError:
LOG.error("Remapping error")
if self.exit_on_error:
raise
continue
LOG.debug(
"Running nearest neighbor for the following products:\n\t%s",
"\n\t".join(product_names))
edge_res = swath_def.get("limb_resolution", None)
if kwargs.get("distance_upper_bound", None) is None:
if edge_res is not None:
if grid_def.is_latlong:
distance_upper_bound = (edge_res /
2) / grid_def.cell_width_meters
else:
distance_upper_bound = (edge_res /
2) / grid_def["cell_width"]
LOG.debug("Distance upper bound dynamically set to %f",
distance_upper_bound)
else:
distance_upper_bound = 3.0
kwargs["distance_upper_bound"] = distance_upper_bound
try:
grid_x, grid_y = numpy.mgrid[:grid_def["height"], :
grid_def["width"]]
# we need flattened versions of these
shape = (swath_def["swath_rows"] *
swath_def["swath_columns"], )
cols_array = numpy.memmap(cols_fn,
shape=shape,
dtype=swath_def["data_type"])
rows_array = numpy.memmap(rows_fn,
shape=shape,
dtype=swath_def["data_type"])
good_mask = ~mask_helper(cols_array, swath_def["fill_value"])
if share_remap_mask:
for product_name in product_names:
LOG.debug(
"Combining data masks before building KDTree for nearest neighbor: %s",
product_name)
good_mask &= ~swath_scene[product_name].get_data_mask(
).ravel()
x = _ndim_coords_from_arrays(
(cols_array[good_mask], rows_array[good_mask]))
xi = _ndim_coords_from_arrays((grid_y, grid_x))
dist, i = cKDTree(x).query(
xi, distance_upper_bound=kwargs["distance_upper_bound"])
except StandardError:
LOG.debug("Remapping exception: ", exc_info=True)
LOG.error("Remapping error")
if self.exit_on_error:
self._clear_ll2cr_cache()
raise
continue
product_filepaths = swath_scene.get_data_filepaths(product_names)
output_filepaths = self._add_prefix("grid_%s_" % (grid_name, ),
*product_filepaths)
# Prepare the products
fill_value = numpy.nan
for product_name, output_fn in izip(product_names,
output_filepaths):
LOG.debug(
"Running nearest neighbor on '%s' with search distance %f",
product_name, kwargs["distance_upper_bound"])
if os.path.isfile(output_fn):
if not self.overwrite_existing:
LOG.error(
"Intermediate remapping file already exists: %s" %
(output_fn, ))
raise RuntimeError(
"Intermediate remapping file already exists: %s" %
(output_fn, ))
else:
LOG.warning(
"Intermediate remapping file already exists, will overwrite: %s",
output_fn)
try:
image_array = swath_scene[product_name].get_data_array(
).ravel()
values = numpy.append(image_array[good_mask],
image_array.dtype.type(fill_value))
output_array = values[i]
output_array.tofile(output_fn)
# Give the gridded product ownership of the remapped data
swath_product = swath_scene[product_name]
gridded_product = GriddedProduct()
gridded_product.from_swath_product(swath_product)
gridded_product["grid_definition"] = grid_def
gridded_product["fill_value"] = fill_value
gridded_product["grid_data"] = output_fn
# Check grid coverage
valid_points = numpy.count_nonzero(
~gridded_product.get_data_mask())
grid_covered_ratio = valid_points / float(
grid_def["width"] * grid_def["height"])
grid_covered = grid_covered_ratio > grid_coverage
if not grid_covered:
msg = "Nearest neighbor resampling only found %f%% of the grid covered (need %f%%) for %s" % (
grid_covered_ratio * 100, grid_coverage * 100,
product_name)
LOG.warning(msg)
continue
LOG.debug(
"Nearest neighbor resampling found %f%% of the grid covered for %s"
% (grid_covered_ratio * 100, product_name))
gridded_scene[product_name] = gridded_product
# hopefully force garbage collection
del output_array
except StandardError:
LOG.debug("Remapping exception: ", exc_info=True)
LOG.error("Remapping error")
self._safe_remove(output_fn)
if self.exit_on_error:
self._clear_ll2cr_cache()
raise
continue
LOG.debug("Done running nearest neighbor on '%s'",
product_name)
# Remove ll2cr files now that we are done with them
self._clear_ll2cr_cache()
if not gridded_scene:
raise RuntimeError(
"Nearest neighbor resampling could not remap any of the data to grid '%s'"
% (grid_name, ))
return gridded_scene
def _remap_scene_sensor(self, swath_scene, grid_def, **kwargs):
if not isinstance(swath_scene, Scene):
raise ValueError("'sensor' resampling only supports SatPy scenes")
new_scn = None
for area_obj, ds_list in swath_scene.iter_by_area():
_new_scn = swath_scene.resample(area_obj, datasets=ds_list)
if new_scn is None:
new_scn = _new_scn
for ds in _new_scn:
new_scn[ds.info["id"]] = ds
return new_scn
def remap_product(self, product, grid_name):
raise NotImplementedError(
"Single product remapping is not implemented yet")
def main(argv=sys.argv[1:]):
global LOG
from satpy import Scene
from satpy.resample import get_area_def
from satpy.writers import compute_writer_results
from dask.diagnostics import ProgressBar
from polar2grid.core.script_utils import (
setup_logging, rename_log_file, create_exc_handler)
import argparse
prog = os.getenv('PROG_NAME', sys.argv[0])
# "usage: " will be printed at the top of this:
usage = """
%(prog)s -h
see available products:
%(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...]
basic processing:
%(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...]
basic processing with limited products:
%(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...]
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage,
description="Load, composite, resample, and save datasets.")
parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0,
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
parser.add_argument('-l', '--log', dest="log_fn", default=None,
help="specify the log filename")
parser.add_argument('--progress', action='store_true',
help="show processing progress bar (not recommended for logged output)")
parser.add_argument('--num-workers', type=int, default=4,
help="specify number of worker threads to use (default: 4)")
parser.add_argument('--match-resolution', dest='preserve_resolution', action='store_false',
help="When using the 'native' resampler for composites, don't save data "
"at its native resolution, use the resolution used to create the "
"composite.")
parser.add_argument('-w', '--writers', nargs='+',
help='writers to save datasets with')
parser.add_argument("--list-products", dest="list_products", action="store_true",
help="List available reader products and exit")
subgroups = add_scene_argument_groups(parser)
subgroups += add_resample_argument_groups(parser)
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
args, remaining_args = parser.parse_known_args(argv_without_help)
# get the logger if we know the readers and writers that will be used
if args.reader is not None and args.writers is not None:
glue_name = args.reader + "_" + "-".join(args.writers or [])
LOG = logging.getLogger(glue_name)
# add writer arguments
if args.writers is not None:
for writer in (args.writers or []):
parser_func = WRITER_PARSER_FUNCTIONS.get(writer)
if parser_func is None:
continue
subgroups += parser_func(parser)
args = parser.parse_args(argv)
if args.reader is None:
parser.print_usage()
parser.exit(1, "\nERROR: Reader must be provided (-r flag).\n"
"Supported readers:\n\t{}\n".format('\n\t'.join(['abi_l1b', 'ahi_hsd', 'hrit_ahi'])))
if args.writers is None:
parser.print_usage()
parser.exit(1, "\nERROR: Writer must be provided (-w flag) with one or more writer.\n"
"Supported writers:\n\t{}\n".format('\n\t'.join(['geotiff'])))
def _args_to_dict(group_actions):
return {ga.dest: getattr(args, ga.dest) for ga in group_actions if hasattr(args, ga.dest)}
scene_args = _args_to_dict(subgroups[0]._group_actions)
load_args = _args_to_dict(subgroups[1]._group_actions)
resample_args = _args_to_dict(subgroups[2]._group_actions)
writer_args = {}
for idx, writer in enumerate(args.writers):
sgrp1, sgrp2 = subgroups[3 + idx * 2: 5 + idx * 2]
wargs = _args_to_dict(sgrp1._group_actions)
if sgrp2 is not None:
wargs.update(_args_to_dict(sgrp2._group_actions))
writer_args[writer] = wargs
# get default output filename
if 'filename' in wargs and wargs['filename'] is None:
wargs['filename'] = get_default_output_filename(args.reader, writer)
if not args.filenames:
parser.print_usage()
parser.exit(1, "\nERROR: No data files provided (-f flag)\n")
# Prepare logging
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn)
logging.getLogger('rasterio').setLevel(levels[min(2, args.verbosity)])
sys.excepthook = create_exc_handler(LOG.name)
if levels[min(3, args.verbosity)] > logging.DEBUG:
import warnings
warnings.filterwarnings("ignore")
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Set up dask and the number of workers
if args.num_workers:
from multiprocessing.pool import ThreadPool
dask.config.set(pool=ThreadPool(args.num_workers))
# Parse provided files and search for files if provided directories
scene_args['filenames'] = get_input_files(scene_args['filenames'])
# Create a Scene, analyze the provided files
LOG.info("Sorting and reading input files...")
try:
scn = Scene(**scene_args)
except ValueError as e:
LOG.error("{} | Enable debug message (-vvv) or see log file for details.".format(str(e)))
LOG.debug("Further error information: ", exc_info=True)
return -1
except OSError:
LOG.error("Could not open files. Enable debug message (-vvv) or see log file for details.")
LOG.debug("Further error information: ", exc_info=True)
return -1
if args.list_products:
print("\n".join(sorted(scn.available_dataset_names(composites=True))))
return 0
# Rename the log file
if rename_log:
rename_log_file(glue_name + scn.attrs['start_time'].strftime("_%Y%m%d_%H%M%S.log"))
# Load the actual data arrays and metadata (lazy loaded as dask arrays)
if load_args['products'] is None:
try:
reader_mod = importlib.import_module('polar2grid.readers.' + scene_args['reader'])
load_args['products'] = reader_mod.DEFAULT_PRODUCTS
LOG.info("Using default product list: {}".format(load_args['products']))
except (ImportError, AttributeError):
LOG.error("No default products list set, please specify with `--products`.")
return -1
LOG.info("Loading product metadata from files...")
scn.load(load_args['products'])
resample_kwargs = resample_args.copy()
areas_to_resample = resample_kwargs.pop('grids')
grid_configs = resample_kwargs.pop('grid_configs')
resampler = resample_kwargs.pop('resampler')
if areas_to_resample is None and resampler in [None, 'native']:
# no areas specified
areas_to_resample = ['MAX']
elif areas_to_resample is None:
raise ValueError("Resampling method specified (--method) without any destination grid/area (-g flag).")
elif not areas_to_resample:
# they don't want any resampling (they used '-g' with no args)
areas_to_resample = [None]
has_custom_grid = any(g not in ['MIN', 'MAX', None] for g in areas_to_resample)
if has_custom_grid and resampler == 'native':
LOG.error("Resampling method 'native' can only be used with 'MIN' or 'MAX' grids "
"(use 'nearest' method instead).")
return -1
p2g_grid_configs = [x for x in grid_configs if x.endswith('.conf')]
pyresample_area_configs = [x for x in grid_configs if not x.endswith('.conf')]
if not grid_configs or p2g_grid_configs:
# if we were given p2g grid configs or we weren't given any to choose from
from polar2grid.grids import GridManager
grid_manager = GridManager(*p2g_grid_configs)
else:
grid_manager = {}
if pyresample_area_configs:
from pyresample.utils import parse_area_file
custom_areas = parse_area_file(pyresample_area_configs)
custom_areas = {x.area_id: x for x in custom_areas}
else:
custom_areas = {}
ll_bbox = resample_kwargs.pop('ll_bbox')
if ll_bbox:
scn = scn.crop(ll_bbox=ll_bbox)
wishlist = scn.wishlist.copy()
preserve_resolution = get_preserve_resolution(args, resampler, areas_to_resample)
if preserve_resolution:
preserved_products = set(wishlist) & set(scn.datasets.keys())
resampled_products = set(wishlist) - preserved_products
# original native scene
to_save = write_scene(scn, args.writers, writer_args, preserved_products)
else:
preserved_products = set()
resampled_products = set(wishlist)
to_save = []
LOG.debug("Products to preserve resolution for: {}".format(preserved_products))
LOG.debug("Products to use new resolution for: {}".format(resampled_products))
for area_name in areas_to_resample:
if area_name is None:
# no resampling
area_def = None
elif area_name == 'MAX':
area_def = scn.max_area()
elif area_name == 'MIN':
area_def = scn.min_area()
elif area_name in custom_areas:
area_def = custom_areas[area_name]
elif area_name in grid_manager:
from pyresample.geometry import DynamicAreaDefinition
p2g_def = grid_manager[area_name]
area_def = p2g_def.to_satpy_area()
if isinstance(area_def, DynamicAreaDefinition) and p2g_def['cell_width'] is not None:
area_def = area_def.freeze(scn.max_area(),
resolution=(abs(p2g_def['cell_width']), abs(p2g_def['cell_height'])))
else:
area_def = get_area_def(area_name)
if resampler is None and area_def is not None:
rs = 'native' if area_name in ['MIN', 'MAX'] else 'nearest'
LOG.debug("Setting default resampling to '{}' for grid '{}'".format(rs, area_name))
else:
rs = resampler
if area_def is not None:
LOG.info("Resampling data to '%s'", area_name)
new_scn = scn.resample(area_def, resampler=rs, **resample_kwargs)
elif not preserve_resolution:
# the user didn't want to resample to any areas
# the user also requested that we don't preserve resolution
# which means we have to save this Scene's datasets
# because they won't be saved
new_scn = scn
to_save = write_scene(new_scn, args.writers, writer_args, resampled_products, to_save=to_save)
if args.progress:
pbar = ProgressBar()
pbar.register()
LOG.info("Computing products and saving data to writers...")
compute_writer_results(to_save)
LOG.info("SUCCESS")
return 0
class Remapper(object):
def __init__(self, grid_configs=[],
overwrite_existing=False, keep_intermediate=False, exit_on_error=True, **kwargs):
self.grid_manager = GridManager(*grid_configs)
self.overwrite_existing = overwrite_existing
self.keep_intermediate = keep_intermediate
self.exit_on_error = exit_on_error
self.methods = {
"ewa": self._remap_scene_ewa,
"nearest": self._remap_scene_nearest,
"sensor": self._remap_scene_sensor,
}
self.ll2cr_cache = {}
def highest_resolution_swath_definition(self, swath_scene_or_product):
if isinstance(swath_scene_or_product, SwathScene):
swath_defs = [product_def["swath_definition"] for product_def in swath_scene_or_product.values()]
# choose the highest resolution swath definition navigation (the one with the most columns)
swath_def = max(swath_defs, key=lambda d: d["swath_columns"])
else:
# given a single product
swath_def = swath_scene_or_product["swath_definition"]
return swath_def
def remap_scene(self, swath_scene, grid_name, **kwargs):
method = kwargs.pop("remap_method", "ewa")
LOG.debug("Remap scene being run with method '%s'", method)
if method not in self.methods:
LOG.error("Unknown remapping method '%s'", method)
raise ValueError("Unknown remapping method '%s'" % (method,))
if method == "sensor":
if grid_name != "sensor":
raise ValueError("'sensor' resampling only supports the 'sensor' grid")
else:
# grid def isn't used by 'sensor' resampling
grid_def = None
else:
grid_def = self.grid_manager.get_grid_definition(grid_name)
func = self.methods[method]
# FUTURE: Make this a keyword and add the logic to support it
if kwargs.get("share_dynamic_grids", True) and method != "sensor":
# Let's run ll2cr to fill in any parameters we need to and decide if the data fits in the grid
best_swath_def = self.highest_resolution_swath_definition(swath_scene)
LOG.debug("Running ll2cr on the highest resolution swath to determine if it fits")
try:
self.run_ll2cr(best_swath_def, grid_def, swath_usage=kwargs.get("swath_usage", SWATH_USAGE))
grid_str = str(grid_def).replace("\n", "\n\t")
LOG.info("Grid information:\n\t%s", grid_str)
except StandardError:
LOG.error("Remapping error")
raise
return func(swath_scene, grid_def, **kwargs)
def run_ll2cr(self, swath_definition, grid_definition, swath_usage=SWATH_USAGE):
geo_id = swath_definition["swath_name"]
grid_name = grid_definition["grid_name"]
if (geo_id, grid_name) in self.ll2cr_cache:
return self.ll2cr_cache[(geo_id, grid_name)]
LOG.debug("Swath '%s' -> Grid '%s'", geo_id, grid_name)
rows_fn = "ll2cr_rows_%s_%s.dat" % (grid_name, geo_id)
cols_fn = "ll2cr_cols_%s_%s.dat" % (grid_name, geo_id)
# lon_arr = swath_definition.get_longitude_array()
# lat_arr = swath_definition.get_latitude_array()
if os.path.isfile(rows_fn):
if not self.overwrite_existing:
LOG.error("Intermediate remapping file already exists: %s" % (rows_fn,))
raise RuntimeError("Intermediate remapping file already exists: %s" % (rows_fn,))
else:
LOG.warning("Intermediate remapping file already exists, will overwrite: %s", rows_fn)
if os.path.isfile(cols_fn):
if not self.overwrite_existing:
LOG.error("Intermediate remapping file already exists: %s" % (cols_fn,))
raise RuntimeError("Intermediate remapping file already exists: %s" % (cols_fn,))
else:
LOG.warning("Intermediate remapping file already exists, will overwrite: %s", cols_fn)
try:
rows_arr = swath_definition.copy_latitude_array(filename=rows_fn, read_only=False)
cols_arr = swath_definition.copy_longitude_array(filename=cols_fn, read_only=False)
points_in_grid, _, _ = ll2cr.ll2cr(cols_arr, rows_arr, grid_definition,
fill_in=swath_definition["fill_value"])
grid_str = str(grid_definition).replace("\n", "\n\t")
LOG.debug("Grid information:\n\t%s", grid_str)
except StandardError:
LOG.error("Unexpected error encountered during ll2cr gridding for %s -> %s", geo_id, grid_name)
LOG.debug("ll2cr error exception: ", exc_info=True)
self._safe_remove(rows_fn, cols_fn)
raise
# if 5% of the grid will have data in it then it fits
fraction_in = points_in_grid / float(rows_arr.size)
swath_used = fraction_in > swath_usage
if not swath_used:
self._safe_remove(rows_fn, cols_fn)
LOG.error("Data does not fit in grid %s because it only %f%% of the swath is used" % (grid_name, fraction_in * 100))
raise RuntimeError("Data does not fit in grid %s" % (grid_name,))
else:
LOG.debug("Data fits in grid %s and uses %f%% of the swath", grid_name, fraction_in * 100)
self.ll2cr_cache[(geo_id, grid_name)] = (cols_fn, rows_fn)
return cols_fn, rows_fn
def _add_prefix(self, prefix, *filepaths):
return [os.path.join(os.path.dirname(x), prefix + os.path.basename(x)) for x in filepaths]
def _safe_remove(self, *filepaths):
if not self.keep_intermediate:
for fp in filepaths:
if os.path.isfile(fp):
try:
LOG.debug("Removing intermediate file '%s'...", fp)
os.remove(fp)
except OSError:
LOG.warning("Could not remove intermediate files that aren't needed anymore.")
LOG.debug("Intermediate output file remove exception:", exc_info=True)
def _clear_ll2cr_cache(self):
# Remove ll2cr files now that we are done with them
for cols_fn, rows_fn in self.ll2cr_cache.values():
self._safe_remove(rows_fn, cols_fn)
self.ll2cr_cache = {}
def _remap_scene_ewa(self, swath_scene, grid_def, share_dynamic_grids=True, **kwargs):
# TODO: Make methods more flexible than just a function call
gridded_scene = GriddedScene()
grid_name = grid_def["grid_name"]
# Group products together that shared the same geolocation
product_groups = defaultdict(list)
for product_name, swath_product in swath_scene.items():
swath_def = swath_product["swath_definition"]
is_cat = swath_product.get('flag_meanings') is not None
geo_id = swath_def["swath_name"]
product_groups[(is_cat, geo_id)].append(product_name)
# keep a copy of the original grid definition
# if a shared grid definition isn't used then
# we start from the original
orig_grid_def = grid_def
for (is_cat, geo_id), product_names in product_groups.items():
try:
LOG.debug("Running ll2cr on the geolocation data for the following products:\n\t%s", "\n\t".join(sorted(product_names)))
swath_def = swath_scene[product_names[0]]["swath_definition"]
if not share_dynamic_grids:
grid_def = orig_grid_def.copy()
cols_fn, rows_fn = self.run_ll2cr(swath_def, grid_def,
swath_usage=kwargs.get("swath_usage", SWATH_USAGE))
except StandardError:
LOG.error("Remapping error")
if self.exit_on_error:
raise
continue
# Run fornav for all of the products at once
LOG.debug("Running fornav for the following products:\n\t%s", "\n\t".join(sorted(product_names)))
# XXX: May have to do something smarter if there are float products and integer products together (is_category property on SwathProduct?)
product_filepaths = list(swath_scene.get_data_filepaths(product_names))
fornav_filepaths = self._add_prefix("grid_%s_" % (grid_name,), *product_filepaths)
for fp in fornav_filepaths:
if os.path.isfile(fp):
if not self.overwrite_existing:
LOG.error("Intermediate remapping file already exists: %s" % (fp,))
raise RuntimeError("Intermediate remapping file already exists: %s" % (fp,))
else:
LOG.warning("Intermediate remapping file already exists, will overwrite: %s", fp)
rows_per_scan = swath_def.get("rows_per_scan", 0)
if rows_per_scan < 2:
LOG.warning("Data has less than 2 rows per scan, this is not optimal for the EWA resampling algorithm. All rows will be used as one scan")
rows_per_scan = swath_def['swath_rows']
edge_res = swath_def.get("limb_resolution", None)
fornav_D = kwargs.get("fornav_D", None)
if fornav_D is None:
if edge_res is not None:
if grid_def.is_latlong:
fornav_D = (edge_res / 2) / grid_def.cell_width_meters
else:
fornav_D = (edge_res / 2) / grid_def["cell_width"]
LOG.debug("Fornav 'D' option dynamically set to %f", fornav_D)
else:
fornav_D = 10.0
mwm = kwargs.get('maximum_weight_mode', False)
if is_cat and not mwm:
LOG.debug("Turning on maximum weight mode in EWA resampling for category products")
mwm = True
try:
# fornav.ms2gt_fornav(
# len(product_filepaths),
# swath_def["swath_columns"],
# swath_def["swath_rows"]/rows_per_scan,
# rows_per_scan,
# cols_fn,
# rows_fn,
# product_filepaths,
# grid_def["width"],
# grid_def["height"],
# fornav_filepaths,
# swath_data_type_1="f4",
# swath_fill_1=swath_scene.get_fill_value(product_names),
# grid_fill_1=numpy.nan,
# weight_delta_max=fornav_D,
# weight_distance_max=kwargs.get("fornav_d", None),
# maximum_weight_mode=kwargs.get("maximum_weight_mode", None),
# start_scan=(0, 0),
# )
cols_array = numpy.memmap(cols_fn, dtype=numpy.float32, mode='r', shape=(swath_def["swath_rows"], swath_def["swath_columns"]))
rows_array = numpy.memmap(rows_fn, dtype=numpy.float32, mode='r', shape=(swath_def["swath_rows"], swath_def["swath_columns"]))
# Assumed that all share the same fill value and data type
input_dtype = [swath_scene[pn]["data_type"] for pn in product_names]
input_fill = [swath_scene[pn]["fill_value"] for pn in product_names]
LOG.debug("Running fornav with D={} and d={}".format(fornav_D, kwargs.get('fornav_d', 1.0)))
valid_list = fornav.fornav(cols_array,
rows_array,
rows_per_scan,
product_filepaths,
input_dtype=input_dtype,
input_fill=input_fill,
output_arrays=fornav_filepaths,
grid_cols=grid_def["width"],
grid_rows=grid_def["height"],
weight_delta_max=fornav_D,
weight_distance_max=kwargs.get("fornav_d", 1.0),
maximum_weight_mode=mwm,
use_group_size=True
)
except StandardError:
LOG.debug("Remapping exception: ", exc_info=True)
LOG.error("Remapping error")
self._safe_remove(*fornav_filepaths)
if self.exit_on_error:
self._clear_ll2cr_cache()
raise
continue
# Give the gridded product ownership of the remapped data
for product_name, fornav_fp, valid_points in zip(product_names, fornav_filepaths, valid_list):
swath_product = swath_scene[product_name]
gridded_product = GriddedProduct()
gridded_product.from_swath_product(swath_product)
gridded_product["grid_definition"] = grid_def
gridded_product["fill_value"] = numpy.nan
gridded_product["grid_data"] = fornav_fp
grid_coverage = kwargs.get("grid_coverage", GRID_COVERAGE)
grid_covered_ratio = valid_points / float(grid_def["width"] * grid_def["height"])
grid_covered = grid_covered_ratio > grid_coverage
if not grid_covered:
msg = "EWA resampling only found %f%% of the grid covered (need %f%%) for %s" % (grid_covered_ratio * 100, grid_coverage * 100, product_name)
LOG.warning(msg)
continue
LOG.debug("EWA resampling found %f%% of the grid covered for %s" % (grid_covered_ratio * 100, product_name))
gridded_scene[product_name] = gridded_product
self._clear_ll2cr_cache()
if not gridded_scene:
self._safe_remove(*fornav_filepaths)
raise RuntimeError("EWA resampling could not remap any of the data to grid '%s'" % (grid_name,))
return gridded_scene
def _remap_scene_nearest(self, swath_scene, grid_def, share_dynamic_grids=True, share_remap_mask=True, **kwargs):
# TODO: Make methods more flexible than just a function call
gridded_scene = GriddedScene()
grid_name = grid_def["grid_name"]
# Group products together that shared the same geolocation
product_groups = defaultdict(list)
for product_name, swath_product in swath_scene.items():
swath_def = swath_product["swath_definition"]
geo_id = swath_def["swath_name"]
product_groups[geo_id].append(product_name)
grid_coverage = kwargs.get("grid_coverage", GRID_COVERAGE)
orig_grid_def = grid_def
for geo_id, product_names in product_groups.items():
pp_names = "\n\t".join(product_names)
LOG.debug("Running ll2cr on the geolocation data for the following products:\n\t%s", pp_names)
LOG.debug("Swath name: %s", geo_id)
# TODO: Move into it's own function if this gets complicated
# TODO: Add some multiprocessing
try:
swath_def = swath_scene[product_names[0]]["swath_definition"]
if not share_dynamic_grids:
grid_def = orig_grid_def.copy()
cols_fn, rows_fn = self.run_ll2cr(swath_def, grid_def)
except StandardError:
LOG.error("Remapping error")
if self.exit_on_error:
raise
continue
LOG.debug("Running nearest neighbor for the following products:\n\t%s", "\n\t".join(product_names))
edge_res = swath_def.get("limb_resolution", None)
if kwargs.get("distance_upper_bound", None) is None:
if edge_res is not None:
if grid_def.is_latlong:
distance_upper_bound = (edge_res / 2) / grid_def.cell_width_meters
else:
distance_upper_bound = (edge_res / 2) / grid_def["cell_width"]
LOG.debug("Distance upper bound dynamically set to %f", distance_upper_bound)
else:
distance_upper_bound = 3.0
kwargs["distance_upper_bound"] = distance_upper_bound
try:
grid_x, grid_y = numpy.mgrid[:grid_def["height"], :grid_def["width"]]
# we need flattened versions of these
shape = (swath_def["swath_rows"] * swath_def["swath_columns"],)
cols_array = numpy.memmap(cols_fn, shape=shape, dtype=swath_def["data_type"])
rows_array = numpy.memmap(rows_fn, shape=shape, dtype=swath_def["data_type"])
good_mask = ~mask_helper(cols_array, swath_def["fill_value"])
if share_remap_mask:
for product_name in product_names:
LOG.debug("Combining data masks before building KDTree for nearest neighbor: %s", product_name)
good_mask &= ~swath_scene[product_name].get_data_mask().ravel()
x = _ndim_coords_from_arrays((cols_array[good_mask], rows_array[good_mask]))
xi = _ndim_coords_from_arrays((grid_y, grid_x))
dist, i = cKDTree(x).query(xi, distance_upper_bound=kwargs["distance_upper_bound"])
except StandardError:
LOG.debug("Remapping exception: ", exc_info=True)
LOG.error("Remapping error")
if self.exit_on_error:
self._clear_ll2cr_cache()
raise
continue
product_filepaths = swath_scene.get_data_filepaths(product_names)
output_filepaths = self._add_prefix("grid_%s_" % (grid_name,), *product_filepaths)
# Prepare the products
fill_value = numpy.nan
for product_name, output_fn in izip(product_names, output_filepaths):
LOG.debug("Running nearest neighbor on '%s' with search distance %f", product_name, kwargs["distance_upper_bound"])
if os.path.isfile(output_fn):
if not self.overwrite_existing:
LOG.error("Intermediate remapping file already exists: %s" % (output_fn,))
raise RuntimeError("Intermediate remapping file already exists: %s" % (output_fn,))
else:
LOG.warning("Intermediate remapping file already exists, will overwrite: %s", output_fn)
try:
image_array = swath_scene[product_name].get_data_array().ravel()
values = numpy.append(image_array[good_mask], image_array.dtype.type(fill_value))
output_array = values[i]
output_array.tofile(output_fn)
# Give the gridded product ownership of the remapped data
swath_product = swath_scene[product_name]
gridded_product = GriddedProduct()
gridded_product.from_swath_product(swath_product)
gridded_product["grid_definition"] = grid_def
gridded_product["fill_value"] = fill_value
gridded_product["grid_data"] = output_fn
# Check grid coverage
valid_points = numpy.count_nonzero(~gridded_product.get_data_mask())
grid_covered_ratio = valid_points / float(grid_def["width"] * grid_def["height"])
grid_covered = grid_covered_ratio > grid_coverage
if not grid_covered:
msg = "Nearest neighbor resampling only found %f%% of the grid covered (need %f%%) for %s" % (grid_covered_ratio * 100, grid_coverage * 100, product_name)
LOG.warning(msg)
continue
LOG.debug("Nearest neighbor resampling found %f%% of the grid covered for %s" % (grid_covered_ratio * 100, product_name))
gridded_scene[product_name] = gridded_product
# hopefully force garbage collection
del output_array
except StandardError:
LOG.debug("Remapping exception: ", exc_info=True)
LOG.error("Remapping error")
self._safe_remove(output_fn)
if self.exit_on_error:
self._clear_ll2cr_cache()
raise
continue
LOG.debug("Done running nearest neighbor on '%s'", product_name)
# Remove ll2cr files now that we are done with them
self._clear_ll2cr_cache()
if not gridded_scene:
raise RuntimeError("Nearest neighbor resampling could not remap any of the data to grid '%s'" % (grid_name,))
return gridded_scene
def _remap_scene_sensor(self, swath_scene, grid_def, **kwargs):
if not isinstance(swath_scene, Scene):
raise ValueError("'sensor' resampling only supports SatPy scenes")
new_scn = None
for area_obj, ds_list in swath_scene.iter_by_area():
_new_scn = swath_scene.resample(area_obj, datasets=ds_list)
if new_scn is None:
new_scn = _new_scn
for ds in _new_scn:
new_scn[ds.info["id"]] = ds
return new_scn
def remap_product(self, product, grid_name):
raise NotImplementedError("Single product remapping is not implemented yet")
def test_init_basic1(self):
gm = GridManager()
def test_grid_manager_basic(builtin_test_grids_conf):
"""Test basic parsing of .conf files."""
GridManager(*builtin_test_grids_conf)
