def filter_daytime(self, scene):
if self.fraction_day_scene is None:
self._calc_percent_day(scene)
# make a copy of the scene list so we can edit it later
for ds in list(scene):
if ds.attrs['standard_name'] in ('toa_bidirectional_reflectance', 'false_color', 'true_color') and \
self.fraction_day_scene <= self.day_fraction:
ds_id = DatasetID.from_dict(ds.attrs)
LOG.info("Will not create product '%s' because there is less than %f%% of day data",
ds.attrs['name'], self.day_fraction * 100.)
del scene[ds_id]
def filter_nighttime(self, scene):
if self.fraction_day_scene is None:
self._calc_percent_day(scene)
# make a copy of the scene list so we can edit it later
for ds in list(scene):
if ds.attrs['name'] in ('ifog',) and \
self.fraction_night_scene <= self.night_fraction:
ds_id = DatasetID.from_dict(ds.attrs)
LOG.info("Will not create product '%s' because there is less than %f%% of night data",
ds.attrs['name'], self.night_fraction * 100.)
del scene[ds_id]
def filter_nighttime(self, scene):
if self.fraction_day_scene is None:
self._calc_percent_day(scene)
# make a copy of the scene list so we can edit it later
for ds in list(scene):
if ds.attrs['name'] in ('ifog',) and \
self.fraction_night_scene <= self.night_fraction:
ds_id = DatasetID.from_dict(ds.attrs)
LOG.info("Will not create product '%s' because there is less than %f%% of night data",
ds.attrs['name'], self.night_fraction * 100.)
del scene[ds_id]
def filter_daytime(self, scene):
if self.fraction_day_scene is None:
self._calc_percent_day(scene)
# make a copy of the scene list so we can edit it later
for ds in list(scene):
if ds.attrs['standard_name'] in ('toa_bidirectional_reflectance',) and \
self.fraction_day_scene <= self.day_fraction:
ds_id = DatasetID.from_dict(ds.attrs)
LOG.info("Will not create product '%s' because there is less than %f%% of day data",
ds.attrs['name'], self.day_fraction * 100.)
del scene[ds_id]
def __call__(self, datasets, optional_datasets, **info):
if self.attrs['optional_prerequisites']:
for opt_dep in self.attrs['optional_prerequisites']:
if 'NOPE' in opt_dep or 'fail' in opt_dep:
continue
assert optional_datasets is not None and \
len(optional_datasets)
resolution = DatasetID.from_dict(datasets[0].attrs).resolution
if name == 'res_change' and resolution is not None:
i = datasets[0].attrs.copy()
i['resolution'] *= 5
elif 'incomp_areas' in name:
raise IncompatibleAreas(
"Test modifier 'incomp_areas' always raises IncompatibleAreas")
else:
i = datasets[0].attrs
info = datasets[0].attrs.copy()
self.apply_modifier_info(i, info)
return DataArray(np.ma.MaskedArray(datasets[0]), attrs=info)
def __call__(self, datasets, optional_datasets, **info):
if self.attrs['optional_prerequisites']:
for opt_dep in self.attrs['optional_prerequisites']:
if 'NOPE' in opt_dep or 'fail' in opt_dep:
continue
assert optional_datasets is not None and \
len(optional_datasets)
resolution = DatasetID.from_dict(datasets[0].attrs).resolution
if name == 'res_change' and resolution is not None:
i = datasets[0].attrs.copy()
i['resolution'] *= 5
elif name == 'incomp_areas':
raise IncompatibleAreas(
"Test modifier 'incomp_areas' always raises IncompatibleAreas")
else:
i = datasets[0].attrs
info = datasets[0].attrs.copy()
self.apply_modifier_info(i, info)
return DataArray(np.ma.MaskedArray(datasets[0]), attrs=info)
def _update_modifier_key(self, orig_key, dep_key):
"""Update a key based on the dataset it will modified (dep).
Typical use case is requesting a modified dataset (orig_key). This
modified dataset most likely depends on a less-modified
dataset (dep_key). The less-modified dataset must come from a reader
(at least for now) or will eventually depend on a reader dataset.
The original request key may be limited like
(wavelength=0.67, modifiers=('a', 'b')) while the reader-based key
should have all of its properties specified. This method updates the
original request key so it is fully specified and should reduce the
chance of Node's not being unique.
"""
orig_dict = orig_key._asdict()
dep_dict = dep_key._asdict()
# don't change the modifiers
for k in DATASET_KEYS[:-1]:
orig_dict[k] = dep_dict[k]
return DatasetID.from_dict(orig_dict)
def _update_modifier_key(self, orig_key, dep_key):
"""Update a key based on the dataset it will modified (dep).
Typical use case is requesting a modified dataset (orig_key). This
modified dataset most likely depends on a less-modified
dataset (dep_key). The less-modified dataset must come from a reader
(at least for now) or will eventually depend on a reader dataset.
The original request key may be limited like
(wavelength=0.67, modifiers=('a', 'b')) while the reader-based key
should have all of its properties specified. This method updates the
original request key so it is fully specified and should reduce the
chance of Node's not being unique.
"""
orig_dict = orig_key._asdict()
dep_dict = dep_key._asdict()
# don't change the modifiers
for k in DATASET_KEYS[:-1]:
orig_dict[k] = dep_dict[k]
return DatasetID.from_dict(orig_dict)
def main(argv=sys.argv[1:]):
from polar2grid.core.script_utils import setup_logging, create_basic_parser, create_exc_handler, rename_log_file, ExtendAction
from polar2grid.compositors import CompositorManager
frontends = available_frontends()
backends = available_backends()
parser = create_basic_parser(
description=
"Extract swath data, remap it, and write it to a new file format")
parser.add_argument(
"frontend",
choices=sorted(frontends.keys()),
help=
"Specify the swath extractor to use to read data (additional arguments are determined after this is specified)"
)
parser.add_argument(
"backend",
choices=sorted(backends.keys()),
help=
"Specify the backend to use to write data output (additional arguments are determined after this is specified)"
)
parser.add_argument(
"--compositor-configs",
nargs="*",
default=None,
help="Specify alternative configuration file(s) for compositors")
# don't include the help flag
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
args, remaining_args = parser.parse_known_args(argv_without_help)
glue_name = args.frontend + "2" + args.backend
LOG = logging.getLogger(glue_name)
# Load compositor information (we can't know the compositor choices until we've loaded the configuration)
compositor_manager = CompositorManager(
config_files=args.compositor_configs)
# Hack: argparse doesn't let you use choices and nargs=* on a positional argument
parser.add_argument(
"compositors",
choices=list(compositor_manager.keys()) + [[]],
nargs="*",
help=
"Specify the compositors to apply to the provided scene (additional arguments are determined after this is specified)"
)
# load the actual components we need
farg_func = get_frontend_argument_func(frontends, args.frontend)
fcls = get_frontend_class(frontends, args.frontend)
barg_func = get_backend_argument_func(backends, args.backend)
bcls = get_backend_class(backends, args.backend)
# add_frontend_arguments(parser)
subgroup_titles = []
subgroup_titles += farg_func(parser)
subgroup_titles += add_remap_argument_groups(parser)
subgroup_titles += barg_func(parser)
parser.add_argument(
'-f',
dest='data_files',
nargs="+",
default=[],
action=ExtendAction,
help="List of files or directories to extract data from")
parser.add_argument(
'-d',
dest='data_files',
nargs="+",
default=[],
action=ExtendAction,
help="Data directories to look for input data files (equivalent to -f)"
)
global_keywords = ("keep_intermediate", "overwrite_existing",
"exit_on_error")
args = parser.parse_args(argv,
global_keywords=global_keywords,
subgroup_titles=subgroup_titles)
if not args.data_files:
# FUTURE: When the -d flag is removed this won't be needed because -f will be required
parser.print_usage()
parser.exit(1, "ERROR: No data files provided (-f flag)\n")
# Logs are renamed once data the provided start date is known
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)],
log_filename=args.log_fn)
sys.excepthook = create_exc_handler(LOG.name)
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Keep track of things going wrong to tell the user what went wrong (we want to create as much as possible)
status_to_return = STATUS_SUCCESS
# Compositor validation
# XXX: Hack to make `polar2grid.sh crefl gtiff` work like legacy crefl2gtiff.sh script
if args.subgroup_args['Frontend Swath Extraction'].get('no_compositors'):
LOG.debug("Removing all compositors")
args.compositors = []
elif args.frontend == 'crefl':
if args.backend in ['awips', 'scmi']:
LOG.debug("Adding 'crefl_sharpen' compositor")
args.compositors.append('crefl_sharpen' if args.backend ==
'scmi' else 'crefl_sharpen_awips')
else:
LOG.debug("Adding 'true_color' compositor")
args.compositors.append('true_color')
if '--true-color' in sys.argv and 'true_color' not in args.compositors:
LOG.debug("Adding 'true_color' compositor")
args.compositors.append('true_color')
if '--false-color' in sys.argv and 'false_color' not in args.compositors:
LOG.debug("Adding 'false_color' compositor")
args.compositors.append('false_color')
# if "--true-color" in
for c in args.compositors:
if c not in compositor_manager:
LOG.error("Compositor '%s' is unknown" % (c, ))
raise RuntimeError("Compositor '%s' is unknown" % (c, ))
# Frontend
try:
LOG.info("Initializing reader...")
list_products = args.subgroup_args["Frontend Initialization"].pop(
"list_products")
f = fcls(search_paths=args.data_files,
**args.subgroup_args["Frontend Initialization"])
except (ValueError, KeyError):
LOG.debug("Frontend exception: ", exc_info=True)
LOG.error(
"%s frontend failed to load and sort data files (see log for details)",
args.frontend)
return STATUS_FRONTEND_FAIL
# Rename the log file
if rename_log:
rename_log_file(glue_name +
f.begin_time.strftime("_%Y%m%d_%H%M%S.log"))
if list_products:
print("\n".join(sorted(f.available_product_names)))
return STATUS_SUCCESS
try:
LOG.info("Initializing remapping...")
remapper = Remapper(**args.subgroup_args["Remapping Initialization"])
remap_kwargs = args.subgroup_args["Remapping"]
except (ValueError, KeyError):
LOG.debug("Remapping initialization exception: ", exc_info=True)
LOG.error("Remapping initialization failed (see log for details)")
return STATUS_REMAP_FAIL
try:
LOG.info("Initializing backend...")
backend = bcls(**args.subgroup_args["Backend Initialization"])
except (ValueError, KeyError):
LOG.debug("Writer initialization exception: ", exc_info=True)
LOG.error("Writer initialization failed (see log for details)")
return STATUS_BACKEND_FAIL
try:
LOG.info("Initializing compositor objects...")
compositor_objects = {}
for c in args.compositors:
compositor_objects[c] = compositor_manager.get_compositor(
c, **args.global_kwargs)
except (ValueError, KeyError):
LOG.debug("Compositor initialization exception: ", exc_info=True)
LOG.error("Compositor initialization failed (see log for details)")
return STATUS_COMP_FAIL
try:
LOG.info("Extracting swaths from data files available...")
scene = f.create_scene(
**args.subgroup_args["Frontend Swath Extraction"])
# Determine if we have a satpy scene if we should convert it to
# a P2G Scene to continue processing
resample_method = args.subgroup_args["Remapping"].get("remap_method")
is_satpy_resample_method = resample_method in SATPY_RESAMPLERS
if is_satpy_resample_method and not isinstance(scene, Scene):
raise RuntimeError(
"Resampling method '{}' only supports 'satpy' readers".format(
resample_method))
elif not is_satpy_resample_method and isinstance(scene, Scene):
# convert satpy scene to P2G Scene to be compatible with old P2G resamplers
scene = convert_satpy_to_p2g_swath(f, scene)
if isinstance(scene, Scene):
if not scene.datasets:
LOG.error("No products were returned by the frontend")
raise RuntimeError("No products were returned by the frontend")
if args.keep_intermediate:
raise RuntimeError(
"satpy readers do not currently support saving intermediate files"
)
else:
if (isinstance(scene, Scene) and not scene.datasets) or not scene:
LOG.error("No products were returned by the frontend")
raise RuntimeError("No products were returned by the frontend")
if args.keep_intermediate:
filename = glue_name + "_swath_scene.json"
LOG.info("Saving intermediate swath scene as '%s'", filename)
scene.save(filename)
except (ValueError, KeyError):
LOG.debug("Frontend data extraction exception: ", exc_info=True)
LOG.error("Frontend data extraction failed (see log for details)")
return STATUS_FRONTEND_FAIL
# What grids should we remap to (the user should tell us or the backend should have a good set of defaults)
known_grids = backend.known_grids
LOG.debug("Writer known grids: %r", known_grids)
grids = remap_kwargs.pop("forced_grids", None)
LOG.debug("Forced Grids: %r", grids)
if resample_method == "sensor" and grids != ["sensor"]:
LOG.error("'sensor' resampling method only supports the 'sensor' grid")
return STATUS_GDETER_FAIL
if not grids and not known_grids:
# the user didn't ask for any grids and the backend doesn't have specific defaults
LOG.error("No grids specified and no known defaults")
return STATUS_GDETER_FAIL
elif not grids:
# the user didn't tell us what to do, so let's try everything the backend knows how to do
grids = known_grids
elif known_grids is not None:
# the user told us what to do, let's make sure the backend can do it
grids = list(set(grids) & set(known_grids))
if not grids:
LOG.error(
"%s backend doesn't know how to handle any of the grids specified",
args.backend)
return STATUS_GDETER_FAIL
LOG.debug("Grids that will be mapped to: %r", grids)
# Remap
for grid_name in grids:
LOG.info("Remapping to grid %s", grid_name)
try:
gridded_scene = remapper.remap_scene(scene, grid_name,
**remap_kwargs)
if args.keep_intermediate:
filename = glue_name + "_gridded_scene_" + grid_name + ".json"
LOG.debug("saving intermediate gridded scene as '%s'",
filename)
gridded_scene.save(filename)
except (ValueError, KeyError):
LOG.debug("Remapping data exception: ", exc_info=True)
LOG.error("Remapping data failed")
status_to_return |= STATUS_REMAP_FAIL
if args.exit_on_error:
return status_to_return
continue
if not isinstance(scene, Scene):
# Composition
for c, comp in compositor_objects.items():
try:
LOG.info("Running gridded scene through '%s' compositor",
c)
gridded_scene = comp.modify_scene(
gridded_scene,
**args.subgroup_args[c + " Modification"])
if args.keep_intermediate:
filename = glue_name + "_gridded_scene_" + grid_name + ".json"
LOG.debug(
"Updating saved intermediate gridded scene (%s) after compositor",
filename)
gridded_scene.save(filename)
except (KeyError, ValueError):
LOG.debug("Compositor Error: ", exc_info=True)
LOG.error(
"Could not properly modify scene using compositor '%s'"
% (c, ))
if args.exit_on_error:
raise RuntimeError(
"Could not properly modify scene using compositor '%s'"
% (c, ))
if isinstance(f,
ReaderWrapper) and not isinstance(gridded_scene, Scene):
this_grid_definition = None
# HACK: Create SatPy composites that were either separated before
# resampling or needed resampling to be created
rgbs = {}
for product_name in gridded_scene.keys():
rgb_name = product_name[:-6]
# Keep track of one of the grid definitions
if this_grid_definition is None:
this_grid_definition = gridded_scene[product_name][
"grid_definition"]
if product_name.endswith("rgb_0") or product_name.endswith(
"rgb_1") or product_name.endswith("rgb_2"):
if rgb_name not in rgbs:
rgbs[rgb_name] = [None, None, None]
chn_idx = int(product_name[-1])
rgbs[rgb_name][chn_idx] = product_name
LOG.debug("Putting RGBs back together again")
for rgb_name, v in rgbs.items():
r = gridded_scene.pop(v[0])
g = gridded_scene.pop(v[1])
b = gridded_scene.pop(v[2])
new_info = r.copy()
new_info["grid_data"] = new_info["grid_data"].replace(
v[0], rgb_name)
new_info["product_name"] = rgb_name
data = np.memmap(new_info["grid_data"],
dtype=new_info["data_type"],
mode="w+",
shape=(3,
new_info["grid_definition"]["height"],
new_info["grid_definition"]["width"]))
data[0] = r.get_data_array()[:]
data[1] = g.get_data_array()[:]
data[2] = b.get_data_array()[:]
gridded_scene[rgb_name] = new_info
del data, new_info
# Create composites that satpy couldn't complete until after remapping
composite_names = [
x for x in f.wishlist if not isinstance(x, DatasetID)
]
if composite_names:
tmp_scene = Scene()
for k, v in gridded_scene.items():
if not isinstance(v["sensor"], set):
v["sensor"] = set([
v["sensor"]
]) # turn sensor back in to a set to match satpy usage
tmp_scene[v["id"]] = DataArray(v.get_data_array(), attrs=v)
tmp_scene[v["id"]].attrs[
"area"] = this_grid_definition.to_satpy_area()
# tmp_scene[v["id"]].info = {}
if v["sensor"] not in tmp_scene.attrs["sensor"]:
tmp_scene.attrs["sensor"].extend(v["sensor"])
# Overwrite the wishlist that will include the above assigned datasets
tmp_scene.wishlist = f.wishlist
for cname in composite_names:
tmp_scene.compositors[
cname] = tmp_scene.cpl.load_compositor(
cname, tmp_scene.attrs["sensor"])
tmp_scene.compute()
tmp_scene.unload()
# Add any new Datasets to our P2G Scene if SatPy created them
for ds in tmp_scene:
ds_id = DatasetID.from_dict(ds.attrs)
if ds_id.name not in gridded_scene:
LOG.debug("Adding Dataset from SatPy Commpositing: %s",
ds_id)
gridded_scene[
ds_id.name] = dataarray_to_gridded_product(ds)
gridded_scene[ds_id.name][
"grid_definition"] = this_grid_definition
# Remove any Products from P2G Scene that SatPy decided it didn't need anymore
for k, v in list(gridded_scene.items()):
if v["id"].name not in tmp_scene:
LOG.debug(
"Removing Dataset that is no longer used: %s", k)
del gridded_scene[k]
del tmp_scene, v
if isinstance(gridded_scene, Scene):
LOG.debug("Converting satpy Scene to P2G Gridded Scene")
# Convert it to P2G Gridded Scene
gridded_scene = convert_satpy_to_p2g_gridded(f, gridded_scene)
# Writer
try:
LOG.info("Creating output from data mapped to grid %s", grid_name)
backend.create_output_from_scene(
gridded_scene, **args.subgroup_args["Backend Output Creation"])
except (ValueError, KeyError):
LOG.debug("Writer output creation exception: ", exc_info=True)
LOG.error("Writer output creation failed (see log for details)")
status_to_return |= STATUS_BACKEND_FAIL
if args.exit_on_error:
return status_to_return
continue
LOG.info("Processing data for grid %s complete", grid_name)
# Force deletion and eventual garbage collection of the scene objects
del gridded_scene
del scene
return status_to_return
def main(argv=sys.argv[1:]):
from polar2grid.core.script_utils import setup_logging, create_basic_parser, create_exc_handler, rename_log_file, ExtendAction
from polar2grid.compositors import CompositorManager
frontends = available_frontends()
backends = available_backends()
parser = create_basic_parser(description="Extract swath data, remap it, and write it to a new file format")
parser.add_argument("frontend", choices=sorted(frontends.keys()),
help="Specify the swath extractor to use to read data (additional arguments are determined after this is specified)")
parser.add_argument("backend", choices=sorted(backends.keys()),
help="Specify the backend to use to write data output (additional arguments are determined after this is specified)")
parser.add_argument("--compositor-configs", nargs="*", default=None,
help="Specify alternative configuration file(s) for compositors")
# don't include the help flag
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
args, remaining_args = parser.parse_known_args(argv_without_help)
glue_name = args.frontend + "2" + args.backend
LOG = logging.getLogger(glue_name)
# Load compositor information (we can't know the compositor choices until we've loaded the configuration)
compositor_manager = CompositorManager(config_files=args.compositor_configs)
# Hack: argparse doesn't let you use choices and nargs=* on a positional argument
parser.add_argument("compositors", choices=list(compositor_manager.keys()) + [[]], nargs="*",
help="Specify the compositors to apply to the provided scene (additional arguments are determined after this is specified)")
# load the actual components we need
farg_func = get_frontend_argument_func(frontends, args.frontend)
fcls = get_frontend_class(frontends, args.frontend)
barg_func = get_backend_argument_func(backends, args.backend)
bcls = get_backend_class(backends, args.backend)
# add_frontend_arguments(parser)
subgroup_titles = []
subgroup_titles += farg_func(parser)
subgroup_titles += add_remap_argument_groups(parser)
subgroup_titles += barg_func(parser)
parser.add_argument('-f', dest='data_files', nargs="+", default=[], action=ExtendAction,
help="List of files or directories to extract data from")
parser.add_argument('-d', dest='data_files', nargs="+", default=[], action=ExtendAction,
help="Data directories to look for input data files (equivalent to -f)")
global_keywords = ("keep_intermediate", "overwrite_existing", "exit_on_error")
args = parser.parse_args(argv, global_keywords=global_keywords, subgroup_titles=subgroup_titles)
if not args.data_files:
# FUTURE: When the -d flag is removed this won't be needed because -f will be required
parser.print_usage()
parser.exit(1, "ERROR: No data files provided (-f flag)\n")
# Logs are renamed once data the provided start date is known
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn)
sys.excepthook = create_exc_handler(LOG.name)
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Keep track of things going wrong to tell the user what went wrong (we want to create as much as possible)
status_to_return = STATUS_SUCCESS
# Compositor validation
# XXX: Hack to make `polar2grid.sh crefl gtiff` work like legacy crefl2gtiff.sh script
if args.subgroup_args['Frontend Swath Extraction'].get('no_compositors'):
LOG.debug("Removing all compositors")
args.compositors = []
elif args.frontend == 'crefl':
if args.backend in ['awips', 'scmi']:
LOG.debug("Adding 'crefl_sharpen' compositor")
args.compositors.append('crefl_sharpen' if args.backend == 'scmi' else 'crefl_sharpen_awips')
else:
LOG.debug("Adding 'true_color' compositor")
args.compositors.append('true_color')
if '--true-color' in sys.argv and 'true_color' not in args.compositors:
LOG.debug("Adding 'true_color' compositor")
args.compositors.append('true_color')
if '--false-color' in sys.argv and 'false_color' not in args.compositors:
LOG.debug("Adding 'false_color' compositor")
args.compositors.append('false_color')
# if "--true-color" in
for c in args.compositors:
if c not in compositor_manager:
LOG.error("Compositor '%s' is unknown" % (c,))
raise RuntimeError("Compositor '%s' is unknown" % (c,))
# Frontend
try:
LOG.info("Initializing reader...")
list_products = args.subgroup_args["Frontend Initialization"].pop("list_products")
f = fcls(search_paths=args.data_files, **args.subgroup_args["Frontend Initialization"])
except (ValueError, KeyError):
LOG.debug("Frontend exception: ", exc_info=True)
LOG.error("%s frontend failed to load and sort data files (see log for details)", args.frontend)
return STATUS_FRONTEND_FAIL
# Rename the log file
if rename_log:
rename_log_file(glue_name + f.begin_time.strftime("_%Y%m%d_%H%M%S.log"))
if list_products:
print("\n".join(sorted(f.available_product_names)))
return STATUS_SUCCESS
try:
LOG.info("Initializing remapping...")
remapper = Remapper(**args.subgroup_args["Remapping Initialization"])
remap_kwargs = args.subgroup_args["Remapping"]
except (ValueError, KeyError):
LOG.debug("Remapping initialization exception: ", exc_info=True)
LOG.error("Remapping initialization failed (see log for details)")
return STATUS_REMAP_FAIL
try:
LOG.info("Initializing backend...")
backend = bcls(**args.subgroup_args["Backend Initialization"])
except (ValueError, KeyError):
LOG.debug("Writer initialization exception: ", exc_info=True)
LOG.error("Writer initialization failed (see log for details)")
return STATUS_BACKEND_FAIL
try:
LOG.info("Initializing compositor objects...")
compositor_objects = {}
for c in args.compositors:
compositor_objects[c] = compositor_manager.get_compositor(c, **args.global_kwargs)
except (ValueError, KeyError):
LOG.debug("Compositor initialization exception: ", exc_info=True)
LOG.error("Compositor initialization failed (see log for details)")
return STATUS_COMP_FAIL
try:
LOG.info("Extracting swaths from data files available...")
scene = f.create_scene(**args.subgroup_args["Frontend Swath Extraction"])
# Determine if we have a satpy scene if we should convert it to
# a P2G Scene to continue processing
resample_method = args.subgroup_args["Remapping"].get("remap_method")
is_satpy_resample_method = resample_method in SATPY_RESAMPLERS
if is_satpy_resample_method and not isinstance(scene, Scene):
raise RuntimeError("Resampling method '{}' only supports 'satpy' readers".format(resample_method))
elif not is_satpy_resample_method and isinstance(scene, Scene):
# convert satpy scene to P2G Scene to be compatible with old P2G resamplers
scene = convert_satpy_to_p2g_swath(f, scene)
if isinstance(scene, Scene):
if not scene.datasets:
LOG.error("No products were returned by the frontend")
raise RuntimeError("No products were returned by the frontend")
if args.keep_intermediate:
raise RuntimeError("satpy readers do not currently support saving intermediate files")
else:
if (isinstance(scene, Scene) and not scene.datasets) or not scene:
LOG.error("No products were returned by the frontend")
raise RuntimeError("No products were returned by the frontend")
if args.keep_intermediate:
filename = glue_name + "_swath_scene.json"
LOG.info("Saving intermediate swath scene as '%s'", filename)
scene.save(filename)
except (ValueError, KeyError):
LOG.debug("Frontend data extraction exception: ", exc_info=True)
LOG.error("Frontend data extraction failed (see log for details)")
return STATUS_FRONTEND_FAIL
# What grids should we remap to (the user should tell us or the backend should have a good set of defaults)
known_grids = backend.known_grids
LOG.debug("Writer known grids: %r", known_grids)
grids = remap_kwargs.pop("forced_grids", None)
LOG.debug("Forced Grids: %r", grids)
if resample_method == "sensor" and grids != ["sensor"]:
LOG.error("'sensor' resampling method only supports the 'sensor' grid")
return STATUS_GDETER_FAIL
if not grids and not known_grids:
# the user didn't ask for any grids and the backend doesn't have specific defaults
LOG.error("No grids specified and no known defaults")
return STATUS_GDETER_FAIL
elif not grids:
# the user didn't tell us what to do, so let's try everything the backend knows how to do
grids = known_grids
elif known_grids is not None:
# the user told us what to do, let's make sure the backend can do it
grids = list(set(grids) & set(known_grids))
if not grids:
LOG.error("%s backend doesn't know how to handle any of the grids specified", args.backend)
return STATUS_GDETER_FAIL
LOG.debug("Grids that will be mapped to: %r", grids)
# Remap
for grid_name in grids:
LOG.info("Remapping to grid %s", grid_name)
try:
gridded_scene = remapper.remap_scene(scene, grid_name, **remap_kwargs)
if args.keep_intermediate:
filename = glue_name + "_gridded_scene_" + grid_name + ".json"
LOG.debug("saving intermediate gridded scene as '%s'", filename)
gridded_scene.save(filename)
except (ValueError, KeyError):
LOG.debug("Remapping data exception: ", exc_info=True)
LOG.error("Remapping data failed")
status_to_return |= STATUS_REMAP_FAIL
if args.exit_on_error:
return status_to_return
continue
if not isinstance(scene, Scene):
# Composition
for c, comp in compositor_objects.items():
try:
LOG.info("Running gridded scene through '%s' compositor", c)
gridded_scene = comp.modify_scene(gridded_scene, **args.subgroup_args[c + " Modification"])
if args.keep_intermediate:
filename = glue_name + "_gridded_scene_" + grid_name + ".json"
LOG.debug("Updating saved intermediate gridded scene (%s) after compositor", filename)
gridded_scene.save(filename)
except (KeyError, ValueError):
LOG.debug("Compositor Error: ", exc_info=True)
LOG.error("Could not properly modify scene using compositor '%s'" % (c,))
if args.exit_on_error:
raise RuntimeError("Could not properly modify scene using compositor '%s'" % (c,))
if isinstance(f, ReaderWrapper) and not isinstance(gridded_scene, Scene):
this_grid_definition = None
# HACK: Create SatPy composites that were either separated before
# resampling or needed resampling to be created
rgbs = {}
for product_name in gridded_scene.keys():
rgb_name = product_name[:-6]
# Keep track of one of the grid definitions
if this_grid_definition is None:
this_grid_definition = gridded_scene[product_name]["grid_definition"]
if product_name.endswith("rgb_0") or product_name.endswith("rgb_1") or product_name.endswith("rgb_2"):
if rgb_name not in rgbs:
rgbs[rgb_name] = [None, None, None]
chn_idx = int(product_name[-1])
rgbs[rgb_name][chn_idx] = product_name
LOG.debug("Putting RGBs back together again")
for rgb_name, v in rgbs.items():
r = gridded_scene.pop(v[0])
g = gridded_scene.pop(v[1])
b = gridded_scene.pop(v[2])
new_info = r.copy()
new_info["grid_data"] = new_info["grid_data"].replace(v[0], rgb_name)
new_info["product_name"] = rgb_name
data = np.memmap(new_info["grid_data"], dtype=new_info["data_type"],
mode="w+", shape=(3, new_info["grid_definition"]["height"], new_info["grid_definition"]["width"]))
data[0] = r.get_data_array()[:]
data[1] = g.get_data_array()[:]
data[2] = b.get_data_array()[:]
gridded_scene[rgb_name] = new_info
del data, new_info
# Create composites that satpy couldn't complete until after remapping
composite_names = [x for x in f.wishlist if not isinstance(x, DatasetID)]
if composite_names:
tmp_scene = Scene()
for k, v in gridded_scene.items():
if not isinstance(v["sensor"], set):
v["sensor"] = set([v["sensor"]]) # turn sensor back in to a set to match satpy usage
tmp_scene[v["id"]] = DataArray(v.get_data_array(), attrs=v)
tmp_scene[v["id"]].attrs["area"] = this_grid_definition.to_satpy_area()
# tmp_scene[v["id"]].info = {}
if v["sensor"] not in tmp_scene.attrs["sensor"]:
tmp_scene.attrs["sensor"].extend(v["sensor"])
# Overwrite the wishlist that will include the above assigned datasets
tmp_scene.wishlist = f.wishlist
for cname in composite_names:
tmp_scene.compositors[cname] = tmp_scene.cpl.load_compositor(cname, tmp_scene.attrs["sensor"])
tmp_scene.compute()
tmp_scene.unload()
# Add any new Datasets to our P2G Scene if SatPy created them
for ds in tmp_scene:
ds_id = DatasetID.from_dict(ds.attrs)
if ds_id.name not in gridded_scene:
LOG.debug("Adding Dataset from SatPy Commpositing: %s", ds_id)
gridded_scene[ds_id.name] = dataarray_to_gridded_product(ds)
gridded_scene[ds_id.name]["grid_definition"] = this_grid_definition
# Remove any Products from P2G Scene that SatPy decided it didn't need anymore
for k, v in list(gridded_scene.items()):
if v["id"].name not in tmp_scene:
LOG.debug("Removing Dataset that is no longer used: %s", k)
del gridded_scene[k]
del tmp_scene, v
if isinstance(gridded_scene, Scene):
LOG.debug("Converting satpy Scene to P2G Gridded Scene")
# Convert it to P2G Gridded Scene
gridded_scene = convert_satpy_to_p2g_gridded(f, gridded_scene)
# Writer
try:
LOG.info("Creating output from data mapped to grid %s", grid_name)
backend.create_output_from_scene(gridded_scene, **args.subgroup_args["Backend Output Creation"])
except (ValueError, KeyError):
LOG.debug("Writer output creation exception: ", exc_info=True)
LOG.error("Writer output creation failed (see log for details)")
status_to_return |= STATUS_BACKEND_FAIL
if args.exit_on_error:
return status_to_return
continue
LOG.info("Processing data for grid %s complete", grid_name)
# Force deletion and eventual garbage collection of the scene objects
del gridded_scene
del scene
return status_to_return
