def test_partial_filter(viirs_sdr_i01_scene): """Test that resampling completes even when coverage filters some datasets.""" # make another DataArray that is shifted away from the target area # orig_lons, orig_lats = viirs_sdr_i01_scene['I01'].attrs['area'].get_lonlats() new_i01 = viirs_sdr_i01_scene["I01"].copy() orig_lons = new_i01.attrs["area"].lons orig_lats = new_i01.attrs["area"].lats new_lons = orig_lons + 180.0 new_swath_def = SwathDefinition(new_lons, orig_lats) new_i01.attrs["name"] = "I01_2" new_i01.attrs["area"] = new_swath_def new_scn = Scene() new_scn["I01"] = viirs_sdr_i01_scene["I01"] new_scn["I01_2"] = new_i01 with dask.config.set(scheduler=CustomScheduler(2)): scenes_to_save = resample_scene( new_scn, ["211e"], ["grids.conf"], None, is_polar2grid=True, grid_coverage=0.05, ) assert len(scenes_to_save) == 1 new_scn, data_ids = scenes_to_save[0] assert len(new_scn.keys()) == 1 # I01
def filter_scene(self, scene: Scene): """Create a new Scene with filtered DataArrays removed.""" _cache = {} remaining_ids = [] filtered_ids = [] for data_id in scene.keys(): logger.debug("Analyzing '{}' for filtering...".format(data_id)) if not self._filter_data_array(scene[data_id], _cache): remaining_ids.append(data_id) else: logger.debug(self.FILTER_MSG.format(data_id)) filtered_ids.append(data_id) if not remaining_ids: return None new_scn = scene.copy(remaining_ids) new_scn._wishlist = scene.wishlist.copy() return new_scn
def apply_p2g_name_to_scene( self, scn: Scene, ): """Assign a new 'p2g_name' string attribute to every DataArray. This is typically done just before writing the 'Scene' output to disk so that the `filename` pattern used in `Scene.save_datasets` has access to the user-facing "P2G name" rather than the internal Satpy name or DataID. """ all_ids = list(scn.keys()) all_p2g_names = list(self.convert_satpy_to_p2g_name(all_ids)) for data_id, p2g_name in zip(all_ids, all_p2g_names): if p2g_name is None: # the Satpy ID doesn't have a Polar2Grid compatible name logger.debug("Satpy DataID %s does not have a compatible polar2grid name.", data_id) continue scn[data_id].attrs["p2g_name"] = p2g_name logger.debug("Mapping Satpy ID to P2G name: %s -> %s", data_id, p2g_name)
global_scene = Scene(filenames=files) global_scene.available_dataset_names() #!!# print(global_scene['overview']) ### this one does only work in the develop version print("") print("available_composite_names") print(global_scene.available_composite_names()) if make_images: # this will load RGBs ready to plot global_scene.load(nwcsaf.product[p_]) #global_scene.load([ 'cloud_top_height', 'cloud_top_pressure', 'cloud_top_temperature']) #global_scene.load(['cloudtype']) print("global_scene.keys()", global_scene.keys()) #[DatasetID(name='cloud_top_height', wavelength=None, resolution=None, polarization=None, calibration=None, level=None, modifiers=None)] # 'cloud_top_height' is loaded in RGB mode already #print(global_scene['cloud_top_height'].data.shape) #print(global_scene['cloud_top_height'].data.compute()) #print(global_scene['cloud_top_height'].values) else: # work with scientific data for p__ in nwcsaf.product[p_]: global_scene.load(product_ids[p__]) #global_scene.load(['ctth_alti']) print("") print("global_scene")
def main(argv=sys.argv[1:]): global LOG from satpy import Scene from satpy.resample import get_area_def from satpy.writers import compute_writer_results from dask.diagnostics import ProgressBar from polar2grid.core.script_utils import (setup_logging, rename_log_file, create_exc_handler) import argparse prog = os.getenv('PROG_NAME', sys.argv[0]) # "usage: " will be printed at the top of this: usage = """ %(prog)s -h see available products: %(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...] basic processing: %(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...] basic processing with limited products: %(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...] """ parser = argparse.ArgumentParser( prog=prog, usage=usage, fromfile_prefix_chars="@", description="Load, composite, resample, and save datasets.") parser.add_argument( '-v', '--verbose', dest='verbosity', action="count", default=0, help= 'each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)' ) parser.add_argument('-l', '--log', dest="log_fn", default=None, help="specify the log filename") parser.add_argument( '--progress', action='store_true', help="show processing progress bar (not recommended for logged output)" ) parser.add_argument( '--num-workers', type=int, default=os.getenv('DASK_NUM_WORKERS', 4), help="specify number of worker threads to use (default: 4)") parser.add_argument( '--match-resolution', dest='preserve_resolution', action='store_false', help="When using the 'native' resampler for composites, don't save data " "at its native resolution, use the resolution used to create the " "composite.") parser.add_argument('-w', '--writers', nargs='+', help='writers to save datasets with') parser.add_argument("--list-products", dest="list_products", action="store_true", help="List available reader products and exit") subgroups = add_scene_argument_groups(parser) subgroups += add_resample_argument_groups(parser) argv_without_help = [x for x in argv if x not in ["-h", "--help"]] args, remaining_args = parser.parse_known_args(argv_without_help) os.environ['DASK_NUM_WORKERS'] = str(args.num_workers) # get the logger if we know the readers and writers that will be used if args.reader is not None and args.writers is not None: glue_name = args.reader + "_" + "-".join(args.writers or []) LOG = logging.getLogger(glue_name) # add writer arguments if args.writers is not None: for writer in (args.writers or []): parser_func = WRITER_PARSER_FUNCTIONS.get(writer) if parser_func is None: continue subgroups += parser_func(parser) args = parser.parse_args(argv) if args.reader is None: parser.print_usage() parser.exit( 1, "\nERROR: Reader must be provided (-r flag).\n" "Supported readers:\n\t{}\n".format('\n\t'.join( ['abi_l1b', 'ahi_hsd', 'hrit_ahi']))) if args.writers is None: parser.print_usage() parser.exit( 1, "\nERROR: Writer must be provided (-w flag) with one or more writer.\n" "Supported writers:\n\t{}\n".format('\n\t'.join(['geotiff']))) def _args_to_dict(group_actions): return { ga.dest: getattr(args, ga.dest) for ga in group_actions if hasattr(args, ga.dest) } scene_args = _args_to_dict(subgroups[0]._group_actions) load_args = _args_to_dict(subgroups[1]._group_actions) resample_args = _args_to_dict(subgroups[2]._group_actions) writer_args = {} for idx, writer in enumerate(args.writers): sgrp1, sgrp2 = subgroups[3 + idx * 2:5 + idx * 2] wargs = _args_to_dict(sgrp1._group_actions) if sgrp2 is not None: wargs.update(_args_to_dict(sgrp2._group_actions)) writer_args[writer] = wargs # get default output filename if 'filename' in wargs and wargs['filename'] is None: wargs['filename'] = get_default_output_filename( args.reader, writer) if not args.filenames: parser.print_usage() parser.exit(1, "\nERROR: No data files provided (-f flag)\n") # Prepare logging rename_log = False if args.log_fn is None: rename_log = True args.log_fn = glue_name + "_fail.log" levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn) logging.getLogger('rasterio').setLevel(levels[min(2, args.verbosity)]) sys.excepthook = create_exc_handler(LOG.name) if levels[min(3, args.verbosity)] > logging.DEBUG: import warnings warnings.filterwarnings("ignore") LOG.debug("Starting script with arguments: %s", " ".join(sys.argv)) # Set up dask and the number of workers if args.num_workers: from multiprocessing.pool import ThreadPool dask.config.set(pool=ThreadPool(args.num_workers)) # Parse provided files and search for files if provided directories scene_args['filenames'] = get_input_files(scene_args['filenames']) # Create a Scene, analyze the provided files LOG.info("Sorting and reading input files...") try: scn = Scene(**scene_args) except ValueError as e: LOG.error( "{} | Enable debug message (-vvv) or see log file for details.". format(str(e))) LOG.debug("Further error information: ", exc_info=True) return -1 except OSError: LOG.error( "Could not open files. Enable debug message (-vvv) or see log file for details." ) LOG.debug("Further error information: ", exc_info=True) return -1 if args.list_products: print("\n".join(sorted(scn.available_dataset_names(composites=True)))) return 0 # Rename the log file if rename_log: rename_log_file(glue_name + scn.attrs['start_time'].strftime("_%Y%m%d_%H%M%S.log")) # Load the actual data arrays and metadata (lazy loaded as dask arrays) if load_args['products'] is None: try: reader_mod = importlib.import_module('polar2grid.readers.' + scene_args['reader']) load_args['products'] = reader_mod.DEFAULT_PRODUCTS LOG.info("Using default product list: {}".format( load_args['products'])) except (ImportError, AttributeError): LOG.error( "No default products list set, please specify with `--products`." ) return -1 LOG.info("Loading product metadata from files...") scn.load(load_args['products']) resample_kwargs = resample_args.copy() areas_to_resample = resample_kwargs.pop('grids') grid_configs = resample_kwargs.pop('grid_configs') resampler = resample_kwargs.pop('resampler') if areas_to_resample is None and resampler in [None, 'native']: # no areas specified areas_to_resample = ['MAX'] elif areas_to_resample is None: raise ValueError( "Resampling method specified (--method) without any destination grid/area (-g flag)." ) elif not areas_to_resample: # they don't want any resampling (they used '-g' with no args) areas_to_resample = [None] p2g_grid_configs = [x for x in grid_configs if x.endswith('.conf')] pyresample_area_configs = [ x for x in grid_configs if not x.endswith('.conf') ] if not grid_configs or p2g_grid_configs: # if we were given p2g grid configs or we weren't given any to choose from from polar2grid.grids import GridManager grid_manager = GridManager(*p2g_grid_configs) else: grid_manager = {} if pyresample_area_configs: from pyresample.utils import parse_area_file custom_areas = parse_area_file(pyresample_area_configs) custom_areas = {x.area_id: x for x in custom_areas} else: custom_areas = {} ll_bbox = resample_kwargs.pop('ll_bbox') if ll_bbox: scn = scn.crop(ll_bbox=ll_bbox) wishlist = scn.wishlist.copy() preserve_resolution = get_preserve_resolution(args, resampler, areas_to_resample) if preserve_resolution: preserved_products = set(wishlist) & set(scn.keys()) resampled_products = set(wishlist) - preserved_products # original native scene to_save = write_scene(scn, args.writers, writer_args, preserved_products) else: preserved_products = set() resampled_products = set(wishlist) to_save = [] LOG.debug( "Products to preserve resolution for: {}".format(preserved_products)) LOG.debug( "Products to use new resolution for: {}".format(resampled_products)) for area_name in areas_to_resample: if area_name is None: # no resampling area_def = None elif area_name == 'MAX': area_def = scn.max_area() elif area_name == 'MIN': area_def = scn.min_area() elif area_name in custom_areas: area_def = custom_areas[area_name] elif area_name in grid_manager: p2g_def = grid_manager[area_name] area_def = p2g_def.to_satpy_area() if isinstance(area_def, DynamicAreaDefinition ) and p2g_def['cell_width'] is not None: area_def = area_def.freeze( scn.max_area(), resolution=(abs(p2g_def['cell_width']), abs(p2g_def['cell_height']))) else: area_def = get_area_def(area_name) if resampler is None and area_def is not None: rs = 'native' if area_name in ['MIN', 'MAX'] or is_native_grid( area_def, scn.max_area()) else 'nearest' LOG.debug( "Setting default resampling to '{}' for grid '{}'".format( rs, area_name)) else: rs = resampler if area_def is not None: LOG.info("Resampling data to '%s'", area_name) new_scn = scn.resample(area_def, resampler=rs, **resample_kwargs) elif not preserve_resolution: # the user didn't want to resample to any areas # the user also requested that we don't preserve resolution # which means we have to save this Scene's datasets # because they won't be saved new_scn = scn to_save = write_scene(new_scn, args.writers, writer_args, resampled_products, to_save=to_save) if args.progress: pbar = ProgressBar() pbar.register() LOG.info("Computing products and saving data to writers...") compute_writer_results(to_save) LOG.info("SUCCESS") return 0