def test_scene_load_available_datasets(self): """Test that all datasets are available.""" from satpy import Scene fname = os.path.join(self.base_dir, FILENAME) scn = Scene(reader='ascat_l2_soilmoisture_bufr', filenames=[fname]) self.assertTrue( 'surface_soil_moisture' in scn.available_dataset_names()) scn.load(scn.available_dataset_names()) loaded = [dataset.name for dataset in scn] self.assertTrue( sorted(loaded) == sorted(scn.available_dataset_names()))
def test_scene_available_datasets(self, input_files, expected_names, expected_data_res, expected_geo_res): """Test that datasets are available.""" scene = Scene(reader='modis_l1b', filenames=input_files) available_datasets = scene.available_dataset_names() assert len(available_datasets) > 0 assert 'longitude' in available_datasets assert 'latitude' in available_datasets for chan_name in expected_names: assert chan_name in available_datasets available_data_ids = scene.available_dataset_ids() available_datas = {x: [] for x in expected_data_res} available_geos = {x: [] for x in expected_geo_res} # Make sure that every resolution from the reader is what we expect for data_id in available_data_ids: res = data_id['resolution'] if data_id['name'] in ['longitude', 'latitude']: assert res in expected_geo_res available_geos[res].append(data_id) else: assert res in expected_data_res available_datas[res].append(data_id) # Make sure that every resolution we expect has at least one dataset for exp_res, avail_id in available_datas.items(): assert avail_id, f"Missing datasets for data resolution {exp_res}" for exp_res, avail_id in available_geos.items(): assert avail_id, f"Missing geo datasets for geo resolution {exp_res}"
def test_scene_dataset_values(self): """Test loading data.""" from satpy import Scene fname = os.path.join(self.base_dir, FILENAME) scn = Scene(reader='iasi_l2_so2_bufr', filenames=[fname]) for name in scn.available_dataset_names(): scn.load([name]) loaded_values = scn.datasets[name].values fill_value = scn.datasets[name].attrs['fill_value'] # replace nans in data loaded from file with the fill value defined in the .yaml # to make them comparable loaded_values_nan_filled = np.nan_to_num(loaded_values, nan=fill_value) key = scn.datasets[name].attrs['key'] original_values = msg[key] # this makes each assertion below a separate test from unittest's point of view # (note: if all subtests pass, they will count as one test) with self.subTest(msg="Test failed for dataset: " + name): self.assertTrue( np.allclose(original_values, loaded_values_nan_filled))
def test_scene_load_available_datasets(self): """Test that all datasets are available""" from satpy import Scene fname = os.path.join(self.base_dir, FNAME) scn = Scene(reader='iasi_l2', filenames=[ fname, ]) scn.load(scn.available_dataset_names())
#else: # print ("NWCSAF data product " + product + " is missing or is not readable") print("... search files in " + base_dir) files_nwc = find_files_and_readers(sensor='seviri', start_time=start_time, end_time=end_time, base_dir=base_dir, reader='nwcsaf-geo') #print (files_nwc) #files = dict(files_sat.items() + files_nwc.items()) files = dict(list(files_nwc.items())) global_scene = Scene(filenames=files) global_scene.available_dataset_names() #!!# print(global_scene['overview']) ### this one does only work in the develop version print("") print("available_composite_names") print(global_scene.available_composite_names()) if make_images: # this will load RGBs ready to plot global_scene.load(nwcsaf.product[p_]) #global_scene.load([ 'cloud_top_height', 'cloud_top_pressure', 'cloud_top_temperature']) #global_scene.load(['cloudtype']) print("global_scene.keys()", global_scene.keys()) #[DatasetID(name='cloud_top_height', wavelength=None, resolution=None, polarization=None, calibration=None, level=None, modifiers=None)]
def main(argv=sys.argv[1:]): global LOG from satpy import Scene from satpy.resample import get_area_def from satpy.writers import compute_writer_results from dask.diagnostics import ProgressBar from polar2grid.core.script_utils import ( setup_logging, rename_log_file, create_exc_handler) import argparse prog = os.getenv('PROG_NAME', sys.argv[0]) # "usage: " will be printed at the top of this: usage = """ %(prog)s -h see available products: %(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...] basic processing: %(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...] basic processing with limited products: %(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...] """ parser = argparse.ArgumentParser(prog=prog, usage=usage, description="Load, composite, resample, and save datasets.") parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0, help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)') parser.add_argument('-l', '--log', dest="log_fn", default=None, help="specify the log filename") parser.add_argument('--progress', action='store_true', help="show processing progress bar (not recommended for logged output)") parser.add_argument('--num-workers', type=int, default=4, help="specify number of worker threads to use (default: 4)") parser.add_argument('--match-resolution', dest='preserve_resolution', action='store_false', help="When using the 'native' resampler for composites, don't save data " "at its native resolution, use the resolution used to create the " "composite.") parser.add_argument('-w', '--writers', nargs='+', help='writers to save datasets with') parser.add_argument("--list-products", dest="list_products", action="store_true", help="List available reader products and exit") subgroups = add_scene_argument_groups(parser) subgroups += add_resample_argument_groups(parser) argv_without_help = [x for x in argv if x not in ["-h", "--help"]] args, remaining_args = parser.parse_known_args(argv_without_help) # get the logger if we know the readers and writers that will be used if args.reader is not None and args.writers is not None: glue_name = args.reader + "_" + "-".join(args.writers or []) LOG = logging.getLogger(glue_name) # add writer arguments if args.writers is not None: for writer in (args.writers or []): parser_func = WRITER_PARSER_FUNCTIONS.get(writer) if parser_func is None: continue subgroups += parser_func(parser) args = parser.parse_args(argv) if args.reader is None: parser.print_usage() parser.exit(1, "\nERROR: Reader must be provided (-r flag).\n" "Supported readers:\n\t{}\n".format('\n\t'.join(['abi_l1b', 'ahi_hsd', 'hrit_ahi']))) if args.writers is None: parser.print_usage() parser.exit(1, "\nERROR: Writer must be provided (-w flag) with one or more writer.\n" "Supported writers:\n\t{}\n".format('\n\t'.join(['geotiff']))) def _args_to_dict(group_actions): return {ga.dest: getattr(args, ga.dest) for ga in group_actions if hasattr(args, ga.dest)} scene_args = _args_to_dict(subgroups[0]._group_actions) load_args = _args_to_dict(subgroups[1]._group_actions) resample_args = _args_to_dict(subgroups[2]._group_actions) writer_args = {} for idx, writer in enumerate(args.writers): sgrp1, sgrp2 = subgroups[3 + idx * 2: 5 + idx * 2] wargs = _args_to_dict(sgrp1._group_actions) if sgrp2 is not None: wargs.update(_args_to_dict(sgrp2._group_actions)) writer_args[writer] = wargs # get default output filename if 'filename' in wargs and wargs['filename'] is None: wargs['filename'] = get_default_output_filename(args.reader, writer) if not args.filenames: parser.print_usage() parser.exit(1, "\nERROR: No data files provided (-f flag)\n") # Prepare logging rename_log = False if args.log_fn is None: rename_log = True args.log_fn = glue_name + "_fail.log" levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn) logging.getLogger('rasterio').setLevel(levels[min(2, args.verbosity)]) sys.excepthook = create_exc_handler(LOG.name) if levels[min(3, args.verbosity)] > logging.DEBUG: import warnings warnings.filterwarnings("ignore") LOG.debug("Starting script with arguments: %s", " ".join(sys.argv)) # Set up dask and the number of workers if args.num_workers: from multiprocessing.pool import ThreadPool dask.config.set(pool=ThreadPool(args.num_workers)) # Parse provided files and search for files if provided directories scene_args['filenames'] = get_input_files(scene_args['filenames']) # Create a Scene, analyze the provided files LOG.info("Sorting and reading input files...") try: scn = Scene(**scene_args) except ValueError as e: LOG.error("{} | Enable debug message (-vvv) or see log file for details.".format(str(e))) LOG.debug("Further error information: ", exc_info=True) return -1 except OSError: LOG.error("Could not open files. Enable debug message (-vvv) or see log file for details.") LOG.debug("Further error information: ", exc_info=True) return -1 if args.list_products: print("\n".join(sorted(scn.available_dataset_names(composites=True)))) return 0 # Rename the log file if rename_log: rename_log_file(glue_name + scn.attrs['start_time'].strftime("_%Y%m%d_%H%M%S.log")) # Load the actual data arrays and metadata (lazy loaded as dask arrays) if load_args['products'] is None: try: reader_mod = importlib.import_module('polar2grid.readers.' + scene_args['reader']) load_args['products'] = reader_mod.DEFAULT_PRODUCTS LOG.info("Using default product list: {}".format(load_args['products'])) except (ImportError, AttributeError): LOG.error("No default products list set, please specify with `--products`.") return -1 LOG.info("Loading product metadata from files...") scn.load(load_args['products']) resample_kwargs = resample_args.copy() areas_to_resample = resample_kwargs.pop('grids') grid_configs = resample_kwargs.pop('grid_configs') resampler = resample_kwargs.pop('resampler') if areas_to_resample is None and resampler in [None, 'native']: # no areas specified areas_to_resample = ['MAX'] elif areas_to_resample is None: raise ValueError("Resampling method specified (--method) without any destination grid/area (-g flag).") elif not areas_to_resample: # they don't want any resampling (they used '-g' with no args) areas_to_resample = [None] has_custom_grid = any(g not in ['MIN', 'MAX', None] for g in areas_to_resample) if has_custom_grid and resampler == 'native': LOG.error("Resampling method 'native' can only be used with 'MIN' or 'MAX' grids " "(use 'nearest' method instead).") return -1 p2g_grid_configs = [x for x in grid_configs if x.endswith('.conf')] pyresample_area_configs = [x for x in grid_configs if not x.endswith('.conf')] if not grid_configs or p2g_grid_configs: # if we were given p2g grid configs or we weren't given any to choose from from polar2grid.grids import GridManager grid_manager = GridManager(*p2g_grid_configs) else: grid_manager = {} if pyresample_area_configs: from pyresample.utils import parse_area_file custom_areas = parse_area_file(pyresample_area_configs) custom_areas = {x.area_id: x for x in custom_areas} else: custom_areas = {} ll_bbox = resample_kwargs.pop('ll_bbox') if ll_bbox: scn = scn.crop(ll_bbox=ll_bbox) wishlist = scn.wishlist.copy() preserve_resolution = get_preserve_resolution(args, resampler, areas_to_resample) if preserve_resolution: preserved_products = set(wishlist) & set(scn.datasets.keys()) resampled_products = set(wishlist) - preserved_products # original native scene to_save = write_scene(scn, args.writers, writer_args, preserved_products) else: preserved_products = set() resampled_products = set(wishlist) to_save = [] LOG.debug("Products to preserve resolution for: {}".format(preserved_products)) LOG.debug("Products to use new resolution for: {}".format(resampled_products)) for area_name in areas_to_resample: if area_name is None: # no resampling area_def = None elif area_name == 'MAX': area_def = scn.max_area() elif area_name == 'MIN': area_def = scn.min_area() elif area_name in custom_areas: area_def = custom_areas[area_name] elif area_name in grid_manager: from pyresample.geometry import DynamicAreaDefinition p2g_def = grid_manager[area_name] area_def = p2g_def.to_satpy_area() if isinstance(area_def, DynamicAreaDefinition) and p2g_def['cell_width'] is not None: area_def = area_def.freeze(scn.max_area(), resolution=(abs(p2g_def['cell_width']), abs(p2g_def['cell_height']))) else: area_def = get_area_def(area_name) if resampler is None and area_def is not None: rs = 'native' if area_name in ['MIN', 'MAX'] else 'nearest' LOG.debug("Setting default resampling to '{}' for grid '{}'".format(rs, area_name)) else: rs = resampler if area_def is not None: LOG.info("Resampling data to '%s'", area_name) new_scn = scn.resample(area_def, resampler=rs, **resample_kwargs) elif not preserve_resolution: # the user didn't want to resample to any areas # the user also requested that we don't preserve resolution # which means we have to save this Scene's datasets # because they won't be saved new_scn = scn to_save = write_scene(new_scn, args.writers, writer_args, resampled_products, to_save=to_save) if args.progress: pbar = ProgressBar() pbar.register() LOG.info("Computing products and saving data to writers...") compute_writer_results(to_save) LOG.info("SUCCESS") return 0
def main(argv=sys.argv[1:]): global LOG import satpy from satpy import Scene from satpy.writers import compute_writer_results from dask.diagnostics import ProgressBar from polar2grid.core.script_utils import (setup_logging, rename_log_file, create_exc_handler) import argparse dist = pkg_resources.get_distribution('polar2grid') if dist_is_editable(dist): p2g_etc = os.path.join(dist.module_path, 'etc') else: p2g_etc = os.path.join(sys.prefix, 'etc', 'polar2grid') config_path = satpy.config.get('config_path') if p2g_etc not in config_path: satpy.config.set(config_path=config_path + [p2g_etc]) USE_POLAR2GRID_DEFAULTS = bool( int(os.environ.setdefault("USE_POLAR2GRID_DEFAULTS", "1"))) prog = os.getenv('PROG_NAME', sys.argv[0]) # "usage: " will be printed at the top of this: usage = """ %(prog)s -h see available products: %(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...] basic processing: %(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...] basic processing with limited products: %(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...] """ parser = argparse.ArgumentParser( prog=prog, usage=usage, fromfile_prefix_chars="@", description="Load, composite, resample, and save datasets.") parser.add_argument( '-v', '--verbose', dest='verbosity', action="count", default=0, help='each occurrence increases verbosity 1 level through ' 'ERROR-WARNING-INFO-DEBUG (default INFO)') parser.add_argument('-l', '--log', dest="log_fn", default=None, help="specify the log filename") parser.add_argument( '--progress', action='store_true', help="show processing progress bar (not recommended for logged output)" ) parser.add_argument( '--num-workers', type=int, default=os.getenv('DASK_NUM_WORKERS', 4), help="specify number of worker threads to use (default: 4)") parser.add_argument( '--match-resolution', dest='preserve_resolution', action='store_false', help="When using the 'native' resampler for composites, don't save data " "at its native resolution, use the resolution used to create the " "composite.") parser.add_argument("--list-products", dest="list_products", action="store_true", help="List available reader products and exit") reader_group = add_scene_argument_groups( parser, is_polar2grid=USE_POLAR2GRID_DEFAULTS)[0] resampling_group = add_resample_argument_groups( parser, is_polar2grid=USE_POLAR2GRID_DEFAULTS)[0] writer_group = add_writer_argument_groups(parser)[0] subgroups = [reader_group, resampling_group, writer_group] argv_without_help = [x for x in argv if x not in ["-h", "--help"]] _retitle_optional_arguments(parser) args, remaining_args = parser.parse_known_args(argv_without_help) os.environ['DASK_NUM_WORKERS'] = str(args.num_workers) # get the logger if we know the readers and writers that will be used if args.readers is not None and args.writers is not None: glue_name = args.readers[0] + "_" + "-".join(args.writers or []) LOG = logging.getLogger(glue_name) # add writer arguments for writer in (args.writers or []): parser_func = WRITER_PARSER_FUNCTIONS.get(writer) if parser_func is None: continue subgroups += parser_func(parser) args = parser.parse_args(argv) if args.readers is None: parser.print_usage() parser.exit( 1, "\nERROR: Reader must be provided (-r flag).\n" "Supported readers:\n\t{}\n".format('\n\t'.join( ['abi_l1b', 'ahi_hsd', 'hrit_ahi']))) elif len(args.readers) > 1: parser.print_usage() parser.exit( 1, "\nMultiple readers is not currently supported. Got:\n\t" "{}\n".format('\n\t'.join(args.readers))) return -1 if args.writers is None: parser.print_usage() parser.exit( 1, "\nERROR: Writer must be provided (-w flag) with one or more writer.\n" "Supported writers:\n\t{}\n".format('\n\t'.join(['geotiff']))) def _args_to_dict(group_actions, exclude=None): if exclude is None: exclude = [] return { ga.dest: getattr(args, ga.dest) for ga in group_actions if hasattr(args, ga.dest) and ga.dest not in exclude } reader_args = _args_to_dict(reader_group._group_actions) reader_names = reader_args.pop('readers') scene_creation = { 'filenames': reader_args.pop('filenames'), 'reader': reader_names[0], } load_args = { 'products': reader_args.pop('products'), } # anything left in 'reader_args' is a reader-specific kwarg resample_args = _args_to_dict(resampling_group._group_actions) writer_args = _args_to_dict(writer_group._group_actions) # writer_args = {} subgroup_idx = 3 for idx, writer in enumerate(writer_args['writers']): sgrp1, sgrp2 = subgroups[subgroup_idx + idx * 2:subgroup_idx + 2 + idx * 2] wargs = _args_to_dict(sgrp1._group_actions) if sgrp2 is not None: wargs.update(_args_to_dict(sgrp2._group_actions)) writer_args[writer] = wargs # get default output filename if 'filename' in wargs and wargs['filename'] is None: wargs['filename'] = get_default_output_filename( args.readers[0], writer) if not args.filenames: parser.print_usage() parser.exit(1, "\nERROR: No data files provided (-f flag)\n") # Prepare logging rename_log = False if args.log_fn is None: rename_log = True args.log_fn = glue_name + "_fail.log" levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn) logging.getLogger('rasterio').setLevel(levels[min(2, args.verbosity)]) sys.excepthook = create_exc_handler(LOG.name) if levels[min(3, args.verbosity)] > logging.DEBUG: import warnings warnings.filterwarnings("ignore") LOG.debug("Starting script with arguments: %s", " ".join(sys.argv)) # Set up dask and the number of workers if args.num_workers: dask.config.set(num_workers=args.num_workers) # Parse provided files and search for files if provided directories scene_creation['filenames'] = get_input_files(scene_creation['filenames']) # Create a Scene, analyze the provided files LOG.info("Sorting and reading input files...") try: scn = Scene(**scene_creation) except ValueError as e: LOG.error( "{} | Enable debug message (-vvv) or see log file for details.". format(str(e))) LOG.debug("Further error information: ", exc_info=True) return -1 except OSError: LOG.error( "Could not open files. Enable debug message (-vvv) or see log file for details." ) LOG.debug("Further error information: ", exc_info=True) return -1 if args.list_products: print("\n".join(sorted(scn.available_dataset_names(composites=True)))) return 0 # Rename the log file if rename_log: rename_log_file(glue_name + scn.attrs['start_time'].strftime("_%Y%m%d_%H%M%S.log")) # Load the actual data arrays and metadata (lazy loaded as dask arrays) LOG.info("Loading product metadata from files...") load_args['products'] = _apply_default_products_and_aliases( scn, scene_creation['reader'], load_args['products']) if not load_args['products']: return -1 scn.load(load_args['products']) ll_bbox = resample_args.pop('ll_bbox') if ll_bbox: scn = scn.crop(ll_bbox=ll_bbox) scn = filter_scene( scn, reader_names, sza_threshold=reader_args['sza_threshold'], day_fraction=reader_args['filter_day_products'], night_fraction=reader_args['filter_night_products'], ) if scn is None: LOG.info("No remaining products after filtering.") return 0 to_save = [] areas_to_resample = resample_args.pop("grids") if 'ewa_persist' in resample_args: resample_args['persist'] = resample_args.pop('ewa_persist') scenes_to_save = resample_scene( scn, areas_to_resample, preserve_resolution=args.preserve_resolution, is_polar2grid=USE_POLAR2GRID_DEFAULTS, **resample_args) for scene_to_save, products_to_save in scenes_to_save: overwrite_platform_name_with_aliases(scene_to_save) to_save = write_scene(scene_to_save, writer_args['writers'], writer_args, products_to_save, to_save=to_save) if args.progress: pbar = ProgressBar() pbar.register() LOG.info("Computing products and saving data to writers...") compute_writer_results(to_save) LOG.info("SUCCESS") return 0
def set_of_photos(hrit_files, central_lat, central_lon, elevation, time, dpi, photo_path, all_lines=False, load_photo=['VIS006']): """Return path of photo Parameters ---------- parameter_name : parameter_type Quick description of it. Returns ------- photos: array Array of saved on disc files. """ # Loading decompressed previously photos from satpy import Scene import numpy as np # from glob import glob import matplotlib.pyplot as plt from satpy import find_files_and_readers from datetime import datetime import cartopy.crs as ccrs import matplotlib as mpl mpl.rcParams['figure.dpi'] = dpi # load_photo = 'VIS006' first = 0 last = len(time) - 1 # print(len(time),last) yearF, monthF, dayF, hourF, minuteF, secondF = time[first].tt_calendar() yearL, monthL, dayL, hourL, minuteL, secondL = time[last].tt_calendar() # IT IS NOT WORKING FIND SOLUTION - Adding a minute in case there is only one point on map if len(time) == 1: time[last].tt = time[last].tt + 1 / 3600 yearL, monthL, dayL, hourL, minuteL, secondL = time[last].tt_calendar() # print("udalosie") # print(yearF, monthF, dayF, hourF, minuteF, secondF ) # print(yearL, monthL, dayL, hourL, minuteL, secondL) # time[0].tt_calendar()[0] files = find_files_and_readers(base_dir=hrit_files, start_time=datetime(yearF, monthF, dayF, hourF, minuteF), end_time=datetime(yearL, monthL, dayL, hourL, minuteL), reader='seviri_l1b_hrit') scn = Scene(filenames=files) load_photo = scn.available_dataset_names() # HRV does not work properly load_photo.remove('HRV') # print(load_photo) # print(scn.available_dataset_names()) div = 30 name_img = [] for photo_type in load_photo: # print(photo_type) scn.load([photo_type]) # print(scn) for i in range(len(time)): new_scn = scn # print(new_scn) crs = new_scn[photo_type].attrs['area'].to_cartopy_crs() y, x = polygon_surrounding_sat(central_lat[i], central_lon[i], elevation[i], div) extent = calculate_extent(x, y) ax = plt.axes( projection=ccrs.Sinusoidal(central_lon[i], central_lat[i])) ax.set_extent(extent) if (all_lines): ax.gridlines() ax.coastlines(resolution='50m', color='red') ax.coastlines() # ax.gridlines() # ax.set_global() plt.imshow(new_scn[photo_type], transform=crs, extent=crs.bounds, origin='upper', cmap='gray') # cbar = plt.colorbar() # cbar.set_label("Kelvin") # plt.imsave('imsave.png', im, cmap='gray') name = photo_path + photo_type + "_" + time[i].utc_iso() + '.png' plt.savefig(name, dpi=dpi) name_img = np.append(name_img, name) return (name_img)