def load_sat_geo(in_reader, globber, chans, latpos, lonpos, deller):
"""Load data for a given sensor, find BT at particular lat/lon."""
storm_bbox = [
lonpos - deller, latpos - deller, lonpos + deller, latpos + deller
]
files = glob(globber)
if len(files) < 1:
return None
scn = Scene(files, reader=in_reader)
scn.load(chans)
scn2 = scn.crop(ll_bbox=storm_bbox)
output_dict = {}
for chan in chans:
try:
btdata = np.array(scn2[chan])
output_dict[chan] = [
np.nanmin(btdata), scn[chan].wavelength.min,
scn[chan].wavelength.central, scn[chan].wavelength.max
]
except KeyError:
output_dict[chan] = [None, None, None, None]
return output_dict
def main(argv=sys.argv[1:]):
global LOG
from satpy import Scene
from satpy.writers import compute_writer_results
from dask.diagnostics import ProgressBar
from polar2grid.core.script_utils import (
setup_logging,
rename_log_file,
create_exc_handler,
)
import argparse
add_polar2grid_config_paths()
USE_POLAR2GRID_DEFAULTS = bool(
int(os.environ.setdefault("USE_POLAR2GRID_DEFAULTS", "1")))
BINARY_NAME = "polar2grid" if USE_POLAR2GRID_DEFAULTS else "geo2grid"
prog = os.getenv("PROG_NAME", sys.argv[0])
# "usage: " will be printed at the top of this:
usage = """
%(prog)s -h
see available products:
%(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...]
basic processing:
%(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...]
basic processing with limited products:
%(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...]
"""
parser = argparse.ArgumentParser(
prog=prog,
usage=usage,
fromfile_prefix_chars="@",
description="Load, composite, resample, and save datasets.",
)
parser.add_argument(
"-v",
"--verbose",
dest="verbosity",
action="count",
default=0,
help="each occurrence increases verbosity 1 level through "
"ERROR-WARNING-INFO-DEBUG (default INFO)",
)
parser.add_argument("-l",
"--log",
dest="log_fn",
default=None,
help="specify the log filename")
parser.add_argument(
"--progress",
action="store_true",
help="show processing progress bar (not recommended for logged output)",
)
parser.add_argument(
"--num-workers",
type=int,
default=os.getenv("DASK_NUM_WORKERS", 4),
help="specify number of worker threads to use (default: 4)",
)
parser.add_argument(
"--match-resolution",
dest="preserve_resolution",
action="store_false",
help="When using the 'native' resampler for composites, don't save data "
"at its native resolution, use the resolution used to create the "
"composite.",
)
parser.add_argument(
"--list-products",
dest="list_products",
action="store_true",
help="List available {} products and exit".format(BINARY_NAME),
)
parser.add_argument(
"--list-products-all",
dest="list_products_all",
action="store_true",
help="List available {} products and custom/Satpy products and exit".
format(BINARY_NAME),
)
reader_group = add_scene_argument_groups(
parser, is_polar2grid=USE_POLAR2GRID_DEFAULTS)[0]
resampling_group = add_resample_argument_groups(
parser, is_polar2grid=USE_POLAR2GRID_DEFAULTS)[0]
writer_group = add_writer_argument_groups(parser)[0]
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
_retitle_optional_arguments(parser)
args, remaining_args = parser.parse_known_args(argv_without_help)
os.environ["DASK_NUM_WORKERS"] = str(args.num_workers)
# get the logger if we know the readers and writers that will be used
if args.readers is not None and args.writers is not None:
glue_name = args.readers[0] + "_" + "-".join(args.writers or [])
LOG = logging.getLogger(glue_name)
reader_subgroups = _add_component_parser_args(parser, "readers",
args.readers or [])
writer_subgroups = _add_component_parser_args(parser, "writers",
args.writers or [])
args = parser.parse_args(argv)
if args.readers is None:
parser.print_usage()
parser.exit(
1,
"\nERROR: Reader must be provided (-r flag).\n"
"Supported readers:\n\t{}\n".format("\n\t".join(
["abi_l1b", "ahi_hsd", "hrit_ahi"])),
)
elif len(args.readers) > 1:
parser.print_usage()
parser.exit(
1,
"\nMultiple readers is not currently supported. Got:\n\t"
"{}\n".format("\n\t".join(args.readers)),
)
return -1
if args.writers is None:
parser.print_usage()
parser.exit(
1,
"\nERROR: Writer must be provided (-w flag) with one or more writer.\n"
"Supported writers:\n\t{}\n".format("\n\t".join(["geotiff"])),
)
reader_args = _args_to_dict(args, reader_group._group_actions)
reader_names = reader_args.pop("readers")
scene_creation, load_args = _get_scene_init_load_args(
args, reader_args, reader_names, reader_subgroups)
resample_args = _args_to_dict(args, resampling_group._group_actions)
writer_args = _args_to_dict(args, writer_group._group_actions)
writer_specific_args = _parse_writer_args(writer_args["writers"],
writer_subgroups, reader_names,
args)
writer_args.update(writer_specific_args)
if not args.filenames:
parser.print_usage()
parser.exit(1, "\nERROR: No data files provided (-f flag)\n")
# Prepare logging
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)],
log_filename=args.log_fn)
logging.getLogger("rasterio").setLevel(levels[min(2, args.verbosity)])
sys.excepthook = create_exc_handler(LOG.name)
if levels[min(3, args.verbosity)] > logging.DEBUG:
import warnings
warnings.filterwarnings("ignore")
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Set up dask and the number of workers
if args.num_workers:
dask.config.set(num_workers=args.num_workers)
# Create a Scene, analyze the provided files
LOG.info("Sorting and reading input files...")
try:
scn = Scene(**scene_creation)
except ValueError as e:
LOG.error(
"{} | Enable debug message (-vvv) or see log file for details.".
format(str(e)))
LOG.debug("Further error information: ", exc_info=True)
return -1
except OSError:
LOG.error(
"Could not open files. Enable debug message (-vvv) or see log file for details."
)
LOG.debug("Further error information: ", exc_info=True)
return -1
# Rename the log file
if rename_log:
rename_log_file(glue_name +
scn.attrs["start_time"].strftime("_%Y%m%d_%H%M%S.log"))
# Load the actual data arrays and metadata (lazy loaded as dask arrays)
LOG.info("Loading product metadata from files...")
reader_info = ReaderProxyBase.from_reader_name(scene_creation["reader"],
scn, load_args["products"])
if args.list_products or args.list_products_all:
_print_list_products(reader_info, p2g_only=not args.list_products_all)
return 0
load_args["products"] = reader_info.get_satpy_products_to_load()
if not load_args["products"]:
return -1
scn.load(load_args["products"])
ll_bbox = resample_args.pop("ll_bbox")
if ll_bbox:
scn = scn.crop(ll_bbox=ll_bbox)
scn = filter_scene(
scn,
reader_names,
sza_threshold=reader_args["sza_threshold"],
day_fraction=reader_args["filter_day_products"],
night_fraction=reader_args["filter_night_products"],
)
if scn is None:
LOG.info("No remaining products after filtering.")
return 0
to_save = []
areas_to_resample = resample_args.pop("grids")
if "ewa_persist" in resample_args:
resample_args["persist"] = resample_args.pop("ewa_persist")
scenes_to_save = resample_scene(
scn,
areas_to_resample,
preserve_resolution=args.preserve_resolution,
is_polar2grid=USE_POLAR2GRID_DEFAULTS,
**resample_args)
for scene_to_save, products_to_save in scenes_to_save:
overwrite_platform_name_with_aliases(scene_to_save)
reader_info.apply_p2g_name_to_scene(scene_to_save)
to_save = write_scene(
scene_to_save,
writer_args["writers"],
writer_args,
products_to_save,
to_save=to_save,
)
if args.progress:
pbar = ProgressBar()
pbar.register()
LOG.info("Computing products and saving data to writers...")
compute_writer_results(to_save)
LOG.info("SUCCESS")
return 0
def main(argv=sys.argv[1:]):
global LOG
from satpy import Scene
from satpy.resample import get_area_def
from satpy.writers import compute_writer_results
from dask.diagnostics import ProgressBar
from polar2grid.core.script_utils import (
setup_logging, rename_log_file, create_exc_handler)
import argparse
prog = os.getenv('PROG_NAME', sys.argv[0])
# "usage: " will be printed at the top of this:
usage = """
%(prog)s -h
see available products:
%(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...]
basic processing:
%(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...]
basic processing with limited products:
%(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...]
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage,
description="Load, composite, resample, and save datasets.")
parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0,
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
parser.add_argument('-l', '--log', dest="log_fn", default=None,
help="specify the log filename")
parser.add_argument('--progress', action='store_true',
help="show processing progress bar (not recommended for logged output)")
parser.add_argument('--num-workers', type=int, default=4,
help="specify number of worker threads to use (default: 4)")
parser.add_argument('--match-resolution', dest='preserve_resolution', action='store_false',
help="When using the 'native' resampler for composites, don't save data "
"at its native resolution, use the resolution used to create the "
"composite.")
parser.add_argument('-w', '--writers', nargs='+',
help='writers to save datasets with')
parser.add_argument("--list-products", dest="list_products", action="store_true",
help="List available reader products and exit")
subgroups = add_scene_argument_groups(parser)
subgroups += add_resample_argument_groups(parser)
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
args, remaining_args = parser.parse_known_args(argv_without_help)
# get the logger if we know the readers and writers that will be used
if args.reader is not None and args.writers is not None:
glue_name = args.reader + "_" + "-".join(args.writers or [])
LOG = logging.getLogger(glue_name)
# add writer arguments
if args.writers is not None:
for writer in (args.writers or []):
parser_func = WRITER_PARSER_FUNCTIONS.get(writer)
if parser_func is None:
continue
subgroups += parser_func(parser)
args = parser.parse_args(argv)
if args.reader is None:
parser.print_usage()
parser.exit(1, "\nERROR: Reader must be provided (-r flag).\n"
"Supported readers:\n\t{}\n".format('\n\t'.join(['abi_l1b', 'ahi_hsd', 'hrit_ahi'])))
if args.writers is None:
parser.print_usage()
parser.exit(1, "\nERROR: Writer must be provided (-w flag) with one or more writer.\n"
"Supported writers:\n\t{}\n".format('\n\t'.join(['geotiff'])))
def _args_to_dict(group_actions):
return {ga.dest: getattr(args, ga.dest) for ga in group_actions if hasattr(args, ga.dest)}
scene_args = _args_to_dict(subgroups[0]._group_actions)
load_args = _args_to_dict(subgroups[1]._group_actions)
resample_args = _args_to_dict(subgroups[2]._group_actions)
writer_args = {}
for idx, writer in enumerate(args.writers):
sgrp1, sgrp2 = subgroups[3 + idx * 2: 5 + idx * 2]
wargs = _args_to_dict(sgrp1._group_actions)
if sgrp2 is not None:
wargs.update(_args_to_dict(sgrp2._group_actions))
writer_args[writer] = wargs
# get default output filename
if 'filename' in wargs and wargs['filename'] is None:
wargs['filename'] = get_default_output_filename(args.reader, writer)
if not args.filenames:
parser.print_usage()
parser.exit(1, "\nERROR: No data files provided (-f flag)\n")
# Prepare logging
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn)
logging.getLogger('rasterio').setLevel(levels[min(2, args.verbosity)])
sys.excepthook = create_exc_handler(LOG.name)
if levels[min(3, args.verbosity)] > logging.DEBUG:
import warnings
warnings.filterwarnings("ignore")
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Set up dask and the number of workers
if args.num_workers:
from multiprocessing.pool import ThreadPool
dask.config.set(pool=ThreadPool(args.num_workers))
# Parse provided files and search for files if provided directories
scene_args['filenames'] = get_input_files(scene_args['filenames'])
# Create a Scene, analyze the provided files
LOG.info("Sorting and reading input files...")
try:
scn = Scene(**scene_args)
except ValueError as e:
LOG.error("{} | Enable debug message (-vvv) or see log file for details.".format(str(e)))
LOG.debug("Further error information: ", exc_info=True)
return -1
except OSError:
LOG.error("Could not open files. Enable debug message (-vvv) or see log file for details.")
LOG.debug("Further error information: ", exc_info=True)
return -1
if args.list_products:
print("\n".join(sorted(scn.available_dataset_names(composites=True))))
return 0
# Rename the log file
if rename_log:
rename_log_file(glue_name + scn.attrs['start_time'].strftime("_%Y%m%d_%H%M%S.log"))
# Load the actual data arrays and metadata (lazy loaded as dask arrays)
if load_args['products'] is None:
try:
reader_mod = importlib.import_module('polar2grid.readers.' + scene_args['reader'])
load_args['products'] = reader_mod.DEFAULT_PRODUCTS
LOG.info("Using default product list: {}".format(load_args['products']))
except (ImportError, AttributeError):
LOG.error("No default products list set, please specify with `--products`.")
return -1
LOG.info("Loading product metadata from files...")
scn.load(load_args['products'])
resample_kwargs = resample_args.copy()
areas_to_resample = resample_kwargs.pop('grids')
grid_configs = resample_kwargs.pop('grid_configs')
resampler = resample_kwargs.pop('resampler')
if areas_to_resample is None and resampler in [None, 'native']:
# no areas specified
areas_to_resample = ['MAX']
elif areas_to_resample is None:
raise ValueError("Resampling method specified (--method) without any destination grid/area (-g flag).")
elif not areas_to_resample:
# they don't want any resampling (they used '-g' with no args)
areas_to_resample = [None]
has_custom_grid = any(g not in ['MIN', 'MAX', None] for g in areas_to_resample)
if has_custom_grid and resampler == 'native':
LOG.error("Resampling method 'native' can only be used with 'MIN' or 'MAX' grids "
"(use 'nearest' method instead).")
return -1
p2g_grid_configs = [x for x in grid_configs if x.endswith('.conf')]
pyresample_area_configs = [x for x in grid_configs if not x.endswith('.conf')]
if not grid_configs or p2g_grid_configs:
# if we were given p2g grid configs or we weren't given any to choose from
from polar2grid.grids import GridManager
grid_manager = GridManager(*p2g_grid_configs)
else:
grid_manager = {}
if pyresample_area_configs:
from pyresample.utils import parse_area_file
custom_areas = parse_area_file(pyresample_area_configs)
custom_areas = {x.area_id: x for x in custom_areas}
else:
custom_areas = {}
ll_bbox = resample_kwargs.pop('ll_bbox')
if ll_bbox:
scn = scn.crop(ll_bbox=ll_bbox)
wishlist = scn.wishlist.copy()
preserve_resolution = get_preserve_resolution(args, resampler, areas_to_resample)
if preserve_resolution:
preserved_products = set(wishlist) & set(scn.datasets.keys())
resampled_products = set(wishlist) - preserved_products
# original native scene
to_save = write_scene(scn, args.writers, writer_args, preserved_products)
else:
preserved_products = set()
resampled_products = set(wishlist)
to_save = []
LOG.debug("Products to preserve resolution for: {}".format(preserved_products))
LOG.debug("Products to use new resolution for: {}".format(resampled_products))
for area_name in areas_to_resample:
if area_name is None:
# no resampling
area_def = None
elif area_name == 'MAX':
area_def = scn.max_area()
elif area_name == 'MIN':
area_def = scn.min_area()
elif area_name in custom_areas:
area_def = custom_areas[area_name]
elif area_name in grid_manager:
from pyresample.geometry import DynamicAreaDefinition
p2g_def = grid_manager[area_name]
area_def = p2g_def.to_satpy_area()
if isinstance(area_def, DynamicAreaDefinition) and p2g_def['cell_width'] is not None:
area_def = area_def.freeze(scn.max_area(),
resolution=(abs(p2g_def['cell_width']), abs(p2g_def['cell_height'])))
else:
area_def = get_area_def(area_name)
if resampler is None and area_def is not None:
rs = 'native' if area_name in ['MIN', 'MAX'] else 'nearest'
LOG.debug("Setting default resampling to '{}' for grid '{}'".format(rs, area_name))
else:
rs = resampler
if area_def is not None:
LOG.info("Resampling data to '%s'", area_name)
new_scn = scn.resample(area_def, resampler=rs, **resample_kwargs)
elif not preserve_resolution:
# the user didn't want to resample to any areas
# the user also requested that we don't preserve resolution
# which means we have to save this Scene's datasets
# because they won't be saved
new_scn = scn
to_save = write_scene(new_scn, args.writers, writer_args, resampled_products, to_save=to_save)
if args.progress:
pbar = ProgressBar()
pbar.register()
LOG.info("Computing products and saving data to writers...")
compute_writer_results(to_save)
LOG.info("SUCCESS")
return 0
#read the data
#-------------
#specify reader for the respective file type (here CM-SAF Claas 2 reader)
#available from satpy 0.22 onwards, has to be installed using conda forge channel
scnFile = {"cmsaf-claas2_l2_nc": filename}
scn = Scene(filenames=scnFile)
#show all available channels
#scn.all_dataset_names()
channel = scn.all_dataset_names()[3]
#load the ct channel
scn.load([channel])
#scn[channel]
#scn.show(channel)
scn.available_composite_names()
#crop the scene using the coordinates of the RADOLAN boundary (DWD 2004)
#more in northern and southern direction because no perfect overlap after reprojection
#crop function: ll_bbox=(xmin, ymin, xmax, ymax)
#https://satpy.readthedocs.io/en/latest/api/satpy.html?highlight=crop#satpy.scene.Scene.crop
scn_cropped = scn.crop(ll_bbox=(3.6, 46.7, 15.7,
55.2)) #y_original:ymin:47, ymax:54.7
scn_cropped.show(channel)
scn_cropped.save_datasets(writer="geotiff",
datasets=["ct"],
base_dir=outputDir,
filename="CT_" + files[5:-17] +
"_geotiff.tif",
enhance=False,
dtype=np.float32)
def main(argv=sys.argv[1:]):
global LOG
import satpy
from satpy import Scene
from satpy.writers import compute_writer_results
from dask.diagnostics import ProgressBar
from polar2grid.core.script_utils import (setup_logging, rename_log_file,
create_exc_handler)
import argparse
dist = pkg_resources.get_distribution('polar2grid')
if dist_is_editable(dist):
p2g_etc = os.path.join(dist.module_path, 'etc')
else:
p2g_etc = os.path.join(sys.prefix, 'etc', 'polar2grid')
config_path = satpy.config.get('config_path')
if p2g_etc not in config_path:
satpy.config.set(config_path=config_path + [p2g_etc])
USE_POLAR2GRID_DEFAULTS = bool(
int(os.environ.setdefault("USE_POLAR2GRID_DEFAULTS", "1")))
prog = os.getenv('PROG_NAME', sys.argv[0])
# "usage: " will be printed at the top of this:
usage = """
%(prog)s -h
see available products:
%(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...]
basic processing:
%(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...]
basic processing with limited products:
%(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...]
"""
parser = argparse.ArgumentParser(
prog=prog,
usage=usage,
fromfile_prefix_chars="@",
description="Load, composite, resample, and save datasets.")
parser.add_argument(
'-v',
'--verbose',
dest='verbosity',
action="count",
default=0,
help='each occurrence increases verbosity 1 level through '
'ERROR-WARNING-INFO-DEBUG (default INFO)')
parser.add_argument('-l',
'--log',
dest="log_fn",
default=None,
help="specify the log filename")
parser.add_argument(
'--progress',
action='store_true',
help="show processing progress bar (not recommended for logged output)"
)
parser.add_argument(
'--num-workers',
type=int,
default=os.getenv('DASK_NUM_WORKERS', 4),
help="specify number of worker threads to use (default: 4)")
parser.add_argument(
'--match-resolution',
dest='preserve_resolution',
action='store_false',
help="When using the 'native' resampler for composites, don't save data "
"at its native resolution, use the resolution used to create the "
"composite.")
parser.add_argument("--list-products",
dest="list_products",
action="store_true",
help="List available reader products and exit")
reader_group = add_scene_argument_groups(
parser, is_polar2grid=USE_POLAR2GRID_DEFAULTS)[0]
resampling_group = add_resample_argument_groups(
parser, is_polar2grid=USE_POLAR2GRID_DEFAULTS)[0]
writer_group = add_writer_argument_groups(parser)[0]
subgroups = [reader_group, resampling_group, writer_group]
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
_retitle_optional_arguments(parser)
args, remaining_args = parser.parse_known_args(argv_without_help)
os.environ['DASK_NUM_WORKERS'] = str(args.num_workers)
# get the logger if we know the readers and writers that will be used
if args.readers is not None and args.writers is not None:
glue_name = args.readers[0] + "_" + "-".join(args.writers or [])
LOG = logging.getLogger(glue_name)
# add writer arguments
for writer in (args.writers or []):
parser_func = WRITER_PARSER_FUNCTIONS.get(writer)
if parser_func is None:
continue
subgroups += parser_func(parser)
args = parser.parse_args(argv)
if args.readers is None:
parser.print_usage()
parser.exit(
1, "\nERROR: Reader must be provided (-r flag).\n"
"Supported readers:\n\t{}\n".format('\n\t'.join(
['abi_l1b', 'ahi_hsd', 'hrit_ahi'])))
elif len(args.readers) > 1:
parser.print_usage()
parser.exit(
1, "\nMultiple readers is not currently supported. Got:\n\t"
"{}\n".format('\n\t'.join(args.readers)))
return -1
if args.writers is None:
parser.print_usage()
parser.exit(
1,
"\nERROR: Writer must be provided (-w flag) with one or more writer.\n"
"Supported writers:\n\t{}\n".format('\n\t'.join(['geotiff'])))
def _args_to_dict(group_actions, exclude=None):
if exclude is None:
exclude = []
return {
ga.dest: getattr(args, ga.dest)
for ga in group_actions
if hasattr(args, ga.dest) and ga.dest not in exclude
}
reader_args = _args_to_dict(reader_group._group_actions)
reader_names = reader_args.pop('readers')
scene_creation = {
'filenames': reader_args.pop('filenames'),
'reader': reader_names[0],
}
load_args = {
'products': reader_args.pop('products'),
}
# anything left in 'reader_args' is a reader-specific kwarg
resample_args = _args_to_dict(resampling_group._group_actions)
writer_args = _args_to_dict(writer_group._group_actions)
# writer_args = {}
subgroup_idx = 3
for idx, writer in enumerate(writer_args['writers']):
sgrp1, sgrp2 = subgroups[subgroup_idx + idx * 2:subgroup_idx + 2 +
idx * 2]
wargs = _args_to_dict(sgrp1._group_actions)
if sgrp2 is not None:
wargs.update(_args_to_dict(sgrp2._group_actions))
writer_args[writer] = wargs
# get default output filename
if 'filename' in wargs and wargs['filename'] is None:
wargs['filename'] = get_default_output_filename(
args.readers[0], writer)
if not args.filenames:
parser.print_usage()
parser.exit(1, "\nERROR: No data files provided (-f flag)\n")
# Prepare logging
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)],
log_filename=args.log_fn)
logging.getLogger('rasterio').setLevel(levels[min(2, args.verbosity)])
sys.excepthook = create_exc_handler(LOG.name)
if levels[min(3, args.verbosity)] > logging.DEBUG:
import warnings
warnings.filterwarnings("ignore")
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Set up dask and the number of workers
if args.num_workers:
dask.config.set(num_workers=args.num_workers)
# Parse provided files and search for files if provided directories
scene_creation['filenames'] = get_input_files(scene_creation['filenames'])
# Create a Scene, analyze the provided files
LOG.info("Sorting and reading input files...")
try:
scn = Scene(**scene_creation)
except ValueError as e:
LOG.error(
"{} | Enable debug message (-vvv) or see log file for details.".
format(str(e)))
LOG.debug("Further error information: ", exc_info=True)
return -1
except OSError:
LOG.error(
"Could not open files. Enable debug message (-vvv) or see log file for details."
)
LOG.debug("Further error information: ", exc_info=True)
return -1
if args.list_products:
print("\n".join(sorted(scn.available_dataset_names(composites=True))))
return 0
# Rename the log file
if rename_log:
rename_log_file(glue_name +
scn.attrs['start_time'].strftime("_%Y%m%d_%H%M%S.log"))
# Load the actual data arrays and metadata (lazy loaded as dask arrays)
LOG.info("Loading product metadata from files...")
load_args['products'] = _apply_default_products_and_aliases(
scn, scene_creation['reader'], load_args['products'])
if not load_args['products']:
return -1
scn.load(load_args['products'])
ll_bbox = resample_args.pop('ll_bbox')
if ll_bbox:
scn = scn.crop(ll_bbox=ll_bbox)
scn = filter_scene(
scn,
reader_names,
sza_threshold=reader_args['sza_threshold'],
day_fraction=reader_args['filter_day_products'],
night_fraction=reader_args['filter_night_products'],
)
if scn is None:
LOG.info("No remaining products after filtering.")
return 0
to_save = []
areas_to_resample = resample_args.pop("grids")
if 'ewa_persist' in resample_args:
resample_args['persist'] = resample_args.pop('ewa_persist')
scenes_to_save = resample_scene(
scn,
areas_to_resample,
preserve_resolution=args.preserve_resolution,
is_polar2grid=USE_POLAR2GRID_DEFAULTS,
**resample_args)
for scene_to_save, products_to_save in scenes_to_save:
overwrite_platform_name_with_aliases(scene_to_save)
to_save = write_scene(scene_to_save,
writer_args['writers'],
writer_args,
products_to_save,
to_save=to_save)
if args.progress:
pbar = ProgressBar()
pbar.register()
LOG.info("Computing products and saving data to writers...")
compute_writer_results(to_save)
LOG.info("SUCCESS")
return 0
