def get_enhanced_image(dataset,
enhancer=None,
fill_value=None,
ppp_config_dir=None,
enhancement_config_file=None):
mode = _determine_mode(dataset)
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
if enhancer is None:
enhancer = Enhancer(ppp_config_dir, enhancement_config_file)
if enhancer.enhancement_tree is None:
raise RuntimeError(
"No enhancement configuration files found or specified, cannot"
" automatically enhance dataset")
if dataset.info.get("sensor", None):
enhancer.add_sensor_enhancements(dataset.info["sensor"])
# Create an image for enhancement
img = to_image(dataset, mode=mode, fill_value=fill_value)
enhancer.apply(img, **dataset.info)
img.info.update(dataset.info)
return img
def __init__(self, ppp_config_dir=None, enhancement_config_file=None):
"""
Args:
ppp_config_dir: Points to the base configuration directory
enhancement_config_file: The enhancement configuration to
apply, False to leave as is.
"""
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
self.enhancement_config_file = enhancement_config_file
# Set enhancement_config_file to False for no enhancements
if self.enhancement_config_file is None:
# it wasn't specified in the config or in the kwargs, we should
# provide a default
config_fn = os.path.join("enhancements", "generic.cfg")
self.enhancement_config_file = config_search_paths(
config_fn, self.ppp_config_dir)
if not isinstance(self.enhancement_config_file, (list, tuple)):
self.enhancement_config_file = [self.enhancement_config_file]
if self.enhancement_config_file:
self.enhancement_tree = EnhancementDecisionTree(
*self.enhancement_config_file)
else:
# They don't want any automatic enhancements
self.enhancement_tree = None
self.sensor_enhancement_configs = []
def get_enhanced_image(dataset,
enhancer=None,
fill_value=None,
ppp_config_dir=None,
enhancement_config_file=None,
overlay=None):
mode = _determine_mode(dataset)
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
if enhancer is None:
enhancer = Enhancer(ppp_config_dir, enhancement_config_file)
if enhancer.enhancement_tree is None:
raise RuntimeError(
"No enhancement configuration files found or specified, cannot"
" automatically enhance dataset")
if dataset.info.get("sensor", None):
enhancer.add_sensor_enhancements(dataset.info["sensor"])
# Create an image for enhancement
img = to_image(dataset, mode=mode, fill_value=fill_value)
enhancer.apply(img, **dataset.info)
img.info.update(dataset.info)
if overlay is not None:
add_overlay(img, dataset.info['area'], **overlay)
return img
def get_enhanced_image(dataset,
enhancer=None,
fill_value=None,
ppp_config_dir=None,
enhancement_config_file=None,
overlay=None,
decorate=None):
mode = _determine_mode(dataset)
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
if enhancer is None:
enhancer = Enhancer(ppp_config_dir, enhancement_config_file)
# Create an image for enhancement
img = to_image(dataset, mode=mode, fill_value=fill_value)
if enhancer.enhancement_tree is None:
LOG.debug("No enhancement being applied to dataset")
else:
if dataset.attrs.get("sensor", None):
enhancer.add_sensor_enhancements(dataset.attrs["sensor"])
enhancer.apply(img, **dataset.attrs)
if overlay is not None:
add_overlay(img, dataset.attrs['area'], **overlay)
if decorate is not None:
add_decorate(img, **decorate)
return img
def __init__(self,
ppp_config_dir=None,
default_config_filename=None,
config_files=None,
**kwargs):
"""Load configuration files related to this plugin.
This initializes a `self.config` dictionary that can be used to customize the subclass.
Args:
ppp_config_dir (str): Base "etc" directory for all configuration
files.
default_config_filename (str): Configuration filename to use if
no other files have been specified with `config_files`.
config_files (list or str): Configuration files to load instead
of those automatically found in `ppp_config_dir` and other
default configuration locations.
kwargs (dict): Unused keyword arguments.
"""
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
self.default_config_filename = default_config_filename
self.config_files = config_files
if self.config_files is None and self.default_config_filename is not None:
# Specify a default
self.config_files = config_search_paths(
self.default_config_filename, self.ppp_config_dir)
if not isinstance(self.config_files, (list, tuple)):
self.config_files = [self.config_files]
self.config = {}
if self.config_files:
for config_file in self.config_files:
self.load_yaml_config(config_file)
def __init__(self, ppp_config_dir=None, enhancement_config_file=None):
"""Initialize an Enhancer instance.
Args:
ppp_config_dir: Points to the base configuration directory
enhancement_config_file: The enhancement configuration to
apply, False to leave as is.
"""
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
self.enhancement_config_file = enhancement_config_file
# Set enhancement_config_file to False for no enhancements
if self.enhancement_config_file is None:
# it wasn't specified in the config or in the kwargs, we should
# provide a default
config_fn = os.path.join("enhancements", "generic.yaml")
self.enhancement_config_file = config_search_paths(
config_fn, self.ppp_config_dir)
if not self.enhancement_config_file:
# They don't want any automatic enhancements
self.enhancement_tree = None
else:
if not isinstance(self.enhancement_config_file, (list, tuple)):
self.enhancement_config_file = [self.enhancement_config_file]
self.enhancement_tree = EnhancementDecisionTree(
*self.enhancement_config_file)
self.sensor_enhancement_configs = []
def __init__(self,
filenames=None,
ppp_config_dir=get_environ_config_dir(),
reader=None,
base_dir=None,
**metadata):
"""The Scene object constructor.
Args:
filenames: A sequence of files that will be used to load data from.
ppp_config_dir: The directory containing the configuration files for
satpy.
reader: The name of the reader to use for loading the data.
base_dir: The directory to search for files containing the data to
load.
metadata: Free metadata information.
"""
InfoObject.__init__(self, **metadata)
# Set the PPP_CONFIG_DIR in the environment in case it's used elsewhere
# in pytroll
LOG.debug("Setting 'PPP_CONFIG_DIR' to '%s'", ppp_config_dir)
os.environ["PPP_CONFIG_DIR"] = self.ppp_config_dir = ppp_config_dir
self.readers = self.create_reader_instances(filenames=filenames,
base_dir=base_dir,
reader=reader)
self.info.update(self._compute_metadata_from_readers())
self.datasets = DatasetDict()
self.cpl = CompositorLoader(self.ppp_config_dir)
self.compositors = {}
self.wishlist = set()
def get_enhanced_image(dataset,
enhancer=None,
ppp_config_dir=None,
enhancement_config_file=None,
overlay=None,
decorate=None):
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
if enhancer is None:
enhancer = Enhancer(ppp_config_dir, enhancement_config_file)
# Create an image for enhancement
img = to_image(dataset)
if enhancer.enhancement_tree is None:
LOG.debug("No enhancement being applied to dataset")
else:
if dataset.attrs.get("sensor", None):
enhancer.add_sensor_enhancements(dataset.attrs["sensor"])
enhancer.apply(img, **dataset.attrs)
if overlay is not None:
add_overlay(img, dataset.attrs['area'], **overlay)
if decorate is not None:
add_decorate(img, **decorate)
return img
def __init__(self,
filenames=None,
ppp_config_dir=get_environ_config_dir(),
reader=None,
base_dir=None,
**metadata):
"""The Scene object constructor.
Args:
filenames: A sequence of files that will be used to load data from.
ppp_config_dir: The directory containing the configuration files for
satpy.
reader: The name of the reader to use for loading the data.
base_dir: The directory to search for files containing the data to
load.
metadata: Free metadata information.
"""
InfoObject.__init__(self, **metadata)
# Set the PPP_CONFIG_DIR in the environment in case it's used elsewhere
# in pytroll
LOG.debug("Setting 'PPP_CONFIG_DIR' to '%s'", ppp_config_dir)
os.environ["PPP_CONFIG_DIR"] = self.ppp_config_dir = ppp_config_dir
self.readers = self.create_reader_instances(filenames=filenames,
base_dir=base_dir,
reader=reader)
self.info.update(self._compute_metadata_from_readers())
self.datasets = DatasetDict()
self.cpl = CompositorLoader(self.ppp_config_dir)
self.compositors = {}
self.wishlist = set()
def __init__(self,
filenames=None,
ppp_config_dir=None,
reader=None,
base_dir=None,
**info):
"""The Scene object constructor.
"""
# Get PPP_CONFIG_DIR
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
# Set the PPP_CONFIG_DIR in the environment in case it's used elsewhere
# in pytroll
LOG.debug("Setting 'PPP_CONFIG_DIR' to '%s'", self.ppp_config_dir)
os.environ["PPP_CONFIG_DIR"] = self.ppp_config_dir
InfoObject.__init__(self, **info)
self.readers = {}
self.datasets = DatasetDict()
self.cpl = CompositorLoader(self.ppp_config_dir)
self.compositors = {}
self.wishlist = set()
if filenames is not None and not filenames:
raise ValueError("Filenames are specified but empty")
finder = ReaderFinder(
ppp_config_dir=self.ppp_config_dir,
base_dir=base_dir,
start_time=self.info.get('start_time'),
end_time=self.info.get('end_time'),
area=self.info.get('area'),
)
try:
reader_instances = finder(reader=reader,
sensor=self.info.get("sensor"),
filenames=filenames)
except ValueError as err:
if filenames is None and base_dir is None:
LOG.info(
'Neither filenames nor base_dir provided, '
'creating an empty scene (error was %s)', str(err))
reader_instances = []
else:
raise
# reader finder could return multiple readers
sensors = []
for reader_instance in reader_instances:
if reader_instance:
self.readers[reader_instance.name] = reader_instance
sensors.extend(reader_instance.sensor_names)
# if the user didn't tell us what sensors to work with, let's figure it
# out
if not self.info.get("sensor"):
self.info["sensor"] = sensors
def __init__(self,
ppp_config_dir=get_environ_config_dir(),
base_dir=None,
start_time=None,
end_time=None,
area=None):
"""Find readers.
If both *filenames* and *base_dir* are provided, only *filenames* is
used.
"""
self.ppp_config_dir = ppp_config_dir
self.base_dir = base_dir
self.start_time = start_time
self.end_time = end_time
self.area = area
def load_writer(writer, ppp_config_dir=None, **writer_kwargs):
"""Find and load writer `writer` in the available configuration files."""
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
config_fn = writer + ".yaml" if "." not in writer else writer
config_files = config_search_paths(os.path.join("writers", config_fn),
ppp_config_dir)
writer_kwargs.setdefault("config_files", config_files)
if not writer_kwargs['config_files']:
raise ValueError("Unknown writer '{}'".format(writer))
try:
return load_writer_configs(writer_kwargs['config_files'],
ppp_config_dir=ppp_config_dir,
**writer_kwargs)
except ValueError:
raise ValueError("Writer '{}' does not exist or could not be "
"loaded".format(writer))
def load_writer(writer, ppp_config_dir=None, **writer_kwargs):
"""Find and load writer `writer` in the available configuration files."""
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
config_fn = writer + ".yaml" if "." not in writer else writer
config_files = config_search_paths(
os.path.join("writers", config_fn), ppp_config_dir)
writer_kwargs.setdefault("config_files", config_files)
if not writer_kwargs['config_files']:
raise ValueError("Unknown writer '{}'".format(writer))
try:
return load_writer_configs(writer_kwargs['config_files'],
ppp_config_dir=ppp_config_dir,
**writer_kwargs)
except ValueError:
raise ValueError("Writer '{}' does not exist or could not be "
"loaded".format(writer))
def __init__(self,
ppp_config_dir=None,
default_config_filename=None,
config_files=None,
**kwargs):
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
self.default_config_filename = default_config_filename
self.config_files = config_files
if self.config_files is None and self.default_config_filename is not None:
# Specify a default
self.config_files = config_search_paths(
self.default_config_filename, self.ppp_config_dir)
if not isinstance(self.config_files, (list, tuple)):
self.config_files = [self.config_files]
if self.config_files:
conf = configparser.RawConfigParser()
conf.read(self.config_files)
self.load_config(conf)
def __init__(self,
ppp_config_dir=None,
default_config_filename=None,
config_files=None,
**kwargs):
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
self.default_config_filename = default_config_filename
self.config_files = config_files
if self.config_files is None and self.default_config_filename is not None:
# Specify a default
self.config_files = config_search_paths(
self.default_config_filename, self.ppp_config_dir)
if not isinstance(self.config_files, (list, tuple)):
self.config_files = [self.config_files]
self.config = {}
if self.config_files:
for config_file in self.config_files:
self.load_yaml_config(config_file)
def __init__(self,
ppp_config_dir=None,
default_config_filename=None,
config_files=None,
**kwargs):
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
self.default_config_filename = default_config_filename
self.config_files = config_files
if self.config_files is None and self.default_config_filename is not None:
# Specify a default
self.config_files = config_search_paths(
self.default_config_filename, self.ppp_config_dir)
if not isinstance(self.config_files, (list, tuple)):
self.config_files = [self.config_files]
if self.config_files:
conf = configparser.RawConfigParser()
conf.read(self.config_files)
self.load_config(conf)
def __init__(self,
ppp_config_dir=None,
default_config_filename=None,
config_files=None,
**kwargs):
self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
self.default_config_filename = default_config_filename
self.config_files = config_files
if self.config_files is None and self.default_config_filename is not None:
# Specify a default
self.config_files = config_search_paths(
self.default_config_filename, self.ppp_config_dir)
if not isinstance(self.config_files, (list, tuple)):
self.config_files = [self.config_files]
self.config = {}
if self.config_files:
for config_file in self.config_files:
self.load_yaml_config(config_file)
def load_readers(filenames=None,
reader=None,
reader_kwargs=None,
ppp_config_dir=get_environ_config_dir()):
"""Create specified readers and assign files to them.
Args:
filenames (iterable or dict): A sequence of files that will be used to load data from. A ``dict`` object
should map reader names to a list of filenames for that reader.
reader (str or list): The name of the reader to use for loading the data or a list of names.
filter_parameters (dict): Specify loaded file filtering parameters.
Shortcut for `reader_kwargs['filter_parameters']`.
reader_kwargs (dict): Keyword arguments to pass to specific reader instances.
ppp_config_dir (str): The directory containing the configuration files for satpy.
Returns: Dictionary mapping reader name to reader instance
"""
reader_instances = {}
reader_kwargs = reader_kwargs or {}
if not filenames and not reader:
# used for an empty Scene
return {}
elif reader and filenames is not None and not filenames:
# user made a mistake in their glob pattern
raise ValueError("'filenames' was provided but is empty.")
elif not filenames:
LOG.warning("'filenames' required to create readers and load data")
return {}
elif reader is None and isinstance(filenames, dict):
# filenames is a dictionary of reader_name -> filenames
reader = list(filenames.keys())
remaining_filenames = set(f for fl in filenames.values() for f in fl)
elif reader and isinstance(filenames, dict):
# filenames is a dictionary of reader_name -> filenames
# but they only want one of the readers
filenames = filenames[reader]
remaining_filenames = set(filenames or [])
else:
remaining_filenames = set(filenames or [])
for idx, reader_configs in enumerate(
configs_for_reader(reader, ppp_config_dir)):
if isinstance(filenames, dict):
readers_files = set(filenames[reader[idx]])
else:
readers_files = remaining_filenames
try:
reader_instance = load_reader(reader_configs, **reader_kwargs)
except (KeyError, IOError, yaml.YAMLError) as err:
LOG.info('Cannot use %s', str(reader_configs))
LOG.debug(str(err))
continue
if readers_files:
loadables = reader_instance.select_files_from_pathnames(
readers_files)
if loadables:
reader_instance.create_filehandlers(loadables)
reader_instances[reader_instance.name] = reader_instance
remaining_filenames -= set(loadables)
if not remaining_filenames:
break
if remaining_filenames:
LOG.warning("Don't know how to open the following files: {}".format(
str(remaining_filenames)))
if not reader_instances:
raise ValueError("No supported files found")
return reader_instances
def find_files_and_readers(start_time=None,
end_time=None,
base_dir=None,
reader=None,
sensor=None,
ppp_config_dir=get_environ_config_dir(),
filter_parameters=None,
reader_kwargs=None):
"""Find on-disk files matching the provided parameters.
Use `start_time` and/or `end_time` to limit found filenames by the times
in the filenames (not the internal file metadata). Files are matched if
they fall anywhere within the range specified by these parameters.
Searching is **NOT** recursive.
The returned dictionary can be passed directly to the `Scene` object
through the `filenames` keyword argument.
The behaviour of time-based filtering depends on whether or not the filename
contains information about the end time of the data or not:
- if the end time is not present in the filename, the start time of the filename
is used and has to fall between (inclusive) the requested start and end times
- otherwise, the timespan of the filename has to overlap the requested timespan
Args:
start_time (datetime): Limit used files by starting time.
end_time (datetime): Limit used files by ending time.
base_dir (str): The directory to search for files containing the
data to load. Defaults to the current directory.
reader (str or list): The name of the reader to use for loading the data or a list of names.
sensor (str or list): Limit used files by provided sensors.
ppp_config_dir (str): The directory containing the configuration
files for SatPy.
filter_parameters (dict): Filename pattern metadata to filter on. `start_time` and `end_time` are
automatically added to this dictionary. Shortcut for
`reader_kwargs['filter_parameters']`.
reader_kwargs (dict): Keyword arguments to pass to specific reader
instances to further configure file searching.
Returns: Dictionary mapping reader name string to list of filenames
"""
reader_files = {}
reader_kwargs = reader_kwargs or {}
filter_parameters = filter_parameters or reader_kwargs.get(
'filter_parameters', {})
sensor_supported = False
if start_time or end_time:
filter_parameters['start_time'] = start_time
filter_parameters['end_time'] = end_time
reader_kwargs['filter_parameters'] = filter_parameters
for reader_configs in configs_for_reader(reader, ppp_config_dir):
try:
reader_instance = load_reader(reader_configs, **reader_kwargs)
except (KeyError, IOError, yaml.YAMLError) as err:
LOG.info('Cannot use %s', str(reader_configs))
LOG.debug(str(err))
if reader and (isinstance(reader, str) or len(reader) == 1):
# if it is a single reader then give a more usable error
raise
continue
if not reader_instance.supports_sensor(sensor):
continue
elif sensor is not None:
# sensor was specified and a reader supports it
sensor_supported = True
loadables = reader_instance.select_files_from_directory(base_dir)
if loadables:
loadables = list(
reader_instance.filter_selected_filenames(loadables))
if loadables:
reader_files[reader_instance.name] = list(loadables)
if sensor and not sensor_supported:
raise ValueError(
"Sensor '{}' not supported by any readers".format(sensor))
if not reader_files:
raise ValueError("No supported files found")
return reader_files
def find_files_and_readers(start_time=None, end_time=None, base_dir=None,
reader=None, sensor=None, ppp_config_dir=get_environ_config_dir(),
filter_parameters=None, reader_kwargs=None):
"""Find on-disk files matching the provided parameters.
Use `start_time` and/or `end_time` to limit found filenames by the times
in the filenames (not the internal file metadata). Files are matched if
they fall anywhere within the range specified by these parameters.
Searching is **NOT** recursive.
The returned dictionary can be passed directly to the `Scene` object
through the `filenames` keyword argument.
Args:
start_time (datetime): Limit used files by starting time.
end_time (datetime): Limit used files by ending time.
base_dir (str): The directory to search for files containing the
data to load. Defaults to the current directory.
reader (str or list): The name of the reader to use for loading the data or a list of names.
sensor (str or list): Limit used files by provided sensors.
ppp_config_dir (str): The directory containing the configuration
files for SatPy.
filter_parameters (dict): Filename pattern metadata to filter on. `start_time` and `end_time` are
automatically added to this dictionary. Shortcut for
`reader_kwargs['filter_parameters']`.
reader_kwargs (dict): Keyword arguments to pass to specific reader
instances to further configure file searching.
Returns: Dictionary mapping reader name string to list of filenames
"""
reader_files = {}
reader_kwargs = reader_kwargs or {}
filter_parameters = filter_parameters or reader_kwargs.get('filter_parameters', {})
sensor_supported = False
if start_time or end_time:
filter_parameters['start_time'] = start_time
filter_parameters['end_time'] = end_time
reader_kwargs['filter_parameters'] = filter_parameters
for reader_configs in configs_for_reader(reader, ppp_config_dir):
try:
reader_instance = load_reader(reader_configs, **reader_kwargs)
except (KeyError, IOError, yaml.YAMLError) as err:
LOG.info('Cannot use %s', str(reader_configs))
LOG.debug(str(err))
continue
if not reader_instance.supports_sensor(sensor):
continue
elif sensor is not None:
# sensor was specified and a reader supports it
sensor_supported = True
loadables = reader_instance.select_files_from_directory(base_dir)
if loadables:
loadables = list(
reader_instance.filter_selected_filenames(loadables))
if loadables:
reader_files[reader_instance.name] = list(loadables)
if sensor and not sensor_supported:
raise ValueError("Sensor '{}' not supported by any readers".format(sensor))
if not reader_files:
raise ValueError("No supported files found")
return reader_files
def get_enhanced_image(dataset,
ppp_config_dir=None,
enhance=None,
enhancement_config_file=None,
overlay=None,
decorate=None,
fill_value=None):
"""Get an enhanced version of `dataset` as an :class:`~trollimage.xrimage.XRImage` instance.
Args:
dataset (xarray.DataArray): Data to be enhanced and converted to an image.
ppp_config_dir (str): Root configuration directory.
enhance (bool or Enhancer): Whether to automatically enhance
data to be more visually useful and to fit inside the file
format being saved to. By default this will default to using
the enhancement configuration files found using the default
:class:`~satpy.writers.Enhancer` class. This can be set to
`False` so that no enhancments are performed. This can also
be an instance of the :class:`~satpy.writers.Enhancer` class
if further custom enhancement is needed.
enhancement_config_file (str): Deprecated.
overlay (dict): Options for image overlays. See :func:`add_overlay`
for available options.
decorate (dict): Options for decorating the image. See
:func:`add_decorate` for available options.
fill_value (int or float): Value to use when pixels are masked or
invalid. Default of `None` means to create an alpha channel.
See :meth:`~trollimage.xrimage.XRImage.finalize` for more
details. Only used when adding overlays or decorations. Otherwise
it is up to the caller to "finalize" the image before using it
except if calling ``img.show()`` or providing the image to
a writer as these will finalize the image.
.. versionchanged:: 0.10
Deprecated `enhancement_config_file` and 'enhancer' in favor of
`enhance`. Pass an instance of the `Enhancer` class to `enhance`
instead.
"""
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
if enhancement_config_file is not None:
warnings.warn(
"'enhancement_config_file' has been deprecated. Pass an instance of the "
"'Enhancer' class to the 'enhance' keyword argument instead.",
DeprecationWarning)
if enhance is False:
# no enhancement
enhancer = None
elif enhance is None or enhance is True:
# default enhancement
enhancer = Enhancer(ppp_config_dir, enhancement_config_file)
else:
# custom enhancer
enhancer = enhance
# Create an image for enhancement
img = to_image(dataset)
if enhancer is None or enhancer.enhancement_tree is None:
LOG.debug("No enhancement being applied to dataset")
else:
if dataset.attrs.get("sensor", None):
enhancer.add_sensor_enhancements(dataset.attrs["sensor"])
enhancer.apply(img, **dataset.attrs)
if overlay is not None:
img = add_overlay(img,
dataset.attrs['area'],
fill_value=fill_value,
**overlay)
if decorate is not None:
img = add_decorate(img, fill_value=fill_value, **decorate)
return img
def __init__(self,
filenames=None,
ppp_config_dir=get_environ_config_dir(),
reader=None,
base_dir=None,
sensor=None,
start_time=None,
end_time=None,
area=None,
reader_kwargs=None,
**metadata):
"""The Scene object constructor.
Note to load data either `filenames`, `reader`, or a 'base_dir' must
be specified. If `filenames` is not specified then `reader` must be
specified to search the current directory or `base_dir` if specified.
If neither `filenames` nor `reader` is specified then `base_dir` will
be used to find any files matching the file patterns for any
configured readers. Otherwise the Scene is created with no Readers
available meaning Datasets must be added manually:
scn = Scene(sensor='viirs', start_time=start_time)
scn['my_dataset'] = Dataset(my_data_array, **my_info)
Args:
filenames (iterable): A sequence of files that will be used to load
data from.
ppp_config_dir (str): The directory containing the configuration
files for satpy.
reader: The name of the reader to use for loading the data.
base_dir (str): The directory to search for files containing the
data to load. If *filenames* is also provided,
this is ignored.
sensor (list or str): Limit used files by provided sensors.
area (AreaDefinition): Limit used files by geographic area.
start_time (datetime): Limit used files by starting time.
end_time (datetime): Limit used files by ending time.
reader_kwargs (dict): Keyword arguments to pass to specific reader
instances.
metadata: Other metadata to assign to the Scene's ``.info``.
"""
InfoObject.__init__(self,
sensor=sensor or set(),
area=area,
start_time=start_time,
end_time=end_time,
**metadata)
# Set the PPP_CONFIG_DIR in the environment in case it's used elsewhere
# in pytroll
LOG.debug("Setting 'PPP_CONFIG_DIR' to '%s'", ppp_config_dir)
os.environ["PPP_CONFIG_DIR"] = self.ppp_config_dir = ppp_config_dir
self.readers = self.create_reader_instances(
filenames=filenames,
base_dir=base_dir,
reader=reader,
reader_kwargs=reader_kwargs)
self.info.update(self._compute_metadata_from_readers())
self.datasets = DatasetDict()
self.cpl = CompositorLoader(self.ppp_config_dir)
comps, mods = self.cpl.load_compositors(self.info['sensor'])
self.wishlist = set()
self.dep_tree = DependencyTree(self.readers, comps, mods)
def find_files_and_readers(start_time=None,
end_time=None,
base_dir=None,
reader=None,
sensor=None,
ppp_config_dir=None,
filter_parameters=None,
reader_kwargs=None,
missing_ok=False,
fs=None):
"""Find files matching the provided parameters.
Use `start_time` and/or `end_time` to limit found filenames by the times
in the filenames (not the internal file metadata). Files are matched if
they fall anywhere within the range specified by these parameters.
Searching is **NOT** recursive.
Files may be either on-disk or on a remote file system. By default,
files are searched for locally. Users can search on remote filesystems by
passing an instance of an implementation of
`fsspec.spec.AbstractFileSystem` (strictly speaking, any object of a class
implementing a ``glob`` method works).
If locating files on a local file system, the returned dictionary
can be passed directly to the `Scene` object through the `filenames`
keyword argument. If it points to a remote file system, it is the
responsibility of the user to download the files first (directly
reading from cloud storage is not currently available in Satpy).
The behaviour of time-based filtering depends on whether or not the filename
contains information about the end time of the data or not:
- if the end time is not present in the filename, the start time of the filename
is used and has to fall between (inclusive) the requested start and end times
- otherwise, the timespan of the filename has to overlap the requested timespan
Example usage for querying a s3 filesystem using the s3fs module:
>>> import s3fs, satpy.readers, datetime
>>> satpy.readers.find_files_and_readers(
... base_dir="s3://noaa-goes16/ABI-L1b-RadF/2019/321/14/",
... fs=s3fs.S3FileSystem(anon=True),
... reader="abi_l1b",
... start_time=datetime.datetime(2019, 11, 17, 14, 40))
{'abi_l1b': [...]}
Args:
start_time (datetime): Limit used files by starting time.
end_time (datetime): Limit used files by ending time.
base_dir (str): The directory to search for files containing the
data to load. Defaults to the current directory.
reader (str or list): The name of the reader to use for loading the data or a list of names.
sensor (str or list): Limit used files by provided sensors.
ppp_config_dir (str): The directory containing the configuration
files for Satpy.
filter_parameters (dict): Filename pattern metadata to filter on. `start_time` and `end_time` are
automatically added to this dictionary. Shortcut for
`reader_kwargs['filter_parameters']`.
reader_kwargs (dict): Keyword arguments to pass to specific reader
instances to further configure file searching.
missing_ok (bool): If False (default), raise ValueError if no files
are found. If True, return empty dictionary if no
files are found.
fs (FileSystem): Optional, instance of implementation of
fsspec.spec.AbstractFileSystem (strictly speaking,
any object of a class implementing ``.glob`` is
enough). Defaults to searching the local filesystem.
Returns: Dictionary mapping reader name string to list of filenames
"""
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
reader_files = {}
reader_kwargs = reader_kwargs or {}
filter_parameters = filter_parameters or reader_kwargs.get(
'filter_parameters', {})
sensor_supported = False
if start_time or end_time:
filter_parameters['start_time'] = start_time
filter_parameters['end_time'] = end_time
reader_kwargs['filter_parameters'] = filter_parameters
for reader_configs in configs_for_reader(reader, ppp_config_dir):
try:
reader_instance = load_reader(reader_configs, **reader_kwargs)
except (KeyError, IOError, yaml.YAMLError) as err:
LOG.info('Cannot use %s', str(reader_configs))
LOG.debug(str(err))
if reader and (isinstance(reader, str) or len(reader) == 1):
# if it is a single reader then give a more usable error
raise
continue
if not reader_instance.supports_sensor(sensor):
continue
elif sensor is not None:
# sensor was specified and a reader supports it
sensor_supported = True
loadables = reader_instance.select_files_from_directory(base_dir, fs)
if loadables:
loadables = list(
reader_instance.filter_selected_filenames(loadables))
if loadables:
reader_files[reader_instance.name] = list(loadables)
if sensor and not sensor_supported:
raise ValueError(
"Sensor '{}' not supported by any readers".format(sensor))
if not (reader_files or missing_ok):
raise ValueError("No supported files found")
return reader_files
def load_readers(filenames=None, reader=None, reader_kwargs=None,
ppp_config_dir=get_environ_config_dir()):
"""Create specified readers and assign files to them.
Args:
filenames (iterable or dict): A sequence of files that will be used to load data from. A ``dict`` object
should map reader names to a list of filenames for that reader.
reader (str or list): The name of the reader to use for loading the data or a list of names.
filter_parameters (dict): Specify loaded file filtering parameters.
Shortcut for `reader_kwargs['filter_parameters']`.
reader_kwargs (dict): Keyword arguments to pass to specific reader instances.
ppp_config_dir (str): The directory containing the configuration files for satpy.
Returns: Dictionary mapping reader name to reader instance
"""
reader_instances = {}
reader_kwargs = reader_kwargs or {}
if not filenames and not reader:
# used for an empty Scene
return {}
elif reader and filenames is not None and not filenames:
# user made a mistake in their glob pattern
raise ValueError("'filenames' was provided but is empty.")
elif not filenames:
LOG.warning("'filenames' required to create readers and load data")
return {}
elif reader is None and isinstance(filenames, dict):
# filenames is a dictionary of reader_name -> filenames
reader = list(filenames.keys())
remaining_filenames = set(f for fl in filenames.values() for f in fl)
else:
remaining_filenames = set(filenames or [])
for idx, reader_configs in enumerate(configs_for_reader(reader, ppp_config_dir)):
if isinstance(filenames, dict):
readers_files = set(filenames[reader[idx]])
else:
readers_files = remaining_filenames
try:
reader_instance = load_reader(reader_configs, **reader_kwargs)
except (KeyError, IOError, yaml.YAMLError) as err:
LOG.info('Cannot use %s', str(reader_configs))
LOG.debug(str(err))
continue
if readers_files:
loadables = reader_instance.select_files_from_pathnames(readers_files)
if loadables:
reader_instance.create_filehandlers(loadables)
reader_instances[reader_instance.name] = reader_instance
remaining_filenames -= set(loadables)
if not remaining_filenames:
break
if remaining_filenames:
LOG.warning(
"Don't know how to open the following files: {}".format(str(
remaining_filenames)))
if not reader_instances:
raise ValueError("No supported files found")
return reader_instances
def __init__(self,
filenames=None,
reader=None,
filter_parameters=None,
reader_kwargs=None,
ppp_config_dir=get_environ_config_dir(),
base_dir=None,
sensor=None,
start_time=None,
end_time=None,
area=None):
"""Initialize Scene with Reader and Compositor objects.
To load data `filenames` and preferably `reader` must be specified. If `filenames` is provided without `reader`
then the available readers will be searched for a Reader that can support the provided files. This can take
a considerable amount of time so it is recommended that `reader` always be provided. Note without `filenames`
the Scene is created with no Readers available requiring Datasets to be added manually::
scn = Scene()
scn['my_dataset'] = Dataset(my_data_array, **my_info)
Args:
filenames (iterable or dict): A sequence of files that will be used to load data from. A ``dict`` object
should map reader names to a list of filenames for that reader.
reader (str or list): The name of the reader to use for loading the data or a list of names.
filter_parameters (dict): Specify loaded file filtering parameters.
Shortcut for `reader_kwargs['filter_parameters']`.
reader_kwargs (dict): Keyword arguments to pass to specific reader instances.
ppp_config_dir (str): The directory containing the configuration files for satpy.
base_dir (str): (DEPRECATED) The directory to search for files containing the
data to load. If *filenames* is also provided,
this is ignored.
sensor (list or str): (DEPRECATED: Use `find_files_and_readers` function) Limit used files by provided
sensors.
area (AreaDefinition): (DEPRECATED: Use `filter_parameters`) Limit used files by geographic area.
start_time (datetime): (DEPRECATED: Use `filter_parameters`) Limit used files by starting time.
end_time (datetime): (DEPRECATED: Use `filter_parameters`) Limit used files by ending time.
"""
super(Scene, self).__init__()
# Set the PPP_CONFIG_DIR in the environment in case it's used elsewhere in pytroll
LOG.debug("Setting 'PPP_CONFIG_DIR' to '%s'", ppp_config_dir)
os.environ["PPP_CONFIG_DIR"] = self.ppp_config_dir = ppp_config_dir
if not filenames and (start_time or end_time or base_dir):
import warnings
warnings.warn(
"Deprecated: Use " +
"'from satpy import find_files_and_readers' to find files")
from satpy import find_files_and_readers
filenames = find_files_and_readers(
start_time=start_time,
end_time=end_time,
base_dir=base_dir,
reader=reader,
sensor=sensor,
ppp_config_dir=self.ppp_config_dir,
reader_kwargs=reader_kwargs,
)
elif start_time or end_time or area:
import warnings
warnings.warn(
"Deprecated: Use " +
"'filter_parameters' to filter loaded files by 'start_time', "
+ "'end_time', or 'area'.")
fp = filter_parameters if filter_parameters else {}
fp.update({
'start_time': start_time,
'end_time': end_time,
'area': area,
})
filter_parameters = fp
if filter_parameters:
if reader_kwargs is None:
reader_kwargs = {}
reader_kwargs.setdefault('filter_parameters',
{}).update(filter_parameters)
if filenames and isinstance(filenames, str):
raise ValueError(
"'filenames' must be a list of files: Scene(filenames=[filename])"
)
self.readers = self.create_reader_instances(
filenames=filenames, reader=reader, reader_kwargs=reader_kwargs)
self.attrs.update(self._compute_metadata_from_readers())
self.datasets = DatasetDict()
self.cpl = CompositorLoader(self.ppp_config_dir)
comps, mods = self.cpl.load_compositors(self.attrs['sensor'])
self.wishlist = set()
self.dep_tree = DependencyTree(self.readers, comps, mods)
self.resamplers = {}
def load_readers(filenames=None,
reader=None,
reader_kwargs=None,
ppp_config_dir=None):
"""Create specified readers and assign files to them.
Args:
filenames (iterable or dict): A sequence of files that will be used to load data from. A ``dict`` object
should map reader names to a list of filenames for that reader.
reader (str or list): The name of the reader to use for loading the data or a list of names.
reader_kwargs (dict): Keyword arguments to pass to specific reader instances.
This can either be a single dictionary that will be passed to all
reader instances, or a mapping of reader names to dictionaries. If
the keys of ``reader_kwargs`` match exactly the list of strings in
``reader`` or the keys of filenames, each reader instance will get its
own keyword arguments accordingly.
ppp_config_dir (str): The directory containing the configuration files for satpy.
Returns: Dictionary mapping reader name to reader instance
"""
reader_instances = {}
if ppp_config_dir is None:
ppp_config_dir = get_environ_config_dir()
if not filenames and not reader:
# used for an empty Scene
return {}
elif reader and filenames is not None and not filenames:
# user made a mistake in their glob pattern
raise ValueError("'filenames' was provided but is empty.")
elif not filenames:
LOG.warning("'filenames' required to create readers and load data")
return {}
elif reader is None and isinstance(filenames, dict):
# filenames is a dictionary of reader_name -> filenames
reader = list(filenames.keys())
remaining_filenames = set(f for fl in filenames.values() for f in fl)
elif reader and isinstance(filenames, dict):
# filenames is a dictionary of reader_name -> filenames
# but they only want one of the readers
filenames = filenames[reader]
remaining_filenames = set(filenames or [])
else:
remaining_filenames = set(filenames or [])
(reader_kwargs,
reader_kwargs_without_filter) = _get_reader_kwargs(reader, reader_kwargs)
for idx, reader_configs in enumerate(
configs_for_reader(reader, ppp_config_dir)):
if isinstance(filenames, dict):
readers_files = set(filenames[reader[idx]])
else:
readers_files = remaining_filenames
try:
reader_instance = load_reader(
reader_configs,
**reader_kwargs[None if reader is None else reader[idx]])
except (KeyError, IOError, yaml.YAMLError) as err:
LOG.info('Cannot use %s', str(reader_configs))
LOG.debug(str(err))
continue
if not readers_files:
# we weren't given any files for this reader
continue
loadables = reader_instance.select_files_from_pathnames(readers_files)
if loadables:
reader_instance.create_filehandlers(
loadables,
fh_kwargs=reader_kwargs_without_filter[
None if reader is None else reader[idx]])
reader_instances[reader_instance.name] = reader_instance
remaining_filenames -= set(loadables)
if not remaining_filenames:
break
if remaining_filenames:
LOG.warning("Don't know how to open the following files: {}".format(
str(remaining_filenames)))
if not reader_instances:
raise ValueError("No supported files found")
elif not any(
list(r.available_dataset_ids) for r in reader_instances.values()):
raise ValueError(
"No dataset could be loaded. Either missing "
"requirements (such as Epilog, Prolog) or none of the "
"provided files match the filter parameters.")
return reader_instances
