def test_no_reader(self):
"""Test that reader does not need to be provided."""
from satpy.readers import group_files
# without files it's going to be an empty result
assert group_files([]) == []
groups = group_files(self.g16_files)
self.assertEqual(6, len(groups))
def test_viirs_override_keys(self):
"""Test overriding a group keys to add 'start_time'."""
from satpy.readers import group_files
groups = group_files(self.noaa20_files + self.npp_files,
reader='viirs_sdr',
group_keys=('start_time', 'orbit',
'platform_shortname'))
self.assertEqual(8, len(groups))
self.assertEqual(2, len(groups[0]['viirs_sdr'])) # NPP
self.assertEqual(2, len(groups[1]['viirs_sdr'])) # NPP
self.assertEqual(2, len(groups[2]['viirs_sdr'])) # NPP
self.assertEqual(3, len(groups[3]['viirs_sdr'])) # N20
self.assertEqual(3, len(groups[4]['viirs_sdr'])) # N20
self.assertEqual(3, len(groups[5]['viirs_sdr'])) # N20
self.assertEqual(3, len(groups[6]['viirs_sdr'])) # N20
self.assertEqual(3, len(groups[7]['viirs_sdr'])) # N20
# Ask for a larger time span with our groups
groups = group_files(self.noaa20_files + self.npp_files,
reader='viirs_sdr',
time_threshold=60 * 60 * 2,
group_keys=('start_time', 'orbit',
'platform_shortname'))
self.assertEqual(2, len(groups))
# NPP is first because it has an earlier time
# 3 granules * 2 file types
self.assertEqual(6, len(groups[0]['viirs_sdr']))
# 5 granules * 3 file types
self.assertEqual(5 * 3, len(groups[1]['viirs_sdr']))
def test_multi_readers_invalid_parameter(self):
"""Verify that invalid missing parameter raises ValueError."""
from satpy.readers import group_files
with pytest.raises(ValueError):
group_files(
self._filenames_abi_glm,
reader=["abi_l1b", "glm_l2"],
group_keys=("start_time",),
time_threshold=35,
missing="hopkin green frog")
def from_files(cls, files_to_sort, reader=None,
ensure_all_readers=False, **kwargs):
"""Create multiple Scene objects from multiple files.
Args:
files_to_sort (Collection[str]): files to read
reader (str or Collection[str]): reader or readers to use
ensure_all_readers (bool): If True, limit to scenes where all
readers have at least one file. If False (default), include
all scenes where at least one reader has at least one file.
This uses the :func:`satpy.readers.group_files` function to group
files. See this function for more details on additional possible
keyword arguments. In particular, it is strongly recommended to pass
`"group_keys"` when using multiple instruments.
.. versionadded:: 0.12
"""
from satpy.readers import group_files
file_groups = group_files(files_to_sort, reader=reader, **kwargs)
if ensure_all_readers:
file_groups = [fg for fg in file_groups if all(fg.values())]
scenes = (Scene(filenames=fg) for fg in file_groups)
return cls(scenes)
def test_two_instruments_files_split(self):
"""Test the default behavior when two instruments files are provided and split.
Tell the sorting to include the platform identifier as another field
to use for grouping.
"""
from satpy.readers import group_files
groups = group_files(self.g16_files + self.g17_files, reader='abi_l1b',
group_keys=('start_time', 'platform_shortname'))
self.assertEqual(12, len(groups))
self.assertEqual(2, len(groups[0]['abi_l1b']))
# default for abi_l1b should also behave like this
groups = group_files(self.g16_files + self.g17_files, reader='abi_l1b')
self.assertEqual(12, len(groups))
self.assertEqual(2, len(groups[0]['abi_l1b']))
def _group_files(self, reader) -> bool:
"""Group provided files by time step."""
# the files haven't changed since we were last run
if not self._filelist_changed and self._previous_reader == reader:
return False
self._filelist_changed = False
self.file_groups = {} # reset the list, just in case
selected_files = self._all_filenames.copy()
self._previous_reader = reader
file_groups = group_files(selected_files, reader=reader)
if not file_groups:
self.unknown_files = selected_files
self.file_groups = {}
return True
scenes = {} # recreate Scene dictionary
file_group_map = {}
known_files = set()
for file_group in file_groups:
# file_group includes what reader to use
# NOTE: We only allow a single reader at a time
group_id = tuple(sorted(fn for group_list in file_group.values() for fn in group_list))
known_files.update(group_id)
if group_id not in self.scenes:
# never seen this exact group of files before
scenes[group_id] = None # filled in later
else:
scenes[group_id] = self.scenes[group_id]
file_group_map[group_id] = file_group
self.scenes = scenes
self.file_groups = file_group_map
self.unknown_files = selected_files - known_files
return True
def test_multi_readers_empty_groups_raises_filenotfounderror(self):
"""Test behaviour on empty groups passing multiple readers.
Make sure it raises an exception, for there will be groups
containing GLM but not ABI.
"""
from satpy.readers import group_files
with pytest.raises(
FileNotFoundError, match="when grouping files, group at index 1 "
"had no files for readers: abi_l1b"):
group_files(
self._filenames_abi_glm,
reader=["abi_l1b", "glm_l2"],
group_keys=("start_time",),
time_threshold=35,
missing="raise")
def test_non_datetime_group_key(self):
"""Test what happens when the start_time isn't used for grouping."""
from satpy.readers import group_files
groups = group_files(self.g16_files,
reader='abi_l1b',
group_keys=('platform_shortname', ))
self.assertEqual(1, len(groups))
self.assertEqual(12, len(groups[0]['abi_l1b']))
def test_large_time_threshold(self):
"""Test what happens when the time threshold holds multiple files."""
from satpy.readers import group_files
groups = group_files(self.g16_files,
reader='abi_l1b',
time_threshold=60 * 8)
self.assertEqual(3, len(groups))
self.assertEqual(4, len(groups[0]['abi_l1b']))
def test_viirs_orbits(self):
"""Test a reader that doesn't use 'start_time' for default grouping."""
from satpy.readers import group_files
groups = group_files(self.noaa20_files + self.npp_files, reader='viirs_sdr')
self.assertEqual(2, len(groups))
# the noaa-20 files will be first because the orbit number is smaller
# 5 granules * 3 file types
self.assertEqual(5 * 3, len(groups[0]['viirs_sdr']))
# 3 granules * 2 file types
self.assertEqual(6, len(groups[1]['viirs_sdr']))
def test_default_behavior_set(self):
"""Test the default behavior with the 'abi_l1b' reader."""
from satpy.readers import group_files
files = set(self.g16_files)
num_files = len(files)
groups = group_files(files, reader='abi_l1b')
# we didn't modify it
self.assertEqual(len(files), num_files)
self.assertEqual(6, len(groups))
self.assertEqual(2, len(groups[0]['abi_l1b']))
def test_multi_readers(self):
"""Test passing multiple readers."""
from satpy.readers import group_files
groups = group_files(self.g16_files + self.noaa20_files,
reader=("abi_l1b", "viirs_sdr"))
assert len(groups) == 11
# test that they're grouped together when time threshold is huge and
# only time is used to group
groups = group_files(self.g16_files + self.noaa20_files,
reader=("abi_l1b", "viirs_sdr"),
group_keys=("start_time", ),
time_threshold=10**9)
assert len(groups) == 1
# test that a warning is raised when a string is passed (meaning no
# group keys found in common)
with pytest.warns(UserWarning):
groups = group_files(self.g16_files + self.noaa20_files,
reader=("abi_l1b", "viirs_sdr"),
group_keys=("start_time"),
time_threshold=10**9)
def test_multi_readers_empty_groups_passed(self):
"""Verify that all groups are there, resulting in some that are empty."""
from satpy.readers import group_files
groups = group_files(
self._filenames_abi_glm,
reader=["abi_l1b", "glm_l2"],
group_keys=("start_time",),
time_threshold=35,
missing="pass")
assert len(groups) == 17
assert not groups[1]["abi_l1b"] # should be empty
assert groups[1]["glm_l2"] # should not be empty
def test_two_instruments_files(self):
"""Test the behavior when two instruments files are provided.
This is undesired from a user point of view since we don't want G16
and G17 files in the same Scene. Readers (like abi_l1b) are or can be
configured to have specific group keys for handling these situations.
Due to that this test forces the fallback group keys of
('start_time',).
"""
from satpy.readers import group_files
groups = group_files(self.g16_files + self.g17_files, reader='abi_l1b', group_keys=('start_time',))
self.assertEqual(6, len(groups))
self.assertEqual(4, len(groups[0]['abi_l1b']))
def from_files(cls, files_to_sort, reader=None, **kwargs):
"""Create multiple Scene objects from multiple files.
This uses the :func:`satpy.readers.group_files` function to group
files. See this function for more details on possible keyword
arguments.
.. versionadded:: 0.12
"""
from satpy.readers import group_files
file_groups = group_files(files_to_sort, reader=reader, **kwargs)
scenes = (Scene(filenames=fg) for fg in file_groups)
return cls(scenes)
def test_multi_readers_empty_groups_missing_skip(self):
"""Verify empty groups are skipped.
Verify that all groups lacking ABI are skipped, resulting in only
three groups that are all non-empty for both instruments.
"""
from satpy.readers import group_files
groups = group_files(
self._filenames_abi_glm,
reader=["abi_l1b", "glm_l2"],
group_keys=("start_time",),
time_threshold=35,
missing="skip")
assert len(groups) == 2
for g in groups:
assert g["abi_l1b"]
assert g["glm_l2"]
def bgnd_merge_new_file_metadata_into_mdb(self):
todo, self._scheduled_files = self._scheduled_files, []
ntodo = len(todo)
LOG.debug('collecting metadata from {} potential new files'.format(ntodo))
yield {TASK_DOING: 'collecting metadata 0/{}'.format(ntodo), TASK_PROGRESS: 0.0}
changed_uuids = set()
readers_and_files = group_files(todo, reader='abi_l1b')
num_seen = 0
for reader_and_files in readers_and_files:
for reader_name, filenames in reader_and_files.items():
product_infos = self._ws.collect_product_metadata_for_paths(filenames, reader=reader_name)
for num_prods, product_info in product_infos:
changed_uuids.add(product_info[Info.UUID])
num_seen += len(filenames)
status = {TASK_DOING: 'collecting metadata {}/{}'.format(num_seen, ntodo),
TASK_PROGRESS: float(num_seen) / ntodo + 1}
yield status
yield {TASK_DOING: 'collecting metadata done', TASK_PROGRESS: 1.0}
if changed_uuids:
LOG.debug('{} changed UUIDs, signaling product updates'.format(len(changed_uuids)))
# FUTURE: decide whether signals for metadatabase should belong to metadatabase
self._ws.didUpdateProductsMetadata.emit(changed_uuids)
def test_filename_grouping(self):
"""Test that filenames are grouped properly."""
from satpy.readers import group_files
filenames = [
'gk2a_ami_le1b_ir087_fd020ge_201909300300.nc',
'gk2a_ami_le1b_ir096_fd020ge_201909300300.nc',
'gk2a_ami_le1b_ir105_fd020ge_201909300300.nc',
'gk2a_ami_le1b_ir112_fd020ge_201909300300.nc',
'gk2a_ami_le1b_ir123_fd020ge_201909300300.nc',
'gk2a_ami_le1b_ir133_fd020ge_201909300300.nc',
'gk2a_ami_le1b_nr013_fd020ge_201909300300.nc',
'gk2a_ami_le1b_nr016_fd020ge_201909300300.nc',
'gk2a_ami_le1b_sw038_fd020ge_201909300300.nc',
'gk2a_ami_le1b_vi004_fd010ge_201909300300.nc',
'gk2a_ami_le1b_vi005_fd010ge_201909300300.nc',
'gk2a_ami_le1b_vi006_fd005ge_201909300300.nc',
'gk2a_ami_le1b_vi008_fd010ge_201909300300.nc',
'gk2a_ami_le1b_wv063_fd020ge_201909300300.nc',
'gk2a_ami_le1b_wv069_fd020ge_201909300300.nc',
'gk2a_ami_le1b_wv073_fd020ge_201909300300.nc']
groups = group_files(filenames, reader='ami_l1b')
self.assertEqual(len(groups), 1)
self.assertEqual(len(groups[0]['ami_l1b']), 16)
def from_files(cls,
files_to_sort,
reader=None,
ensure_all_readers=False,
scene_kwargs=None,
**kwargs):
"""Create multiple Scene objects from multiple files.
Args:
files_to_sort (Collection[str]): files to read
reader (str or Collection[str]): reader or readers to use
ensure_all_readers (bool): If True, limit to scenes where all
readers have at least one file. If False (default), include
all scenes where at least one reader has at least one file.
scene_kwargs (Mapping): additional arguments to pass on to
:func:`Scene.__init__` for each created scene.
This uses the :func:`satpy.readers.group_files` function to group
files. See this function for more details on additional possible
keyword arguments. In particular, it is strongly recommended to pass
`"group_keys"` when using multiple instruments.
.. versionadded:: 0.12
"""
from satpy.readers import group_files
if scene_kwargs is None:
scene_kwargs = {}
file_groups = group_files(files_to_sort, reader=reader, **kwargs)
if ensure_all_readers:
warnings.warn(
"Argument ensure_all_readers is deprecated. Use "
"missing='skip' instead.", DeprecationWarning)
file_groups = [fg for fg in file_groups if all(fg.values())]
scenes = (Scene(filenames=fg, **scene_kwargs) for fg in file_groups)
return cls(scenes)
def test_unknown_files(self):
"""Test that error is raised on unknown files."""
from satpy.readers import group_files
with pytest.raises(ValueError):
group_files(self.unknown_files, "abi_l1b")
def test_default_behavior(self):
"""Test the default behavior with the 'abi_l1b' reader."""
from satpy.readers import group_files
groups = group_files(self.g16_files, reader='abi_l1b')
self.assertEqual(6, len(groups))
self.assertEqual(2, len(groups[0]['abi_l1b']))
