def test_load_all_lcc(self, pg):
"""Test loading all test datasets with lcc projections"""
lons = np.array([
[12.19, 0, 0, 0, 14.34208538],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[54.56534318, 0, 0, 0, 57.32843565]])
lats = np.array([
[-133.459, 0, 0, 0, -65.12555139],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[-152.8786225, 0, 0, 0, -49.41598659]])
pg.open.return_value = FakeGRIB(
proj_params={
'a': 6371229, 'b': 6371229, 'proj': 'lcc',
'lon_0': 265.0, 'lat_0': 25.0,
'lat_1': 25.0, 'lat_2': 25.0},
latlons=(lats, lons))
from satpy.readers import load_reader
from satpy import DatasetID
r = load_reader(self.reader_configs)
loadables = r.select_files_from_pathnames([
'gfs.t18z.sfluxgrbf106.grib2',
])
r.create_filehandlers(loadables)
datasets = r.load([
DatasetID(name='t', level=100),
DatasetID(name='t', level=200),
DatasetID(name='t', level=300)])
self.assertEqual(len(datasets), 3)
for v in datasets.values():
self.assertEqual(v.attrs['units'], 'K')
self.assertIsInstance(v, xr.DataArray)
def test_excs(self, reader_configs, caplog):
"""Test that exceptions are raised where expected."""
from satpy import DatasetID
from satpy.readers import load_reader
filenames = [
"W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--"
"CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_"
"20170410113925_20170410113934_N__C_0070_0067.nc",
]
reader = load_reader(reader_configs)
loadables = reader.select_files_from_pathnames(filenames)
fhs = reader.create_filehandlers(loadables)
with pytest.raises(ValueError):
fhs["fci_l1c_fdhsi"][0].get_dataset(DatasetID(name="invalid"), {})
with pytest.raises(ValueError):
fhs["fci_l1c_fdhsi"][0]._get_dataset_quality(
DatasetID(name="invalid"), {})
with caplog.at_level(logging.ERROR):
fhs["fci_l1c_fdhsi"][0].get_dataset(
DatasetID(name="ir_123", calibration="unknown"),
{"units": "unknown"})
assert "unknown calibration key" in caplog.text
def test_fy3d_1km_resolutions(self):
"""Test loading data when only 1km resolutions are available."""
from satpy import DatasetID
from satpy.readers import load_reader, get_key
filenames = [
'tf2019071182739.FY3D-X_MERSI_1000M_L1B.HDF',
'tf2019071182739.FY3D-X_MERSI_GEO1K_L1B.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertTrue(4, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
# Verify that we have multiple resolutions for:
# - Bands 1-4 (visible)
# - Bands 24-25 (IR)
available_datasets = reader.available_dataset_ids
for band_name in ('1', '2', '3', '4', '24', '25'):
if band_name in ('24', '25'):
# don't know how to get radiance for IR bands
num_results = 2
else:
num_results = 3
ds_id = DatasetID(name=band_name, resolution=250)
res = get_key(ds_id, available_datasets,
num_results=num_results, best=False)
self.assertEqual(0, len(res))
ds_id = DatasetID(name=band_name, resolution=1000)
res = get_key(ds_id, available_datasets,
num_results=num_results, best=False)
self.assertEqual(num_results, len(res))
res = reader.load(['1', '2', '3', '4', '5', '20', '24', '25'])
self.assertEqual(8, len(res))
self.assertEqual((2 * 10, 2048), res['1'].shape)
self.assertEqual('reflectance', res['1'].attrs['calibration'])
self.assertEqual('%', res['1'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['2'].shape)
self.assertEqual('reflectance', res['2'].attrs['calibration'])
self.assertEqual('%', res['2'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['3'].shape)
self.assertEqual('reflectance', res['3'].attrs['calibration'])
self.assertEqual('%', res['3'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['4'].shape)
self.assertEqual('reflectance', res['4'].attrs['calibration'])
self.assertEqual('%', res['4'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['5'].shape)
self.assertEqual('reflectance', res['5'].attrs['calibration'])
self.assertEqual('%', res['5'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['20'].shape)
self.assertEqual('brightness_temperature', res['20'].attrs['calibration'])
self.assertEqual('K', res['20'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['24'].shape)
self.assertEqual('brightness_temperature', res['24'].attrs['calibration'])
self.assertEqual('K', res['24'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['25'].shape)
self.assertEqual('brightness_temperature', res['25'].attrs['calibration'])
self.assertEqual('K', res['25'].attrs['units'])
def _get_test_dataset_three_bands_two_prereq(self, bands=3):
"""Helper function to create a single test dataset."""
import xarray as xr
import dask.array as da
from datetime import datetime
from pyresample.geometry import AreaDefinition
from pyresample.utils import proj4_str_to_dict
from satpy import DatasetID
area_def = AreaDefinition(
'test',
'test',
'test',
proj4_str_to_dict('+proj=stere +datum=WGS84 +ellps=WGS84 '
'+lon_0=0. +lat_0=90 +lat_ts=60 +units=km'),
100,
200,
(-1000., -1500., 1000., 1500.),
)
ds1 = xr.DataArray(
da.zeros((bands, 100, 200), chunks=50),
coords=[['R', 'G', 'B'], list(range(100)), list(range(200))],
dims=('bands', 'y', 'x'),
attrs={'name': 'test',
'start_time': datetime.utcnow(),
'platform_name': "TEST_PLATFORM_NAME",
'sensor': 'TEST_SENSOR_NAME',
'area': area_def,
'prerequisites': [DatasetID(name='1', calibration='reflectance'),
DatasetID(name='2', calibration='reflectance')]}
)
return ds1
def test_load_longitude_latitude(self):
"""Test that longitude and latitude datasets are loaded correctly."""
from satpy import DatasetID
def test_func(dname, x, y):
if dname == 'longitude':
# assert less
np.testing.assert_array_less(x, y)
else:
# assert greater
# np.testing.assert_equal(x > y, True)
np.testing.assert_array_less(y, x)
scene = Scene(reader='modis_l2', filenames=[self.file_name])
for dataset_name in ['longitude', 'latitude']:
# Default resolution should be the interpolated 1km
scene.load([dataset_name])
longitude_1km_id = DatasetID(name=dataset_name, resolution=1000)
longitude_1km = scene[longitude_1km_id]
self.assertEqual(longitude_1km.shape, (5*SCAN_WIDTH, 5*SCAN_LEN+4))
test_func(dataset_name, longitude_1km.values, 0)
# Specify original 5km scale
scene.load([dataset_name], resolution=5000)
longitude_5km_id = DatasetID(name=dataset_name, resolution=5000)
longitude_5km = scene[longitude_5km_id]
self.assertEqual(longitude_5km.shape, TEST_DATA[dataset_name.capitalize()]['data'].shape)
test_func(dataset_name, longitude_5km.values, 0)
def test_properties(self):
"""Test basic properties/attributes of the MultiScene."""
from satpy import MultiScene, DatasetID
area = _create_test_area()
scenes = _create_test_scenes(area=area)
ds1_id = DatasetID(name='ds1')
ds2_id = DatasetID(name='ds2')
ds3_id = DatasetID(name='ds3')
ds4_id = DatasetID(name='ds4')
# Add a dataset to only one of the Scenes
scenes[1]['ds3'] = _create_test_dataset('ds3')
mscn = MultiScene(scenes)
self.assertSetEqual(mscn.loaded_dataset_ids, {ds1_id, ds2_id, ds3_id})
self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id})
self.assertTrue(mscn.all_same_area)
bigger_area = _create_test_area(shape=(20, 40))
scenes[0]['ds4'] = _create_test_dataset('ds4',
shape=(20, 40),
area=bigger_area)
self.assertSetEqual(mscn.loaded_dataset_ids,
{ds1_id, ds2_id, ds3_id, ds4_id})
self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id})
self.assertFalse(mscn.all_same_area)
def test_load_all_i_radiances(self):
"""Load all I band radiances."""
from satpy.readers import load_reader
from satpy import DatasetID
r = load_reader(self.reader_configs)
loadables = r.select_files_from_pathnames([
'SVI01_npp_d20120225_t1801245_e1802487_b01708_c20120226002130255476_noaa_ops.h5',
'SVI02_npp_d20120225_t1801245_e1802487_b01708_c20120226002130255476_noaa_ops.h5',
'SVI03_npp_d20120225_t1801245_e1802487_b01708_c20120226002130255476_noaa_ops.h5',
'SVI04_npp_d20120225_t1801245_e1802487_b01708_c20120226002130255476_noaa_ops.h5',
'SVI05_npp_d20120225_t1801245_e1802487_b01708_c20120226002130255476_noaa_ops.h5',
'GITCO_npp_d20120225_t1801245_e1802487_b01708_c20120226002130255476_noaa_ops.h5',
])
r.create_filehandlers(loadables)
ds = r.load([
DatasetID(name='I01', calibration='radiance', modifiers=None),
DatasetID(name='I02', calibration='radiance', modifiers=None),
DatasetID(name='I03', calibration='radiance', modifiers=None),
DatasetID(name='I04', calibration='radiance', modifiers=None),
DatasetID(name='I05', calibration='radiance', modifiers=None),
])
self.assertEqual(len(ds), 5)
for d in ds.values():
self.assertEqual(d.attrs['calibration'], 'radiance')
self.assertEqual(d.attrs['units'], 'W m-2 um-1 sr-1')
self.assertEqual(d.attrs['rows_per_scan'], 32)
self.assertIn('area', d.attrs)
self.assertIsNotNone(d.attrs['area'])
def test_instantiate(self, mocked_dataset):
"""Test initialization of file handlers."""
from satpy.readers.olci_nc import (NCOLCIBase, NCOLCICal, NCOLCIGeo,
NCOLCIChannelBase, NCOLCI1B,
NCOLCI2)
from satpy import DatasetID
import xarray as xr
cal_data = xr.Dataset(
{
'solar_flux': (('bands'), [0, 1, 2]),
'detector_index': (('bands'), [0, 1, 2]),
},
{
'bands': [0, 1, 2],
},
)
ds_id = DatasetID(name='Oa01', calibration='reflectance')
ds_id2 = DatasetID(name='wsqf', calibration='reflectance')
filename_info = {
'mission_id': 'S3A',
'dataset_name': 'Oa01',
'start_time': 0,
'end_time': 0
}
test = NCOLCIBase('somedir/somefile.nc', filename_info, 'c')
test.get_dataset(ds_id, filename_info)
mocked_dataset.assert_called()
mocked_dataset.reset_mock()
test = NCOLCICal('somedir/somefile.nc', filename_info, 'c')
test.get_dataset(ds_id, filename_info)
mocked_dataset.assert_called()
mocked_dataset.reset_mock()
test = NCOLCIGeo('somedir/somefile.nc', filename_info, 'c')
test.get_dataset(ds_id, filename_info)
mocked_dataset.assert_called()
mocked_dataset.reset_mock()
test = NCOLCIChannelBase('somedir/somefile.nc', filename_info, 'c')
test.get_dataset(ds_id, filename_info)
mocked_dataset.assert_called()
mocked_dataset.reset_mock()
cal = mock.Mock()
cal.nc = cal_data
test = NCOLCI1B('somedir/somefile.nc', filename_info, 'c', cal)
test.get_dataset(ds_id, filename_info)
mocked_dataset.assert_called()
mocked_dataset.reset_mock()
test = NCOLCI2('somedir/somefile.nc', filename_info, 'c')
test.get_dataset(ds_id, {'nc_key': 'the_key'})
test.get_dataset(ds_id2, {'nc_key': 'the_key'})
mocked_dataset.assert_called()
mocked_dataset.reset_mock()
def test_fy4a_all_resolutions(self):
"""Test loading data when all resolutions are available."""
from satpy import DatasetID
from satpy.readers import load_reader, get_key
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_0500M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_1000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_2000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_4000M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertTrue(4, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
available_datasets = reader.available_dataset_ids
# 500m
band_names = ['C' + '%02d' % ch for ch in np.linspace(2, 2, 1)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=500)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
# 1km
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 3, 3)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=1000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
# 2km
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 7, 7)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=2000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
if band_name < 'C07':
self.assertEqual(2, len(res))
else:
self.assertEqual(3, len(res))
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 14, 14)]
res = reader.load(band_names)
self.assertEqual(14, len(res))
for band_name in band_names:
self.assertEqual((2, 5), res[band_name].shape)
if band_name < 'C07':
self.assertEqual('reflectance',
res[band_name].attrs['calibration'])
else:
self.assertEqual('brightness_temperature',
res[band_name].attrs['calibration'])
if band_name < 'C07':
self.assertEqual('%', res[band_name].attrs['units'])
else:
self.assertEqual('K', res[band_name].attrs['units'])
def test_fy4a_1km_resolutions(self):
"""Test loading data when only 1km resolutions are available."""
from satpy import DatasetID
from satpy.readers import load_reader, get_key
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_1000M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertTrue(1, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
# Verify that the resolution is only 1km
available_datasets = reader.available_dataset_ids
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 3, 3)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=500)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
ds_id = DatasetID(name=band_name, resolution=1000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
ds_id = DatasetID(name=band_name, resolution=2000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
ds_id = DatasetID(name=band_name, resolution=4000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
res = reader.load(band_names)
self.assertEqual(3, len(res))
expected = {
1:
np.array([[2.01, 2.02, 2.03, 2.04, 2.05],
[2.06, 2.07, 2.08, 2.09, 2.1]]),
2:
np.array([[4.03, 4.06, 4.09, 4.12, 4.15],
[4.18, 4.21, 4.24, 4.27, 4.3]]),
3:
np.array([[6.05, 6.1, 6.15, 6.2, 6.25],
[6.3, 6.35, 6.4, 6.45, 6.5]])
}
for index, band_name in enumerate(band_names):
self.assertEqual(1, res[band_name].attrs['sensor'].islower())
self.assertEqual((2, 5), res[band_name].shape)
self.assertEqual('reflectance',
res[band_name].attrs['calibration'])
self.assertEqual('%', res[band_name].attrs['units'])
self.assertTrue(
np.allclose(res[band_name].values,
expected[index + 1],
equal_nan=True))
def _create_dataset_ids(self, keys):
from itertools import product
ordered_keys = [k for k in keys.keys() if 'id_key' in keys[k]]
for id_vals in product(*[keys[k]['values'] for k in ordered_keys]):
id_keys = [keys[k]['id_key'] for k in ordered_keys]
msg_info = dict(zip(ordered_keys, id_vals))
ds_info = dict(zip(id_keys, id_vals))
msg_id = DatasetID(**ds_info)
ds_info = msg_id.to_dict()
ds_info.update(msg_info)
ds_info['file_type'] = self.filetype_info['file_type']
self._msg_datasets[msg_id] = ds_info
def _create_dataset_ids(self, keys):
from itertools import product
ordered_keys = [k for k in keys.keys() if 'id_key' in keys[k]]
for id_vals in product(*[keys[k]['values'] for k in ordered_keys]):
id_keys = [keys[k]['id_key'] for k in ordered_keys]
msg_info = dict(zip(ordered_keys, id_vals))
ds_info = dict(zip(id_keys, id_vals))
msg_id = DatasetID(**ds_info)
ds_info = msg_id.to_dict()
ds_info.update(msg_info)
ds_info['file_type'] = self.filetype_info['file_type']
self._msg_datasets[msg_id] = ds_info
def test_read_geo(self):
"""Test read_geo() function"""
import h5py
from satpy.readers.iasi_l2 import read_geo
from satpy import DatasetID
with h5py.File(self.fname, 'r') as fid:
key = DatasetID(name='sensing_time')
data = read_geo(fid, key).compute()
self.assertEqual(data.shape, (NUM_SCANLINES, SCAN_WIDTH))
key = DatasetID(name='latitude')
data = read_geo(fid, key).compute()
self.assertEqual(data.shape, (NUM_SCANLINES, SCAN_WIDTH))
def test_get_dataset_coords(self):
"""Test whether coordinates returned by get_dataset() are correct"""
lon = self.reader.get_dataset(key=DatasetID(name='longitude',
calibration=None),
info={})
lat = self.reader.get_dataset(key=DatasetID(name='latitude',
calibration=None),
info={})
# ... this only compares the valid (unmasked) elements
self.assertTrue(np.all(lat.to_masked_array() == self.lat),
msg='get_dataset() returns invalid latitude')
self.assertTrue(np.all(lon.to_masked_array() == self.lon),
msg='get_dataset() returns invalid longitude')
def test_get_dataset(self):
"""Test get_dataset() for different datasets"""
from satpy import DatasetID
info = {'eggs': 'spam'}
key = DatasetID(name='pressure')
data = self.reader.get_dataset(key, info).compute()
self.check_pressure(data)
self.assertTrue('eggs' in data.attrs)
self.assertEqual(data.attrs['eggs'], 'spam')
key = DatasetID(name='emissivity')
data = self.reader.get_dataset(key, info).compute()
self.check_emissivity(data)
key = DatasetID(name='sensing_time')
data = self.reader.get_dataset(key, info).compute()
self.assertEqual(data.shape, (NUM_SCANLINES, SCAN_WIDTH))
def test_load_longitude_latitude(self):
"""Test that longitude and latitude datasets are loaded correctly."""
from satpy import DatasetID
scene = Scene(reader='modis_l2', filenames=[self.file_name])
for dataset_name in ['longitude', 'latitude']:
# Default resolution should be the interpolated 1km
scene.load([dataset_name])
longitude_1km_id = DatasetID(name=dataset_name, resolution=1000)
longitude_1km = scene[longitude_1km_id]
self.assertEqual(longitude_1km.shape, (5*SCAN_WIDTH, 5*SCAN_LEN+4))
# Specify original 5km scale
longitude_5km = scene.load([dataset_name], resolution=5000)
longitude_5km_id = DatasetID(name=dataset_name, resolution=5000)
longitude_5km = scene[longitude_5km_id]
self.assertEqual(longitude_5km.shape, TEST_DATA[dataset_name.capitalize()]['data'].shape)
def test_get_dataset_invalid(self):
"""Test handling of invalid calibrations"""
# VIS -> BT
args = dict(key=DatasetID(name='00_7',
calibration='brightness_temperature'),
info={})
self.assertRaises(ValueError, self.reader.get_dataset, **args)
# IR -> Reflectance
args = dict(key=DatasetID(name='10_7', calibration='reflectance'),
info={})
self.assertRaises(ValueError, self.reader.get_dataset, **args)
# Unsupported calibration
args = dict(key=DatasetID(name='10_7', calibration='invalid'), info={})
self.assertRaises(ValueError, self.reader.get_dataset, **args)
def _init_summary_page(self):
# we are going to use the id table to get our summary
# so make sure the values are correct
if self.ui.selectByTabWidget.currentIndex() != BY_ID_TAB:
self.ui.selectByTabWidget.setCurrentIndex(BY_ID_TAB)
selected_text = []
selected_ids = []
id_format = "| {name:<20s} | {level:>8s} |"
header_format = "| {name:<20s} | {level:>8s} |"
header_line = "|-{0:-^20s}-|-{0:-^8s}-|".format('-')
for item_idx in range(self.ui.selectIDTable.rowCount()):
name_item = self.ui.selectIDTable.item(item_idx, 0)
level_item = self.ui.selectIDTable.item(item_idx, 1)
if name_item.checkState():
name = name_item.data(QtCore.Qt.UserRole)
level = level_item.data(QtCore.Qt.UserRole)
selected_ids.append(DatasetID(name=name, level=level))
selected_text.append(id_format.format(
name=name_item.text(),
level=level_item.text(),
))
self.selected_ids = selected_ids
summary_text = """Products to be loaded: {}
""".format(len(selected_ids))
header = header_format.format(name="Name", level="Level")
summary_text += "\n".join([header, header_line] + selected_text)
self.ui.productSummaryText.setText(summary_text)
def test_read_dataset(self):
"""Test read_dataset() function"""
import h5py
from satpy.readers.iasi_l2 import read_dataset
from satpy import DatasetID
with h5py.File(self.fname, 'r') as fid:
key = DatasetID(name='pressure')
data = read_dataset(fid, key).compute()
self.check_pressure(data)
key = DatasetID(name='emissivity')
data = read_dataset(fid, key).compute()
self.check_emissivity(data)
# This dataset doesn't have any attributes
key = DatasetID(name='ozone_total_column')
data = read_dataset(fid, key).compute()
self.assertEqual(len(data.attrs), 0)
def test_get_dataset(self):
"""Test the get_dataset method."""
from satpy import DatasetID
key = DatasetID(name='Rad', calibration='radiance')
res = self.reader.get_dataset(key, {'info': 'info'})
exp = {'calibration': 'radiance',
'instrument_ID': None,
'modifiers': (),
'name': 'Rad',
'observation_type': 'Rad',
'orbital_parameters': {'projection_altitude': 1.0,
'projection_latitude': 0.0,
'projection_longitude': -90.0,
'satellite_nominal_altitude': 35786020.,
'satellite_nominal_latitude': 0.0,
'satellite_nominal_longitude': -89.5,
'yaw_flip': True},
'orbital_slot': None,
'platform_name': 'GOES-16',
'platform_shortname': 'G16',
'production_site': None,
'scan_mode': 'M3',
'scene_abbr': 'C',
'scene_id': None,
'sensor': 'abi',
'timeline_ID': None,
'units': 'W m-2 um-1 sr-1'}
self.assertDictEqual(res.attrs, exp)
# we remove any time dimension information
self.assertNotIn('t', res.coords)
self.assertNotIn('t', res.dims)
self.assertNotIn('time', res.coords)
self.assertNotIn('time', res.dims)
def test_save_array(self):
from satpy import Scene
import xarray as xr
import tempfile
scn = Scene()
start_time = datetime(2018, 5, 30, 10, 0)
end_time = datetime(2018, 5, 30, 10, 15)
scn['test-array'] = xr.DataArray([1, 2, 3],
attrs=dict(
start_time=start_time,
end_time=end_time,
prerequisites=[DatasetID('hej')]))
try:
handle, filename = tempfile.mkstemp()
os.close(handle)
scn.save_datasets(filename=filename, writer='cf')
import h5netcdf as nc4
with nc4.File(filename) as f:
self.assertTrue(all(f['test-array'][:] == [1, 2, 3]))
expected_prereq = (
"DatasetID(name='hej', wavelength=None, "
"resolution=None, polarization=None, "
"calibration=None, level=None, modifiers=())")
self.assertEqual(f['test-array'].attrs['prerequisites'][0],
np.string_(expected_prereq))
finally:
os.remove(filename)
def test_get_dataset_counts(self):
"""Test whether counts returned by get_dataset() are correct"""
from satpy.readers.goes_imager_nc import ALTITUDE, UNKNOWN_SECTOR
self.reader.meta.update({'lon0': -75.0,
'lat0': 0.0,
'sector': UNKNOWN_SECTOR,
'nadir_row': 1,
'nadir_col': 2,
'area_def_uni': 'some_area'})
attrs_exp = {'orbital_parameters': {'projection_longitude': -75.0,
'projection_latitude': 0.0,
'projection_altitude': ALTITUDE,
'yaw_flip': True},
'satellite_longitude': -75.0,
'satellite_latitude': 0.0,
'satellite_altitude': ALTITUDE,
'platform_name': 'GOES-15',
'sensor': 'goes_imager',
'sector': UNKNOWN_SECTOR,
'nadir_row': 1,
'nadir_col': 2,
'area_def_uniform_sampling': 'some_area'}
for ch in self.channels:
counts = self.reader.get_dataset(
key=DatasetID(name=ch, calibration='counts'), info={})
# ... this only compares the valid (unmasked) elements
self.assertTrue(np.all(self.counts/32. == counts.to_masked_array()),
msg='get_dataset() returns invalid counts for '
'channel {}'.format(ch))
# Check attributes
self.assertDictEqual(counts.attrs, attrs_exp)
def test_get_dataset(self):
"""Test basic L2 load."""
from satpy import DatasetID
key = DatasetID(name='HT')
res = self.reader.get_dataset(key, {'file_key': 'HT'})
exp_data = np.array([[2 * 0.3052037, np.nan],
[32768 * 0.3052037, 32767 * 0.3052037]])
exp_attrs = {
'instrument_ID': None,
'modifiers': (),
'name': 'HT',
'orbital_slot': None,
'platform_name': 'GOES-16',
'platform_shortname': 'G16',
'production_site': None,
'satellite_altitude': 35786020.,
'satellite_latitude': 0.0,
'satellite_longitude': -89.5,
'scan_mode': 'M3',
'scene_id': None,
'sensor': 'abi',
'timeline_ID': None,
'units': 'm'
}
self.assertTrue(np.allclose(res.data, exp_data, equal_nan=True))
self.assertDictEqual(dict(res.attrs), exp_attrs)
def test_load_bt(self):
"""Test loading with bt
"""
from satpy import DatasetID
from satpy.readers import load_reader
filenames = [
"W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--"
"CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_"
"20170410113925_20170410113934_N__C_0070_0067.nc",
]
reader = load_reader(self.reader_configs)
loadables = reader.select_files_from_pathnames(filenames)
reader.create_filehandlers(loadables)
res = reader.load([
DatasetID(name=name, calibration="brightness_temperature")
for name in self._chans["terran"]
])
self.assertEqual(8, len(res))
for ch in self._chans["terran"]:
self.assertEqual(res[ch].shape, (200, 11136))
self.assertEqual(res[ch].dtype, np.float64)
self.assertEqual(res[ch].attrs["calibration"],
"brightness_temperature")
self.assertEqual(res[ch].attrs["units"], "K")
numpy.testing.assert_array_almost_equal(res[ch], 181.917084)
def test_get_dataset(self, mocked_dataset):
"""Test reading datasets."""
from satpy.readers.olci_nc import NCOLCI2
from satpy import DatasetID
import numpy as np
import xarray as xr
mocked_dataset.return_value = xr.Dataset(
{
'mask':
(['rows', 'columns'], np.array([1 << x for x in range(30)
]).reshape(5, 6))
},
coords={
'rows': np.arange(5),
'columns': np.arange(6)
})
ds_id = DatasetID(name='mask')
filename_info = {
'mission_id': 'S3A',
'dataset_name': 'mask',
'start_time': 0,
'end_time': 0
}
test = NCOLCI2('somedir/somefile.nc', filename_info, 'c')
res = test.get_dataset(ds_id, {'nc_key': 'mask'})
self.assertEqual(res.dtype, np.dtype('bool'))
def test_get_area_def(self):
"""Get the area definition."""
from satpy.readers.nwcsaf_msg2013_hdf5 import Hdf5NWCSAF
from satpy import DatasetID
filename_info = {}
filetype_info = {}
dsid = DatasetID(name="ct")
test = Hdf5NWCSAF(self.filename_ct, filename_info, filetype_info)
area_def = test.get_area_def(dsid)
aext_res = AREA_DEF_DICT['area_extent']
for i in range(4):
self.assertAlmostEqual(area_def.area_extent[i], aext_res[i], 4)
proj_dict = AREA_DEF_DICT['proj_dict']
self.assertEqual(proj_dict['proj'], area_def.proj_dict['proj'])
# Not all elements passed on Appveyor, so skip testing every single element of the proj-dict:
# for key in proj_dict:
# self.assertEqual(proj_dict[key], area_def.proj_dict[key])
self.assertEqual(AREA_DEF_DICT['x_size'], area_def.width)
self.assertEqual(AREA_DEF_DICT['y_size'], area_def.height)
self.assertEqual(AREA_DEF_DICT['area_id'], area_def.area_id)
def test_navigation(self):
"""Test reading the lon and lats."""
with tempfile.TemporaryFile() as tmpfile:
self._header.tofile(tmpfile)
tmpfile.seek(22016, 0)
self._data.tofile(tmpfile)
fh = AVHRRAAPPL1BFile(tmpfile, self.filename_info,
self.filetype_info)
info = {}
key = DatasetID(name='longitude')
res = fh.get_dataset(key, info)
assert (np.all(res == 0))
key = DatasetID(name='latitude')
res = fh.get_dataset(key, info)
assert (np.all(res == 0))
def test_load_bt(self, reader_configs, caplog):
"""Test loading with bt."""
from satpy import DatasetID
from satpy.readers import load_reader
filenames = [
"W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--"
"CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_"
"20170410113925_20170410113934_N__C_0070_0067.nc",
]
reader = load_reader(reader_configs)
loadables = reader.select_files_from_pathnames(filenames)
reader.create_filehandlers(loadables)
with caplog.at_level(logging.WARNING):
res = reader.load([
DatasetID(name=name, calibration="brightness_temperature")
for name in self._chans["terran"]
])
assert caplog.text == ""
for ch in self._chans["terran"]:
assert res[ch].shape == (200, 11136)
assert res[ch].dtype == np.float64
assert res[ch].attrs["calibration"] == "brightness_temperature"
assert res[ch].attrs["units"] == "K"
if ch == 'ir_38':
numpy.testing.assert_array_almost_equal(
res[ch][~0], 209.68274099)
numpy.testing.assert_array_almost_equal(
res[ch][0], 1888.851296)
else:
numpy.testing.assert_array_almost_equal(res[ch], 209.68274099)
def test_fy4a_counts_calib(self):
"""Test loading data at counts calibration."""
from satpy import DatasetID
from satpy.readers import load_reader
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_0500M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_1000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_2000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_4000M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertTrue(4, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
ds_ids = []
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 14, 14)]
for band_name in band_names:
ds_ids.append(DatasetID(name=band_name, calibration='counts'))
res = reader.load(ds_ids)
self.assertEqual(14, len(res))
for band_name in band_names:
self.assertEqual((2, 5), res[band_name].shape)
self.assertEqual('counts', res[band_name].attrs['calibration'])
self.assertEqual(res[band_name].dtype, np.uint16)
self.assertEqual('1', res[band_name].attrs['units'])
def test_load_reflectance(self, reader_configs):
"""Test loading with reflectance."""
from satpy import DatasetID
from satpy.readers import load_reader
filenames = [
"W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--"
"CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_"
"20170410113925_20170410113934_N__C_0070_0067.nc",
]
reader = load_reader(reader_configs)
loadables = reader.select_files_from_pathnames(filenames)
reader.create_filehandlers(loadables)
res = reader.load([
DatasetID(name=name, calibration="reflectance")
for name in self._chans["solar"]
])
assert 8 == len(res)
for ch in self._chans["solar"]:
assert res[ch].shape == (200, 11136)
assert res[ch].dtype == np.float64
assert res[ch].attrs["calibration"] == "reflectance"
assert res[ch].attrs["units"] == "%"
numpy.testing.assert_array_equal(res[ch],
100 * 15 * 1 * np.pi / 50)
def test_load_counts(self, reader_configs):
"""Test loading with counts."""
from satpy import DatasetID
from satpy.readers import load_reader
# testing two filenames to test correctly combined
filenames = [
"W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--"
"CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_"
"20170410113925_20170410113934_N__C_0070_0067.nc",
"W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--"
"CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_"
"20170410113934_20170410113942_N__C_0070_0068.nc",
]
reader = load_reader(reader_configs)
loadables = reader.select_files_from_pathnames(filenames)
reader.create_filehandlers(loadables)
res = reader.load([
DatasetID(name=name, calibration="counts")
for name in self._chans["solar"] + self._chans["terran"]
])
assert 16 == len(res)
for ch in self._chans["solar"] + self._chans["terran"]:
assert res[ch].shape == (200 * 2, 11136)
assert res[ch].dtype == np.uint16
assert res[ch].attrs["calibration"] == "counts"
assert res[ch].attrs["units"] == "1"
if ch == 'ir_38':
numpy.testing.assert_array_equal(res[ch][~0], 1)
numpy.testing.assert_array_equal(res[ch][0], 5000)
else:
numpy.testing.assert_array_equal(res[ch], 1)
def filter_nighttime(self, scene):
if self.fraction_day_scene is None:
self._calc_percent_day(scene)
# make a copy of the scene list so we can edit it later
for ds in list(scene):
if ds.attrs['name'] in ('ifog',) and \
self.fraction_night_scene <= self.night_fraction:
ds_id = DatasetID.from_dict(ds.attrs)
LOG.info("Will not create product '%s' because there is less than %f%% of night data",
ds.attrs['name'], self.night_fraction * 100.)
del scene[ds_id]
def filter_daytime(self, scene):
if self.fraction_day_scene is None:
self._calc_percent_day(scene)
# make a copy of the scene list so we can edit it later
for ds in list(scene):
if ds.attrs['standard_name'] in ('toa_bidirectional_reflectance', 'false_color', 'true_color') and \
self.fraction_day_scene <= self.day_fraction:
ds_id = DatasetID.from_dict(ds.attrs)
LOG.info("Will not create product '%s' because there is less than %f%% of day data",
ds.attrs['name'], self.day_fraction * 100.)
del scene[ds_id]
def __call__(self, datasets, optional_datasets, **info):
if self.attrs['optional_prerequisites']:
for opt_dep in self.attrs['optional_prerequisites']:
if 'NOPE' in opt_dep or 'fail' in opt_dep:
continue
assert optional_datasets is not None and \
len(optional_datasets)
resolution = DatasetID.from_dict(datasets[0].attrs).resolution
if name == 'res_change' and resolution is not None:
i = datasets[0].attrs.copy()
i['resolution'] *= 5
elif name == 'incomp_areas':
raise IncompatibleAreas(
"Test modifier 'incomp_areas' always raises IncompatibleAreas")
else:
i = datasets[0].attrs
info = datasets[0].attrs.copy()
self.apply_modifier_info(i, info)
return DataArray(np.ma.MaskedArray(datasets[0]), attrs=info)
def _update_modifier_key(self, orig_key, dep_key):
"""Update a key based on the dataset it will modified (dep).
Typical use case is requesting a modified dataset (orig_key). This
modified dataset most likely depends on a less-modified
dataset (dep_key). The less-modified dataset must come from a reader
(at least for now) or will eventually depend on a reader dataset.
The original request key may be limited like
(wavelength=0.67, modifiers=('a', 'b')) while the reader-based key
should have all of its properties specified. This method updates the
original request key so it is fully specified and should reduce the
chance of Node's not being unique.
"""
orig_dict = orig_key._asdict()
dep_dict = dep_key._asdict()
# don't change the modifiers
for k in DATASET_KEYS[:-1]:
orig_dict[k] = dep_dict[k]
return DatasetID.from_dict(orig_dict)
def main(argv=sys.argv[1:]):
from polar2grid.core.script_utils import setup_logging, create_basic_parser, create_exc_handler, rename_log_file, ExtendAction
from polar2grid.compositors import CompositorManager
frontends = available_frontends()
backends = available_backends()
parser = create_basic_parser(description="Extract swath data, remap it, and write it to a new file format")
parser.add_argument("frontend", choices=sorted(frontends.keys()),
help="Specify the swath extractor to use to read data (additional arguments are determined after this is specified)")
parser.add_argument("backend", choices=sorted(backends.keys()),
help="Specify the backend to use to write data output (additional arguments are determined after this is specified)")
parser.add_argument("--compositor-configs", nargs="*", default=None,
help="Specify alternative configuration file(s) for compositors")
# don't include the help flag
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
args, remaining_args = parser.parse_known_args(argv_without_help)
glue_name = args.frontend + "2" + args.backend
LOG = logging.getLogger(glue_name)
# Load compositor information (we can't know the compositor choices until we've loaded the configuration)
compositor_manager = CompositorManager(config_files=args.compositor_configs)
# Hack: argparse doesn't let you use choices and nargs=* on a positional argument
parser.add_argument("compositors", choices=list(compositor_manager.keys()) + [[]], nargs="*",
help="Specify the compositors to apply to the provided scene (additional arguments are determined after this is specified)")
# load the actual components we need
farg_func = get_frontend_argument_func(frontends, args.frontend)
fcls = get_frontend_class(frontends, args.frontend)
barg_func = get_backend_argument_func(backends, args.backend)
bcls = get_backend_class(backends, args.backend)
# add_frontend_arguments(parser)
subgroup_titles = []
subgroup_titles += farg_func(parser)
subgroup_titles += add_remap_argument_groups(parser)
subgroup_titles += barg_func(parser)
parser.add_argument('-f', dest='data_files', nargs="+", default=[], action=ExtendAction,
help="List of files or directories to extract data from")
parser.add_argument('-d', dest='data_files', nargs="+", default=[], action=ExtendAction,
help="Data directories to look for input data files (equivalent to -f)")
global_keywords = ("keep_intermediate", "overwrite_existing", "exit_on_error")
args = parser.parse_args(argv, global_keywords=global_keywords, subgroup_titles=subgroup_titles)
if not args.data_files:
# FUTURE: When the -d flag is removed this won't be needed because -f will be required
parser.print_usage()
parser.exit(1, "ERROR: No data files provided (-f flag)\n")
# Logs are renamed once data the provided start date is known
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn)
sys.excepthook = create_exc_handler(LOG.name)
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Keep track of things going wrong to tell the user what went wrong (we want to create as much as possible)
status_to_return = STATUS_SUCCESS
# Compositor validation
# XXX: Hack to make `polar2grid.sh crefl gtiff` work like legacy crefl2gtiff.sh script
if args.subgroup_args['Frontend Swath Extraction'].get('no_compositors'):
LOG.debug("Removing all compositors")
args.compositors = []
elif args.frontend == 'crefl':
if args.backend in ['awips', 'scmi']:
LOG.debug("Adding 'crefl_sharpen' compositor")
args.compositors.append('crefl_sharpen' if args.backend == 'scmi' else 'crefl_sharpen_awips')
else:
LOG.debug("Adding 'true_color' compositor")
args.compositors.append('true_color')
if '--true-color' in sys.argv and 'true_color' not in args.compositors:
LOG.debug("Adding 'true_color' compositor")
args.compositors.append('true_color')
if '--false-color' in sys.argv and 'false_color' not in args.compositors:
LOG.debug("Adding 'false_color' compositor")
args.compositors.append('false_color')
# if "--true-color" in
for c in args.compositors:
if c not in compositor_manager:
LOG.error("Compositor '%s' is unknown" % (c,))
raise RuntimeError("Compositor '%s' is unknown" % (c,))
# Frontend
try:
LOG.info("Initializing reader...")
list_products = args.subgroup_args["Frontend Initialization"].pop("list_products")
f = fcls(search_paths=args.data_files, **args.subgroup_args["Frontend Initialization"])
except (ValueError, KeyError):
LOG.debug("Frontend exception: ", exc_info=True)
LOG.error("%s frontend failed to load and sort data files (see log for details)", args.frontend)
return STATUS_FRONTEND_FAIL
# Rename the log file
if rename_log:
rename_log_file(glue_name + f.begin_time.strftime("_%Y%m%d_%H%M%S.log"))
if list_products:
print("\n".join(sorted(f.available_product_names)))
return STATUS_SUCCESS
try:
LOG.info("Initializing remapping...")
remapper = Remapper(**args.subgroup_args["Remapping Initialization"])
remap_kwargs = args.subgroup_args["Remapping"]
except (ValueError, KeyError):
LOG.debug("Remapping initialization exception: ", exc_info=True)
LOG.error("Remapping initialization failed (see log for details)")
return STATUS_REMAP_FAIL
try:
LOG.info("Initializing backend...")
backend = bcls(**args.subgroup_args["Backend Initialization"])
except (ValueError, KeyError):
LOG.debug("Writer initialization exception: ", exc_info=True)
LOG.error("Writer initialization failed (see log for details)")
return STATUS_BACKEND_FAIL
try:
LOG.info("Initializing compositor objects...")
compositor_objects = {}
for c in args.compositors:
compositor_objects[c] = compositor_manager.get_compositor(c, **args.global_kwargs)
except (ValueError, KeyError):
LOG.debug("Compositor initialization exception: ", exc_info=True)
LOG.error("Compositor initialization failed (see log for details)")
return STATUS_COMP_FAIL
try:
LOG.info("Extracting swaths from data files available...")
scene = f.create_scene(**args.subgroup_args["Frontend Swath Extraction"])
# Determine if we have a satpy scene if we should convert it to
# a P2G Scene to continue processing
resample_method = args.subgroup_args["Remapping"].get("remap_method")
is_satpy_resample_method = resample_method in SATPY_RESAMPLERS
if is_satpy_resample_method and not isinstance(scene, Scene):
raise RuntimeError("Resampling method '{}' only supports 'satpy' readers".format(resample_method))
elif not is_satpy_resample_method and isinstance(scene, Scene):
# convert satpy scene to P2G Scene to be compatible with old P2G resamplers
scene = convert_satpy_to_p2g_swath(f, scene)
if isinstance(scene, Scene):
if not scene.datasets:
LOG.error("No products were returned by the frontend")
raise RuntimeError("No products were returned by the frontend")
if args.keep_intermediate:
raise RuntimeError("satpy readers do not currently support saving intermediate files")
else:
if (isinstance(scene, Scene) and not scene.datasets) or not scene:
LOG.error("No products were returned by the frontend")
raise RuntimeError("No products were returned by the frontend")
if args.keep_intermediate:
filename = glue_name + "_swath_scene.json"
LOG.info("Saving intermediate swath scene as '%s'", filename)
scene.save(filename)
except (ValueError, KeyError):
LOG.debug("Frontend data extraction exception: ", exc_info=True)
LOG.error("Frontend data extraction failed (see log for details)")
return STATUS_FRONTEND_FAIL
# What grids should we remap to (the user should tell us or the backend should have a good set of defaults)
known_grids = backend.known_grids
LOG.debug("Writer known grids: %r", known_grids)
grids = remap_kwargs.pop("forced_grids", None)
LOG.debug("Forced Grids: %r", grids)
if resample_method == "sensor" and grids != ["sensor"]:
LOG.error("'sensor' resampling method only supports the 'sensor' grid")
return STATUS_GDETER_FAIL
if not grids and not known_grids:
# the user didn't ask for any grids and the backend doesn't have specific defaults
LOG.error("No grids specified and no known defaults")
return STATUS_GDETER_FAIL
elif not grids:
# the user didn't tell us what to do, so let's try everything the backend knows how to do
grids = known_grids
elif known_grids is not None:
# the user told us what to do, let's make sure the backend can do it
grids = list(set(grids) & set(known_grids))
if not grids:
LOG.error("%s backend doesn't know how to handle any of the grids specified", args.backend)
return STATUS_GDETER_FAIL
LOG.debug("Grids that will be mapped to: %r", grids)
# Remap
for grid_name in grids:
LOG.info("Remapping to grid %s", grid_name)
try:
gridded_scene = remapper.remap_scene(scene, grid_name, **remap_kwargs)
if args.keep_intermediate:
filename = glue_name + "_gridded_scene_" + grid_name + ".json"
LOG.debug("saving intermediate gridded scene as '%s'", filename)
gridded_scene.save(filename)
except (ValueError, KeyError):
LOG.debug("Remapping data exception: ", exc_info=True)
LOG.error("Remapping data failed")
status_to_return |= STATUS_REMAP_FAIL
if args.exit_on_error:
return status_to_return
continue
if not isinstance(scene, Scene):
# Composition
for c, comp in compositor_objects.items():
try:
LOG.info("Running gridded scene through '%s' compositor", c)
gridded_scene = comp.modify_scene(gridded_scene, **args.subgroup_args[c + " Modification"])
if args.keep_intermediate:
filename = glue_name + "_gridded_scene_" + grid_name + ".json"
LOG.debug("Updating saved intermediate gridded scene (%s) after compositor", filename)
gridded_scene.save(filename)
except (KeyError, ValueError):
LOG.debug("Compositor Error: ", exc_info=True)
LOG.error("Could not properly modify scene using compositor '%s'" % (c,))
if args.exit_on_error:
raise RuntimeError("Could not properly modify scene using compositor '%s'" % (c,))
if isinstance(f, ReaderWrapper) and not isinstance(gridded_scene, Scene):
this_grid_definition = None
# HACK: Create SatPy composites that were either separated before
# resampling or needed resampling to be created
rgbs = {}
for product_name in gridded_scene.keys():
rgb_name = product_name[:-6]
# Keep track of one of the grid definitions
if this_grid_definition is None:
this_grid_definition = gridded_scene[product_name]["grid_definition"]
if product_name.endswith("rgb_0") or product_name.endswith("rgb_1") or product_name.endswith("rgb_2"):
if rgb_name not in rgbs:
rgbs[rgb_name] = [None, None, None]
chn_idx = int(product_name[-1])
rgbs[rgb_name][chn_idx] = product_name
LOG.debug("Putting RGBs back together again")
for rgb_name, v in rgbs.items():
r = gridded_scene.pop(v[0])
g = gridded_scene.pop(v[1])
b = gridded_scene.pop(v[2])
new_info = r.copy()
new_info["grid_data"] = new_info["grid_data"].replace(v[0], rgb_name)
new_info["product_name"] = rgb_name
data = np.memmap(new_info["grid_data"], dtype=new_info["data_type"],
mode="w+", shape=(3, new_info["grid_definition"]["height"], new_info["grid_definition"]["width"]))
data[0] = r.get_data_array()[:]
data[1] = g.get_data_array()[:]
data[2] = b.get_data_array()[:]
gridded_scene[rgb_name] = new_info
del data, new_info
# Create composites that satpy couldn't complete until after remapping
composite_names = [x for x in f.wishlist if not isinstance(x, DatasetID)]
if composite_names:
tmp_scene = Scene()
for k, v in gridded_scene.items():
if not isinstance(v["sensor"], set):
v["sensor"] = set([v["sensor"]]) # turn sensor back in to a set to match satpy usage
tmp_scene[v["id"]] = DataArray(v.get_data_array(), attrs=v)
tmp_scene[v["id"]].attrs["area"] = this_grid_definition.to_satpy_area()
# tmp_scene[v["id"]].info = {}
if v["sensor"] not in tmp_scene.attrs["sensor"]:
tmp_scene.attrs["sensor"].extend(v["sensor"])
# Overwrite the wishlist that will include the above assigned datasets
tmp_scene.wishlist = f.wishlist
for cname in composite_names:
tmp_scene.compositors[cname] = tmp_scene.cpl.load_compositor(cname, tmp_scene.attrs["sensor"])
tmp_scene.compute()
tmp_scene.unload()
# Add any new Datasets to our P2G Scene if SatPy created them
for ds in tmp_scene:
ds_id = DatasetID.from_dict(ds.attrs)
if ds_id.name not in gridded_scene:
LOG.debug("Adding Dataset from SatPy Commpositing: %s", ds_id)
gridded_scene[ds_id.name] = dataarray_to_gridded_product(ds)
gridded_scene[ds_id.name]["grid_definition"] = this_grid_definition
# Remove any Products from P2G Scene that SatPy decided it didn't need anymore
for k, v in list(gridded_scene.items()):
if v["id"].name not in tmp_scene:
LOG.debug("Removing Dataset that is no longer used: %s", k)
del gridded_scene[k]
del tmp_scene, v
if isinstance(gridded_scene, Scene):
LOG.debug("Converting satpy Scene to P2G Gridded Scene")
# Convert it to P2G Gridded Scene
gridded_scene = convert_satpy_to_p2g_gridded(f, gridded_scene)
# Writer
try:
LOG.info("Creating output from data mapped to grid %s", grid_name)
backend.create_output_from_scene(gridded_scene, **args.subgroup_args["Backend Output Creation"])
except (ValueError, KeyError):
LOG.debug("Writer output creation exception: ", exc_info=True)
LOG.error("Writer output creation failed (see log for details)")
status_to_return |= STATUS_BACKEND_FAIL
if args.exit_on_error:
return status_to_return
continue
LOG.info("Processing data for grid %s complete", grid_name)
# Force deletion and eventual garbage collection of the scene objects
del gridded_scene
del scene
return status_to_return
