class TestHRITMSGFileHandler(unittest.TestCase):
"""Test the HRITFileHandler."""
@mock.patch('satpy.readers.seviri_l1b_hrit.np.fromfile')
def setUp(self, fromfile):
"""Setup the hrit file handler for testing."""
m = mock.mock_open()
fromfile.return_value = np.array([(1, 2)],
dtype=[('total_header_length', int),
('hdr_id', int)])
with mock.patch('satpy.readers.hrit_base.open', m,
create=True) as newopen:
with mock.patch('satpy.readers.seviri_l1b_hrit.CHANNEL_NAMES'):
with mock.patch.object(HRITMSGFileHandler,
'_get_hd',
new=new_get_hd):
newopen.return_value.__enter__.return_value.tell.return_value = 1
prologue = mock.MagicMock()
prologue.prologue = {
"SatelliteStatus": {
"SatelliteDefinition": {
"SatelliteId": 324,
"NominalLongitude": 47
}
},
'GeometricProcessing': {
'EarthModel': {
'TypeOfEarthModel': 2,
'NorthPolarRadius': 10,
'SouthPolarRadius': 10,
'EquatorialRadius': 10
}
},
'ImageDescription': {
'ProjectionDescription': {
'LongitudeOfSSP': 0.0
}
}
}
prologue.get_satpos.return_value = None, None, None
prologue.get_earth_radii.return_value = None, None
self.reader = HRITMSGFileHandler(
'filename', {
'platform_shortname': 'MSG3',
'start_time': datetime(2016, 3, 3, 0, 0),
'service': 'MSG'
}, {'filetype': 'info'}, prologue, mock.MagicMock())
ncols = 3712
nlines = 464
nbits = 10
self.reader.mda['number_of_bits_per_pixel'] = nbits
self.reader.mda['number_of_lines'] = nlines
self.reader.mda['number_of_columns'] = ncols
self.reader.mda[
'data_field_length'] = nlines * ncols * nbits
self.reader.mda['cfac'] = 5
self.reader.mda['lfac'] = 5
self.reader.mda['coff'] = 10
self.reader.mda['loff'] = 10
self.reader.mda['projection_parameters'] = {}
self.reader.mda['projection_parameters']['a'] = 6378169.0
self.reader.mda['projection_parameters']['b'] = 6356583.8
self.reader.mda['projection_parameters']['h'] = 35785831.0
self.reader.mda['projection_parameters'][
'SSP_longitude'] = 44
self.reader.mda['projection_parameters'][
'SSP_latitude'] = 0.0
self.reader.mda['orbital_parameters'] = {}
self.reader.mda['orbital_parameters'][
'satellite_nominal_longitude'] = 47
self.reader.mda['orbital_parameters'][
'satellite_nominal_latitude'] = 0.0
self.reader.mda['orbital_parameters'][
'satellite_actual_longitude'] = 47.5
self.reader.mda['orbital_parameters'][
'satellite_actual_latitude'] = -0.5
self.reader.mda['orbital_parameters'][
'satellite_actual_altitude'] = 35783328
tline = np.zeros(nlines,
dtype=[('days', '>u2'),
('milliseconds', '>u4')])
tline['days'][1:-1] = 21246 * np.ones(
nlines - 2) # 2016-03-03
tline['milliseconds'][1:-1] = np.arange(nlines - 2)
self.reader.mda['image_segment_line_quality'] = {
'line_mean_acquisition': tline
}
def test_get_xy_from_linecol(self):
"""Test get_xy_from_linecol."""
x__, y__ = self.reader.get_xy_from_linecol(0, 0, (10, 10), (5, 5))
self.assertEqual(-131072, x__)
self.assertEqual(131072, y__)
x__, y__ = self.reader.get_xy_from_linecol(10, 10, (10, 10), (5, 5))
self.assertEqual(0, x__)
self.assertEqual(0, y__)
x__, y__ = self.reader.get_xy_from_linecol(20, 20, (10, 10), (5, 5))
self.assertEqual(131072, x__)
self.assertEqual(-131072, y__)
def test_get_area_extent(self):
res = self.reader.get_area_extent((20, 20), (10, 10), (5, 5), 33)
exp = (-71717.44995740513, -79266.655216079365, 79266.655216079365,
71717.44995740513)
self.assertTupleEqual(res, exp)
def test_get_area_def(self):
area = self.reader.get_area_def(DatasetID('VIS006'))
self.assertEqual(
area.proj_dict, {
'a': 6378169.0,
'b': 6356583.8,
'h': 35785831.0,
'lon_0': 44.0,
'proj': 'geos',
'units': 'm'
})
self.assertEqual(area.area_extent,
(-77771774058.38356, -3720765401003.719,
30310525626438.438, 77771774058.38356))
# Data shifted by 1.5km to N-W
self.reader.mda['offset_corrected'] = False
area = self.reader.get_area_def(DatasetID('VIS006'))
self.assertEqual(area.area_extent,
(-77771772558.38356, -3720765402503.719,
30310525627938.438, 77771772558.38356))
@mock.patch('satpy.readers.hrit_base.np.memmap')
def test_read_band(self, memmap):
nbits = self.reader.mda['number_of_bits_per_pixel']
memmap.return_value = np.random.randint(0,
256,
size=int(
(464 * 3712 * nbits) / 8),
dtype=np.uint8)
res = self.reader.read_band('VIS006', None)
self.assertEqual(res.compute().shape, (464, 3712))
@mock.patch('satpy.readers.hrit_base.HRITFileHandler.__init__',
return_value=None)
@mock.patch('satpy.readers.seviri_l1b_hrit.HRITMSGFileHandler._get_header',
autospec=True)
@mock.patch(
'satpy.readers.seviri_base.SEVIRICalibrationHandler._convert_to_radiance'
)
def test_calibrate(self, _convert_to_radiance, get_header, *mocks):
"""Test selection of calibration coefficients"""
shp = (10, 10)
counts = xr.DataArray(np.zeros(shp))
nominal_gain = np.arange(1, 13)
nominal_offset = np.arange(-1, -13, -1)
gsics_gain = np.arange(0.1, 1.3, 0.1)
gsics_offset = np.arange(-0.1, -1.3, -0.1)
# Mock prologue & epilogue
pro = mock.MagicMock(
prologue={
'RadiometricProcessing': {
'Level15ImageCalibration': {
'CalSlope': nominal_gain,
'CalOffset': nominal_offset
},
'MPEFCalFeedback': {
'GSICSCalCoeff': gsics_gain,
'GSICSOffsetCount': gsics_offset
}
}
})
epi = mock.MagicMock(epilogue=None)
# Mock header readout
mda = {
'image_segment_line_quality': {
'line_validity': np.zeros(shp[0]),
'line_radiometric_quality': np.zeros(shp[0]),
'line_geometric_quality': np.zeros(shp[0])
}
}
def get_header_patched(self):
self.mda = mda
get_header.side_effect = get_header_patched
# Test selection of calibration coefficients
#
# a) Default: Nominal calibration
reader = HRITMSGFileHandler(filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi)
for ch_id, ch_name in CHANNEL_NAMES.items():
reader.channel_name = ch_name
reader.mda['spectral_channel_id'] = ch_id
reader.calibrate(data=counts, calibration='radiance')
_convert_to_radiance.assert_called_with(mock.ANY,
nominal_gain[ch_id - 1],
nominal_offset[ch_id - 1])
# b) GSICS calibration for IR channels, nominal calibration for VIS channels
reader = HRITMSGFileHandler(filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi,
calib_mode='GSICS')
for ch_id, ch_name in CHANNEL_NAMES.items():
if ch_name in VIS_CHANNELS:
gain, offset = nominal_gain[ch_id - 1], nominal_offset[ch_id -
1]
else:
gain, offset = gsics_gain[ch_id - 1], gsics_offset[ch_id - 1]
reader.channel_name = ch_name
reader.mda['spectral_channel_id'] = ch_id
reader.calibrate(data=counts, calibration='radiance')
_convert_to_radiance.assert_called_with(mock.ANY, gain, offset)
# c) External calibration coefficients for selected channels, GSICS coefs for remaining
# IR channels, nominal coefs for remaining VIS channels
coefs = {
'VIS006': {
'gain': 1.234,
'offset': -0.1
},
'IR_108': {
'gain': 2.345,
'offset': -0.2
}
}
reader = HRITMSGFileHandler(filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi,
ext_calib_coefs=coefs,
calib_mode='GSICS')
for ch_id, ch_name in CHANNEL_NAMES.items():
if ch_name in coefs.keys():
gain, offset = coefs[ch_name]['gain'], coefs[ch_name]['offset']
elif ch_name not in VIS_CHANNELS:
gain, offset = gsics_gain[ch_id - 1], gsics_offset[ch_id - 1]
else:
gain, offset = nominal_gain[ch_id - 1], nominal_offset[ch_id -
1]
reader.channel_name = ch_name
reader.mda['spectral_channel_id'] = ch_id
reader.calibrate(data=counts, calibration='radiance')
_convert_to_radiance.assert_called_with(mock.ANY, gain, offset)
# d) Invalid mode
self.assertRaises(ValueError,
HRITMSGFileHandler,
filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi,
calib_mode='invalid')
@mock.patch(
'satpy.readers.seviri_l1b_hrit.HRITMSGFileHandler._get_timestamps')
@mock.patch('satpy.readers.seviri_l1b_hrit.HRITFileHandler.get_dataset')
@mock.patch('satpy.readers.seviri_l1b_hrit.HRITMSGFileHandler.calibrate')
def test_get_dataset(self, calibrate, parent_get_dataset, _get_timestamps):
key = mock.MagicMock(calibration='calibration')
info = {
'units': 'units',
'wavelength': 'wavelength',
'standard_name': 'standard_name'
}
timestamps = np.array([1, 2, 3], dtype='datetime64[ns]')
parent_get_dataset.return_value = mock.MagicMock()
calibrate.return_value = xr.DataArray(data=np.zeros((3, 3)),
dims=('y', 'x'))
_get_timestamps.return_value = timestamps
res = self.reader.get_dataset(key, info)
# Test method calls
parent_get_dataset.assert_called_with(key, info)
calibrate.assert_called_with(parent_get_dataset(), key.calibration)
# Test attributes (just check if raw metadata is there and then remove it before checking the remaining
# attributes)
attrs_exp = info.copy()
attrs_exp.update({
'platform_name':
self.reader.platform_name,
'sensor':
'seviri',
'satellite_longitude':
self.reader.mda['projection_parameters']['SSP_longitude'],
'satellite_latitude':
self.reader.mda['projection_parameters']['SSP_latitude'],
'satellite_altitude':
self.reader.mda['projection_parameters']['h'],
'orbital_parameters': {
'projection_longitude': 44,
'projection_latitude': 0.,
'projection_altitude': 35785831.0,
'satellite_nominal_longitude': 47,
'satellite_nominal_latitude': 0.0,
'satellite_actual_longitude': 47.5,
'satellite_actual_latitude': -0.5,
'satellite_actual_altitude': 35783328
},
'georef_offset_corrected':
self.reader.mda['offset_corrected']
})
self.assertIn('raw_metadata', res.attrs)
res.attrs.pop('raw_metadata')
self.assertDictEqual(attrs_exp, res.attrs)
# Test timestamps
self.assertTrue(np.all(res['acq_time'] == timestamps))
self.assertEqual(res['acq_time'].attrs['long_name'],
'Mean scanline acquisition time')
def test_get_raw_mda(self):
"""Test provision of raw metadata"""
self.reader.mda = {'segment': 1, 'loff': 123}
self.reader.prologue_.reduce = lambda max_size: {'prologue': 1}
self.reader.epilogue_.reduce = lambda max_size: {'epilogue': 1}
expected = {'prologue': 1, 'epilogue': 1, 'segment': 1}
self.assertDictEqual(self.reader._get_raw_mda(), expected)
# Make sure _get_raw_mda() doesn't modify the original dictionary
self.assertIn('loff', self.reader.mda)
def test_get_timestamps(self):
tline = self.reader._get_timestamps()
# First and last scanline have invalid timestamps (space)
self.assertTrue(np.isnat(tline[0]))
self.assertTrue(np.isnat(tline[-1]))
# Test remaining lines
year = tline.astype('datetime64[Y]').astype(int) + 1970
month = tline.astype('datetime64[M]').astype(int) % 12 + 1
day = (tline.astype('datetime64[D]') - tline.astype('datetime64[M]') +
1).astype(int)
msec = (tline - tline.astype('datetime64[D]')).astype(int)
self.assertTrue(np.all(year[1:-1] == 2016))
self.assertTrue(np.all(month[1:-1] == 3))
self.assertTrue(np.all(day[1:-1] == 3))
self.assertTrue(np.all(msec[1:-1] == np.arange(len(tline) - 2)))
class TestHRITMSGFileHandler(unittest.TestCase):
"""Test the HRITFileHandler."""
@mock.patch('satpy.readers.seviri_l1b_hrit.np.fromfile')
def setUp(self, fromfile):
"""Set up the hrit file handler for testing."""
m = mock.mock_open()
fromfile.return_value = np.array([(1, 2)],
dtype=[('total_header_length', int),
('hdr_id', int)])
with mock.patch('satpy.readers.hrit_base.open', m,
create=True) as newopen:
with mock.patch('satpy.readers.seviri_l1b_hrit.CHANNEL_NAMES'):
with mock.patch.object(HRITMSGFileHandler,
'_get_hd',
new=new_get_hd):
newopen.return_value.__enter__.return_value.tell.return_value = 1
prologue = mock.MagicMock()
prologue.prologue = {
"SatelliteStatus": {
"SatelliteDefinition": {
"SatelliteId": 324,
"NominalLongitude": 47
}
},
'GeometricProcessing': {
'EarthModel': {
'TypeOfEarthModel': 2,
'NorthPolarRadius': 10,
'SouthPolarRadius': 10,
'EquatorialRadius': 10
}
},
'ImageDescription': {
'ProjectionDescription': {
'LongitudeOfSSP': 0.0
},
'Level15ImageProduction': {
'ImageProcDirection': 1
}
}
}
prologue.get_satpos.return_value = None, None, None
prologue.get_earth_radii.return_value = None, None
self.reader = HRITMSGFileHandler(
'filename', {
'platform_shortname': 'MSG3',
'start_time': datetime(2016, 3, 3, 0, 0),
'service': 'MSG'
}, {'filetype': 'info'}, prologue, mock.MagicMock())
ncols = 3712
nlines = 464
nbits = 10
self.reader.mda['number_of_bits_per_pixel'] = nbits
self.reader.mda['number_of_lines'] = nlines
self.reader.mda['number_of_columns'] = ncols
self.reader.mda[
'data_field_length'] = nlines * ncols * nbits
self.reader.mda['cfac'] = 5
self.reader.mda['lfac'] = 5
self.reader.mda['coff'] = 10
self.reader.mda['loff'] = 10
self.reader.mda['projection_parameters'] = {}
self.reader.mda['projection_parameters']['a'] = 6378169.0
self.reader.mda['projection_parameters']['b'] = 6356583.8
self.reader.mda['projection_parameters']['h'] = 35785831.0
self.reader.mda['projection_parameters'][
'SSP_longitude'] = 9.5
self.reader.mda['projection_parameters'][
'SSP_latitude'] = 0.0
self.reader.mda['orbital_parameters'] = {}
self.reader.mda['orbital_parameters'][
'satellite_nominal_longitude'] = 47
self.reader.mda['orbital_parameters'][
'satellite_nominal_latitude'] = 0.0
self.reader.mda['orbital_parameters'][
'satellite_actual_longitude'] = 47.5
self.reader.mda['orbital_parameters'][
'satellite_actual_latitude'] = -0.5
self.reader.mda['orbital_parameters'][
'satellite_actual_altitude'] = 35783328
tline = np.zeros(nlines,
dtype=[('days', '>u2'),
('milliseconds', '>u4')])
tline['days'][1:-1] = 21246 * np.ones(
nlines - 2) # 2016-03-03
tline['milliseconds'][1:-1] = np.arange(nlines - 2)
self.reader.mda['image_segment_line_quality'] = {
'line_mean_acquisition': tline
}
def test_get_area_def(self):
"""Test getting the area def."""
from pyresample.utils import proj4_radius_parameters
area = self.reader.get_area_def(
make_dataid(name='VIS006', resolution=3000))
proj_dict = area.proj_dict
a, b = proj4_radius_parameters(proj_dict)
self.assertEqual(a, 6378169.0)
self.assertEqual(b, 6356583.8)
self.assertEqual(proj_dict['h'], 35785831.0)
self.assertEqual(proj_dict['lon_0'], 9.5)
self.assertEqual(proj_dict['proj'], 'geos')
self.assertEqual(proj_dict['units'], 'm')
self.assertEqual(area.area_extent,
(-77771774058.38356, -3720765401003.719,
30310525626438.438, 77771774058.38356))
# Data shifted by 1.5km to N-W
self.reader.mda['offset_corrected'] = False
area = self.reader.get_area_def(
make_dataid(name='VIS006', resolution=3000))
self.assertEqual(area.area_extent,
(-77771772558.38356, -3720765402503.719,
30310525627938.438, 77771772558.38356))
self.assertEqual(area.area_id, 'msg_seviri_rss_3km')
@mock.patch('satpy.readers.hrit_base.np.memmap')
def test_read_band(self, memmap):
"""Test reading a band."""
nbits = self.reader.mda['number_of_bits_per_pixel']
memmap.return_value = np.random.randint(0,
256,
size=int(
(464 * 3712 * nbits) / 8),
dtype=np.uint8)
res = self.reader.read_band('VIS006', None)
self.assertEqual(res.shape, (464, 3712))
@mock.patch(
'satpy.readers.seviri_l1b_hrit.HRITMSGFileHandler._get_timestamps')
@mock.patch('satpy.readers.seviri_l1b_hrit.HRITFileHandler.get_dataset')
@mock.patch('satpy.readers.seviri_l1b_hrit.HRITMSGFileHandler.calibrate')
def test_get_dataset(self, calibrate, parent_get_dataset, _get_timestamps):
"""Test getting the dataset."""
key = make_dataid(name='VIS006', calibration='reflectance')
info = {
'units': 'units',
'wavelength': 'wavelength',
'standard_name': 'standard_name'
}
timestamps = np.array([1, 2, 3], dtype='datetime64[ns]')
parent_get_dataset.return_value = mock.MagicMock()
calibrate.return_value = xr.DataArray(data=np.zeros((3, 3)),
dims=('y', 'x'))
_get_timestamps.return_value = timestamps
res = self.reader.get_dataset(key, info)
# Test method calls
parent_get_dataset.assert_called_with(key, info)
calibrate.assert_called_with(parent_get_dataset(), key['calibration'])
# Test attributes (just check if raw metadata is there and then remove it before checking the remaining
# attributes)
attrs_exp = info.copy()
attrs_exp.update({
'platform_name':
self.reader.platform_name,
'sensor':
'seviri',
'satellite_longitude':
self.reader.mda['projection_parameters']['SSP_longitude'],
'satellite_latitude':
self.reader.mda['projection_parameters']['SSP_latitude'],
'satellite_altitude':
self.reader.mda['projection_parameters']['h'],
'orbital_parameters': {
'projection_longitude': 9.5,
'projection_latitude': 0.,
'projection_altitude': 35785831.0,
'satellite_nominal_longitude': 47,
'satellite_nominal_latitude': 0.0,
'satellite_actual_longitude': 47.5,
'satellite_actual_latitude': -0.5,
'satellite_actual_altitude': 35783328
},
'georef_offset_corrected':
self.reader.mda['offset_corrected']
})
self.assertIn('raw_metadata', res.attrs)
res.attrs.pop('raw_metadata')
self.assertDictEqual(attrs_exp, res.attrs)
# Test timestamps
self.assertTrue(np.all(res['acq_time'] == timestamps))
self.assertEqual(res['acq_time'].attrs['long_name'],
'Mean scanline acquisition time')
def test_get_raw_mda(self):
"""Test provision of raw metadata."""
self.reader.mda = {'segment': 1, 'loff': 123}
self.reader.prologue_.reduce = lambda max_size: {'prologue': 1}
self.reader.epilogue_.reduce = lambda max_size: {'epilogue': 1}
expected = {'prologue': 1, 'epilogue': 1, 'segment': 1}
self.assertDictEqual(self.reader._get_raw_mda(), expected)
# Make sure _get_raw_mda() doesn't modify the original dictionary
self.assertIn('loff', self.reader.mda)
def test_get_timestamps(self):
"""Test getting the timestamps."""
tline = self.reader._get_timestamps()
# First and last scanline have invalid timestamps (space)
self.assertTrue(np.isnat(tline[0]))
self.assertTrue(np.isnat(tline[-1]))
# Test remaining lines
year = tline.astype('datetime64[Y]').astype(int) + 1970
month = tline.astype('datetime64[M]').astype(int) % 12 + 1
day = (tline.astype('datetime64[D]') - tline.astype('datetime64[M]') +
1).astype(int)
msec = (tline - tline.astype('datetime64[D]')).astype(int)
self.assertTrue(np.all(year[1:-1] == 2016))
self.assertTrue(np.all(month[1:-1] == 3))
self.assertTrue(np.all(day[1:-1] == 3))
self.assertTrue(np.all(msec[1:-1] == np.arange(len(tline) - 2)))
def test_get_header(self):
"""Test getting the header."""
# Make sure that the actual satellite position is only included if available
self.reader.mda['orbital_parameters'] = {}
self.reader.prologue_.get_satpos.return_value = 1, 2, 3
self.reader._get_header()
self.assertIn('satellite_actual_longitude',
self.reader.mda['orbital_parameters'])
self.reader.mda['orbital_parameters'] = {}
self.reader.prologue_.get_satpos.return_value = None, None, None
self.reader._get_header()
self.assertNotIn('satellite_actual_longitude',
self.reader.mda['orbital_parameters'])