def _get_acq_time_visir(self, dataset_id):
band_idx = list(CHANNEL_NAMES.values()).index(dataset_id['name'])
day_key = 'channel_data_visir_data_l10_line_mean_acquisition_time_day'
msec_key = 'channel_data_visir_data_l10_line_mean_acquisition_msec'
days = self.nc[day_key].isel(channels_vis_ir_dim=band_idx)
msecs = self.nc[msec_key].isel(channels_vis_ir_dim=band_idx)
return days, msecs
def get_dataset(self, dataset_id, dataset_info):
channel = dataset_id.name
i = list(CHANNEL_NAMES.values()).index(channel)
if (channel == 'HRV'):
self.nc = self.nc.rename({
'num_columns_hrv': 'x',
'num_rows_hrv': 'y'
})
else:
# the first channel of a composite will rename the dimension variable
# but the later channels will raise a value error as its already been renamed
# we can just ignore these exceptions
try:
self.nc = self.nc.rename({
'num_columns_vis_ir': 'x',
'num_rows_vis_ir': 'y'
})
except ValueError:
pass
dataset = self.nc[dataset_info['nc_key']]
dataset.attrs.update(dataset_info)
# Calibrate the data as needed
# MPEF MSG calibration coeffiencts (gain and count)
offset = dataset.attrs['add_offset'].astype('float32')
gain = dataset.attrs['scale_factor'].astype('float32')
self.platform_id = int(self.nc.attrs['satellite_id'])
cal_type = self.nc['planned_chan_processing'].values[i]
# Correct for the scan line order
dataset = dataset.sel(y=slice(None, None, -1))
if dataset_id.calibration == 'counts':
dataset.attrs['_FillValue'] = 0
if dataset_id.calibration in [
'radiance', 'reflectance', 'brightness_temperature'
]:
dataset = dataset.where(dataset != 0).astype('float32')
dataset = self._convert_to_radiance(dataset, gain, offset)
if dataset_id.calibration == 'reflectance':
solar_irradiance = CALIB[int(self.platform_id)][channel]["F"]
dataset = self._vis_calibrate(dataset, solar_irradiance)
elif dataset_id.calibration == 'brightness_temperature':
dataset = self._ir_calibrate(dataset, channel, cal_type)
dataset.attrs.update(self.nc[dataset_info['nc_key']].attrs)
dataset.attrs.update(dataset_info)
dataset.attrs['platform_name'] = "Meteosat-" + SATNUM[self.platform_id]
dataset.attrs['sensor'] = 'seviri'
return dataset
def calibrate(self, data, dataset_id):
"""Calibrate the data."""
tic = datetime.now()
data15hdr = self.header['15_DATA_HEADER']
calibration = dataset_id['calibration']
channel = dataset_id['name']
# even though all the channels may not be present in the file,
# the header does have calibration coefficients for all the channels
# hence, this channel index needs to refer to full channel list
i = list(CHANNEL_NAMES.values()).index(channel)
if calibration == 'counts':
return data
if calibration in [
'radiance', 'reflectance', 'brightness_temperature'
]:
# determine the required calibration coefficients to use
# for the Level 1.5 Header
if (self.calib_mode.upper() != 'GSICS'
and self.calib_mode.upper() != 'NOMINAL'):
raise NotImplementedError(
'Unknown Calibration mode : Please check')
# NB GSICS doesn't have calibration coeffs for VIS channels
if (self.calib_mode.upper() != 'GSICS' or channel in VIS_CHANNELS):
coeffs = data15hdr['RadiometricProcessing'][
'Level15ImageCalibration']
gain = coeffs['CalSlope'][i]
offset = coeffs['CalOffset'][i]
else:
coeffs = data15hdr['RadiometricProcessing']['MPEFCalFeedback']
gain = coeffs['GSICSCalCoeff'][i]
offset = coeffs['GSICSOffsetCount'][i]
offset = offset * gain
res = self._convert_to_radiance(data, gain, offset)
if calibration == 'reflectance':
solar_irradiance = CALIB[self.platform_id][channel]["F"]
res = self._vis_calibrate(res, solar_irradiance)
elif calibration == 'brightness_temperature':
cal_type = data15hdr['ImageDescription']['Level15ImageProduction'][
'PlannedChanProcessing'][i]
res = self._ir_calibrate(res, channel, cal_type)
logger.debug("Calibration time " + str(datetime.now() - tic))
return res
def calibrate(self, data, dsid):
"""Calibrate the data."""
tic = datetime.now()
data15hdr = self.header['15_DATA_HEADER']
calibration = dsid.calibration
channel = dsid.name
# even though all the channels may not be present in the file,
# the header does have calibration coefficients for all the channels
# hence, this channel index needs to refer to full channel list
i = list(CHANNEL_NAMES.values()).index(channel)
if calibration == 'counts':
return data
if calibration in ['radiance', 'reflectance', 'brightness_temperature']:
# you cant apply GSICS values to the VIS channels
visual_channels = ['HRV', 'VIS006', 'VIS008', 'IR_016']
# determine the required calibration coefficients to use
# for the Level 1.5 Header
calMode = os.environ.get('CAL_MODE', 'NOMINAL')
# NB GSICS doesn't have calibration coeffs for VIS channels
if (calMode.upper() != 'GSICS' or channel in visual_channels):
coeffs = data15hdr[
'RadiometricProcessing']['Level15ImageCalibration']
gain = coeffs['CalSlope'][i]
offset = coeffs['CalOffset'][i]
else:
coeffs = data15hdr[
'RadiometricProcessing']['MPEFCalFeedback']
gain = coeffs['GSICSCalCoeff'][i]
offset = coeffs['GSICSOffsetCount'][i]
offset = offset * gain
res = self._convert_to_radiance(data, gain, offset)
if calibration == 'reflectance':
solar_irradiance = CALIB[self.mda['platform_id']][channel]["F"]
res = self._vis_calibrate(res, solar_irradiance)
elif calibration == 'brightness_temperature':
cal_type = data15hdr['ImageDescription'][
'Level15ImageProduction']['PlannedChanProcessing'][i]
res = self._ir_calibrate(res, channel, cal_type)
logger.debug("Calibration time " + str(datetime.now() - tic))
return res
def _get_calib_coefs(self, dataset, channel):
"""Get coefficients for calibration from counts to radiance."""
band_idx = list(CHANNEL_NAMES.values()).index(channel)
offset = dataset.attrs['add_offset'].astype('float32')
gain = dataset.attrs['scale_factor'].astype('float32')
# Only one calibration available here
return {
'coefs': {
'NOMINAL': {
'gain': gain,
'offset': offset
},
'EXTERNAL': self.ext_calib_coefs.get(channel, {})
},
'radiance_type': self.nc['planned_chan_processing'].values[band_idx]
}
def _get_calib_coefs(self, channel_name):
"""Get coefficients for calibration from counts to radiance."""
# even though all the channels may not be present in the file,
# the header does have calibration coefficients for all the channels
# hence, this channel index needs to refer to full channel list
band_idx = list(CHANNEL_NAMES.values()).index(channel_name)
coefs_nominal = self.header['15_DATA_HEADER']['RadiometricProcessing'][
'Level15ImageCalibration']
coefs_gsics = self.header['15_DATA_HEADER']['RadiometricProcessing'][
'MPEFCalFeedback']
radiance_types = self.header['15_DATA_HEADER']['ImageDescription'][
'Level15ImageProduction']['PlannedChanProcessing']
return create_coef_dict(
coefs_nominal=(coefs_nominal['CalSlope'][band_idx],
coefs_nominal['CalOffset'][band_idx]),
coefs_gsics=(coefs_gsics['GSICSCalCoeff'][band_idx],
coefs_gsics['GSICSOffsetCount'][band_idx]),
ext_coefs=self.ext_calib_coefs.get(channel_name, {}),
radiance_type=radiance_types[band_idx])
def get_dataset(self, dataset_id, dataset_info):
channel = dataset_id.name
i = list(CHANNEL_NAMES.values()).index(channel)
dataset = self.nc[dataset_info['nc_key']]
dataset.attrs.update(dataset_info)
# Calibrate the data as needed
# MPEF MSG calibration coeffiencts (gain and count)
offset = dataset.attrs['add_offset'].astype('float32')
gain = dataset.attrs['scale_factor'].astype('float32')
self.platform_id = int(self.nc.attrs['satellite_id'])
cal_type = self.nc['planned_chan_processing'].values[i]
# Correct for the scan line order
dataset = dataset.sel(y=slice(None, None, -1))
if dataset_id.calibration == 'counts':
dataset.attrs['_FillValue'] = 0
if dataset_id.calibration in [
'radiance', 'reflectance', 'brightness_temperature'
]:
dataset = dataset.where(dataset != 0).astype('float32')
dataset = self._convert_to_radiance(dataset, gain, offset)
if dataset_id.calibration == 'reflectance':
solar_irradiance = CALIB[int(self.platform_id)][channel]["F"]
dataset = self._vis_calibrate(dataset, solar_irradiance)
elif dataset_id.calibration == 'brightness_temperature':
dataset = self._ir_calibrate(dataset, channel, cal_type)
dataset.attrs.update(self.nc[dataset_info['nc_key']].attrs)
dataset.attrs.update(dataset_info)
dataset.attrs['platform_name'] = "Meteosat-" + SATNUM[self.platform_id]
dataset.attrs['sensor'] = 'seviri'
return dataset
def test_calibrate(self, _convert_to_radiance, get_header, *mocks):
"""Test selection of calibration coefficients"""
shp = (10, 10)
counts = xr.DataArray(np.zeros(shp))
nominal_gain = np.arange(1, 13)
nominal_offset = np.arange(-1, -13, -1)
gsics_gain = np.arange(0.1, 1.3, 0.1)
gsics_offset = np.arange(-0.1, -1.3, -0.1)
# Mock prologue & epilogue
pro = mock.MagicMock(
prologue={
'RadiometricProcessing': {
'Level15ImageCalibration': {
'CalSlope': nominal_gain,
'CalOffset': nominal_offset
},
'MPEFCalFeedback': {
'GSICSCalCoeff': gsics_gain,
'GSICSOffsetCount': gsics_offset
}
}
})
epi = mock.MagicMock(epilogue=None)
# Mock header readout
mda = {
'image_segment_line_quality': {
'line_validity': np.zeros(shp[0]),
'line_radiometric_quality': np.zeros(shp[0]),
'line_geometric_quality': np.zeros(shp[0])
}
}
def get_header_patched(self):
self.mda = mda
get_header.side_effect = get_header_patched
# Test selection of calibration coefficients
#
# a) Default: Nominal calibration
reader = HRITMSGFileHandler(filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi)
for ch_id, ch_name in CHANNEL_NAMES.items():
reader.channel_name = ch_name
reader.mda['spectral_channel_id'] = ch_id
reader.calibrate(data=counts, calibration='radiance')
_convert_to_radiance.assert_called_with(mock.ANY,
nominal_gain[ch_id - 1],
nominal_offset[ch_id - 1])
# b) GSICS calibration for IR channels, nominal calibration for VIS channels
reader = HRITMSGFileHandler(filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi,
calib_mode='GSICS')
for ch_id, ch_name in CHANNEL_NAMES.items():
if ch_name in VIS_CHANNELS:
gain, offset = nominal_gain[ch_id - 1], nominal_offset[ch_id -
1]
else:
gain, offset = gsics_gain[ch_id - 1], gsics_offset[ch_id - 1]
reader.channel_name = ch_name
reader.mda['spectral_channel_id'] = ch_id
reader.calibrate(data=counts, calibration='radiance')
_convert_to_radiance.assert_called_with(mock.ANY, gain, offset)
# c) External calibration coefficients for selected channels, GSICS coefs for remaining
# IR channels, nominal coefs for remaining VIS channels
coefs = {
'VIS006': {
'gain': 1.234,
'offset': -0.1
},
'IR_108': {
'gain': 2.345,
'offset': -0.2
}
}
reader = HRITMSGFileHandler(filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi,
ext_calib_coefs=coefs,
calib_mode='GSICS')
for ch_id, ch_name in CHANNEL_NAMES.items():
if ch_name in coefs.keys():
gain, offset = coefs[ch_name]['gain'], coefs[ch_name]['offset']
elif ch_name not in VIS_CHANNELS:
gain, offset = gsics_gain[ch_id - 1], gsics_offset[ch_id - 1]
else:
gain, offset = nominal_gain[ch_id - 1], nominal_offset[ch_id -
1]
reader.channel_name = ch_name
reader.mda['spectral_channel_id'] = ch_id
reader.calibrate(data=counts, calibration='radiance')
_convert_to_radiance.assert_called_with(mock.ANY, gain, offset)
# d) Invalid mode
self.assertRaises(ValueError,
HRITMSGFileHandler,
filename=None,
filename_info=None,
filetype_info=None,
prologue=pro,
epilogue=epi,
calib_mode='invalid')
def _read_header(self):
"""Read the header info."""
data = np.fromfile(self.filename,
dtype=native_header, count=1)
self.header.update(recarray2dict(data))
data15hd = self.header['15_DATA_HEADER']
sec15hd = self.header['15_SECONDARY_PRODUCT_HEADER']
# Set the list of available channels:
self.mda['available_channels'] = get_available_channels(self.header)
self.mda['channel_list'] = [i for i in CHANNEL_NAMES.values()
if self.mda['available_channels'][i]]
self.platform_id = data15hd[
'SatelliteStatus']['SatelliteDefinition']['SatelliteId']
self.mda['platform_name'] = "Meteosat-" + SATNUM[self.platform_id]
equator_radius = data15hd['GeometricProcessing'][
'EarthModel']['EquatorialRadius'] * 1000.
north_polar_radius = data15hd[
'GeometricProcessing']['EarthModel']['NorthPolarRadius'] * 1000.
south_polar_radius = data15hd[
'GeometricProcessing']['EarthModel']['SouthPolarRadius'] * 1000.
polar_radius = (north_polar_radius + south_polar_radius) * 0.5
ssp_lon = data15hd['ImageDescription'][
'ProjectionDescription']['LongitudeOfSSP']
self.mda['projection_parameters'] = {'a': equator_radius,
'b': polar_radius,
'h': 35785831.00,
'ssp_longitude': ssp_lon}
north = int(sec15hd['NorthLineSelectedRectangle']['Value'])
east = int(sec15hd['EastColumnSelectedRectangle']['Value'])
south = int(sec15hd['SouthLineSelectedRectangle']['Value'])
west = int(sec15hd['WestColumnSelectedRectangle']['Value'])
ncolumns = west - east + 1
nrows = north - south + 1
# check if the file has less rows or columns than
# the maximum, if so it is an area of interest file
if (nrows < VISIR_NUM_LINES) or (ncolumns < VISIR_NUM_COLUMNS):
self.mda['is_full_disk'] = False
# If the number of columns in the file is not divisible by 4,
# UMARF will add extra columns to the file
modulo = ncolumns % 4
padding = 0
if modulo > 0:
padding = 4 - modulo
cols_visir = ncolumns + padding
# Check the VISIR calculated column dimension against
# the header information
cols_visir_hdr = int(sec15hd['NumberColumnsVISIR']['Value'])
if cols_visir_hdr != cols_visir:
logger.warning(
"Number of VISIR columns from the header is incorrect!")
logger.warning("Header: %d", cols_visir_hdr)
logger.warning("Calculated: = %d", cols_visir)
# HRV Channel - check if the area is reduced in east west
# direction as this affects the number of columns in the file
cols_hrv_hdr = int(sec15hd['NumberColumnsHRV']['Value'])
if ncolumns < VISIR_NUM_COLUMNS:
cols_hrv = cols_hrv_hdr
else:
cols_hrv = int(cols_hrv_hdr / 2)
# self.mda represents the 16bit dimensions not 10bit
self.mda['number_of_lines'] = int(sec15hd['NumberLinesVISIR']['Value'])
self.mda['number_of_columns'] = cols_visir
self.mda['hrv_number_of_lines'] = int(sec15hd["NumberLinesHRV"]['Value'])
self.mda['hrv_number_of_columns'] = cols_hrv
def get_dataset(self, dataset_id, dataset_info):
"""Get the dataset."""
channel = dataset_id['name']
i = list(CHANNEL_NAMES.values()).index(channel)
if (channel == 'HRV'):
self.nc = self.nc.rename({
'num_columns_hrv': 'x',
'num_rows_hrv': 'y'
})
else:
# the first channel of a composite will rename the dimension variable
# but the later channels will raise a value error as its already been renamed
# we can just ignore these exceptions
try:
self.nc = self.nc.rename({
'num_columns_vis_ir': 'x',
'num_rows_vis_ir': 'y'
})
except ValueError:
pass
dataset = self.nc[dataset_info['nc_key']]
dataset.attrs.update(dataset_info)
# Calibrate the data as needed
# MPEF MSG calibration coeffiencts (gain and count)
offset = dataset.attrs['add_offset'].astype('float32')
gain = dataset.attrs['scale_factor'].astype('float32')
self.platform_id = int(self.nc.attrs['satellite_id'])
cal_type = self.nc['planned_chan_processing'].values[i]
# Correct for the scan line order
dataset = dataset.sel(y=slice(None, None, -1))
if dataset_id['calibration'] == 'counts':
dataset.attrs['_FillValue'] = 0
if dataset_id['calibration'] in [
'radiance', 'reflectance', 'brightness_temperature'
]:
dataset = dataset.where(dataset != 0).astype('float32')
dataset = self._convert_to_radiance(dataset, gain, offset)
if dataset_id['calibration'] == 'reflectance':
solar_irradiance = CALIB[int(self.platform_id)][channel]["F"]
dataset = self._vis_calibrate(dataset, solar_irradiance)
elif dataset_id['calibration'] == 'brightness_temperature':
dataset = self._ir_calibrate(dataset, channel, cal_type)
dataset.attrs.update(self.nc[dataset_info['nc_key']].attrs)
dataset.attrs.update(dataset_info)
dataset.attrs['platform_name'] = "Meteosat-" + SATNUM[self.platform_id]
dataset.attrs['sensor'] = 'seviri'
dataset.attrs['orbital_parameters'] = {
'projection_longitude':
self.mda['projection_parameters']['ssp_longitude'],
'projection_latitude':
0.,
'projection_altitude':
self.mda['projection_parameters']['h']
}
# remove attributes from original file which don't apply anymore
strip_attrs = [
"comment", "long_name", "nc_key", "scale_factor", "add_offset",
"valid_min", "valid_max"
]
for a in strip_attrs:
dataset.attrs.pop(a)
return dataset
