def _get_acq_time_visir(self, dataset_id): band_idx = list(CHANNEL_NAMES.values()).index(dataset_id['name']) day_key = 'channel_data_visir_data_l10_line_mean_acquisition_time_day' msec_key = 'channel_data_visir_data_l10_line_mean_acquisition_msec' days = self.nc[day_key].isel(channels_vis_ir_dim=band_idx) msecs = self.nc[msec_key].isel(channels_vis_ir_dim=band_idx) return days, msecs
def get_dataset(self, dataset_id, dataset_info): channel = dataset_id.name i = list(CHANNEL_NAMES.values()).index(channel) if (channel == 'HRV'): self.nc = self.nc.rename({ 'num_columns_hrv': 'x', 'num_rows_hrv': 'y' }) else: # the first channel of a composite will rename the dimension variable # but the later channels will raise a value error as its already been renamed # we can just ignore these exceptions try: self.nc = self.nc.rename({ 'num_columns_vis_ir': 'x', 'num_rows_vis_ir': 'y' }) except ValueError: pass dataset = self.nc[dataset_info['nc_key']] dataset.attrs.update(dataset_info) # Calibrate the data as needed # MPEF MSG calibration coeffiencts (gain and count) offset = dataset.attrs['add_offset'].astype('float32') gain = dataset.attrs['scale_factor'].astype('float32') self.platform_id = int(self.nc.attrs['satellite_id']) cal_type = self.nc['planned_chan_processing'].values[i] # Correct for the scan line order dataset = dataset.sel(y=slice(None, None, -1)) if dataset_id.calibration == 'counts': dataset.attrs['_FillValue'] = 0 if dataset_id.calibration in [ 'radiance', 'reflectance', 'brightness_temperature' ]: dataset = dataset.where(dataset != 0).astype('float32') dataset = self._convert_to_radiance(dataset, gain, offset) if dataset_id.calibration == 'reflectance': solar_irradiance = CALIB[int(self.platform_id)][channel]["F"] dataset = self._vis_calibrate(dataset, solar_irradiance) elif dataset_id.calibration == 'brightness_temperature': dataset = self._ir_calibrate(dataset, channel, cal_type) dataset.attrs.update(self.nc[dataset_info['nc_key']].attrs) dataset.attrs.update(dataset_info) dataset.attrs['platform_name'] = "Meteosat-" + SATNUM[self.platform_id] dataset.attrs['sensor'] = 'seviri' return dataset
def calibrate(self, data, dataset_id): """Calibrate the data.""" tic = datetime.now() data15hdr = self.header['15_DATA_HEADER'] calibration = dataset_id['calibration'] channel = dataset_id['name'] # even though all the channels may not be present in the file, # the header does have calibration coefficients for all the channels # hence, this channel index needs to refer to full channel list i = list(CHANNEL_NAMES.values()).index(channel) if calibration == 'counts': return data if calibration in [ 'radiance', 'reflectance', 'brightness_temperature' ]: # determine the required calibration coefficients to use # for the Level 1.5 Header if (self.calib_mode.upper() != 'GSICS' and self.calib_mode.upper() != 'NOMINAL'): raise NotImplementedError( 'Unknown Calibration mode : Please check') # NB GSICS doesn't have calibration coeffs for VIS channels if (self.calib_mode.upper() != 'GSICS' or channel in VIS_CHANNELS): coeffs = data15hdr['RadiometricProcessing'][ 'Level15ImageCalibration'] gain = coeffs['CalSlope'][i] offset = coeffs['CalOffset'][i] else: coeffs = data15hdr['RadiometricProcessing']['MPEFCalFeedback'] gain = coeffs['GSICSCalCoeff'][i] offset = coeffs['GSICSOffsetCount'][i] offset = offset * gain res = self._convert_to_radiance(data, gain, offset) if calibration == 'reflectance': solar_irradiance = CALIB[self.platform_id][channel]["F"] res = self._vis_calibrate(res, solar_irradiance) elif calibration == 'brightness_temperature': cal_type = data15hdr['ImageDescription']['Level15ImageProduction'][ 'PlannedChanProcessing'][i] res = self._ir_calibrate(res, channel, cal_type) logger.debug("Calibration time " + str(datetime.now() - tic)) return res
def calibrate(self, data, dsid): """Calibrate the data.""" tic = datetime.now() data15hdr = self.header['15_DATA_HEADER'] calibration = dsid.calibration channel = dsid.name # even though all the channels may not be present in the file, # the header does have calibration coefficients for all the channels # hence, this channel index needs to refer to full channel list i = list(CHANNEL_NAMES.values()).index(channel) if calibration == 'counts': return data if calibration in ['radiance', 'reflectance', 'brightness_temperature']: # you cant apply GSICS values to the VIS channels visual_channels = ['HRV', 'VIS006', 'VIS008', 'IR_016'] # determine the required calibration coefficients to use # for the Level 1.5 Header calMode = os.environ.get('CAL_MODE', 'NOMINAL') # NB GSICS doesn't have calibration coeffs for VIS channels if (calMode.upper() != 'GSICS' or channel in visual_channels): coeffs = data15hdr[ 'RadiometricProcessing']['Level15ImageCalibration'] gain = coeffs['CalSlope'][i] offset = coeffs['CalOffset'][i] else: coeffs = data15hdr[ 'RadiometricProcessing']['MPEFCalFeedback'] gain = coeffs['GSICSCalCoeff'][i] offset = coeffs['GSICSOffsetCount'][i] offset = offset * gain res = self._convert_to_radiance(data, gain, offset) if calibration == 'reflectance': solar_irradiance = CALIB[self.mda['platform_id']][channel]["F"] res = self._vis_calibrate(res, solar_irradiance) elif calibration == 'brightness_temperature': cal_type = data15hdr['ImageDescription'][ 'Level15ImageProduction']['PlannedChanProcessing'][i] res = self._ir_calibrate(res, channel, cal_type) logger.debug("Calibration time " + str(datetime.now() - tic)) return res
def _get_calib_coefs(self, dataset, channel): """Get coefficients for calibration from counts to radiance.""" band_idx = list(CHANNEL_NAMES.values()).index(channel) offset = dataset.attrs['add_offset'].astype('float32') gain = dataset.attrs['scale_factor'].astype('float32') # Only one calibration available here return { 'coefs': { 'NOMINAL': { 'gain': gain, 'offset': offset }, 'EXTERNAL': self.ext_calib_coefs.get(channel, {}) }, 'radiance_type': self.nc['planned_chan_processing'].values[band_idx] }
def _get_calib_coefs(self, channel_name): """Get coefficients for calibration from counts to radiance.""" # even though all the channels may not be present in the file, # the header does have calibration coefficients for all the channels # hence, this channel index needs to refer to full channel list band_idx = list(CHANNEL_NAMES.values()).index(channel_name) coefs_nominal = self.header['15_DATA_HEADER']['RadiometricProcessing'][ 'Level15ImageCalibration'] coefs_gsics = self.header['15_DATA_HEADER']['RadiometricProcessing'][ 'MPEFCalFeedback'] radiance_types = self.header['15_DATA_HEADER']['ImageDescription'][ 'Level15ImageProduction']['PlannedChanProcessing'] return create_coef_dict( coefs_nominal=(coefs_nominal['CalSlope'][band_idx], coefs_nominal['CalOffset'][band_idx]), coefs_gsics=(coefs_gsics['GSICSCalCoeff'][band_idx], coefs_gsics['GSICSOffsetCount'][band_idx]), ext_coefs=self.ext_calib_coefs.get(channel_name, {}), radiance_type=radiance_types[band_idx])
def get_dataset(self, dataset_id, dataset_info): channel = dataset_id.name i = list(CHANNEL_NAMES.values()).index(channel) dataset = self.nc[dataset_info['nc_key']] dataset.attrs.update(dataset_info) # Calibrate the data as needed # MPEF MSG calibration coeffiencts (gain and count) offset = dataset.attrs['add_offset'].astype('float32') gain = dataset.attrs['scale_factor'].astype('float32') self.platform_id = int(self.nc.attrs['satellite_id']) cal_type = self.nc['planned_chan_processing'].values[i] # Correct for the scan line order dataset = dataset.sel(y=slice(None, None, -1)) if dataset_id.calibration == 'counts': dataset.attrs['_FillValue'] = 0 if dataset_id.calibration in [ 'radiance', 'reflectance', 'brightness_temperature' ]: dataset = dataset.where(dataset != 0).astype('float32') dataset = self._convert_to_radiance(dataset, gain, offset) if dataset_id.calibration == 'reflectance': solar_irradiance = CALIB[int(self.platform_id)][channel]["F"] dataset = self._vis_calibrate(dataset, solar_irradiance) elif dataset_id.calibration == 'brightness_temperature': dataset = self._ir_calibrate(dataset, channel, cal_type) dataset.attrs.update(self.nc[dataset_info['nc_key']].attrs) dataset.attrs.update(dataset_info) dataset.attrs['platform_name'] = "Meteosat-" + SATNUM[self.platform_id] dataset.attrs['sensor'] = 'seviri' return dataset
def _read_header(self): """Read the header info.""" data = np.fromfile(self.filename, dtype=native_header, count=1) self.header.update(recarray2dict(data)) data15hd = self.header['15_DATA_HEADER'] sec15hd = self.header['15_SECONDARY_PRODUCT_HEADER'] # Set the list of available channels: self.mda['available_channels'] = get_available_channels(self.header) self.mda['channel_list'] = [i for i in CHANNEL_NAMES.values() if self.mda['available_channels'][i]] self.platform_id = data15hd[ 'SatelliteStatus']['SatelliteDefinition']['SatelliteId'] self.mda['platform_name'] = "Meteosat-" + SATNUM[self.platform_id] equator_radius = data15hd['GeometricProcessing'][ 'EarthModel']['EquatorialRadius'] * 1000. north_polar_radius = data15hd[ 'GeometricProcessing']['EarthModel']['NorthPolarRadius'] * 1000. south_polar_radius = data15hd[ 'GeometricProcessing']['EarthModel']['SouthPolarRadius'] * 1000. polar_radius = (north_polar_radius + south_polar_radius) * 0.5 ssp_lon = data15hd['ImageDescription'][ 'ProjectionDescription']['LongitudeOfSSP'] self.mda['projection_parameters'] = {'a': equator_radius, 'b': polar_radius, 'h': 35785831.00, 'ssp_longitude': ssp_lon} north = int(sec15hd['NorthLineSelectedRectangle']['Value']) east = int(sec15hd['EastColumnSelectedRectangle']['Value']) south = int(sec15hd['SouthLineSelectedRectangle']['Value']) west = int(sec15hd['WestColumnSelectedRectangle']['Value']) ncolumns = west - east + 1 nrows = north - south + 1 # check if the file has less rows or columns than # the maximum, if so it is an area of interest file if (nrows < VISIR_NUM_LINES) or (ncolumns < VISIR_NUM_COLUMNS): self.mda['is_full_disk'] = False # If the number of columns in the file is not divisible by 4, # UMARF will add extra columns to the file modulo = ncolumns % 4 padding = 0 if modulo > 0: padding = 4 - modulo cols_visir = ncolumns + padding # Check the VISIR calculated column dimension against # the header information cols_visir_hdr = int(sec15hd['NumberColumnsVISIR']['Value']) if cols_visir_hdr != cols_visir: logger.warning( "Number of VISIR columns from the header is incorrect!") logger.warning("Header: %d", cols_visir_hdr) logger.warning("Calculated: = %d", cols_visir) # HRV Channel - check if the area is reduced in east west # direction as this affects the number of columns in the file cols_hrv_hdr = int(sec15hd['NumberColumnsHRV']['Value']) if ncolumns < VISIR_NUM_COLUMNS: cols_hrv = cols_hrv_hdr else: cols_hrv = int(cols_hrv_hdr / 2) # self.mda represents the 16bit dimensions not 10bit self.mda['number_of_lines'] = int(sec15hd['NumberLinesVISIR']['Value']) self.mda['number_of_columns'] = cols_visir self.mda['hrv_number_of_lines'] = int(sec15hd["NumberLinesHRV"]['Value']) self.mda['hrv_number_of_columns'] = cols_hrv
def get_dataset(self, dataset_id, dataset_info): """Get the dataset.""" channel = dataset_id['name'] i = list(CHANNEL_NAMES.values()).index(channel) if (channel == 'HRV'): self.nc = self.nc.rename({ 'num_columns_hrv': 'x', 'num_rows_hrv': 'y' }) else: # the first channel of a composite will rename the dimension variable # but the later channels will raise a value error as its already been renamed # we can just ignore these exceptions try: self.nc = self.nc.rename({ 'num_columns_vis_ir': 'x', 'num_rows_vis_ir': 'y' }) except ValueError: pass dataset = self.nc[dataset_info['nc_key']] dataset.attrs.update(dataset_info) # Calibrate the data as needed # MPEF MSG calibration coeffiencts (gain and count) offset = dataset.attrs['add_offset'].astype('float32') gain = dataset.attrs['scale_factor'].astype('float32') self.platform_id = int(self.nc.attrs['satellite_id']) cal_type = self.nc['planned_chan_processing'].values[i] # Correct for the scan line order dataset = dataset.sel(y=slice(None, None, -1)) if dataset_id['calibration'] == 'counts': dataset.attrs['_FillValue'] = 0 if dataset_id['calibration'] in [ 'radiance', 'reflectance', 'brightness_temperature' ]: dataset = dataset.where(dataset != 0).astype('float32') dataset = self._convert_to_radiance(dataset, gain, offset) if dataset_id['calibration'] == 'reflectance': solar_irradiance = CALIB[int(self.platform_id)][channel]["F"] dataset = self._vis_calibrate(dataset, solar_irradiance) elif dataset_id['calibration'] == 'brightness_temperature': dataset = self._ir_calibrate(dataset, channel, cal_type) dataset.attrs.update(self.nc[dataset_info['nc_key']].attrs) dataset.attrs.update(dataset_info) dataset.attrs['platform_name'] = "Meteosat-" + SATNUM[self.platform_id] dataset.attrs['sensor'] = 'seviri' dataset.attrs['orbital_parameters'] = { 'projection_longitude': self.mda['projection_parameters']['ssp_longitude'], 'projection_latitude': 0., 'projection_altitude': self.mda['projection_parameters']['h'] } # remove attributes from original file which don't apply anymore strip_attrs = [ "comment", "long_name", "nc_key", "scale_factor", "add_offset", "valid_min", "valid_max" ] for a in strip_attrs: dataset.attrs.pop(a) return dataset