def cl_azimuth_zenith(self, msg_channel, gmst_degree, asc_radian, dec_radian, sat_sub_lon=0.0): self.simpleProfiler.start("cl_azimuth_zenith") print("cl_azimuth_zenith") scale_x = msg_channel.geotransform[1] scale_y = msg_channel.geotransform[5] offset_x = msg_channel.geotransform[0] offset_y = msg_channel.geotransform[3] band = msg_channel.data azimuth = np.zeros_like(band, dtype=np.float32) zenith = np.zeros_like(band, dtype=np.float32) kernel_shape = band.shape[:: -1] # this is needed as numpy uses z:y:x instead of x:y:z!!!! mf = cl.mem_flags azimuth_buffer = cl.Buffer(self.cl_ctx, mf.WRITE_ONLY | mf.USE_HOST_PTR, hostbuf=azimuth) zenith_buffer = cl.Buffer(self.cl_ctx, mf.WRITE_ONLY | mf.USE_HOST_PTR, hostbuf=zenith) self.cl_program.azimuthZenithKernel(self.cl_queue, kernel_shape, None, azimuth_buffer, zenith_buffer, np.float64(scale_x), np.float64(scale_y), np.float64(offset_x), np.float64(offset_y), np.float64(_to_view_angle_fac), np.float64(gmst_degree), np.float64(asc_radian), np.float64(dec_radian), np.float64(sat_sub_lon)) cl.enqueue_copy(self.cl_queue, azimuth, azimuth_buffer) cl.enqueue_copy(self.cl_queue, zenith, zenith_buffer) self.simpleProfiler.stop("cl_azimuth_zenith") return MsgChannel(data=azimuth, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name='azimuth'), MsgChannel( data=zenith, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name='zenith')
def channel_raw_to_temperature(msg_channel, name_suffix='_RAD_TEMP'): calibration_slope = msg_channel.metadata['calibration_slope'] calibration_offset = msg_channel.metadata['calibration_offset'] if calibration_slope is None or calibration_offset is None: return None co = calibration_offset cs = calibration_slope calibrated_data = raw_to_radiance(cs, co, msg_channel.data) if msg_channel.metadata is None: return None channel_number = msg_channel.metadata['channel_number'] wavenumber = msg_channel.satellite.vc[channel_number - 1] alpha = msg_channel.satellite.alpha[channel_number - 1] beta = msg_channel.satellite.beta[channel_number - 1] temperature_data = radiance_to_temperature(wavenumber, alpha, beta, calibrated_data) return MsgChannel(data=temperature_data, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name=msg_channel.name + name_suffix)
def cl_channel_raw_to_radiance(self, msg_channel, convert=False, no_data_value_out=_FLOAT_NO_DATA_VALUE, name_suffix='_RAD'): calibration_slope = np.float32( msg_channel.metadata['calibration_slope']) calibration_offset = np.float32( msg_channel.metadata['calibration_offset']) conversion_factor = 1.0 no_data_value_in = _INT_NO_DATA_VALUE if msg_channel.no_data_value is not None: no_data_value_in = msg_channel.no_data_value if calibration_slope is None or calibration_offset is None: return None if convert: channel_number = msg_channel.metadata['channel_number'] cwl = msg_channel.satellite.cwl[channel_number - 1] conversion_factor = 10.0 / (cwl * cwl) conversion_factor = np.float32(conversion_factor) band = msg_channel.data calibrated_data = np.zeros_like(band, dtype=np.float32) kernel_shape = band.shape[:: -1] # this is needed as numpy uses z:y:x instead of x:y:z!!!! mf = cl.mem_flags band_buffer = cl.Buffer(self.cl_ctx, mf.READ_ONLY | mf.USE_HOST_PTR, hostbuf=band) cl.enqueue_copy(self.cl_queue, band_buffer, band) # TODO: Needed? calibrated_data_buffer = cl.Buffer(self.cl_ctx, mf.WRITE_ONLY | mf.USE_HOST_PTR, hostbuf=calibrated_data) self.cl_program.rawToRadianceKernel( self.cl_queue, kernel_shape, None, band_buffer, calibrated_data_buffer, calibration_offset, calibration_slope, conversion_factor, np.int16(no_data_value_in), np.float32(no_data_value_out), ) cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer) return MsgChannel(data=calibrated_data, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name=msg_channel.name + name_suffix, no_data_value=no_data_value_out)
def channel_raw_to_temperature_optimized(msg_channel, name_suffix='_RAD_TEMP'): lookup = channel_raw_to_temperature_lut(msg_channel) lookup_fn = np.vectorize(lambda r: lookup[r]) temperature_data = lookup_fn(msg_channel.data) return MsgChannel(data=temperature_data, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name=msg_channel.name + name_suffix)
def co2_correction(self, bt039_channel, bt108_channel, bt134_channel, no_data_value_out=_FLOAT_NO_DATA_VALUE, name_suffix='_CO2CORR'): no_data_value_in = _FLOAT_NO_DATA_VALUE if bt039_channel.no_data_value is not None: no_data_value_in = bt039_channel.no_data_value calibrated_data = np.zeros_like(bt039_channel.data, dtype=np.float32) kernel_shape = bt039_channel.data.shape[:: -1] # this is needed as numpy uses z:y:x instead of x:y:z!!!! mf = cl.mem_flags bt039_channel_buffer = cl.Buffer(self.cl_ctx, mf.READ_ONLY | mf.USE_HOST_PTR, hostbuf=bt039_channel.data) bt108_channel_buffer = cl.Buffer(self.cl_ctx, mf.READ_ONLY | mf.USE_HOST_PTR, hostbuf=bt108_channel.data) bt134_channel_buffer = cl.Buffer(self.cl_ctx, mf.READ_ONLY | mf.USE_HOST_PTR, hostbuf=bt134_channel.data) calibrated_data_buffer = cl.Buffer(self.cl_ctx, mf.WRITE_ONLY | mf.USE_HOST_PTR, hostbuf=calibrated_data) cl.enqueue_copy(self.cl_queue, bt039_channel_buffer, bt039_channel.data) # TODO: Needed? cl.enqueue_copy(self.cl_queue, bt108_channel_buffer, bt108_channel.data) # TODO: Needed? cl.enqueue_copy(self.cl_queue, bt134_channel_buffer, bt134_channel.data) # TODO: Needed? self.cl_program.co2CorrectionKernel(self.cl_queue, kernel_shape, None, bt039_channel_buffer, bt108_channel_buffer, bt134_channel_buffer, calibrated_data_buffer, np.float32(no_data_value_in), np.float32(no_data_value_out)) cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer) return MsgChannel(data=calibrated_data, metadata=bt039_channel.metadata.copy(), geotransform=bt039_channel.geotransform, name=bt039_channel.name + name_suffix, no_data_value=no_data_value_out)
def channel_raw_to_radiance(msg_channel, name_suffix='_RAD'): calibration_slope = msg_channel.metadata['calibration_slope'] calibration_offset = msg_channel.metadata['calibration_offset'] if calibration_slope is None or calibration_offset is None: return None co = calibration_offset cs = calibration_slope calibrated_data = raw_to_radiance(cs, co, msg_channel.data) return MsgChannel(data=calibrated_data, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name=msg_channel.name + name_suffix)
def channel_radiance_to_temperature(msg_channel, name_suffix='_TEMP'): if msg_channel.metadata is None: return None channel_number = msg_channel.metadata['channel_number'] wavenumber = msg_channel.satellite.vc[channel_number - 1] alpha = msg_channel.satellite.alpha[channel_number - 1] beta = msg_channel.satellite.beta[channel_number - 1] temperature_data = radiance_to_temperature(wavenumber, alpha, beta, msg_channel.data) return MsgChannel(data=temperature_data, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name=msg_channel.name + name_suffix)
def cl_channel_raw_to_bbt(self, msg_channel, no_data_value_out=_FLOAT_NO_DATA_VALUE, name_suffix='_RAD_TEMP'): lookup = channel_raw_to_temperature_lut(msg_channel) no_data_value_in = _INT_NO_DATA_VALUE if msg_channel.no_data_value is not None: no_data_value_in = msg_channel.no_data_value band = msg_channel.data calibrated_data = np.zeros_like(band, dtype=np.float32) kernel_shape = band.shape[:: -1] # this is needed as numpy uses z:y:x instead of x:y:z!!!! mf = cl.mem_flags band_buffer = cl.Buffer(self.cl_ctx, mf.READ_ONLY | mf.USE_HOST_PTR, hostbuf=band) lookup_buffer = cl.Buffer(self.cl_ctx, mf.READ_ONLY | mf.USE_HOST_PTR, hostbuf=lookup) calibrated_data_buffer = cl.Buffer(self.cl_ctx, mf.WRITE_ONLY | mf.USE_HOST_PTR, hostbuf=calibrated_data) cl.enqueue_copy(self.cl_queue, band_buffer, band) # TODO: Needed? cl.enqueue_copy(self.cl_queue, lookup_buffer, lookup) # TODO: Needed? self.cl_program.rawToBbtKernel(self.cl_queue, kernel_shape, None, band_buffer, calibrated_data_buffer, lookup_buffer, np.int16(no_data_value_in), np.float32(no_data_value_out)) cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer) return MsgChannel(data=calibrated_data, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name=msg_channel.name + name_suffix, no_data_value=no_data_value_out)
def cl_channel_raw_to_reflectance(self, msg_channel, gmst_degree, asc_radian, dec_radian, esd, solar_correction=True, no_data_value_out=_FLOAT_NO_DATA_VALUE, sat_sub_lon=0.0, name_suffix='_RAD_REFL'): calibration_slope = msg_channel.metadata['calibration_slope'] calibration_offset = msg_channel.metadata['calibration_offset'] channel_number = msg_channel.metadata['channel_number'] if calibration_slope is None or calibration_offset is None or channel_number is None: return None no_data_value_in = _INT_NO_DATA_VALUE if msg_channel.no_data_value is not None: no_data_value_in = msg_channel.no_data_value scale_x = msg_channel.geotransform[1] scale_y = msg_channel.geotransform[5] offset_x = msg_channel.geotransform[0] offset_y = msg_channel.geotransform[3] etsr = msg_channel.satellite.etsr[channel_number - 1] / np.pi band = msg_channel.data calibrated_data = np.zeros_like(band, dtype=np.float32) kernel_shape = band.shape[:: -1] # this is needed as numpy uses z:y:x instead of x:y:z!!!! mf = cl.mem_flags band_buffer = cl.Buffer(self.cl_ctx, mf.READ_ONLY | mf.USE_HOST_PTR, hostbuf=band) calibrated_data_buffer = cl.Buffer(self.cl_ctx, mf.WRITE_ONLY | mf.USE_HOST_PTR, hostbuf=calibrated_data) cl.enqueue_copy(self.cl_queue, band_buffer, band) # TODO: Needed? if solar_correction: # __kernel void reflectanceWithSolarCorrectionKernel(__global const int *in_data, __global float *out_data, const float offset, const float slope, const double dETSRconst, const double dESD, const double scale_x, const double scale_y, const double origin_x, const double origin_y, const double projectionCooridnateToViewAngleFactor, const double dGreenwichMeanSiderealTime, const double dRightAscension, const double dDeclination) { self.cl_program.rawToReflectanceWithSolarCorrectionKernel( self.cl_queue, kernel_shape, None, band_buffer, calibrated_data_buffer, np.float32(calibration_offset), np.float32(calibration_slope), np.float64(etsr), np.float64(esd), np.float64(scale_x), np.float64(scale_y), np.float64(offset_x), np.float64(offset_y), np.float64(_to_view_angle_fac), np.float64(gmst_degree), np.float64(asc_radian), np.float64(dec_radian), np.int16(no_data_value_in), np.float32(no_data_value_out), np.float64(sat_sub_lon), ) else: # __kernel void rawToReflectanceWithoutSolarCorrectionKernel(__global const int *in_data, __global float *out_data, const float offset, const float slope, const double dETSRconst, const double dESD) { self.cl_program.rawToReflectanceWithoutSolarCorrectionKernel( self.cl_queue, kernel_shape, None, band_buffer, calibrated_data_buffer, np.float32(calibration_offset), np.float32(calibration_slope), np.float64(etsr), np.float64(esd.value), np.int16(no_data_value_in), np.float32(no_data_value_out), ) cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer) return MsgChannel(data=calibrated_data, metadata=msg_channel.metadata.copy(), geotransform=msg_channel.geotransform, name=msg_channel.name + name_suffix, no_data_value=no_data_value_out)
def load_scene(self, date_time, channel_names=CHANNEL_NAMES, pixel_area=(0, 0, 3712, 3712), geos_area=None, overwrite_dataset_geos_area=False): if (self.simpleProfiler is not None): self.simpleProfiler.start("GdalMsgLoader_load_scene") channel_name_list = [] x_min, y_min, x_max, y_max = (None, None, None, None) if pixel_area is not None: x_min, y_min, x_max, y_max = pixel_area if geos_area is None and pixel_area is not None: geos_area = geos_area_from_pixel_area(pixel_area) if geos_area is not None and pixel_area is None: x_min, y_min, x_max, y_max = pixel_area_from_geos_area(geos_area) pixel_area = (np.int(x_min), np.int(y_min), np.int(x_max), np.int(y_max)) if x_min is None: raise AreaError(pixel_area, geos_area) x_off = np.int(np.floor(x_min)) y_off = np.int(np.floor(y_min)) x_size = np.int(np.absolute(np.floor(x_max) - np.floor(x_min))) y_size = np.int(np.absolute(np.floor(y_max) - np.floor(y_min))) #print(x_off, y_off, x_size, y_size) wkt = None satellite = None cropped_geotransform = None path_with_prefix = None for pre in self.prefixes: new_path_with_prefix = self.base_path + "/" + datetime.strftime( date_time, pre) if os.path.isdir(new_path_with_prefix): path_with_prefix = new_path_with_prefix break if path_with_prefix is None: return None for channel_number, channel_name in enumerate(channel_names): filename = path_with_prefix + "/" + get_xrit_filename( date_time, channel_name) try: dataset = gdal.Open(filename, gdalconst.GA_ReadOnly) # get dataset information from the first channel only if channel_number == 0: # print('Driver: ', dataset.GetDriver().ShortName, '/', dataset.GetDriver().LongName) # print('Size is ', dataset.RasterXSize, 'x', dataset.RasterYSize, 'x', dataset.RasterCount) # print('Projection is ', dataset.GetProjection()) # print('GeoTransform', dataset.GetGeoTransform()) satellite_number = dataset.GetMetadata( "msg")['satellite_number'] if satellite_number is not None: satellite = MSG_SATELLITES[int(satellite_number)] wkt = dataset.GetProjectionRef() geotransform = dataset.GetGeoTransform() cropped_geotransform = geotransform if geotransform is not None: top_left_x, we_pixel_resolution, _, top_left_y, _, ns_pixel_resolution = geotransform cropped_geotransform = (top_left_x + (x_off * we_pixel_resolution), we_pixel_resolution, 0.0, top_left_y + (y_off * ns_pixel_resolution), 0.0, ns_pixel_resolution) if overwrite_dataset_geos_area is True: print("Overwriting data geotransform", cropped_geotransform) cropped_geotransform = (geos_area[0], we_pixel_resolution, 0.0, geos_area[1], 0.0, ns_pixel_resolution) print("Overwriting with geotransform", cropped_geotransform) # get the raster band. It's always the first one... (MSG/HRIT) band = dataset.GetRasterBand(1) #print("rasterband") raw_metadata = band.GetMetadata("msg") metadata = { 'calibration_offset': float(raw_metadata['calibration_offset']), 'calibration_slope': float(raw_metadata['calibration_slope']), 'channel_number': int(raw_metadata['channel_number']), 'date_time': date_time } #print("metadata") data = band.ReadAsArray( xoff=x_off, yoff=y_off, win_xsize=x_size, win_ysize=y_size).astype( np.int16) # TODO read only the needed area! #print("data") channel_name_list.append((channel_name, MsgChannel(channel_name, data, cropped_geotransform, metadata, satellite, no_data_value=0))) del band del dataset except RuntimeError as e: print(e) #raise GdalError(e) #print(channel_name_list) if len(channel_name_list) <= 0: return None if (self.simpleProfiler is not None): self.simpleProfiler.stop("GdalMsgLoader_load_scene") return MsgScene(channel_name_list, date_time, wkt, cropped_geotransform, geos_area, pixel_area)
def load_scene(self, date_time, channel_names=CHANNEL_NAMES, pixel_area=(0, 0, 3712, 3712), geos_area=None): channel_name_list = [] x_min, y_min, x_max, y_max = (None, None, None, None) if geos_area is None and pixel_area is not None: x_min, y_min, x_max, y_max = pixel_area geos_area = geos_area_from_pixel_area(pixel_area) if geos_area is not None: x_min, y_min, x_max, y_max = pixel_area_from_geos_area(geos_area) pixel_area = (x_min, y_min, x_max, y_max) if x_min is None: raise AreaError(pixel_area, geos_area) x_off = np.floor(x_min) y_off = np.floor(y_min) x_size = np.floor(x_max) - np.floor(x_min) y_size = np.floor(y_max) - np.floor(y_min) #print(x_off, y_off, x_size, y_size) path_with_prefix = None for pre in self.prefixes: new_path_with_prefix = self.base_path + "/" + datetime.strftime( date_time, pre) if os.path.isdir(new_path_with_prefix): path_with_prefix = new_path_with_prefix break if path_with_prefix is None: return None filename, metadata_filename = self.find_filename(date_time) if filename is None: return None h5file = h5py.File(path_with_prefix + '/' + filename, "r") metadata_image_description = dict(h5file[ "/U-MARF/MSG/Level1.5/METADATA/HEADER/ImageDescription/ImageDescription_DESCR"] [:]) metadata_calibration_slope_offset = h5file[ '/U-MARF/MSG/Level1.5/METADATA/HEADER/RadiometricProcessing/Level15ImageCalibration_ARRAY'][:] # print(metadata_calibration_slope_offset) we_pixel_resolution = np.float( metadata_image_description['ReferenceGridVIS_IR-LineDirGridStep'] ) * 1000.0 # TODO: negative? ns_pixel_resolution = np.float( metadata_image_description['ReferenceGridVIS_IR-ColumnDirGridStep'] ) * -1000.0 sub_satellite_point_lon = np.float( metadata_image_description['ProjectionDescription-LongitudeOfSSP']) satellite_number = np.int(metadata_filename['msg_id']) # print('we_pixel_resolution, ns_pixel_resolution, sub_sat_lon, satellite_number', we_pixel_resolution, ns_pixel_resolution, sub_sat_lon, satellite_number) satellite = MSG_SATELLITES[int(satellite_number)] wkt = get_geos_wkt(str(sub_satellite_point_lon)) top_left_x, top_left_y, _, _ = geos_area cropped_geotransform = (top_left_x, we_pixel_resolution, 0.0, top_left_y, 0.0, ns_pixel_resolution) print('cropped_geotransform', cropped_geotransform) for channel_name in channel_names: channel_number = get_channel_number_for_channel_name(channel_name) h5_inner_path = '/U-MARF/MSG/Level1.5/DATA/Channel ' + '{i:02d}'.format( i=channel_number) + '/IMAGE_DATA' slope, offset = metadata_calibration_slope_offset[channel_number] try: metadata = { 'calibration_offset': offset, 'calibration_slope': slope, 'channel_number': get_channel_number_for_channel_name(channel_name), 'date_time': date_time } # NOW WE HAVE TO FLIP THE Y-AXIS (MSG DATA IS FLIPPED WHEN ORIGIN = 2) flipped_y_min = 3712 - y_max flipped_y_max = 3712 - y_min data = h5file[h5_inner_path][np.int(y_min):np.int(y_max), np.int(x_min):np.int(x_max)] data = np.asarray(data, dtype=np.int16) channel_name_list.append((channel_name, MsgChannel(channel_name, data, cropped_geotransform, metadata, satellite, no_data_value=0.0))) except KeyError as e: print(e) h5file = None if len(channel_name_list) <= 0: return None return MsgScene(channel_name_list, geos_area, pixel_area, date_time, wkt, cropped_geotransform, sub_satellite_point_lon=sub_satellite_point_lon)