def cl_azimuth_zenith(self,
msg_channel,
gmst_degree,
asc_radian,
dec_radian,
sat_sub_lon=0.0):
self.simpleProfiler.start("cl_azimuth_zenith")
print("cl_azimuth_zenith")
scale_x = msg_channel.geotransform[1]
scale_y = msg_channel.geotransform[5]
offset_x = msg_channel.geotransform[0]
offset_y = msg_channel.geotransform[3]
band = msg_channel.data
azimuth = np.zeros_like(band, dtype=np.float32)
zenith = np.zeros_like(band, dtype=np.float32)
kernel_shape = band.shape[::
-1] # this is needed as numpy uses z:y:x instead of x:y:z!!!!
mf = cl.mem_flags
azimuth_buffer = cl.Buffer(self.cl_ctx,
mf.WRITE_ONLY | mf.USE_HOST_PTR,
hostbuf=azimuth)
zenith_buffer = cl.Buffer(self.cl_ctx,
mf.WRITE_ONLY | mf.USE_HOST_PTR,
hostbuf=zenith)
self.cl_program.azimuthZenithKernel(self.cl_queue, kernel_shape, None,
azimuth_buffer, zenith_buffer,
np.float64(scale_x),
np.float64(scale_y),
np.float64(offset_x),
np.float64(offset_y),
np.float64(_to_view_angle_fac),
np.float64(gmst_degree),
np.float64(asc_radian),
np.float64(dec_radian),
np.float64(sat_sub_lon))
cl.enqueue_copy(self.cl_queue, azimuth, azimuth_buffer)
cl.enqueue_copy(self.cl_queue, zenith, zenith_buffer)
self.simpleProfiler.stop("cl_azimuth_zenith")
return MsgChannel(data=azimuth,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name='azimuth'), MsgChannel(
data=zenith,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name='zenith')
def channel_raw_to_temperature(msg_channel, name_suffix='_RAD_TEMP'):
calibration_slope = msg_channel.metadata['calibration_slope']
calibration_offset = msg_channel.metadata['calibration_offset']
if calibration_slope is None or calibration_offset is None:
return None
co = calibration_offset
cs = calibration_slope
calibrated_data = raw_to_radiance(cs, co, msg_channel.data)
if msg_channel.metadata is None:
return None
channel_number = msg_channel.metadata['channel_number']
wavenumber = msg_channel.satellite.vc[channel_number - 1]
alpha = msg_channel.satellite.alpha[channel_number - 1]
beta = msg_channel.satellite.beta[channel_number - 1]
temperature_data = radiance_to_temperature(wavenumber, alpha, beta,
calibrated_data)
return MsgChannel(data=temperature_data,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name=msg_channel.name + name_suffix)
def cl_channel_raw_to_radiance(self,
msg_channel,
convert=False,
no_data_value_out=_FLOAT_NO_DATA_VALUE,
name_suffix='_RAD'):
calibration_slope = np.float32(
msg_channel.metadata['calibration_slope'])
calibration_offset = np.float32(
msg_channel.metadata['calibration_offset'])
conversion_factor = 1.0
no_data_value_in = _INT_NO_DATA_VALUE
if msg_channel.no_data_value is not None:
no_data_value_in = msg_channel.no_data_value
if calibration_slope is None or calibration_offset is None:
return None
if convert:
channel_number = msg_channel.metadata['channel_number']
cwl = msg_channel.satellite.cwl[channel_number - 1]
conversion_factor = 10.0 / (cwl * cwl)
conversion_factor = np.float32(conversion_factor)
band = msg_channel.data
calibrated_data = np.zeros_like(band, dtype=np.float32)
kernel_shape = band.shape[::
-1] # this is needed as numpy uses z:y:x instead of x:y:z!!!!
mf = cl.mem_flags
band_buffer = cl.Buffer(self.cl_ctx,
mf.READ_ONLY | mf.USE_HOST_PTR,
hostbuf=band)
cl.enqueue_copy(self.cl_queue, band_buffer, band) # TODO: Needed?
calibrated_data_buffer = cl.Buffer(self.cl_ctx,
mf.WRITE_ONLY | mf.USE_HOST_PTR,
hostbuf=calibrated_data)
self.cl_program.rawToRadianceKernel(
self.cl_queue,
kernel_shape,
None,
band_buffer,
calibrated_data_buffer,
calibration_offset,
calibration_slope,
conversion_factor,
np.int16(no_data_value_in),
np.float32(no_data_value_out),
)
cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer)
return MsgChannel(data=calibrated_data,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name=msg_channel.name + name_suffix,
no_data_value=no_data_value_out)
def channel_raw_to_temperature_optimized(msg_channel, name_suffix='_RAD_TEMP'):
lookup = channel_raw_to_temperature_lut(msg_channel)
lookup_fn = np.vectorize(lambda r: lookup[r])
temperature_data = lookup_fn(msg_channel.data)
return MsgChannel(data=temperature_data,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name=msg_channel.name + name_suffix)
def co2_correction(self,
bt039_channel,
bt108_channel,
bt134_channel,
no_data_value_out=_FLOAT_NO_DATA_VALUE,
name_suffix='_CO2CORR'):
no_data_value_in = _FLOAT_NO_DATA_VALUE
if bt039_channel.no_data_value is not None:
no_data_value_in = bt039_channel.no_data_value
calibrated_data = np.zeros_like(bt039_channel.data, dtype=np.float32)
kernel_shape = bt039_channel.data.shape[::
-1] # this is needed as numpy uses z:y:x instead of x:y:z!!!!
mf = cl.mem_flags
bt039_channel_buffer = cl.Buffer(self.cl_ctx,
mf.READ_ONLY | mf.USE_HOST_PTR,
hostbuf=bt039_channel.data)
bt108_channel_buffer = cl.Buffer(self.cl_ctx,
mf.READ_ONLY | mf.USE_HOST_PTR,
hostbuf=bt108_channel.data)
bt134_channel_buffer = cl.Buffer(self.cl_ctx,
mf.READ_ONLY | mf.USE_HOST_PTR,
hostbuf=bt134_channel.data)
calibrated_data_buffer = cl.Buffer(self.cl_ctx,
mf.WRITE_ONLY | mf.USE_HOST_PTR,
hostbuf=calibrated_data)
cl.enqueue_copy(self.cl_queue, bt039_channel_buffer,
bt039_channel.data) # TODO: Needed?
cl.enqueue_copy(self.cl_queue, bt108_channel_buffer,
bt108_channel.data) # TODO: Needed?
cl.enqueue_copy(self.cl_queue, bt134_channel_buffer,
bt134_channel.data) # TODO: Needed?
self.cl_program.co2CorrectionKernel(self.cl_queue, kernel_shape, None,
bt039_channel_buffer,
bt108_channel_buffer,
bt134_channel_buffer,
calibrated_data_buffer,
np.float32(no_data_value_in),
np.float32(no_data_value_out))
cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer)
return MsgChannel(data=calibrated_data,
metadata=bt039_channel.metadata.copy(),
geotransform=bt039_channel.geotransform,
name=bt039_channel.name + name_suffix,
no_data_value=no_data_value_out)
def channel_raw_to_radiance(msg_channel, name_suffix='_RAD'):
calibration_slope = msg_channel.metadata['calibration_slope']
calibration_offset = msg_channel.metadata['calibration_offset']
if calibration_slope is None or calibration_offset is None:
return None
co = calibration_offset
cs = calibration_slope
calibrated_data = raw_to_radiance(cs, co, msg_channel.data)
return MsgChannel(data=calibrated_data,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name=msg_channel.name + name_suffix)
def channel_radiance_to_temperature(msg_channel, name_suffix='_TEMP'):
if msg_channel.metadata is None:
return None
channel_number = msg_channel.metadata['channel_number']
wavenumber = msg_channel.satellite.vc[channel_number - 1]
alpha = msg_channel.satellite.alpha[channel_number - 1]
beta = msg_channel.satellite.beta[channel_number - 1]
temperature_data = radiance_to_temperature(wavenumber, alpha, beta,
msg_channel.data)
return MsgChannel(data=temperature_data,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name=msg_channel.name + name_suffix)
def cl_channel_raw_to_bbt(self,
msg_channel,
no_data_value_out=_FLOAT_NO_DATA_VALUE,
name_suffix='_RAD_TEMP'):
lookup = channel_raw_to_temperature_lut(msg_channel)
no_data_value_in = _INT_NO_DATA_VALUE
if msg_channel.no_data_value is not None:
no_data_value_in = msg_channel.no_data_value
band = msg_channel.data
calibrated_data = np.zeros_like(band, dtype=np.float32)
kernel_shape = band.shape[::
-1] # this is needed as numpy uses z:y:x instead of x:y:z!!!!
mf = cl.mem_flags
band_buffer = cl.Buffer(self.cl_ctx,
mf.READ_ONLY | mf.USE_HOST_PTR,
hostbuf=band)
lookup_buffer = cl.Buffer(self.cl_ctx,
mf.READ_ONLY | mf.USE_HOST_PTR,
hostbuf=lookup)
calibrated_data_buffer = cl.Buffer(self.cl_ctx,
mf.WRITE_ONLY | mf.USE_HOST_PTR,
hostbuf=calibrated_data)
cl.enqueue_copy(self.cl_queue, band_buffer, band) # TODO: Needed?
cl.enqueue_copy(self.cl_queue, lookup_buffer, lookup) # TODO: Needed?
self.cl_program.rawToBbtKernel(self.cl_queue, kernel_shape, None,
band_buffer,
calibrated_data_buffer, lookup_buffer,
np.int16(no_data_value_in),
np.float32(no_data_value_out))
cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer)
return MsgChannel(data=calibrated_data,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name=msg_channel.name + name_suffix,
no_data_value=no_data_value_out)
def cl_channel_raw_to_reflectance(self,
msg_channel,
gmst_degree,
asc_radian,
dec_radian,
esd,
solar_correction=True,
no_data_value_out=_FLOAT_NO_DATA_VALUE,
sat_sub_lon=0.0,
name_suffix='_RAD_REFL'):
calibration_slope = msg_channel.metadata['calibration_slope']
calibration_offset = msg_channel.metadata['calibration_offset']
channel_number = msg_channel.metadata['channel_number']
if calibration_slope is None or calibration_offset is None or channel_number is None:
return None
no_data_value_in = _INT_NO_DATA_VALUE
if msg_channel.no_data_value is not None:
no_data_value_in = msg_channel.no_data_value
scale_x = msg_channel.geotransform[1]
scale_y = msg_channel.geotransform[5]
offset_x = msg_channel.geotransform[0]
offset_y = msg_channel.geotransform[3]
etsr = msg_channel.satellite.etsr[channel_number - 1] / np.pi
band = msg_channel.data
calibrated_data = np.zeros_like(band, dtype=np.float32)
kernel_shape = band.shape[::
-1] # this is needed as numpy uses z:y:x instead of x:y:z!!!!
mf = cl.mem_flags
band_buffer = cl.Buffer(self.cl_ctx,
mf.READ_ONLY | mf.USE_HOST_PTR,
hostbuf=band)
calibrated_data_buffer = cl.Buffer(self.cl_ctx,
mf.WRITE_ONLY | mf.USE_HOST_PTR,
hostbuf=calibrated_data)
cl.enqueue_copy(self.cl_queue, band_buffer, band) # TODO: Needed?
if solar_correction:
# __kernel void reflectanceWithSolarCorrectionKernel(__global const int *in_data, __global float *out_data, const float offset, const float slope, const double dETSRconst, const double dESD, const double scale_x, const double scale_y, const double origin_x, const double origin_y, const double projectionCooridnateToViewAngleFactor, const double dGreenwichMeanSiderealTime, const double dRightAscension, const double dDeclination) {
self.cl_program.rawToReflectanceWithSolarCorrectionKernel(
self.cl_queue,
kernel_shape,
None,
band_buffer,
calibrated_data_buffer,
np.float32(calibration_offset),
np.float32(calibration_slope),
np.float64(etsr),
np.float64(esd),
np.float64(scale_x),
np.float64(scale_y),
np.float64(offset_x),
np.float64(offset_y),
np.float64(_to_view_angle_fac),
np.float64(gmst_degree),
np.float64(asc_radian),
np.float64(dec_radian),
np.int16(no_data_value_in),
np.float32(no_data_value_out),
np.float64(sat_sub_lon),
)
else:
# __kernel void rawToReflectanceWithoutSolarCorrectionKernel(__global const int *in_data, __global float *out_data, const float offset, const float slope, const double dETSRconst, const double dESD) {
self.cl_program.rawToReflectanceWithoutSolarCorrectionKernel(
self.cl_queue,
kernel_shape,
None,
band_buffer,
calibrated_data_buffer,
np.float32(calibration_offset),
np.float32(calibration_slope),
np.float64(etsr),
np.float64(esd.value),
np.int16(no_data_value_in),
np.float32(no_data_value_out),
)
cl.enqueue_copy(self.cl_queue, calibrated_data, calibrated_data_buffer)
return MsgChannel(data=calibrated_data,
metadata=msg_channel.metadata.copy(),
geotransform=msg_channel.geotransform,
name=msg_channel.name + name_suffix,
no_data_value=no_data_value_out)
def load_scene(self,
date_time,
channel_names=CHANNEL_NAMES,
pixel_area=(0, 0, 3712, 3712),
geos_area=None,
overwrite_dataset_geos_area=False):
if (self.simpleProfiler is not None):
self.simpleProfiler.start("GdalMsgLoader_load_scene")
channel_name_list = []
x_min, y_min, x_max, y_max = (None, None, None, None)
if pixel_area is not None:
x_min, y_min, x_max, y_max = pixel_area
if geos_area is None and pixel_area is not None:
geos_area = geos_area_from_pixel_area(pixel_area)
if geos_area is not None and pixel_area is None:
x_min, y_min, x_max, y_max = pixel_area_from_geos_area(geos_area)
pixel_area = (np.int(x_min), np.int(y_min), np.int(x_max),
np.int(y_max))
if x_min is None:
raise AreaError(pixel_area, geos_area)
x_off = np.int(np.floor(x_min))
y_off = np.int(np.floor(y_min))
x_size = np.int(np.absolute(np.floor(x_max) - np.floor(x_min)))
y_size = np.int(np.absolute(np.floor(y_max) - np.floor(y_min)))
#print(x_off, y_off, x_size, y_size)
wkt = None
satellite = None
cropped_geotransform = None
path_with_prefix = None
for pre in self.prefixes:
new_path_with_prefix = self.base_path + "/" + datetime.strftime(
date_time, pre)
if os.path.isdir(new_path_with_prefix):
path_with_prefix = new_path_with_prefix
break
if path_with_prefix is None:
return None
for channel_number, channel_name in enumerate(channel_names):
filename = path_with_prefix + "/" + get_xrit_filename(
date_time, channel_name)
try:
dataset = gdal.Open(filename, gdalconst.GA_ReadOnly)
# get dataset information from the first channel only
if channel_number == 0:
# print('Driver: ', dataset.GetDriver().ShortName, '/', dataset.GetDriver().LongName)
# print('Size is ', dataset.RasterXSize, 'x', dataset.RasterYSize, 'x', dataset.RasterCount)
# print('Projection is ', dataset.GetProjection())
# print('GeoTransform', dataset.GetGeoTransform())
satellite_number = dataset.GetMetadata(
"msg")['satellite_number']
if satellite_number is not None:
satellite = MSG_SATELLITES[int(satellite_number)]
wkt = dataset.GetProjectionRef()
geotransform = dataset.GetGeoTransform()
cropped_geotransform = geotransform
if geotransform is not None:
top_left_x, we_pixel_resolution, _, top_left_y, _, ns_pixel_resolution = geotransform
cropped_geotransform = (top_left_x +
(x_off * we_pixel_resolution),
we_pixel_resolution, 0.0,
top_left_y +
(y_off * ns_pixel_resolution),
0.0, ns_pixel_resolution)
if overwrite_dataset_geos_area is True:
print("Overwriting data geotransform",
cropped_geotransform)
cropped_geotransform = (geos_area[0],
we_pixel_resolution, 0.0,
geos_area[1], 0.0,
ns_pixel_resolution)
print("Overwriting with geotransform",
cropped_geotransform)
# get the raster band. It's always the first one... (MSG/HRIT)
band = dataset.GetRasterBand(1)
#print("rasterband")
raw_metadata = band.GetMetadata("msg")
metadata = {
'calibration_offset':
float(raw_metadata['calibration_offset']),
'calibration_slope':
float(raw_metadata['calibration_slope']),
'channel_number': int(raw_metadata['channel_number']),
'date_time': date_time
}
#print("metadata")
data = band.ReadAsArray(
xoff=x_off,
yoff=y_off, win_xsize=x_size, win_ysize=y_size).astype(
np.int16) # TODO read only the needed area!
#print("data")
channel_name_list.append((channel_name,
MsgChannel(channel_name,
data,
cropped_geotransform,
metadata,
satellite,
no_data_value=0)))
del band
del dataset
except RuntimeError as e:
print(e)
#raise GdalError(e)
#print(channel_name_list)
if len(channel_name_list) <= 0:
return None
if (self.simpleProfiler is not None):
self.simpleProfiler.stop("GdalMsgLoader_load_scene")
return MsgScene(channel_name_list, date_time, wkt,
cropped_geotransform, geos_area, pixel_area)
def load_scene(self,
date_time,
channel_names=CHANNEL_NAMES,
pixel_area=(0, 0, 3712, 3712),
geos_area=None):
channel_name_list = []
x_min, y_min, x_max, y_max = (None, None, None, None)
if geos_area is None and pixel_area is not None:
x_min, y_min, x_max, y_max = pixel_area
geos_area = geos_area_from_pixel_area(pixel_area)
if geos_area is not None:
x_min, y_min, x_max, y_max = pixel_area_from_geos_area(geos_area)
pixel_area = (x_min, y_min, x_max, y_max)
if x_min is None:
raise AreaError(pixel_area, geos_area)
x_off = np.floor(x_min)
y_off = np.floor(y_min)
x_size = np.floor(x_max) - np.floor(x_min)
y_size = np.floor(y_max) - np.floor(y_min)
#print(x_off, y_off, x_size, y_size)
path_with_prefix = None
for pre in self.prefixes:
new_path_with_prefix = self.base_path + "/" + datetime.strftime(
date_time, pre)
if os.path.isdir(new_path_with_prefix):
path_with_prefix = new_path_with_prefix
break
if path_with_prefix is None:
return None
filename, metadata_filename = self.find_filename(date_time)
if filename is None:
return None
h5file = h5py.File(path_with_prefix + '/' + filename, "r")
metadata_image_description = dict(h5file[
"/U-MARF/MSG/Level1.5/METADATA/HEADER/ImageDescription/ImageDescription_DESCR"]
[:])
metadata_calibration_slope_offset = h5file[
'/U-MARF/MSG/Level1.5/METADATA/HEADER/RadiometricProcessing/Level15ImageCalibration_ARRAY'][:]
# print(metadata_calibration_slope_offset)
we_pixel_resolution = np.float(
metadata_image_description['ReferenceGridVIS_IR-LineDirGridStep']
) * 1000.0 # TODO: negative?
ns_pixel_resolution = np.float(
metadata_image_description['ReferenceGridVIS_IR-ColumnDirGridStep']
) * -1000.0
sub_satellite_point_lon = np.float(
metadata_image_description['ProjectionDescription-LongitudeOfSSP'])
satellite_number = np.int(metadata_filename['msg_id'])
# print('we_pixel_resolution, ns_pixel_resolution, sub_sat_lon, satellite_number', we_pixel_resolution, ns_pixel_resolution, sub_sat_lon, satellite_number)
satellite = MSG_SATELLITES[int(satellite_number)]
wkt = get_geos_wkt(str(sub_satellite_point_lon))
top_left_x, top_left_y, _, _ = geos_area
cropped_geotransform = (top_left_x, we_pixel_resolution, 0.0,
top_left_y, 0.0, ns_pixel_resolution)
print('cropped_geotransform', cropped_geotransform)
for channel_name in channel_names:
channel_number = get_channel_number_for_channel_name(channel_name)
h5_inner_path = '/U-MARF/MSG/Level1.5/DATA/Channel ' + '{i:02d}'.format(
i=channel_number) + '/IMAGE_DATA'
slope, offset = metadata_calibration_slope_offset[channel_number]
try:
metadata = {
'calibration_offset':
offset,
'calibration_slope':
slope,
'channel_number':
get_channel_number_for_channel_name(channel_name),
'date_time':
date_time
}
# NOW WE HAVE TO FLIP THE Y-AXIS (MSG DATA IS FLIPPED WHEN ORIGIN = 2)
flipped_y_min = 3712 - y_max
flipped_y_max = 3712 - y_min
data = h5file[h5_inner_path][np.int(y_min):np.int(y_max),
np.int(x_min):np.int(x_max)]
data = np.asarray(data, dtype=np.int16)
channel_name_list.append((channel_name,
MsgChannel(channel_name,
data,
cropped_geotransform,
metadata,
satellite,
no_data_value=0.0)))
except KeyError as e:
print(e)
h5file = None
if len(channel_name_list) <= 0:
return None
return MsgScene(channel_name_list,
geos_area,
pixel_area,
date_time,
wkt,
cropped_geotransform,
sub_satellite_point_lon=sub_satellite_point_lon)