def make_quickview(qickview_figure_file, raw_image_file, glt_image_file, setting_file):
""" Make a RGB quickview image.
Arguments:
qickview_figure_file: str
Quickview figure filename.
raw_image_file: str
Raw Hyspex image filename.
glt_image_file: str
GLT image filename.
"""
from radiometric import get_cal_data, raw2rdn
# Read Hyspex raw image
raw_header = read_envi_header(raw_image_file[:-len('.hyspex')]+'.hdr')
raw_image = np.memmap(raw_image_file,
dtype='int16',
mode='r',
offset=raw_header['header offset'],
shape=(raw_header['lines'], raw_header['bands'], raw_header['samples']))
cal_data = get_cal_data(raw_image_file, setting_file)
# Read GLT image
glt_header = read_envi_header(glt_image_file+'.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
offset=0,
shape=(2, glt_header['lines'], glt_header['samples']))
# Write RGB image
driver = gdal.GetDriverByName('GTiff')
ds = driver.Create(qickview_figure_file,
glt_header['samples'], glt_header['lines'], 3,
gdal.GDT_Byte)
ds.SetGeoTransform((float(glt_header['map info'][3]),
float(glt_header['map info'][5]),
0, float(glt_header['map info'][4]),
0, -float(glt_header['map info'][6])))
ds.SetProjection(glt_header['coordinate system string'])
quickview_image = np.zeros((glt_header['lines'], glt_header['samples']), dtype='uint8')
I,J = np.where(glt_image[0,:,:]>0)
for output_band_index, rgb_band_index in enumerate(raw_header['default bands']):
quickview_image[:,:] = 0
rdn_image = raw2rdn(raw_image[:,rgb_band_index-1,:], cal_data, rgb_band_index-1)
rdn_image = linear_percent_stretch(rdn_image)
quickview_image[I,J] = rdn_image[glt_image[0,I,J], glt_image[1,I,J]]
ds.GetRasterBand(output_band_index+1).WriteArray(quickview_image)
del rdn_image
raw_image.flush()
glt_image.flush()
ds = None
del cal_data, I, J, glt_header, raw_header
logger.info('Save quickview figure to %s.' %qickview_figure_file)
def get_acquisition_time(header_file, imugps_file):
"""Get Hyspex image acquistion time.
Notes:
(1) The code is adapted from Brendan Heberlein's ([email protected]) script.
Arguments:
header_file: str
Hyspex header filename.
imugps_file: str
Hyspex imugps filename.
Returns:
when: datetime object
Image acquisition time.
"""
from datetime import datetime, timedelta
from envi import read_envi_header
header = read_envi_header(header_file)
week_start = datetime.strptime(f"{header['acquisition date']} 00:00:00",
"%Y-%m-%d %H:%M:%S")
week_seconds = np.loadtxt(imugps_file)[:, 7].mean()
epoch = datetime(1980, 1, 6, 0, 0)
gps_week = (week_start - epoch).days // 7
time_elapsed = timedelta(days=gps_week * 7, seconds=week_seconds)
when = epoch + time_elapsed
return when
def get_avg_scan_angles(sca_image_file):
""" Get average scan angles along each column.
Arguments:
sca_image_file: str
Sensor view angle image filename.
Returns:
avg_vza, avg_raa: 1D array
Averaged view zenith angle, averaged relative azimuth angle.
"""
# Read scan angles
sca_header = read_envi_header(sca_image_file + '.hdr')
sca_image = np.memmap(sca_image_file,
dtype='float32',
mode='r',
offset=0,
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# Get average sensor view zenith angle along each column
avg_vza = sca_image[0, :, :].mean(axis=0)
# Get average relative azimuth angle along each column
saa = float(sca_header['sun azimuth'])
raa = saa - sca_image[1, :, :]
raa[raa < 0] += 360.0
raa[raa > 180] = 360.0 - raa[raa > 180]
avg_raa = raa.mean(axis=0)
sca_image.flush()
del saa, raa
return avg_vza, avg_raa
def plot_image_area(image_area_figure_file, dem_image_file, igm_image_file, imugps_file):
""" Plot image area.
Arguments:
image_area_figure_file: str
Image area figure filename.
dem_image_file: str
DEM image filename.
igm_image_file: str
IGM image filename.
imugps_file: str
IMUGPS filename.
"""
# Read DEM.
ds = gdal.Open(dem_image_file, gdal.GA_ReadOnly)
dem_image = ds.GetRasterBand(1).ReadAsArray()
dem_geotransform = ds.GetGeoTransform()
ds = None
# Read IGM.
igm_header = read_envi_header(igm_image_file+'.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
offset=0,
shape=(2, igm_header['lines'], igm_header['samples']))
cols = (igm_image[0,:,:]-dem_geotransform[0])/dem_geotransform[1]
rows = (igm_image[1,:,:]-dem_geotransform[3])/dem_geotransform[5]
igm_image.flush()
# Read IMUGPS
imugps = np.loadtxt(imugps_file)
# Make a plot
plt.figure(figsize=(10, 10.0*dem_image.shape[0]/dem_image.shape[1]))
plt.imshow(dem_image, cmap='gray', vmin=dem_image.min(), vmax=dem_image.max())
del dem_image
plt.plot(cols[:,0], rows[:,0], '-', color='green', lw=4)
plt.plot(cols[:,-1], rows[:,-1], '-', color='green', lw=4)
plt.plot(cols[0,:], rows[0,:], '-', color='green', lw=4)
plt.plot(cols[-1,:], rows[-1,:], '-', color='green', lw=4)
cols = (imugps[:,1]-dem_geotransform[0])/dem_geotransform[1]
rows = (imugps[:,2]-dem_geotransform[3])/dem_geotransform[5]
plt.plot(cols, rows, '-', color='red', lw=5)
plt.plot(cols[0], rows[0], '.', color='red', ms=25)
plt.arrow(cols[-100], rows[-100],
cols[-1]-cols[-100], rows[-1]-rows[-100],
head_width=10, head_length=10,
fc='red', ec='red')
plt.xticks([])
plt.yticks([])
plt.savefig(image_area_figure_file, dpi=1000)
plt.close()
del cols, rows, imugps
logger.info('Save image area figure to %s.' %image_area_figure_file)
def plot_angle_geometry(angle_geometry_figure_file, sca_image_file):
""" Plot sun and view geometry in a polar coordinate system.
Arguments:
angle_geometry_figure_file: str
Angle geometry figure filename.
sca_image_file: str
Scan angle image filename.
"""
# Read sca image data.
sca_header = read_envi_header(sca_image_file+'.hdr')
sca_image = np.memmap(sca_image_file,
dtype='float32',
mode='r',
offset=0,
shape=(sca_header['bands'], sca_header['lines'], sca_header['samples']))
# Scatter-plot view geometry.
plt.figure(figsize=(10, 10))
ax = plt.subplot(111, projection='polar')
ax.scatter(np.deg2rad(sca_image[1,:,:].flatten()), sca_image[0,:,:].flatten(), color='green')
sca_image.flush()
# Scatter-plot sun geometry.
ax.scatter(np.deg2rad(float(sca_header['sun azimuth'])), float(sca_header['sun zenith']), color='red', marker='*', s=500)
del sca_header
# Flip figure left to right.
ax.set_theta_direction(-1)
# Rotate figure by 90 degrees.
ax.set_theta_zero_location('N')
ax.tick_params(labelsize=20)
plt.savefig(angle_geometry_figure_file)
plt.close()
del ax
logger.info('Save angle geometry figure to %s.' %angle_geometry_figure_file)
def make_atm_lut(config):
""" Make an atmophere look-up-table (LUT).
Notes:
(1) If the `atm_database` exits, then do interpolation to generate a LUT.
Otherwise, use the LibRadTran model to generate it.
Arguments:
config: dictionary
Configuration parameters.
"""
from envi import read_envi_header
from dem import get_avg_elev
# If the LUT has been created, do nothing.
if os.path.exists(config['atm_lut_file']):
logger.info('Write atmospheric LUT to %s.' % config['atm_lut_file'])
return
# Make a directory.
if not os.path.exists(config['atm_lut_dir']):
os.mkdir(config['atm_lut_dir'])
# Make the LUT
""" Notes:
(1) If `atm_database` is None, the LibRadTran model is used to the LUT.
Otherwise, the LUT is interpolated from the `atm_database`.
"""
""" Step 1: Read sun zenith `sza` in degrees,
sun azimuth `saa` in degrees,
elevation `elev` in km,
above-ground flight altitude `zout` in km.
"""
sca_header = read_envi_header(config['vnir']['sca_image_file'] + '.hdr')
sza = float(sca_header['sun zenith'])
saa = float(sca_header['sun azimuth'])
elev = get_avg_elev(config['vnir']['new_dem_image_file']) / 1000.0
imugps = np.loadtxt(config['vnir']['new_imugps_file'])
avg_flight_altitude = imugps[:, 3].mean()
zout = avg_flight_altitude / 1000.0 - elev
del imugps, sca_header, avg_flight_altitude
""" Step 2: Get the LUT grids of
view zenith angle `VZA` in degrees,
relative azimuth angle `RAA` in degrees.
"""
VZA = []
RAA = []
for sensor in ['vnir', 'swir']:
sca_header = read_envi_header(config[sensor]['sca_image_file'] +
'.hdr')
sca_image = np.memmap(config[sensor]['sca_image_file'],
dtype='float32',
mode='r',
offset=0,
shape=(2, sca_header['lines'],
sca_header['samples']))
raa = saa - sca_image[1, :, :]
raa[raa < 0] += 360.0
raa[raa > 180] = 360.0 - raa[raa > 180]
VZA += list(
np.arange(int(np.floor(sca_image[0, :, :].min() / vza_grid)),
int(np.ceil(sca_image[0, :, :].max() / vza_grid)) + .01,
1) * vza_grid)
RAA += list(
np.arange(int(np.floor(raa.min() / raa_grid)),
int(np.ceil(raa.max() / raa_grid)) + .01, 1) * raa_grid)
sca_image.flush()
del raa, sca_header
VZA = sorted(list(set(VZA)))
RAA = sorted(list(set(RAA)))
if config['atm_database'] is None:
""" Step 3: Use the LibRadTran model to make the LUT.
"""
# Initialize rtm configurations
rtm_config = dict()
rtm_config['altitude'] = elev
rtm_config['zout'] = zout
rtm_config['sun_zenith'] = sza
rtm_config['sun_azimuth'] = 0
rtm_config['resolution'] = config['rtm_params']['resolution']
rtm_config['source_file'] = '../data/solar_flux/kurudz_0.1nm.dat'
rtm_config['atmosphere_file'] = config['rtm_params']['atm_mode']
rtm_config['lambda_0'] = 400
rtm_config['lambda_1'] = 2500
rtm_config['o3'] = 331
# Make atmospheric look-up-tables
pool = multiprocessing.Pool(processes=min(
config['rtm_params']['cpu_count'], multiprocessing.cpu_count()))
rtm_config['sensor_zenith'] = VZA
rtm_config['sensor_azimuth'] = RAA
for rho in RHO:
rtm_config['albedo'] = rho
for wvc in WVC:
rtm_config['h2o'] = wvc
for vis in VIS:
rtm_config['aerosol_visibility'] = vis
inp_file = os.path.join(
config['atm_lut_dir'],
'rho_%03d_wvc_%03d_vis_%03d.inp' %
(rho * 100, wvc, vis))
out_file = inp_file[:-len('.inp')] + '.out'
make_inp_file(inp_file, rtm_config)
if os.path.exists(out_file) and os.path.getsize(out_file):
continue
pool.apply_async(
run_rtm,
(os.path.join(config['rtm_params']['install_dir'],
'test'), inp_file, out_file))
pool.close()
pool.join()
# Save all .out files to a binary file
atm_lut = np.memmap(config['atm_lut_file'],
dtype='float32',
mode='w+',
offset=0,
shape=(len(RHO), len(WVC), len(VIS), len(VZA),
len(RAA), len(WAVE)))
for rho_index, rho in enumerate(RHO):
for wvc_index, wvc in enumerate(WVC):
for vis_index, vis in enumerate(VIS):
out_file = os.path.join(
config['atm_lut_dir'],
'rho_%03d_wvc_%03d_vis_%03d.out' %
(rho * 100, wvc, vis))
data = np.loadtxt(
out_file, dtype='float32'
) # data.shape = [len(WAVE), len(SZA)*len(RAA)]
data = np.dstack(
np.split(data, len(VZA), axis=1)
[::-1]) # data.shape = [len(WAVE), len(RAA), len(SZA)]
data = data.swapaxes(
0, 2) # data.shape = [len(SZA), len(RAA), len(WAVE)]
""" Notes:
(1) In the .out file, the radiance uu is arranged as:
umu(0),phi(0) umu(0),phi(1) ... umu(0),phi(N)
umu(1),phi(0) umu(1),phi(1) ... umu(0),phi(N)
...
umu(M),phi(0) umu(M),phi(1) ... umu(M),phi(N)
where umu=cos(sensor_zenith), which is ascendingly ordered.
To ascend SZA, we need to reverse the order of the radiance.
That is why we have [::-1] in the `data = np.dstack(...)` step.
"""
atm_lut[rho_index, wvc_index, vis_index, :, :, :] = data
del data, out_file
atm_lut.flush()
atm_lut_metadata = dict()
atm_lut_metadata['description'] = 'Atmospheric Look-Up-Table Metadata'
atm_lut_metadata['dtype'] = 'float32'
atm_lut_metadata['shape'] = [
len(RHO),
len(WVC),
len(VIS),
len(VZA),
len(RAA),
len(WAVE)
]
atm_lut_metadata['dimension'] = [
'RHO', 'WVC', 'VIS', 'VZA', 'RAA', 'WAVE'
]
atm_lut_metadata['RHO'] = RHO
atm_lut_metadata['WVC'] = WVC
atm_lut_metadata['VIS'] = VIS
atm_lut_metadata['VZA'] = VZA
atm_lut_metadata['RAA'] = RAA
atm_lut_metadata['WAVE'] = WAVE
atm_lut_metadata['SZA'] = sza
atm_lut_metadata['SAA'] = saa
write_atm_lut_metadata(config['atm_lut_file'] + '.meta',
atm_lut_metadata)
#TODO: remove all *.inp and *.out files.
else:
#TODO: do interpolations on `atm_database` to generate the LUT.
pass
logger.info('Write atmospheric LUT to %s.' % config['atm_lut_file'])
def remove_smile_effect(config, atm_lut_file, sun_zenith):
from atmosphere import read_wvc_model
# Read calibration data
cal_data = get_cal_data(config['raw_image_file'], config['setting_file'])
# Read raw image data
raw_header = read_envi_header(
os.path.splitext(config['raw_image_file'])[0] + '.hdr')
raw_image = np.memmap(config['raw_image_file'],
dtype='uint16',
mode='r',
offset=raw_header['header offset'],
shape=(raw_header['lines'], raw_header['bands'],
raw_header['samples']))
samples = raw_header['samples']
# Read mask
mask_header = read_envi_header(
os.path.splitext(config['raw_image_file'])[0] + '.hdr')
mask_image = np.memmap(config['mask_image_file'],
dtype='bool',
mode='r',
shape=(mask_header['bands'], raw_header['lines'],
raw_header['samples']))
# Get average radiance spectra with dark current corrected
avg_rdn_01 = get_avg_rdn_01(raw_image, cal_data, mask_image)
raw_image.flush()
mask_image.flush()
del raw_header, mask_header
# Get average radiance spectra with dark current and gain corrected
avg_rdn_02 = get_avg_rdn_02(avg_rdn_01, cal_data)
plt.figure()
plt.plot(cal_data['spectralVector'], avg_rdn_02, '-')
plt.show()
plt.figure()
plt.plot(cal_data['spectralVector'], avg_rdn_02[:, 10], '-')
plt.show()
# Get average scan angles
avg_vza, avg_raa = get_avg_scan_angles(config['sca_image_file'])
# Build water vapor column estimation model
""" Notes:
(1) The view geometry does not seem to affect the model.
Therefore, a vza=0 and raa=0 are used here.
"""
wvc_model = read_wvc_model(config['wvc_model_file'])
# Estimate water vapor for each column
ratio = avg_rdn_02[wvc_model['Bands'][1], :] / (
avg_rdn_02[wvc_model['Bands'][0], :] * wvc_model['Weights'][0] +
avg_rdn_02[wvc_model['Bands'][2], :] * wvc_model['Weights'][1])
wvc = np.interp(ratio, wvc_model['Ratio'], wvc_model['WVC'])
vis = 80
plt.figure()
plt.plot(wvc_model['Ratio'], wvc_model['WVC'], 'r.-')
plt.plot(ratio, wvc, 'b.')
plt.show()
del ratio
wvc = wvc.mean()
# Interpolate atm lut
lut_wave, lut_rdn_0, lut_rdn_1 = interp_atm_lut_to_angles(
atm_lut_file, wvc, vis, avg_vza, avg_raa)
plt.plot(lut_wave, lut_rdn_0, 'r-')
plt.show()
for wave_range in features.values():
wave_range = [744, 784]
# wave_range = [1255, 1285]
wave_0, band_0 = get_closest_wave(cal_data['spectralVector'],
wave_range[0])
wave_1, band_1 = get_closest_wave(cal_data['spectralVector'],
wave_range[1])
if abs(wave_0 - wave_range[0]) > 10 or abs(wave_1 -
wave_range[1]) > 10:
continue
X = []
lower_bounds = [-5.0, -2.0]
upper_bounds = [5.0, 2.0]
for sample in range(0, samples, 100):
x0 = [0, 0]
p = optimize.least_squares(err,
x0,
bounds=(lower_bounds, upper_bounds),
args=(cal_data, avg_rdn_01, lut_wave,
lut_rdn_0, lut_rdn_1,
[band_0, band_1 + 1], sample))
X.append(p.x)
X = np.array(X)
plt.plot(X[:, 0], '.', color='blue')
plt.show()
def build_mask(mask_image_file, raw_image_file, setting_file, sun_zenith):
""" Mask out bad pixels.
Arguments:
mask_image_file: str
Mask image filename.
raw_image: 3D array
Raw DN image, dimension = [lines, bands, samples]
cal_data: dict
Calibration data.
sun_zenith: float
Sun zenith angle in degrees.
"""
# Read calibration data
cal_data = get_cal_data(raw_image_file, setting_file)
# Read raw image data
raw_header = read_envi_header(os.path.splitext(raw_image_file)[0] + '.hdr')
raw_image = np.memmap(raw_image_file,
dtype='int16',
mode='r',
offset=raw_header['header offset'],
shape=(raw_header['lines'], raw_header['bands'],
raw_header['samples']))
bands = raw_header['bands']
""" Rule 1 for good pixels:
At-sensor blue reflectance>0.10, and nir reflectance>0.10, and swir reflectance>0.10
"""
solar_flux = resample_solar_flux(solar_flux_file,
cal_data['spectralVector'],
cal_data['fwhm'])
solar_flux /= 10 # mW / (m2 nm) -> mW / (cm2 um)
cos_sun_zenith = np.cos(np.deg2rad(sun_zenith))
mask = np.full((raw_image.shape[0], raw_image.shape[2]),
True,
dtype=np.bool)
# if the reflectance at 470 nm is less than 0.10, then mask these pixels.
wave, band = get_closest_wave(cal_data['spectralVector'], 470)
if abs(wave - 470) < 10:
refl = raw2rdn(raw_image[:, band, :], cal_data,
band) * np.pi / (solar_flux[band] * cos_sun_zenith)
mask &= (refl > 0.10)
# if the reflectance at 850 nm is less than 0.10, then mask these pixels.
wave, band = get_closest_wave(cal_data['spectralVector'], 850)
if abs(wave - 850) < 10:
refl = raw2rdn(raw_image[:, band, :], cal_data,
band) * np.pi / (solar_flux[band] * cos_sun_zenith)
mask &= (refl > 0.10)
# if the reflectance at 1600 nm is less than 0.10, then mask these pixels.
wave, band = get_closest_wave(cal_data['spectralVector'], 1600)
if abs(wave - 1600) < 10:
refl = raw2rdn(raw_image[:, band, :], cal_data,
band) * np.pi / (solar_flux[band] * cos_sun_zenith)
mask &= (refl > 0.10)
""" Rule 2:
Mask dark pixels by using radiance values.
"""
mask_image = np.memmap(mask_image_file,
dtype='uint8',
mode='w+',
shape=(raw_image.shape[1], raw_image.shape[0],
raw_image.shape[2]))
log_mesg = 'Band (max=%d): ' % bands
for band in range(bands):
if band % 50 == 0:
log_mesg += '%d, ' % band
rdn = raw2rdn(raw_image[:, band, :], cal_data, band)
mask_image[band, :, :] = mask & (raw_image[:, band, :] <
cal_data['satValue']) & (rdn > 0.0)
del rdn
mask_image.flush()
raw_image.flush()
del cal_data, mask
log_mesg += '%d, Done!' % bands
logging.info(log_mesg)
mask_header = empty_envi_header()
mask_header['description'] = 'Mask 0: bad; 1: good'
mask_header['samples'] = raw_image.shape[2]
mask_header['lines'] = raw_image.shape[0]
mask_header['bands'] = raw_image.shape[1]
mask_header['byte order'] = 0
mask_header['header offset'] = 0
mask_header['interleave'] = 'bsq'
mask_header['data type'] = 1
write_envi_header(mask_image_file + '.hdr', mask_header)