def get_acquisition_time(dn_header_file, raw_imugps_file):
""" Get Hyspex image acquistion time.
Notes
-----
(1) This code is adapted from Brendan Heberlein ([email protected]).
Parameters
----------
header_file: str
Hyspex DN image header filename.
imugps_file: str
Hyspex raw imugps filename.
Returns
-------
when: datetime object
Image acquisition time.
"""
from datetime import datetime, timedelta
from ENVI import read_envi_header
import numpy as np
header = read_envi_header(dn_header_file)
week_start = datetime.strptime(f"{header['acquisition date']} 00:00:00",
"%Y-%m-%d %H:%M:%S")
week_seconds = np.loadtxt(raw_imugps_file)[:, 7].mean()
epoch = datetime(1980, 1, 6, 0, 0)
gps_week = (week_start - epoch).days // 7
time_elapsed = timedelta(days=gps_week * 7, seconds=week_seconds)
when = epoch + time_elapsed
return when
def plot_avg_rdn(avg_rdn_figure_file, avg_rdn_file):
""" Plot average radiance to a figure.
Arguments:
avg_rdn_figure_file: str
Average radiance figure filename.
avg_rdn_file: str
Average radiance filename.
"""
if os.path.exists(avg_rdn_figure_file):
logger.info('Save the average radiance spectra figure to %s.' %avg_rdn_figure_file)
return
from ENVI import read_envi_header
header = read_envi_header(os.path.splitext(avg_rdn_file)[0]+'.hdr')
avg_rdn = np.memmap(avg_rdn_file,
mode='r',
dtype='float32',
shape=(header['lines'],
header['samples'])) # shape=(bands, samples)
wavelength = np.array([float(v) for v in header['waves'].split(',')])
plt.figure(figsize=(10, 6))
plt.plot(wavelength, avg_rdn, lw=1)
plt.xlim(np.floor(wavelength.min()), np.ceil(wavelength.max()))
plt.xlabel('Wavelength (nm)', fontsize=16)
plt.ylabel(r'Radiance $(mW{\cdot}cm^{-2}{\cdot}{\mu}m^{-1}{\cdot}sr^{-1})$', fontsize=16)
plt.xticks(fontsize=16)
plt.yticks(fontsize=16)
plt.savefig(avg_rdn_figure_file, dpi=1000)
plt.close()
avg_rdn.flush()
del avg_rdn
logger.info('Save the average radiance spectra figure to %s.' %avg_rdn_figure_file)
def interpolate_detectors(sensor_dict, mode='bicurvilinear', *args, **kwargs):
''' Interpolate radiance over bad detectors. '''
if mode == 'bicurvilinear':
interpolate = interpolate_detectors__bicurvilinear_
# Initialize radiance cube from disk
hdr = read_envi_header(
os.path.splitext(sensor_dict['raw_rdn_image_file'])[0] + '.hdr')
rdn_image = np.memmap(sensor_dict['raw_rdn_image_file'],
shape=(hdr['lines'], hdr['bands'], hdr['samples']),
offset=hdr['header offset'],
mode='r+',
dtype='float32') #, dtype='int16')
### TODO: Read dtype from header!
# Load bad detectors registry from settings file
bad_detectors = bad_detectors_from_hyspex_set(sensor_dict['setting_file'])
# Interpolate over bad detectors
logger.info('Do interpolation.')
wavelengths = np.array(hdr['wavelength'], dtype=np.float64)
interpolate(rdn_image, bad_detectors, wavelengths, *args, **kwargs)
# Commit changes to disk
del rdn_image
def plot_angle_geometry(angle_geometry_figure_file, sca_image_file):
""" Plot the sun-target-view geometry in a polar coordinate system.
Parameters
----------
angle_geometry_figure_file: str
Angle geometry figure filename.
sca_image_file: str
Scan angle image filename.
"""
if os.path.exists(angle_geometry_figure_file):
logger.info('Save the angle geometry figure to %s.' %
angle_geometry_figure_file)
return
from ENVI import read_envi_header
# Read SCA image data
sca_header = read_envi_header(os.path.splitext(sca_image_file)[0] + '.hdr')
sca_image = np.memmap(sca_image_file,
dtype='float32',
mode='r',
offset=0,
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# Scatter-plot view geometry
plt.figure(figsize=(10, 10))
ax = plt.subplot(111, projection='polar')
ax.scatter(np.deg2rad(sca_image[1, ::10, ::10].flatten()),
sca_image[0, ::10, ::10].flatten(),
color='green',
marker='.',
s=10)
sca_image.flush()
del sca_image
# Scatter-plot solar geometry
ax.scatter(np.deg2rad(float(sca_header['sun azimuth'])),
float(sca_header['sun zenith']),
color='red',
marker='*',
s=500)
ax.set_theta_direction(-1)
ax.set_theta_zero_location('N')
_, _ = ax.set_thetagrids([0, 45, 90, 135, 180, 225, 270, 315],
labels=('0 N', '45', '90 E', '135', '180 S',
'225', '270 W', '315'))
ax.tick_params(labelsize=20)
plt.savefig(angle_geometry_figure_file, dpi=1000)
plt.close()
del sca_header, ax
logger.info('Save the angle geometry figure to %s.' %
angle_geometry_figure_file)
def modify_hyspex_band(img, band, arr):
src_header = read_envi_header(img + '.hdr')
src_image = np.memmap(img,
dtype='int16',
mode='r+',
shape=(src_header['lines'], src_header['samples']),
offset=int(src_header['lines'] *
src_header['samples'] * band * 16 / 8))
src_image[:] = arr
src_image.flush()
del src_image, arr
def plot_wvc_histogram(wvc_histogram_figure_file, water_vapor_column_image_file):
""" Plot water vapor column histogram.
wvc_histogram_figure_file: str
Water vapor column histogram figure filename.
water_vapor_column_image_file: str
Water vapor column image filename.
"""
if os.path.exists(wvc_histogram_figure_file):
logger.info('Save water vapor column histogram figure to %s.' %wvc_histogram_figure_file)
return
from ENVI import read_envi_header
# Read water vapor column image
wvc_header = read_envi_header(os.path.splitext(water_vapor_column_image_file)[0]+'.hdr')
wvc_image = np.memmap(water_vapor_column_image_file,
dtype='float32',
mode='r',
shape=(wvc_header['lines'], wvc_header['samples']))
# Plot water vapor column histogram
wvc_bins = np.arange(0, 51, 1)
freq = []
for bin_index in range(len(wvc_bins)-1):
index = (wvc_image>=wvc_bins[bin_index])&(wvc_image<wvc_bins[bin_index+1])
freq.append(np.sum(index)/wvc_image.size*100)
freq = np.array(freq)
freq_max = 100
plt.figure(figsize=(10,6))
plt.bar(wvc_bins[:-1], freq, width=1, color='darkgreen', edgecolor='black', linewidth=1)
plt.vlines([wvc_image.mean()], 0, freq_max, color='darkred', lw=2, linestyles='solid', label=r'WVC$_{Mean}$')
plt.vlines([wvc_image.mean()-2*wvc_image.std()], 0, freq_max, color='darkred', lw=2, linestyles='dotted', label=r'WVC$_{Mean}$-2WVC$_{SD}$')
plt.vlines([wvc_image.mean()+2*wvc_image.std()], 0, freq_max, color='darkred', lw=2, linestyles='dashed', label=r'WVC$_{Mean}$+2WVC$_{SD}$')
plt.xticks(ticks=np.arange(0, 51, 5), labels=np.arange(0, 51, 5), fontsize=20)
plt.yticks(ticks=np.arange(0,101,10), labels=np.arange(0,101,10), fontsize=20)
plt.xlabel('Water Vapor Column (mm)', fontsize=20)
plt.ylabel('Relative Frequency (%)', fontsize=20)
plt.xlim(0, 50)
plt.ylim(0, freq_max)
plt.legend(fontsize=20)
plt.savefig(wvc_histogram_figure_file, dpi=1000)
plt.close()
wvc_image.flush()
del wvc_image
logger.info('Save water vapor column histogram figure to %s.' %wvc_histogram_figure_file)
def process_sca_file(raw_sca_image_file, processed_sca_image_file):
# Read raw scan angle data.
raw_sca_header = read_envi_header(
os.path.splitext(raw_sca_image_file)[0] + '.hdr')
raw_sca_image = np.memmap(raw_sca_image_file,
mode='r',
dtype='float32',
shape=(raw_sca_header['bands'],
raw_sca_header['lines'],
raw_sca_header['samples']))
# Initialize processed scan angle data.
processed_sca_image = np.memmap(processed_sca_image_file,
mode='w+',
dtype='int16',
shape=(raw_sca_header['bands'],
raw_sca_header['lines'],
raw_sca_header['samples']))
# zenith*100
angle = raw_sca_image[0, :, :] * 100
angle[angle < 0] = 9100
processed_sca_image[0, :, :] = angle.astype('int16')
del angle
# zenith*10
angle = raw_sca_image[1, :, :] * 10
angle[angle < 0] = -1
processed_sca_image[1, :, :] = angle.astype('int16')
del angle
# Clear data
processed_sca_image.flush()
raw_sca_image.flush()
del processed_sca_image, raw_sca_image
# Write header
raw_sca_header['description'] = 'SCA'
raw_sca_header['data type'] = 2
raw_sca_header['band names'] = [
'Sensor Zenith [100*deg]', 'Sensor Azimuth [10*deg]'
]
raw_sca_header['data ignore value'] = None
write_envi_header(
os.path.splitext(processed_sca_image_file)[0] + '.hdr', raw_sca_header)
def create_polygon(dir_in, outshp):
if os.path.exists(outshp):
print("Output file exists!")
sys.exit(1)
# Read IGM.
igm_header = read_envi_header(glob.glob(dir_in + '*IGM.hdr')[0])
igm_image = np.memmap(glob.glob(dir_in + '*IGM')[0],
dtype='float64',
mode='r',
offset=0,
shape=(2, igm_header['lines'],
igm_header['samples']))
lon = igm_image[0, :, :]
lat = igm_image[1, :, :]
igm_image.flush()
del igm_image
# concat outline lon points:
# lons = list(lon[:,0].flatten()) +list(lon[-1,:].flatten()) + list(np.flip(lon[:,-1].flatten()))+ list(np.flip(lon[0,:].flatten()))
lons = np.concatenate(
(lon[:, 0].flatten(), lon[-1, :].flatten(),
np.flip(lon[:, -1].flatten()), np.flip(lon[0, :].flatten())))
# lats = list(lat[:,0].flatten()) +list(lat[-1,:].flatten()) + list(np.flip(lat[:,-1].flatten()))+ list(np.flip(lat[0,:].flatten()))
lats = np.concatenate(
(lat[:, 0].flatten(), lat[-1, :].flatten(),
np.flip(lat[:, -1].flatten()), np.flip(lat[0, :].flatten())))
# construct the geometry:
ring = ogr.Geometry(ogr.wkbLinearRing)
for i in range(0, len(lons)):
ring.AddPoint(lons[i], lats[i])
# Create polygon
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(ring)
return poly.ExportToWkt()
def dn2rdn_Hyspex(rdn_image_file, dn_image_file, radio_cali_file,
acquisition_time):
""" Do Hyspex radiometric calibration.
Parameters
----------
rdn_image_file: str
Radiance image filename.
dn_image_file: str
Hyspex DN image filename.
radio_cali_file: str
Hyspex radiometric calibration coefficients filename.
acquisition_time: datetime object
Acquisition time.
"""
if os.path.exists(rdn_image_file):
logger.info('Write the radiance image to %s.' % rdn_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read calibration coefficients
radio_cali_header = read_envi_header(
os.path.splitext(radio_cali_file)[0] + '.hdr')
radio_cali_coeff = np.memmap(radio_cali_file,
dtype='float64',
mode='r',
shape=(radio_cali_header['bands'],
radio_cali_header['lines'],
radio_cali_header['samples']))
wavelengths = np.array(
[float(v) for v in radio_cali_header['waves'].split(',')])
fwhms = np.array([float(v) for v in radio_cali_header['fwhms'].split(',')])
# Read DN image
dn_header = read_envi_header(os.path.splitext(dn_image_file)[0] + '.hdr')
dn_image = np.memmap(dn_image_file,
dtype='uint16',
mode='r',
offset=dn_header['header offset'],
shape=(dn_header['lines'], dn_header['bands'],
dn_header['samples']))
# Get gain coefficients
gain = radio_cali_coeff[0, :, :] # shape=(bands, samples)
# Do radiometric calibration
info = 'Line (max=%d): ' % dn_header['lines']
fid = open(rdn_image_file, 'wb')
for from_line in range(0, dn_header['lines'], 500):
info += '%d, ' % (from_line + 1)
# Determine chunk size
to_line = min(from_line + 500, dn_header['lines'])
# Get offset coefficients
if radio_cali_header['bands'] == 2:
offset = radio_cali_coeff[1, :, :] # shape=(bands, samples)
else:
background = np.stack([radio_cali_coeff[1, :, :]] *
(to_line - from_line))
backgroundLast = np.stack([radio_cali_coeff[2, :, :]] *
(to_line - from_line))
factor = np.arange(from_line, to_line) / dn_header['lines']
offset = background + (
backgroundLast - background
) * factor[:, np.newaxis,
np.newaxis] # shape=(to_line-from_line, bands, samples)
del background, backgroundLast, factor
# Convert DN to radiance
rdn = (dn_image[from_line:to_line, :, :].astype('float32') -
offset) * gain # shape=(to_line-from_line, bands, samples)
# Write radiance to file
fid.write(rdn.astype('float32').tostring())
# Clear temporary data
del rdn, to_line, offset
fid.close()
info += '%d, Done!' % dn_header['lines']
logger.info(info)
# Clear data
dn_image.flush()
radio_cali_coeff.flush()
del gain, from_line
del dn_image, radio_cali_coeff
# Write header
rdn_header = empty_envi_header()
rdn_header['description'] = 'Hyspex radiance in mW/(cm2*um*sr)'
rdn_header['file type'] = 'ENVI Standard'
rdn_header['samples'] = dn_header['samples']
rdn_header['lines'] = dn_header['lines']
rdn_header['bands'] = dn_header['bands']
rdn_header['byte order'] = 0
rdn_header['header offset'] = 0
rdn_header['interleave'] = 'bil'
rdn_header['data type'] = 4
rdn_header['wavelength'] = list(wavelengths)
rdn_header['fwhm'] = list(fwhms)
rdn_header['wavelength units'] = 'nm'
rdn_header['default bands'] = dn_header['default bands']
rdn_header['acquisition time'] = acquisition_time.strftime(
'%Y-%m-%dT%H:%M:%S.%f')
write_envi_header(os.path.splitext(rdn_image_file)[0] + '.hdr', rdn_header)
del radio_cali_header, dn_header, rdn_header
logger.info('Write the radiance image to %s.' % rdn_image_file)
def orthorectify_dem(ortho_dem_image_file, igm_image_file, glt_image_file):
""" Do orthorectifications on DEM.
Arguments:
ortho_dem_image_file: str
Orthorectified DEM image filename.
igm_image_file: str
IGM image filename.
glt_image_file: str
Geographic look-up table image filename.
"""
if os.path.exists(ortho_dem_image_file):
logger.info('Write the geo-rectified DEM image to %s.' %ortho_dem_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read IGM image (the third band is DEM).
igm_header = read_envi_header(igm_image_file+'.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
shape=(igm_header['bands'],
igm_header['lines'],
igm_header['samples']))
del igm_header
# Read GLT image.
glt_header = read_envi_header(glt_image_file+'.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
shape=(glt_header['bands'],
glt_header['lines'],
glt_header['samples']))
# Get spatial indices.
I, J = np.where((glt_image[0,:,:]>=0)&(glt_image[1,:,:]>=0))
ortho_dem_image = np.zeros((glt_header['lines'], glt_header['samples']), dtype='float32')
# Orthorectify DEM.
fid = open(ortho_dem_image_file, 'wb')
ortho_dem_image[:,:] = -1000.0
ortho_dem_image[I,J] = igm_image[2, glt_image[0,I,J], glt_image[1,I,J]]
fid.write(ortho_dem_image.tostring())
fid.close()
del ortho_dem_image
igm_image.flush()
glt_image.flush()
del igm_image, glt_image
# Write header.
ortho_dem_header = empty_envi_header()
ortho_dem_header['description'] = 'Geo-rectified DEM, in [m]'
ortho_dem_header['file type'] = 'ENVI Standard'
ortho_dem_header['samples'] = glt_header['samples']
ortho_dem_header['lines'] = glt_header['lines']
ortho_dem_header['bands'] = 1
ortho_dem_header['byte order'] = 0
ortho_dem_header['header offset'] = 0
ortho_dem_header['data ignore value'] = -1000.0
ortho_dem_header['interleave'] = 'bsq'
ortho_dem_header['data type'] = 4
ortho_dem_header['map info'] = glt_header['map info']
ortho_dem_header['coordinate system string'] = glt_header['coordinate system string']
write_envi_header(ortho_dem_image_file+'.hdr', ortho_dem_header)
del glt_header, ortho_dem_header
logger.info('Write the geo-rectified DEM image to %s.' %ortho_dem_image_file)
def orthorectify_rdn(ortho_rdn_image_file, rdn_image_file, glt_image_file):
""" Do orthorectifications on radiance.
Arguments:
ortho_rdn_image_file: str
Orthorectified radiance filename.
rdn_image_file: str
Radiance image filename.
glt_image_file: str
Geographic look-up table image filename.
"""
if os.path.exists(ortho_rdn_image_file):
logger.info('Write the geo-rectified radiance image to %s.' %ortho_rdn_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read radiance image.
rdn_header = read_envi_header(rdn_image_file+'.hdr')
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['lines'],
rdn_header['bands'],
rdn_header['samples']))
# Read GLT image.
glt_header = read_envi_header(glt_image_file+'.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
offset=0,
shape=(glt_header['bands'],
glt_header['lines'],
glt_header['samples']))
# Get spatial indices.
I, J = np.where((glt_image[0,:,:]>=0)&(glt_image[1,:,:]>=0))
ortho_image = np.zeros((glt_header['lines'], glt_header['samples']), dtype='float32')
# Orthorectify radiance.
fid = open(ortho_rdn_image_file, 'wb')
info = 'Band (max=%d): ' %rdn_header['bands']
for band in range(rdn_header['bands']):
if band%20==0:
info += '%d, ' %(band+1)
ortho_image[:,:] = 0.0
ortho_image[I,J] = rdn_image[glt_image[0,I,J], band, glt_image[1,I,J]]
fid.write(ortho_image.tostring())
fid.close()
info += 'Done! %s' %rdn_header['bands']
logger.info(info)
del ortho_image
rdn_image.flush()
glt_image.flush()
del rdn_image, glt_image
# Write header.
ortho_rdn_header = empty_envi_header()
ortho_rdn_header['description'] = 'Geo-rectified radiance, in [mW/(cm2*um*sr)]'
ortho_rdn_header['file type'] = 'ENVI Standard'
ortho_rdn_header['samples'] = glt_header['samples']
ortho_rdn_header['lines'] = glt_header['lines']
ortho_rdn_header['bands'] = rdn_header['bands']
ortho_rdn_header['byte order'] = 0
ortho_rdn_header['header offset'] = 0
ortho_rdn_header['interleave'] = 'bsq'
ortho_rdn_header['data type'] = 4
ortho_rdn_header['wavelength'] = rdn_header['wavelength']
ortho_rdn_header['fwhm'] = rdn_header['fwhm']
ortho_rdn_header['wavelength units'] = 'nm'
ortho_rdn_header['default bands'] = rdn_header['default bands']
ortho_rdn_header['acquisition time'] = rdn_header['acquisition time']
ortho_rdn_header['map info'] = glt_header['map info']
ortho_rdn_header['coordinate system string'] = glt_header['coordinate system string']
write_envi_header(ortho_rdn_image_file+'.hdr', ortho_rdn_header)
del glt_header, rdn_header, ortho_rdn_header
logger.info('Write the geo-rectified radiance image to %s.' %ortho_rdn_image_file)
def orthorectify_sca(ortho_sca_image_file, sca_image_file, glt_image_file):
""" Do orthorectifications on SCA.
Arguments:
ortho_sca_image_file: str
Orthorectified SCA image filename.
sca_image_file: str
SCA image filename.
glt_image_file: str
Geographic look-up table image filename.
"""
if os.path.exists(ortho_sca_image_file):
logger.info('Write the geo-rectified SCA image to %s.' %ortho_sca_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read SCA image.
sca_header = read_envi_header(os.path.splitext(sca_image_file)[0]+'.hdr')
# Read GLT image.
glt_header = read_envi_header(os.path.splitext(glt_image_file)[0]+'.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
shape=(glt_header['bands'],
glt_header['lines'],
glt_header['samples']))
# Get spatial indices.
I, J = np.where((glt_image[0,:,:]>=0)&(glt_image[1,:,:]>=0))
ortho_sca_image = np.zeros((glt_header['lines'], glt_header['samples']), dtype='float32')
# Orthorectify SCA.
fid = open(ortho_sca_image_file, 'wb')
for band in range(sca_header['bands']):
ortho_sca_image[:,:] = -1000.0
offset = sca_header['header offset']++4*band*sca_header['lines']*sca_header['samples']
sca_image = np.memmap(sca_image_file,
dtype='float32',
mode='r',
offset=offset,
shape=(sca_header['lines'], sca_header['samples']))
ortho_sca_image[I,J] = sca_image[glt_image[0,I,J], glt_image[1,I,J]]
fid.write(ortho_sca_image.tostring())
sca_image.flush()
del sca_image
fid.close()
del ortho_sca_image
glt_image.flush()
del glt_image
# Write header
ortho_sca_header = empty_envi_header()
ortho_sca_header['description'] = 'Geo-rectified SCA, in [deg]'
ortho_sca_header['file type'] = 'ENVI Standard'
ortho_sca_header['samples'] = glt_header['samples']
ortho_sca_header['lines'] = glt_header['lines']
ortho_sca_header['bands'] = sca_header['bands']
ortho_sca_header['byte order'] = 0
ortho_sca_header['header offset'] = 0
ortho_sca_header['data ignore value'] = -1000.0
ortho_sca_header['interleave'] = 'bsq'
ortho_sca_header['data type'] = 4
ortho_sca_header['band names'] = sca_header['band names']
ortho_sca_header['sun zenith'] = sca_header['sun zenith']
ortho_sca_header['sun azimuth'] = sca_header['sun azimuth']
ortho_sca_header['map info'] = glt_header['map info']
ortho_sca_header['coordinate system string'] = glt_header['coordinate system string']
write_envi_header(ortho_sca_image_file+'.hdr', ortho_sca_header)
del glt_header, sca_header, ortho_sca_header
logger.info('Write the geo-rectified SCA image to %s.' %ortho_sca_image_file)
def merge_rdn(merged_image_file, mask_file, sensors):
""" Merge radiance images.
Arguments:
merged_image_file: str
Merged radiance image filename.
mask_file: str
Background mask filename.
sensors: dict
Sensor dictionaries.
"""
if os.path.exists(merged_image_file):
logger.info('Write the merged refletance image to %s.' %
merged_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read mask.
mask_header = read_envi_header(os.path.splitext(mask_file)[0] + '.hdr')
mask_image = np.memmap(mask_file,
mode='r',
dtype='bool',
shape=(mask_header['lines'],
mask_header['samples']))
# Get the map upper-left coordinates and pixel sizes of VNIR and SWIR images.
ulx, uly, pixel_size = float(mask_header['map info'][3]), float(
mask_header['map info'][4]), float(mask_header['map info'][5])
# Determine regular map grids.
x, y = np.meshgrid(ulx + np.arange(mask_header['samples']) * pixel_size,
uly - np.arange(mask_header['lines']) * pixel_size)
del ulx, uly, pixel_size
# Read radiance header and image.
header_dict = dict()
image_file_dict = dict()
bands_waves_fwhms = []
for sensor_index, sensor_dict in sensors.items():
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_rdn_image_file'])[0] + '.hdr')
for band in range(tmp_header['bands']):
bands_waves_fwhms.append([
'%s_%d' % (sensor_index, band), tmp_header['wavelength'][band],
tmp_header['fwhm'][band]
])
header_dict[sensor_index] = tmp_header
image_file_dict[sensor_index] = sensor_dict['ortho_rdn_image_file']
bands_waves_fwhms.sort(key=lambda x: x[1])
# Merge images.
wavelengths = []
fwhms = []
fid = open(merged_image_file, 'wb')
for v in bands_waves_fwhms:
# Determine which sensor, band to read.
sensor_index, band = v[0].split('_')
band = int(band)
wavelengths.append(v[1])
fwhms.append(v[2])
header = header_dict[sensor_index]
image_file = image_file_dict[sensor_index]
# Write image.
if ((v[1] >= 1339.0) &
(v[1] <= 1438.0)) | ((v[1] >= 1808.0) &
(v[1] <= 1978.0)) | (v[1] >= 2467.0):
# resampled_image = np.zeros(x.shape)
resampled_image = np.zeros(x.shape,
dtype='int16') # in int, by Ting
else:
offset = header[
'header offset'] + 2 * band * header['lines'] * header[
'samples'] # in bytes, int16 consists 2 bytes
rdn_image = np.memmap(
image_file,
# dtype='float32',
dtype='int16', # in int, by Ting
mode='r',
offset=offset,
shape=(header['lines'], header['samples']))
resampled_image = resample_ortho_rdn(
np.copy(rdn_image) /
1000, # divided by 1000 to convert back to original rdn
float(header_dict[sensor_index]['map info'][3]),
float(header_dict[sensor_index]['map info'][4]),
float(header_dict[sensor_index]['map info'][5]),
x,
y)
# resampled_image[mask_image] = 0.0
resampled_image = resampled_image * 1000 # times 1000 to convert to int, by Ting
resampled_image[mask_image] = 0 # in int, by Ting
rdn_image.flush()
del rdn_image
# fid.write(resampled_image.astype('float32').tostring())
fid.write(resampled_image.astype(
'int16').tostring()) # write in int ,by Ting
del resampled_image
fid.close()
del header_dict, image_file_dict, x, y
mask_image.flush()
del mask_image
# Write header.
header = empty_envi_header()
header['description'] = 'Merged radiance, in [mW/(cm2*um*sr)]'
header['file type'] = 'ENVI Standard'
header['samples'] = mask_header['samples']
header['lines'] = mask_header['lines']
header['bands'] = len(wavelengths)
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
# header['data type'] = 4
header['data type'] = 2 # in int, by Ting
header['wavelength'] = wavelengths
header['fwhm'] = fwhms
header['wavelength units'] = 'nm'
header['acquisition time'] = tmp_header['acquisition time']
header['map info'] = mask_header['map info']
header['coordinate system string'] = mask_header[
'coordinate system string']
write_envi_header(os.path.splitext(merged_image_file)[0] + '.hdr', header)
del header, tmp_header
logger.info('Write the merged refletance image to %s.' % merged_image_file)
def main():
# dir_in = r'Z:\townsenduser-rw\HyspexPro\Output\Cheesehead_V3\CHEESEHEAD_20190626\CHEESEHEAD_20190626_02_quicklooks'
dir_in = r'Z:\townsenduser-rw\HyspexPro\Output\Cheesehead_V3\CHEESEHEAD_20190806'
dest_dir = dir_in + '/Deshift'
img = '/CHEESEHEAD_20190806_03_Refl'
# ref_img = dir_in + '/Merge' + img
dest_file = dest_dir + img
header = read_envi_header(dir_in + '/Merge' + img + '.hdr')
band_vnir, band_swir, idx_vnir, idx_swir = band_select(header)
# hyspex_extract(dir_in + '/Merge' + img, [186], dest_dir + img + '_refband')
# hyspex_extract(dir_in + '/Merge' + img, [185], dest_dir + img + '_swir_test')
ref_img = dest_dir + img + '_refband'
tgt_img = dest_dir + img + '_swir_test'
grid_ls = [350] #[300, 350, 400, 450]
windows = [150] #[128, 150, 264, 280]
for i in range(len(grid_ls)):
out = dest_dir + '/shifted_local_03_{}_{}'.format(
grid_ls[i], windows[i])
g = grid_ls[i]
w = windows[i]
kwargs = kwargs = {
'grid_res': g,
'window_size': (w, w),
'path_out': out,
'nodata': (0, 0),
'r_b4match': 1,
's_b4match': 1,
'max_iter': 10,
'max_shift': 10,
# 'min_reliability': 25,
# 'tieP_filter_level': 1,
'resamp_alg_calc': 'nearest',
'q': False
}
dest_img = dest_file + '_deshift'
# calculate the shift info
try:
CRL = COREG_LOCAL(ref_img, tgt_img, **kwargs)
CRL.calculate_spatial_shifts()
# CRL.correct_shifts()
except:
continue
# apply the shift to a single band:
band = 185
# shift_arg = {
# 'band2process': int(band + 1),
# 'resamp_alg': 'nearest',
# 'nodata': 0,
# 'align_grids': True,
# 'q': True
# }
result = DESHIFTER(dir_in + '/Merge' + img,
CRL.coreg_info,
band2process=int(band + 1),
align_grids=True,
match_gsd=True,
nodata=0,
resamp_alg='cubic',
q=True).correct_shifts()
modify_hyspex_band(dest_img, band, result['arr_shifted'])
def make_quicklook(quicklook_figure_file, rdn_image_file, glt_image_file):
""" Make a RGB quicklook image.
Arguments:
quicklook_figure_file: str
Geo-rectified quicklook figure filename.
rdn_image_file: str
Radiance image filename, in BIL format.
glt_image_file: str
GLT image filename.
"""
if os.path.exists(quicklook_figure_file):
logger.info('Save the quicklook figure to %s.' % quicklook_figure_file)
return
from ENVI import read_envi_header
from Spectra import get_closest_wave
# Read radiance image data.
rdn_header = read_envi_header(os.path.splitext(rdn_image_file)[0] + '.hdr')
rdn_image = np.memmap(
rdn_image_file,
# dtype='float32',
dtype='int16', # to int, by Ting
mode='r',
shape=(rdn_header['lines'], rdn_header['bands'],
rdn_header['samples']))
# Get RGB bands.
if rdn_header['default bands'] is not None:
rgb_bands = rdn_header['default bands']
else:
rgb_bands = []
wave, _ = get_closest_wave(rdn_header['wavelength'], 450)
if abs(wave - 450) < 10:
for rgb_wave in [680, 550, 450]:
_, band = get_closest_wave(rdn_header['wavelength'], rgb_wave)
rgb_bands.append(band)
else:
for rgb_wave in [1220, 1656, 2146]:
_, band = get_closest_wave(rdn_header['wavelength'], rgb_wave)
rgb_bands.append(band)
del band, wave
# Read GLT image.
glt_header = read_envi_header(os.path.splitext(glt_image_file)[0] + '.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
shape=(2, glt_header['lines'],
glt_header['samples']))
# Write RGB image.
driver = gdal.GetDriverByName('GTiff')
ds = driver.Create(quicklook_figure_file, glt_header['samples'],
glt_header['lines'], 3, gdal.GDT_Byte)
ds.SetGeoTransform(
(float(glt_header['map info'][3]), float(glt_header['map info'][5]), 0,
float(glt_header['map info'][4]), 0,
-float(glt_header['map info'][6])))
ds.SetProjection(glt_header['coordinate system string'])
image = np.zeros((glt_header['lines'], glt_header['samples']),
dtype='uint8')
I, J = np.where((glt_image[0, :, :] >= 0) & (glt_image[1, :, :] >= 0))
for band_index, rgb_band in enumerate(rgb_bands):
image[:, :] = 0
tmp_image = linear_percent_stretch(rdn_image[:, rgb_band, :])
image[I, J] = tmp_image[glt_image[0, I, J], glt_image[1, I, J]]
ds.GetRasterBand(band_index + 1).WriteArray(image)
del tmp_image
glt_image.flush()
rdn_image.flush()
del glt_image, rdn_image
ds = None
del I, J, glt_header, rdn_header, image
logger.info('Save the quicklook figure to %s.' % quicklook_figure_file)
def calculate_sca(sca_image_file, imugps_file, igm_image_file, sun_angles):
""" Create a scan angle (SCA) image.
Arguments:
sca_image_file: str
Scan angle filename.
imu_gps_file: str
IMU/GPS filename.
igm_image_file: str
IGM image filename.
sun_angles: list
Sun angles [sun zenith, sun azimuth], in degrees.
"""
if os.path.exists(sca_image_file):
logger.info('Write the SCA to %s.' % sca_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read IGM data.
igm_header = read_envi_header(os.path.splitext(igm_image_file)[0] + '.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
offset=0,
shape=(igm_header['bands'], igm_header['lines'],
igm_header['samples']))
# Read GPS data.
imugps = np.loadtxt(imugps_file) # ID, X, Y, Z, R, P, H, ...
# Calculate sensor angles.
DX = igm_image[0, :, :] - np.expand_dims(imugps[:, 1], axis=1)
DY = igm_image[1, :, :] - np.expand_dims(imugps[:, 2], axis=1)
DZ = igm_image[2, :, :] - np.expand_dims(imugps[:, 3], axis=1)
igm_image.flush()
del imugps, igm_image
view_zenith = np.abs(np.arctan(np.sqrt(DX**2 + DY**2) / DZ))
index = view_zenith >= np.deg2rad(40.0)
if np.any(index):
view_zenith[index] = np.deg2rad(39.0)
del index
view_azimuth = np.arcsin(np.abs(DX) / np.sqrt(DX**2 + DY**2))
index = (DX > 0) & (DY < 0)
view_azimuth[index] = np.pi - view_azimuth[index]
index = (DX < 0) & (DY < 0)
view_azimuth[index] = np.pi + view_azimuth[index]
index = (DX < 0) & (DY > 0)
view_azimuth[index] = 2 * np.pi - view_azimuth[index]
del DX, DY, DZ, index
# Save scan angles.
fid = open(sca_image_file, 'wb')
fid.write(np.rad2deg(view_zenith).astype('float32').tostring())
fid.write(np.rad2deg(view_azimuth).astype('float32').tostring())
fid.close()
del view_zenith, view_azimuth
# Write scan angle header file.
sca_header = empty_envi_header()
sca_header['description'] = 'SCA [deg]'
sca_header['file type'] = 'ENVI Standard'
sca_header['samples'] = igm_header['samples']
sca_header['lines'] = igm_header['lines']
sca_header['bands'] = 2
sca_header['byte order'] = 0
sca_header['header offset'] = 0
sca_header['interleave'] = 'bsq'
sca_header['data type'] = 4
sca_header['band names'] = ['Sensor Zenith [deg]', 'Sensor Azimuth [deg]']
sca_header['sun zenith'] = sun_angles[0]
sca_header['sun azimuth'] = sun_angles[1]
sca_header_file = os.path.splitext(sca_image_file)[0] + '.hdr'
write_envi_header(sca_header_file, sca_header)
del sca_header
logger.info('Write the SCA to %s.' % sca_image_file)
def pre_classification(pre_class_image_file,
rdn_image_file,
sun_zenith,
distance,
background_mask_file=None):
""" Pre-classify the image.
Notes
-----
(1) The classification algorithm used here is from ATCOR.
Parameters
----------
pre_class_image_file: str
Pre-classification image filename.
rdn_image_file: str
Radiance image filename, either BIL or BSQ.
sun_zenith: float
Sun zenith angle in degrees.
distance: float
Sun-Earth distance.
background_mask_file: str
Background mask filename.
"""
if os.path.exists(pre_class_image_file):
logger.info('Write the pre-classification map to %s.' %
(pre_class_image_file))
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
from Spectra import get_closest_wave, resample_solar_flux
# Read radiance image data
rdn_header = read_envi_header(os.path.splitext(rdn_image_file)[0] + '.hdr')
if rdn_header['interleave'].lower() == 'bil':
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['lines'], rdn_header['bands'],
rdn_header['samples']))
elif rdn_header['interleave'].lower() == 'bsq':
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['bands'], rdn_header['lines'],
rdn_header['samples']))
else:
logger.error('Cannot read radiance data from %s format file.' %
rdn_header['interleave'])
# Read solar flux
solar_flux = resample_solar_flux(solar_flux_file, rdn_header['wavelength'],
rdn_header['fwhm'])
cos_sun_zenith = np.cos(np.deg2rad(sun_zenith))
d2 = distance**2
# Initialize classification
pre_class_image = np.memmap(pre_class_image_file,
dtype='uint8',
mode='w+',
shape=(rdn_header['lines'],
rdn_header['samples']))
pre_class_image[:, :] = 0
# Define VNIR sensor wavelengths
blue_wave, blue_band = get_closest_wave(rdn_header['wavelength'], 470)
green_wave, green_band = get_closest_wave(rdn_header['wavelength'], 550)
red_wave, red_band = get_closest_wave(rdn_header['wavelength'], 660)
nir_wave, nir_band = get_closest_wave(rdn_header['wavelength'], 850)
# Define SWIR sensor wavelengths
cirrus_wave, cirrus_band = get_closest_wave(rdn_header['wavelength'], 1380)
swir1_wave, swir1_band = get_closest_wave(rdn_header['wavelength'], 1650)
swir2_wave, swir2_band = get_closest_wave(rdn_header['wavelength'], 2200)
# Determine the sensor type
if_vnir = abs(blue_wave - 470) < 20 and abs(green_wave - 550) < 20 and abs(
red_wave - 660) < 20 and abs(nir_wave - 850) < 20
if_swir = abs(cirrus_wave - 1380) < 20 and abs(
swir1_wave - 1650) < 20 and abs(swir2_wave - 2200) < 20
if_whole = if_vnir and if_swir
# Do classification
if if_whole:
# Calculate the reflectance at different bands
if rdn_header['interleave'].lower() == 'bil':
blue_refl = rdn_image[:, blue_band, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[:, green_band, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[:, red_band, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[:, nir_band, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
cirrus_refl = rdn_image[:, cirrus_band, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[:, swir1_band, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[:, swir2_band, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
else:
blue_refl = rdn_image[blue_band, :, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[green_band, :, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[red_band, :, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[nir_band, :, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
cirrus_refl = rdn_image[cirrus_band, :, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[swir1_band, :, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[swir2_band, :, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
rdn_image.flush()
del rdn_image
# Calculate NDSI
ndsi = (green_refl - swir1_refl) / (green_refl + swir1_refl + 1e-10)
# Water
water = (nir_refl < 0.05) & (swir1_refl < 0.03)
# Land
land = ~water
# Shadow
shadow = (water | land) & (green_refl < 0.01)
land = land & (~shadow)
water = water & (~shadow)
# Cloud over land
Tc = 0.20
cloud_over_land = land & (blue_refl > Tc) & (red_refl > 0.15) & (
nir_refl < red_refl * 2.0) & (nir_refl > red_refl * 0.8) & (
nir_refl > swir1_refl) & (ndsi < 0.7)
land = land & (~cloud_over_land)
# Cloud over water
cloud_over_water = water & (blue_refl < 0.40) & (blue_refl > 0.20) & (
green_refl < blue_refl) & (nir_refl < green_refl) & (
swir1_refl < 0.15) & (ndsi < 0.20)
water = water & (~cloud_over_water)
# Cloud shadow
cloud_shadow = (water | land) & (nir_refl < 0.12) & (
nir_refl > 0.04) & (swir1_refl < 0.20)
land = land & (~cloud_shadow)
water = water & (~cloud_shadow)
# Snow & ice
snow_ice = land & (((blue_refl > 0.22) & (ndsi > 0.60)) |
((green_refl > 0.22) & (ndsi > 0.25) &
(swir2_refl < 0.5 * green_refl)))
land = land & (~snow_ice)
# Cirrus over land & water
thin_cirrus = (cirrus_refl < 0.015) & (cirrus_refl > 0.010)
medium_cirrus = (cirrus_refl < 0.025) & (cirrus_refl >= 0.015)
thick_cirrus = (cirrus_refl < 0.040) & (cirrus_refl >= 0.025)
thin_cirrus_over_land = land & thin_cirrus
land = land & (~thin_cirrus_over_land)
medium_cirrus_over_land = land & medium_cirrus
land = land & (~medium_cirrus_over_land)
thick_cirrus_over_land = land & thick_cirrus
land = land & (~thick_cirrus_over_land)
thin_cirrus_over_water = water & thin_cirrus
water = water & (~thin_cirrus_over_water)
medium_cirrus_over_water = water & thin_cirrus
water = water & (~medium_cirrus_over_water)
thick_cirrus_over_water = water & thin_cirrus
water = water & (~thick_cirrus_over_water)
cirrus_cloud = (water
| land) & (cirrus_refl < 0.050) & (cirrus_refl > 0.040)
land = land & (~cirrus_cloud)
water = water & (~cirrus_cloud)
thick_cirrus_cloud = (water | land) & (cirrus_refl > 0.050)
land = land & (~thick_cirrus_cloud)
water = water & (~thick_cirrus_cloud)
# Haze over water
T2 = 0.04
T1 = 0.12
thin_haze_over_water = water & (nir_refl >= T1) & (nir_refl <= 0.06)
water = water & (~thin_haze_over_water)
medium_haze_over_water = water & (nir_refl >= 0.06) & (nir_refl <= T2)
water = water & (~medium_haze_over_water)
# Assign class values
pre_class_image[shadow] = 1
pre_class_image[thin_cirrus_over_water] = 2
pre_class_image[medium_cirrus_over_water] = 3
pre_class_image[thick_cirrus_over_water] = 4
pre_class_image[land] = 5
pre_class_image[snow_ice] = 7
pre_class_image[thin_cirrus_over_land] = 8
pre_class_image[medium_cirrus_over_land] = 9
pre_class_image[thick_cirrus_over_land] = 10
pre_class_image[thin_haze_over_water] = 13
pre_class_image[medium_haze_over_water] = 14
pre_class_image[cloud_over_land] = 15
pre_class_image[cloud_over_water] = 16
pre_class_image[water] = 17
pre_class_image[cirrus_cloud] = 18
pre_class_image[thick_cirrus_cloud] = 19
pre_class_image[cloud_shadow] = 22
# Clear data
del shadow, thin_cirrus_over_water, medium_cirrus_over_water
del thick_cirrus_over_water, land, snow_ice
del thin_cirrus_over_land, medium_cirrus_over_land, thick_cirrus_over_land
del thin_haze_over_water, medium_haze_over_water, cloud_over_land
del cloud_over_water, water, cirrus_cloud
del thick_cirrus_cloud, cloud_shadow
elif if_vnir:
# Calculate the reflectance at different bands
if rdn_header['interleave'].lower() == 'bil':
blue_refl = rdn_image[:, blue_band, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[:, green_band, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[:, red_band, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[:, nir_band, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
else:
blue_refl = rdn_image[blue_band, :, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[green_band, :, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[red_band, :, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[nir_band, :, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
rdn_image.flush()
del rdn_image
# Water
water = nir_refl < 0.05
# Land
land = ~water
# Shadow
shadow = (water | land) & (green_refl < 0.01)
land = land & (~shadow)
water = water & (~shadow)
# Cloud over land
Tc = 0.20
cloud_over_land = land & (blue_refl > Tc) & (red_refl > 0.15) & (
nir_refl < red_refl * 2.0) & (nir_refl > red_refl * 0.8)
land = land & (~cloud_over_land)
# Cloud over water
cloud_over_water = water & (blue_refl < 0.40) & (blue_refl > 0.20) & (
green_refl < blue_refl) & (nir_refl < green_refl)
water = water & (~cloud_over_water)
# Cloud shadow
cloud_shadow = (water | land) & (nir_refl < 0.12) & (nir_refl > 0.04)
land = land & (~cloud_shadow)
water = water & (~cloud_shadow)
# Haze over water
T2 = 0.04
T1 = 0.12
thin_haze_over_water = water & (nir_refl >= T1) & (nir_refl <= 0.06)
water = water & (~thin_haze_over_water)
medium_haze_over_water = water & (nir_refl >= 0.06) & (nir_refl <= T2)
water = water & (~medium_haze_over_water)
# Assign class values
pre_class_image[shadow] = 1
pre_class_image[land] = 5
pre_class_image[thin_haze_over_water] = 13
pre_class_image[medium_haze_over_water] = 14
pre_class_image[cloud_over_land] = 15
pre_class_image[cloud_over_water] = 16
pre_class_image[water] = 17
pre_class_image[cloud_shadow] = 22
# Clear data
del shadow, land, thin_haze_over_water
del medium_haze_over_water, cloud_over_land, cloud_over_water
del water, cloud_shadow
elif if_swir:
# Calculate the reflectance at different bands
if rdn_header['interleave'] == 'bil':
cirrus_refl = rdn_image[:, cirrus_band, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[:, swir1_band, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[:, swir2_band, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
else:
cirrus_refl = rdn_image[cirrus_band, :, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[swir1_band, :, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[swir2_band, :, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
rdn_image.flush()
del rdn_image
# Water
water = swir1_refl < 0.03
# Land
land = ~water
# Cirrus over land & water
thin_cirrus = (cirrus_refl < 0.015) & (cirrus_refl > 0.010)
medium_cirrus = (cirrus_refl < 0.025) & (cirrus_refl >= 0.015)
thick_cirrus = (cirrus_refl < 0.040) & (cirrus_refl >= 0.025)
thin_cirrus_over_land = land & thin_cirrus
land = land & (~thin_cirrus_over_land)
medium_cirrus_over_land = land & medium_cirrus
land = land & (~medium_cirrus_over_land)
thick_cirrus_over_land = land & thick_cirrus
land = land & (~thick_cirrus_over_land)
thin_cirrus_over_water = water & thin_cirrus
water = water & (~thin_cirrus_over_water)
medium_cirrus_over_water = water & thin_cirrus
water = water & (~medium_cirrus_over_water)
thick_cirrus_over_water = water & thin_cirrus
water = water & (~thick_cirrus_over_water)
cirrus_cloud = (water
| land) & (cirrus_refl < 0.050) & (cirrus_refl > 0.040)
land = land & (~cirrus_cloud)
water = water & (~cirrus_cloud)
thick_cirrus_cloud = (water | land) & (cirrus_refl > 0.050)
land = land & (~thick_cirrus_cloud)
water = water & (~thick_cirrus_cloud)
# Assign class values
pre_class_image[thin_cirrus_over_water] = 2
pre_class_image[medium_cirrus_over_water] = 3
pre_class_image[thick_cirrus_over_water] = 4
pre_class_image[land] = 5
pre_class_image[thin_cirrus_over_land] = 8
pre_class_image[medium_cirrus_over_land] = 9
pre_class_image[thick_cirrus_over_land] = 10
pre_class_image[water] = 17
pre_class_image[cirrus_cloud] = 18
pre_class_image[thick_cirrus_cloud] = 19
# Clear data
del thin_cirrus_over_water, medium_cirrus_over_water, thick_cirrus_over_water,
del land, thin_cirrus_over_land, medium_cirrus_over_land, thick_cirrus_over_land
del water, cirrus_cloud, thick_cirrus_cloud
else:
logger.error(
'Cannot find appropriate wavelengths to do pre-classification.')
# Apply background mask
if background_mask_file is not None:
bg_mask_header = read_envi_header(
os.path.splitext(background_mask_file)[0] + '.hdr')
bg_mask_image = np.memmap(background_mask_file,
mode='r',
dtype='bool',
shape=(bg_mask_header['lines'],
bg_mask_header['samples']))
pre_class_image[bg_mask_image] = 0
bg_mask_image.flush()
del bg_mask_image
# Clear data
pre_class_image.flush()
del pre_class_image
# Write header
pre_class_header = empty_envi_header()
pre_class_header['description'] = 'Pre-classification'
pre_class_header['file type'] = 'ENVI Standard'
pre_class_header['samples'] = rdn_header['samples']
pre_class_header['lines'] = rdn_header['lines']
pre_class_header['classes'] = 23
pre_class_header['bands'] = 1
pre_class_header['byte order'] = 0
pre_class_header['header offset'] = 0
pre_class_header['interleave'] = 'bsq'
pre_class_header['data type'] = 1
pre_class_header['class names'] = [
'0: geocoded background', '1: shadow', '2: thin cirrus (water)',
'3: medium cirrus (water)', '4: thick cirrus (water)',
'5: land (clear)', '6: saturated', '7: snow/ice',
'8: thin cirrus (land)', '9: medium cirrus (land)',
'10: thick cirrus (land)', '11: thin haze (land)',
'12: medium haze (land)', '13: thin haze/glint (water)',
'14: med. haze/glint (water)', '15: cloud (land)', '16: cloud (water)',
'17: water', '18: cirrus cloud', '19: cirrus cloud (thick)',
'20: bright', '21: topogr. shadow', '22: cloud shadow'
]
pre_class_header['class lookup'] = [
80, 80, 80, 0, 0, 0, 0, 160, 250, 0, 200, 250, 0, 240, 250, 180, 100,
40, 200, 0, 0, 250, 250, 250, 240, 240, 170, 225, 225, 150, 210, 210,
120, 250, 250, 100, 230, 230, 80, 80, 100, 250, 130, 130, 250, 180,
180, 180, 160, 160, 240, 0, 0, 250, 180, 180, 100, 160, 160, 100, 200,
200, 200, 40, 40, 40, 50, 50, 50
]
pre_class_header['map info'] = rdn_header['map info']
pre_class_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(
os.path.splitext(pre_class_image_file)[0] + '.hdr', pre_class_header)
logger.info('Write the pre-classification map to %s.' %
(pre_class_image_file))
def estimate_vis(vis_file, ddv_file, atm_lut_file, rdn_file, sca_file,
background_mask_file):
""" Estimate visibility.
Arguments:
vis_file: str
Visibility map filename.
ddv_file: str
Dark dense vegetation map filename.
atm_lut_file: str
Atmospheric lookup table filename.
rdn_file: str
Radiance filename.
sca_file: str
Scan angle filename.
background_mask_file:
Background mask filename.
"""
if os.path.exists(ddv_file) and os.path.exists(vis_file):
logger.info('Write the visibility map to %s.' % vis_file)
logger.info('Write the DDV map to %s.' % ddv_file)
return
from ENVI import read_envi_header, empty_envi_header, write_envi_header
from AtmLUT import read_binary_metadata
from Spectra import get_closest_wave
from AtmCorr import atm_corr_band
# Read radiance header.
rdn_header = read_envi_header(os.path.splitext(rdn_file)[0] + '.hdr')
# Find VNIR and SWIR sensor wavelengths.
red_wave, red_band = get_closest_wave(rdn_header['wavelength'], 660)
nir_wave, nir_band = get_closest_wave(rdn_header['wavelength'], 850)
swir1_wave, swir1_band = get_closest_wave(rdn_header['wavelength'], 1650)
swir2_wave, swir2_band = get_closest_wave(rdn_header['wavelength'], 2130)
# Determine the sensor type.
if_vnir = abs(red_wave - 660) < 20 and abs(nir_wave - 850) < 20
if_swir = abs(swir1_wave - 1650) < 20 or abs(swir2_wave - 2130) < 20
if if_vnir and if_swir:
logger.info(
'Both VNIR and SWIR bands are used for estimating visibility.')
elif if_vnir:
logger.info('Only VNIR bands are used for estimating visibility.')
elif if_swir:
logger.info(
'Only SWIR bands are available, a constant visibility (23 km) is used.'
)
else:
logger.error(
'Cannot find appropriate bands for estimating visibility.')
# Read atmospheric lookup table.
atm_lut_metadata = read_binary_metadata(atm_lut_file + '.meta')
atm_lut_metadata['shape'] = tuple(
[int(v) for v in atm_lut_metadata['shape']])
atm_lut_RHO = np.array([float(v) for v in atm_lut_metadata['RHO']])
atm_lut_WVC = np.array([float(v) for v in atm_lut_metadata['WVC']])
atm_lut_VIS = np.array([float(v) for v in atm_lut_metadata['VIS']])
atm_lut_VZA = np.array([float(v) for v in atm_lut_metadata['VZA']])
atm_lut_RAA = np.array([float(v) for v in atm_lut_metadata['RAA']])
atm_lut = np.memmap(atm_lut_file,
dtype=atm_lut_metadata['dtype'],
mode='r',
shape=atm_lut_metadata['shape']
) # shape = (RHO, WVC, VIS, VZA, RAA, WAVE)
# Read radiance image.
rdn_image = np.memmap(
rdn_file,
# dtype='float32',
dtype='int16', # in int, by Ting
mode='r',
shape=(rdn_header['bands'], rdn_header['lines'],
rdn_header['samples']))
# Read VZA and RAA image.
sca_header = read_envi_header(os.path.splitext(sca_file)[0] + '.hdr')
saa = float(sca_header['sun azimuth'])
sca_image = np.memmap(sca_file,
dtype='float32',
offset=0,
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# vza
vza_image = np.copy(sca_image[0, :, :])
# raa
raa_image = saa - sca_image[1, :, :]
raa_image[raa_image < 0] += 360.0
raa_image[raa_image > 180] = 360.0 - raa_image[raa_image > 180]
raa_image[raa_image == 180] = 179.0
# Constrain RAA within bounds of ALT
raa_max = atm_lut_RAA.max()
raa_image[raa_image > raa_max] = raa_max - 5e-2
# clear data
sca_image.flush()
del sca_header, saa, sca_image
# Set visibility and water vapor column values.
metadata = read_binary_metadata(atm_lut_file + '.meta')
tmp_wvc_image = np.ones(
vza_image.shape) * atm_db_wvc_lut[metadata['atm_mode']]
tmp_vis_image = np.ones(vza_image.shape) * 23
del metadata
# Read background mask.
bg_header = read_envi_header(
os.path.splitext(background_mask_file)[0] + '.hdr')
bg_mask = np.memmap(background_mask_file,
dtype='bool',
mode='r',
shape=(bg_header['lines'], bg_header['samples']))
# Calculate NDVI.
red_refl = atm_corr_band(
atm_lut_WVC,
atm_lut_VIS,
atm_lut_VZA,
atm_lut_RAA,
atm_lut[..., red_band],
tmp_wvc_image,
tmp_vis_image,
vza_image,
raa_image,
rdn_image[red_band, :, :] /
1000, # divided by 1000 to convert to original rdn, by Ting
bg_mask)
nir_refl = atm_corr_band(atm_lut_WVC, atm_lut_VIS, atm_lut_VZA,
atm_lut_RAA, atm_lut[...,
nir_band], tmp_wvc_image,
tmp_vis_image, vza_image, raa_image,
rdn_image[nir_band, :, :] / 1000, bg_mask)
ndvi = (nir_refl - red_refl) / (nir_refl + red_refl + 1e-10)
vis_image = np.zeros((rdn_header['lines'], rdn_header['samples']))
if if_vnir and if_swir:
# Decide which SWIR band to use.
if abs(swir2_wave - 2130) < 20:
swir_wave = swir2_wave
swir_band = swir2_band
swir_refl_upper_bounds = [0.05, 0.10, 0.12]
red_swir_ratio = 0.50
else:
swir_wave = swir1_wave
swir_band = swir1_band
swir_refl_upper_bounds = [0.10, 0.15, 0.18]
red_swir_ratio = 0.25
# Calculate swir refletance.
swir_refl = atm_corr_band(
atm_lut_WVC,
atm_lut_VIS,
atm_lut_VZA,
atm_lut_RAA,
atm_lut[..., swir_band],
tmp_wvc_image,
tmp_vis_image,
vza_image,
raa_image,
rdn_image[swir_band, :, :] /
1000, # divided by 1000 to convert to original rdn, by Ting
bg_mask)
# Get DDV mask.
for swir_refl_upper_bound in swir_refl_upper_bounds:
ddv_mask = (ndvi > 0.10) & (swir_refl < swir_refl_upper_bound) & (
swir_refl > 0.01)
percentage = np.sum(ddv_mask[~bg_mask]) / np.sum(~bg_mask)
if percentage > 0.02:
logger.info(
'The SWIR wavelength %.2f is used for detecting dark dense vegetation.'
% swir_wave)
logger.info('The SWIR reflectance upper boundary is %.2f.' %
swir_refl_upper_bound)
logger.info('The number of DDV pixels is %.2f%%.' %
(percentage * 100))
break
# Estimate Visibility.
rows, columns = np.where(ddv_mask)
# Estimate red reflectance.
red_refl[rows, columns] = red_swir_ratio * swir_refl[rows, columns]
del swir_refl
for row, column in zip(rows, columns):
interp_rdn = interp_atm_lut(atm_lut_RHO, atm_lut_WVC, atm_lut_VZA,
atm_lut_RAA, atm_lut[..., red_band],
red_refl[row, column],
tmp_wvc_image[row, column],
vza_image[row,
column], raa_image[row,
column])
vis_image[row, column] = np.interp(
rdn_image[red_band, row, column] / 1000, interp_rdn[::-1],
atm_lut_VIS[::-1]
) # divided by 1000 to convert to original rdn, by Ting
rdn_image.flush()
del rdn_image
logger.info(
'Visibility [km] statistics: min=%.2f, max=%.2f, avg=%.2f, sd=%.2f.'
% (vis_image[ddv_mask].min(), vis_image[ddv_mask].max(),
vis_image[ddv_mask].mean(), vis_image[ddv_mask].std()))
# Fill gaps with average values.
vis_image[~ddv_mask] = vis_image[ddv_mask].mean()
vis_image[bg_mask] = -1000.0
# Write the visibility data.
fid = open(vis_file, 'wb')
fid.write(vis_image.astype('float32').tostring())
fid.close()
del vis_image
vis_header = empty_envi_header()
vis_header['description'] = 'Visibility [km]'
vis_header['samples'] = rdn_header['samples']
vis_header['lines'] = rdn_header['lines']
vis_header['bands'] = 1
vis_header['byte order'] = 0
vis_header['header offset'] = 0
vis_header['interleave'] = 'bsq'
vis_header['data ignore value'] = -1000.0
vis_header['data type'] = 4
vis_header['map info'] = rdn_header['map info']
vis_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(os.path.splitext(vis_file)[0] + '.hdr', vis_header)
logger.info('Write the visibility image to %s.' % vis_file)
# Write the DDV data.
fid = open(ddv_file, 'wb')
fid.write(ddv_mask.tostring())
fid.close()
del ddv_mask
ddv_header = empty_envi_header()
ddv_header[
'description'] = 'DDV mask (1: Dark dense vegetaion; 0: non-dark dense vegetation)'
ddv_header['samples'] = rdn_header['samples']
ddv_header['lines'] = rdn_header['lines']
ddv_header['bands'] = 1
ddv_header['byte order'] = 0
ddv_header['header offset'] = 0
ddv_header['interleave'] = 'bsq'
ddv_header['data type'] = 1
ddv_header['map info'] = rdn_header['map info']
ddv_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(os.path.splitext(ddv_file)[0] + '.hdr', ddv_header)
logger.info('Write the DDV mask to %s.' % ddv_file)
elif if_vnir:
pass
elif if_swir:
pass
# Clear data
del ndvi, red_refl, nir_refl, rdn_header
def resample_rdn(resampled_rdn_image_file, raw_rdn_image_file,
smile_effect_file):
""" Resample radiance spectra.
Parameters
----------
resampled_rdn_image_file: str
Resampled radiance image filename.
raw_rdn_image_file: str
Raw radiance image filename.
smile_effect_file: str
Smile effect filename.
"""
if os.path.exists(resampled_rdn_image_file):
logger.info('Write the spectrally resampled radiance image to %s.' %
resampled_rdn_image_file)
return
from ENVI import read_envi_header, write_envi_header
from scipy import interpolate
# Read raw radiance image
raw_rdn_header = read_envi_header(
os.path.splitext(raw_rdn_image_file)[0] + '.hdr')
raw_rdn_image = np.memmap(raw_rdn_image_file,
dtype='float32',
mode='r',
shape=(raw_rdn_header['lines'],
raw_rdn_header['bands'],
raw_rdn_header['samples']))
# Read spectral center wavelengths & bandwidths
smile_effect_header = read_envi_header(
os.path.splitext(smile_effect_file)[0] + '.hdr')
smile_effect_data = np.memmap(smile_effect_file,
dtype='float32',
mode='r',
shape=(smile_effect_header['bands'],
smile_effect_header['lines'],
smile_effect_header['samples']))
# Do spectral interpolation
info = 'Line (max=%d): ' % smile_effect_header['lines']
fid = open(resampled_rdn_image_file, 'wb')
for from_line in range(0, raw_rdn_header['lines'], 500):
info += '%d, ' % (from_line + 1)
# Determine chunk size
to_line = min(from_line + 500, raw_rdn_header['lines'])
# Initialize
rdn = np.zeros((to_line - from_line, raw_rdn_header['bands'],
raw_rdn_header['samples']
)) # shape=(from_line:to_line, bands, samples)
# Do spectral interpolation
for sample in range(raw_rdn_header['samples']):
# NOTE: No NaN values may be present in the input!
f = interpolate.interp1d(smile_effect_data[0, :, sample],
raw_rdn_image[from_line:to_line, :,
sample],
kind='cubic',
fill_value='extrapolate',
axis=1)
rdn[:, :, sample] = f(raw_rdn_header['wavelength'])
del f
# Write interpolated radiance to file
fid.write(rdn.astype('float32').tostring())
del rdn, to_line
fid.close()
info += '%d, Done!' % smile_effect_header['lines']
logger.info(info)
# Clear data
del smile_effect_data
raw_rdn_image.flush()
del raw_rdn_image
# Write header
write_envi_header(
os.path.splitext(resampled_rdn_image_file)[0] + '.hdr', raw_rdn_header)
del raw_rdn_header
logger.info('Write the spectrally resampled radiance image to %s.' %
resampled_rdn_image_file)
def plot_image_area(image_area_figure_file, dem_image_file, igm_image_file,
imugps_file):
""" Plot image area (DEM is used as the background).
Arguments:
image_area_figure_file: str
Image area figure filename.
dem_image_file: str
DEM image filename.
igm_image_file: str
IGM image filename.
imugps_file: str
IMUGPS filename.
"""
if os.path.exists(image_area_figure_file):
logger.info('Save the image-area figure to %s.' %
image_area_figure_file)
return
from ENVI import read_envi_header
# Read DEM.
ds = gdal.Open(dem_image_file, gdal.GA_ReadOnly)
dem_image = ds.GetRasterBand(1).ReadAsArray()
dem_geotransform = ds.GetGeoTransform()
ds = None
# Read IGM.
igm_header = read_envi_header(os.path.splitext(igm_image_file)[0] + '.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
offset=0,
shape=(2, igm_header['lines'],
igm_header['samples']))
cols = (igm_image[0, :, :] - dem_geotransform[0]) / dem_geotransform[1]
rows = (igm_image[1, :, :] - dem_geotransform[3]) / dem_geotransform[5]
igm_image.flush()
del igm_image
# Read IMUGPS
imugps = np.loadtxt(imugps_file)
# Make a plot
plt.figure(figsize=(10, 10.0 * dem_image.shape[0] / dem_image.shape[1]))
plt.imshow(dem_image,
cmap='gray',
vmin=dem_image.min(),
vmax=dem_image.max())
del dem_image
plt.plot(cols[:, 0],
rows[:, 0],
'-',
color='lime',
lw=2,
label='Image Area')
plt.plot(cols[:, -1], rows[:, -1], '-', color='lime', lw=2)
plt.plot(cols[0, :], rows[0, :], '-', color='lime', lw=2)
plt.plot(cols[-1, :], rows[-1, :], '-', color='lime', lw=2)
cols = (imugps[:, 1] - dem_geotransform[0]) / dem_geotransform[1]
rows = (imugps[:, 2] - dem_geotransform[3]) / dem_geotransform[5]
plt.plot(cols, rows, '--', color='red', lw=2, label='Flight')
plt.scatter(cols[0], rows[0], c='navy', s=20, label='Start')
plt.scatter(cols[-1], rows[-1], c='orange', s=20, label='End')
plt.xticks([])
plt.yticks([])
plt.legend(fontsize=5)
plt.savefig(image_area_figure_file, bbox_inches='tight')
plt.close()
del cols, rows, imugps
logger.info('Save the image-area figure to %s.' % image_area_figure_file)
def hyspex_extract(src_img, bands, out_img):
"""
extract certain bands from hyspex images (note: images are in BSQ)
:param src_img:
:param out_img:
:param bands: list; bands to be extracted, starting from 0
:return:
"""
src_header = read_envi_header(src_img + '.hdr')
# src_image = np.memmap(src_img,
# dtype='int16',
# mode='r',
# offset=src_header['header offset'],
# shape=(src_header['lines'],
# src_header['samples'],
# src_header['bands'],))
# Create empty array for the output image:
# out = np.memmap(out_img, dtype='int16',
# mode='w+',
# shape=(src_header['lines'],
# src_header['samples'],
# len(bands))
# )
# Read the given bands:
i = 0
with open(out_img, 'wb') as outfile:
for b in bands:
print('Now reading band {}'.format(b))
src_image = np.memmap(
src_img,
dtype='int16',
mode='r',
shape=(src_header['lines'], src_header['samples']),
offset=int(src_header['lines'] * src_header['samples'] * b *
16 / 8))
# # Create empty array for the output image:
# out = np.memmap(out_img, dtype='int16',
# mode='w+',
# shape=(src_header['lines'],
# src_header['samples']),
# offset=int(src_header['lines'] * src_header['samples'] * i * 16 / 8))
# out[:, :, i] = src_image[:]
outfile.write(src_image[:])
del src_image
i = i + 1
outfile.close()
# flush memory:
# out.flush()
# src_image.flush()
# Modify the header for out_image:
tgt_header = src_header
# number of bands:
tgt_header['bands'] = len(bands)
# wavelength:
wvl = [src_header['wavelength'][b] for b in bands]
tgt_header['wavelength'] = wvl
# fwhm:
fwhm = [src_header['fwhm'][b] for b in bands]
tgt_header['fwhm'] = fwhm
write_envi_header(out_img + '.hdr', tgt_header)
def average_rdn(avg_rdn_file, rdn_image_file, sca_image_file,
pre_class_image_file):
""" Average radiance along each column.
Parameters
----------
avg_rdn_file: str
Average radiance data filename.
rdn_image_file: 3D array
Radiance image filename, in BIL format.
sca_image_file: 3D array
Scan angle image filename, in BSQ format.
pre_class_image_file: str
Pre-classification image filename.
"""
if os.path.exists(avg_rdn_file):
logger.info('Write the averaged radiance data to %s.' % avg_rdn_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read radiance image data
rdn_header = read_envi_header(os.path.splitext(rdn_image_file)[0] + '.hdr')
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['lines'], rdn_header['bands'],
rdn_header['samples']))
# Read classification map
pre_class_header = read_envi_header(
os.path.splitext(pre_class_image_file)[0] + '.hdr')
pre_class_image = np.memmap(pre_class_image_file,
dtype='uint8',
mode='r',
shape=(pre_class_header['lines'],
pre_class_header['samples']))
mask = pre_class_image == 5 # 5: land (clear)
pre_class_image.flush()
del pre_class_header, pre_class_image
# Average radiance along each column
fid = open(avg_rdn_file, 'wb')
info = 'Band (max=%d): ' % rdn_header['bands']
for band in range(rdn_header['bands']):
if band % 20 == 0:
info += '%d, ' % (band + 1)
# Build temporary mask
tmp_mask = mask & (rdn_image[:, band, :] > 0.0)
# Average radiance
rdn = np.ma.array(rdn_image[:, band, :],
mask=~tmp_mask).mean(axis=0) # shape = (samples,1)
# Interpolate bad radiance values
for bad_sample in np.where(rdn.mask)[0]:
if bad_sample == 0:
rdn[bad_sample] = rdn[bad_sample + 1]
elif bad_sample == rdn_header['samples'] - 1:
rdn[bad_sample] = rdn[bad_sample - 1]
else:
rdn[bad_sample] = (rdn[bad_sample - 1] +
rdn[bad_sample + 1]) / 2.0
# Write average radiance data to file
fid.write(rdn.data.astype('float32'))
# Clear data
del tmp_mask, rdn
fid.close()
info += str(rdn_header['bands'])
logger.info(info)
rdn_image.flush()
del rdn_image
# Read scan angles
sca_header = read_envi_header(os.path.splitext(sca_image_file)[0] + '.hdr')
sca_image = np.memmap(sca_image_file,
dtype='float32',
mode='r',
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
saa = float(sca_header['sun azimuth'])
# Average scan angles
avg_vza = np.ma.array(sca_image[0, :, :], mask=~mask).mean(axis=0)
avg_vaa = np.ma.array(sca_image[1, :, :], mask=~mask).mean(axis=0)
avg_raa = saa - avg_vaa
avg_raa[avg_raa < 0] += 360.0
avg_raa[avg_raa > 180] = 360.0 - avg_raa[avg_raa > 180]
sca_image.flush()
del sca_image
# Write header
avg_rdn_header = empty_envi_header()
avg_rdn_header['description'] = 'Averaged radiance in [mW/(cm2*um*sr)]'
avg_rdn_header['samples'] = rdn_header['samples']
avg_rdn_header['lines'] = rdn_header['bands']
avg_rdn_header['bands'] = 1
avg_rdn_header['byte order'] = 0
avg_rdn_header['header offset'] = 0
avg_rdn_header['interleave'] = 'bsq'
avg_rdn_header['data type'] = 4
avg_rdn_header['waves'] = rdn_header['wavelength']
avg_rdn_header['fwhms'] = rdn_header['fwhm']
avg_rdn_header['VZA'] = list(avg_vza)
avg_rdn_header['RAA'] = list(avg_raa)
write_envi_header(avg_rdn_file + '.hdr', avg_rdn_header)
logger.info('Write the averaged radiance data to %s.' % avg_rdn_file)
def build_glt(glt_image_file, igm_image_file, pixel_size, map_crs):
""" Create a geographic lookup table (GLT) image.
Notes:
(1) This code is adapted from Adam Chlus's ([email protected]) script.
(2) The GLT image consists of two bands:
Band 0: Sample Lookup:
Pixel values indicate the column number of the pixel
in the input geometry file that belongs at the given Y
location in the output image.
Band 1: Line Lookup:
Pixel values indicate the row number of the pixel
in the input geometry file that belongs at the given X
location in the output image.
(3) For more details about GLT, refer to https://www.harrisgeospatial.com/
docs/GeoreferenceFromInputGeometry.html.
Arguments:
glt_image_file: str
Geographic look-up table filename.
igm_image_file: str
Input geometry filename.
pixel_size: float
Output image pixel size.
map_crs: osr object
GLT image map coordinate system.
"""
if os.path.exists(glt_image_file):
logger.info('Write the GLT to %s.' % glt_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read IGM.
igm_header = read_envi_header(os.path.splitext(igm_image_file)[0] + '.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
offset=0,
shape=(igm_header['bands'], igm_header['lines'],
igm_header['samples']))
# Estimate output spatial extent.
X_Min = igm_image[0, :, :].min()
X_Max = igm_image[0, :, :].max()
Y_Min = igm_image[1, :, :].min()
Y_Max = igm_image[1, :, :].max()
X_Min = np.floor(X_Min / pixel_size) * pixel_size - pixel_size
X_Max = np.ceil(X_Max / pixel_size) * pixel_size + pixel_size
Y_Min = np.floor(Y_Min / pixel_size) * pixel_size - pixel_size
Y_Max = np.ceil(Y_Max / pixel_size) * pixel_size + pixel_size
igm_image.flush()
del igm_image
# Build VRT for IGM.
igm_vrt_file = os.path.splitext(igm_image_file)[0] + '.vrt'
igm_vrt = open(igm_vrt_file, 'w')
igm_vrt.write('<VRTDataset rasterxsize="%s" rasterysize="%s">\n' %
(igm_header['samples'], igm_header['lines']))
for band in range(igm_header['bands']):
igm_vrt.write('\t<VRTRasterBand dataType="%s" band="%s">\n' %
("Float64", band + 1))
igm_vrt.write('\t\t<SimpleSource>\n')
igm_vrt.write(
'\t\t\t<SourceFilename relativeToVRT="1">%s</SourceFilename>\n' %
igm_image_file.replace('&', '&'))
igm_vrt.write('\t\t\t<SourceBand>%s</SourceBand>\n' % (band + 1))
igm_vrt.write(
'\t\t\t<SourceProperties RasterXSize="%s" RasterYSize="%s" DataType="%s" BlockXSize="%s" BlockYSize="%s" />\n'
% (igm_header['samples'], igm_header['lines'], "Float64",
igm_header['samples'], 1))
igm_vrt.write(
'\t\t\t<SrcRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(igm_header['samples'], igm_header['lines']))
igm_vrt.write(
'\t\t\t<DstRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(igm_header['samples'], igm_header['lines']))
igm_vrt.write("\t\t</SimpleSource>\n")
igm_vrt.write("\t</VRTRasterBand>\n")
igm_vrt.write("</VRTDataset>\n")
igm_vrt.close()
# Make IGM index image.
index_image_file = igm_image_file + '_Index'
index_image = np.memmap(index_image_file,
dtype='int32',
mode='w+',
offset=0,
shape=(2, igm_header['lines'],
igm_header['samples']))
index_image[0, :, :], index_image[1, :, :] = np.mgrid[
0:igm_header['lines'], 0:igm_header['samples']]
index_image.flush()
del index_image
index_header = empty_envi_header()
index_header['description'] = 'IGM Image Index'
index_header['samples'] = igm_header['samples']
index_header['lines'] = igm_header['lines']
index_header['bands'] = 2
index_header['byte order'] = 0
index_header['header offset'] = 0
index_header['interleave'] = 'bsq'
index_header['data type'] = 3
index_header['band names'] = ['Image Row', 'Image Column']
index_header_file = os.path.splitext(index_image_file)[0] + '.hdr'
write_envi_header(index_header_file, index_header)
# Build VRT for IGM Index.
index_vrt_file = os.path.splitext(index_image_file)[0] + '.vrt'
index_vrt = open(index_vrt_file, 'w')
index_vrt.write('<VRTDataset rasterxsize="%s" rasterysize="%s">\n' %
(index_header['samples'], index_header['lines']))
index_vrt.write('\t<Metadata Domain = "GEOLOCATION">\n')
index_vrt.write('\t\t<mdi key="X_DATASET">%s</mdi>\n' %
igm_image_file.replace('&', '&'))
index_vrt.write('\t\t<mdi key="X_BAND">1</mdi>\n')
index_vrt.write('\t\t<mdi key="Y_DATASET">%s</mdi>\n' %
igm_image_file.replace('&', '&'))
index_vrt.write('\t\t<mdi key="Y_BAND">2</mdi>\n')
index_vrt.write('\t\t<mdi key="PIXEL_OFFSET">0</mdi>\n')
index_vrt.write('\t\t<mdi key="LINE_OFFSET">0</mdi>\n')
index_vrt.write('\t\t<mdi key="PIXEL_STEP">1</mdi>\n')
index_vrt.write('\t\t<mdi key="LINE_STEP">1</mdi>\n')
index_vrt.write('\t</Metadata>\n')
for band in range(index_header['bands']):
index_vrt.write('\t<VRTRasterBand dataType="%s" band="%s">\n' %
("Int16", band + 1))
index_vrt.write('\t\t<SimpleSource>\n')
index_vrt.write(
'\t\t\t<SourceFilename relativeToVRT="1">%s</SourceFilename>\n' %
index_image_file.replace('&', '&'))
index_vrt.write('\t\t\t<SourceBand>%s</SourceBand>\n' % (band + 1))
index_vrt.write(
'\t\t\t<SourceProperties RasterXSize="%s" RasterYSize="%s" DataType="%s" BlockXSize="%s" BlockYSize="%s" />\n'
% (index_header['samples'], index_header['lines'], "Int32",
index_header['samples'], 1))
index_vrt.write(
'\t\t\t<SrcRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(index_header['samples'], index_header['lines']))
index_vrt.write(
'\t\t\t<DstRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(index_header['samples'], index_header['lines']))
index_vrt.write("\t\t</SimpleSource>\n")
index_vrt.write("\t</VRTRasterBand>\n")
index_vrt.write("</VRTDataset>\n")
index_vrt.close()
# Build GLT.
tmp_glt_image_file = glt_image_file + '.tif'
tmp_glt_image = gdal.Warp(tmp_glt_image_file,
index_vrt_file,
outputBounds=(X_Min, Y_Min, X_Max, Y_Max),
xRes=pixel_size,
yRes=pixel_size,
resampleAlg='near',
dstSRS=map_crs.ExportToProj4(),
dstNodata=-1,
multithread=True,
geoloc=True)
del tmp_glt_image
# Convert the .tif file to ENVI format.
ds = gdal.Open(tmp_glt_image_file, gdal.GA_ReadOnly)
lines = ds.RasterYSize
samples = ds.RasterXSize
glt_image = np.memmap(glt_image_file,
dtype='int32',
mode='w+',
offset=0,
shape=(2, lines, samples))
for band in range(ds.RasterCount):
glt_image[band, :, :] = ds.GetRasterBand(band + 1).ReadAsArray()
glt_image.flush()
del glt_image
ds = None
# Write GLT header.
glt_header = empty_envi_header()
glt_header['description'] = 'GLT'
glt_header['file type'] = 'ENVI Standard'
glt_header['samples'] = samples
glt_header['lines'] = lines
glt_header['bands'] = 2
glt_header['byte order'] = 0
glt_header['header offset'] = 0
glt_header['interleave'] = 'bsq'
glt_header['data type'] = 3
glt_header['data ignore value'] = -1
glt_header['band names'] = ['Image Row', 'Image Column']
glt_header['coordinate system string'] = map_crs.ExportToWkt()
glt_header['map info'] = [
map_crs.GetAttrValue('projcs').replace(',', ''), 1, 1, X_Min, Y_Max,
pixel_size, pixel_size, ' ', ' ',
map_crs.GetAttrValue('datum'),
map_crs.GetAttrValue('unit')
]
if map_crs.GetAttrValue('PROJECTION').lower() == 'transverse_mercator':
glt_header['map info'][7] = map_crs.GetUTMZone()
if Y_Max > 0.0:
glt_header['map info'][8] = 'North'
else:
glt_header['map info'][8] = 'South'
glt_header_file = os.path.splitext(glt_image_file)[0] + '.hdr'
write_envi_header(glt_header_file, glt_header)
# Remove temporary files.
os.remove(index_image_file)
os.remove(index_vrt_file)
os.remove(index_header_file)
os.remove(igm_vrt_file)
os.remove(tmp_glt_image_file)
logger.info('Write the GLT to %s.' % glt_image_file)
def merge_dem_sca(background_mask_file, merged_dem_file, merged_sca_file,
sensors):
""" Merge DEM and SCA images.
Arguments:
background_mask_file: str
Background mask filename.
merged_dem_file: str
Merged DEM file.
merged_sca_file: str
Merged SCA file.
sensors: dict
Sensor dictionaries.
"""
if os.path.exists(background_mask_file) and os.path.exists(
merged_dem_file) and os.path.exists(merged_sca_file):
logger.info('Write the background mask to %s.' % background_mask_file)
logger.info('Write the merged DEM image to %s.' % merged_dem_file)
logger.info('Write the merged SCA image to %s.' % merged_sca_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
"""
Get spatial extent and pixel size.
"""
ulx, uly, lrx, lry, pixel_size = -np.inf, np.inf, np.inf, -np.inf, np.inf
for sensor_index, sensor_dict in sensors.items():
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_dem_image_file'])[0] + '.hdr')
tmp_ulx, tmp_uly = float(tmp_header['map info'][3]), float(
tmp_header['map info'][4])
tmp_pixel_size = float(tmp_header['map info'][5])
tmp_lrx, tmp_lry = tmp_ulx + tmp_header[
'samples'] * tmp_pixel_size, tmp_uly - tmp_header[
'lines'] * tmp_pixel_size
if tmp_ulx > ulx:
ulx = tmp_ulx
if tmp_uly < uly:
uly = tmp_uly
if tmp_lrx < lrx:
lrx = tmp_lrx
if tmp_lry > lry:
lry = tmp_lry
if tmp_pixel_size < pixel_size:
pixel_size = tmp_pixel_size
del tmp_ulx, tmp_uly, tmp_pixel_size, tmp_lrx, tmp_lry
# Process at 2x the smallest pixel size
pixel_size = 2 * pixel_size
# Find lower right corners of first CRS-aligned pixels at the
# upper left & lower right corners which are fully in the image
ulx, uly = np.ceil(ulx / pixel_size +
1) * pixel_size, np.floor(uly / pixel_size -
1) * pixel_size
lrx, lry = np.floor(lrx / pixel_size -
1) * pixel_size, np.ceil(lry / pixel_size +
1) * pixel_size
map_info = [
tmp_header['map info'][0], 1, 1, ulx, uly, pixel_size, pixel_size
] + tmp_header['map info'][7:]
crs = tmp_header['coordinate system string']
del tmp_header
logger.info('The spatial range and pixel size of merged images:')
logger.info('Map x = %.2f - %.2f' % (ulx, lrx))
logger.info('Map y = %.2f - %.2f' % (lry, uly))
logger.info('Pixel size = %.2f' % pixel_size)
"""
Determine regular map grids.
"""
x, y = np.meshgrid(np.arange(ulx, lrx, pixel_size),
np.arange(uly, lry, -pixel_size))
mask = np.full(x.shape, True, dtype='bool')
"""
Build a background mask.
"""
# Use DEM and SCA images to build this mask.
for sensor_index, sensor_dict in sensors.items():
# Use dem.
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_dem_image_file'])[0] + '.hdr')
tmp_image = np.memmap(sensor_dict['ortho_dem_image_file'],
dtype='float32',
mode='r',
shape=(tmp_header['lines'],
tmp_header['samples']))
tmp_ulx, tmp_uly = float(tmp_header['map info'][3]), float(
tmp_header['map info'][4])
tmp_pixel_size = float(tmp_header['map info'][5])
resampled_image = resample_ortho_dem(np.copy(tmp_image), tmp_ulx,
tmp_uly, tmp_pixel_size, x, y)
mask = mask & (resampled_image > 0.0)
# Use sca.
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_sca_image_file'])[0] + '.hdr')
tmp_image = np.memmap(sensor_dict['ortho_sca_image_file'],
dtype='float32',
mode='r',
shape=(tmp_header['bands'], tmp_header['lines'],
tmp_header['samples']))
tmp_ulx, tmp_uly = float(tmp_header['map info'][3]), float(
tmp_header['map info'][4])
tmp_pixel_size = float(tmp_header['map info'][5])
resampled_image = resample_ortho_sca(np.copy(tmp_image[0, :, :]),
tmp_ulx, tmp_uly, tmp_pixel_size,
x, y)
mask = mask & (resampled_image > 0.0)
# Clear data.
del tmp_header, tmp_ulx, tmp_uly, tmp_pixel_size, resampled_image
tmp_image.flush()
del tmp_image
mask = ~mask # 1: background pixels; 0: non-background pixels.
# Write the mask to a file.
fid = open(background_mask_file, 'wb')
fid.write(mask.tostring())
fid.close()
# Write the mask header to a file.
header = empty_envi_header()
header[
'description'] = 'Background mask (0: non-background; 1: background)'
header['file type'] = 'ENVI Standard'
header['samples'] = mask.shape[1]
header['lines'] = mask.shape[0]
header['bands'] = 1
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
header['data type'] = 1
header['map info'] = map_info
header['coordinate system string'] = crs
write_envi_header(
os.path.splitext(background_mask_file)[0] + '.hdr', header)
del header
logger.info('Write the background mask to %s.' % background_mask_file)
"""
Merge DEM.
"""
# Read the first DEM.
sensor_index = list(sensors.keys())[0]
sensor_dict = sensors[sensor_index]
raw_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_dem_image_file'])[0] + '.hdr')
raw_image = np.memmap(sensor_dict['ortho_dem_image_file'],
dtype='float32',
mode='r',
shape=(raw_header['lines'], raw_header['samples']))
resampled_image = resample_ortho_dem(np.copy(raw_image),
float(raw_header['map info'][3]),
float(raw_header['map info'][4]),
float(raw_header['map info'][5]), x,
y)
resampled_image[mask] = -1000.0
# Write the merged DEM to a file.
fid = open(merged_dem_file, 'wb')
fid.write(resampled_image.astype('float32').tostring())
fid.close()
del raw_header, sensor_index, sensor_dict, resampled_image
raw_image.flush()
del raw_image
# Write the merged DEM header to a file.
header = empty_envi_header()
header['description'] = 'Merged DEM, in [m]'
header['file type'] = 'ENVI Standard'
header['samples'] = mask.shape[1]
header['lines'] = mask.shape[0]
header['bands'] = 1
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
header['data type'] = 4
header['data ignore value'] = -1000.0
header['map info'] = map_info
header['coordinate system string'] = crs
write_envi_header(os.path.splitext(merged_dem_file)[0] + '.hdr', header)
del header
logger.info('Write the merged DEM image to %s.' % merged_dem_file)
"""
Merge SCA.
"""
# Read the first SCA.
sensor_index = list(sensors.keys())[0]
sensor_dict = sensors[sensor_index]
raw_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_sca_image_file'])[0] + '.hdr')
raw_image = np.memmap(sensor_dict['ortho_sca_image_file'],
dtype='float32',
mode='r',
shape=(raw_header['bands'], raw_header['lines'],
raw_header['samples']))
# Write data.
fid = open(merged_sca_file, 'wb')
for band in range(raw_header['bands']):
resampled_image = resample_ortho_sca(np.copy(raw_image[band, :, :]),
float(raw_header['map info'][3]),
float(raw_header['map info'][4]),
float(raw_header['map info'][5]),
x, y)
resampled_image[mask] = -1000.0
fid.write(resampled_image.astype('float32').tostring())
del resampled_image
fid.close()
del sensor_index, sensor_dict
raw_image.flush()
del raw_image
# Write header.
header = empty_envi_header()
header['description'] = 'Merged SCA, in [deg]'
header['file type'] = 'ENVI Standard'
header['samples'] = mask.shape[1]
header['lines'] = mask.shape[0]
header['bands'] = 2
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
header['data type'] = 4
header['band names'] = ['Sensor Zenith [deg]', 'Sensor Azimuth [deg]']
header['sun azimuth'] = raw_header['sun azimuth']
header['sun zenith'] = raw_header['sun zenith']
header['data ignore value'] = -1000.0
header['map info'] = map_info
header['coordinate system string'] = crs
write_envi_header(os.path.splitext(merged_sca_file)[0] + '.hdr', header)
del raw_header, header
logger.info('Write the merged SCA image to %s.' % merged_sca_file)
def plot_smile_effect(smile_effect_at_atm_features_figure_file,
smile_effect_at_atm_features_file):
""" Plot smile effects at different atmosphere absorption features.
Arguments:
smile_effect_figure_file: str
Smile effect figure filename.
smile_effect_at_atm_features_file: str
Smile effect at atm features filename.
"""
if os.path.exists(smile_effect_at_atm_features_figure_file):
logger.info(
'Save the smile effect at atmosphere absorption features figure to %s.'
% smile_effect_at_atm_features_figure_file)
return
from ENVI import read_envi_header
header = read_envi_header(
os.path.splitext(smile_effect_at_atm_features_file)[0] + '.hdr')
center_waves = [
float(v) for v in header['spectral center wavelengths'].split(',')
]
fwhms = [float(v) for v in header['spectral bandwiths'].split(',')]
shifts = np.memmap(smile_effect_at_atm_features_file,
dtype='float32',
mode='r',
shape=(header['bands'], header['lines'],
header['samples']))
sample_indices = np.arange(header['samples'])
x_lim = [1, header['samples']]
if x_lim[1] - x_lim[0] > 1000:
x_ticks = np.arange(x_lim[0], x_lim[1] + 200, 200)
else:
x_ticks = np.arange(x_lim[0], x_lim[1] + 50, 50)
fig, axs = plt.subplots(nrows=3, ncols=3, figsize=(20, 20))
for index, ax in enumerate(axs.flatten()):
if index >= header['lines']:
ax.axis('off')
continue
# Title
ax.set_title(r'$\lambda$=%.2fnm, FWHM=%.2fnm' %
(center_waves[index], fwhms[index]),
fontsize=10)
# Left y-axis
y = np.copy(shifts[0, index, :])
i = np.abs(y - y.mean()) > 3 * y.std()
y[i] = np.nan
ax.plot(sample_indices,
y,
'-',
color='blue',
alpha=0.3,
lw=1,
label=r'$\Delta\lambda$')
p = np.poly1d(np.polyfit(sample_indices[~i], y[~i], 6))
ax.plot(sample_indices, p(sample_indices), '-', color='blue', lw=4)
y_lim = ax.get_ylim()
y_lim = [np.floor(y_lim[0] / 0.5) * 0.5, np.ceil(y_lim[1] / 0.5) * 0.5]
if y_lim[1] - y_lim[0] <= 1.0:
y_ticks = np.arange(y_lim[0], y_lim[1] + 0.25, 0.25)
elif y_lim[1] - y_lim[0] <= 3.0:
y_ticks = np.arange(y_lim[0], y_lim[1] + 0.50, 0.50)
else:
y_ticks = np.arange(y_lim[0], y_lim[1] + 1.00, 1.00)
y_lim = [y_ticks[0].min(), y_ticks[-1].max()]
ax.set_yticks(y_ticks)
y_ticklabels = ['%.2f' % v for v in y_ticks]
ax.set_yticklabels(y_ticklabels, fontsize=10, color='blue')
ax.set_ylim(y_lim)
ax.set_xlim(x_lim)
# Right y-axis
twin_ax = ax.twinx()
y = np.copy(shifts[1, index, :])
i = np.abs(y - y.mean()) > 3 * y.std()
y[i] = np.nan
twin_ax.plot(sample_indices,
y,
'-',
color='red',
alpha=0.3,
lw=1,
label=r'$\Delta$FWHM')
p = np.poly1d(np.polyfit(sample_indices[~i], y[~i], 6))
twin_ax.plot(sample_indices, p(sample_indices), '-', color='red', lw=4)
y_lim = twin_ax.get_ylim()
y_lim = [np.floor(y_lim[0] / 0.5) * 0.5, np.ceil(y_lim[1] / 0.5) * 0.5]
if y_lim[1] - y_lim[0] <= 1.0:
y_ticks = np.arange(y_lim[0], y_lim[1] + 0.01, 0.25)
elif y_lim[1] - y_lim[0] <= 3.0:
y_ticks = np.arange(y_lim[0], y_lim[1] + 0.50, 0.50)
else:
y_ticks = np.arange(y_lim[0], y_lim[1] + 1.00, 1.00)
y_lim = [y_ticks[0].min(), y_ticks[-1].max()]
twin_ax.set_yticks(y_ticks)
y_ticklabels = ['%.2f' % v for v in y_ticks]
twin_ax.set_yticklabels(y_ticklabels, fontsize=10, color='red')
twin_ax.set_xlim(x_lim)
twin_ax.set_ylim(y_lim)
twin_ax.set_xticks(x_ticks)
twin_ax.set_xticklabels(x_ticks, fontsize=10)
# X-axis
ax.set_xticks(x_ticks)
ax.set_xticklabels(x_ticks, fontsize=10)
ax.set_xlabel('Across-track Pixel', fontsize=10)
if index == 0:
lines, labels = ax.get_legend_handles_labels()
lines2, labels2 = twin_ax.get_legend_handles_labels()
ax.legend(lines + lines2, labels + labels2)
fig.savefig(smile_effect_at_atm_features_figure_file,
dpi=600,
bbox_inches='tight')
plt.close()
del fig, ax, axs, header
shifts.flush()
del shifts
logger.info(
'Save smile effect at atmosphere absorption features figure to %s.' %
smile_effect_at_atm_features_figure_file)
def estimate_wvc(wvc_file, rdn_file, sensors, sun_zenith, distance,
background_mask_file):
""" Estimate water vapor column.
Parameters
----------
wvc_file: str
Water vapor column image filename.
rdn_file: str
Radiance image filename.
sensors: dict
Sensors.
mask_file: str
Mask image filename.
sun_zenith: float
Sun zenith angle.
distance: float
Earth-to-sun distance.
"""
if os.path.exists(wvc_file):
logger.info('Save the WVC image to %s.' % (wvc_file))
return
from ENVI import read_envi_header, empty_envi_header, write_envi_header
from Spectra import get_closest_wave, resample_solar_flux
import json
# Read radiance image
rdn_header = read_envi_header(rdn_file + '.hdr')
rdn_image = np.memmap(rdn_file,
dtype='float32',
mode='r',
shape=(rdn_header['bands'], rdn_header['lines'],
rdn_header['samples']))
# Read boundary mask
bg_mask_header = read_envi_header(background_mask_file + '.hdr')
bg_mask_image = np.memmap(background_mask_file,
mode='r',
dtype='bool',
shape=(bg_mask_header['lines'],
bg_mask_header['samples']))
# Mask out dark pixels
good_mask_image = np.full((rdn_header['lines'], rdn_header['samples']),
True)
solar_flux = resample_solar_flux(solar_flux_file, rdn_header['wavelength'],
rdn_header['fwhm'])
cos_sun_zenith = np.cos(np.deg2rad(sun_zenith))
d2 = distance**2
# If reflectance at 850 nm is less than 0.01, mask out these pixels
wave, band = get_closest_wave(rdn_header['wavelength'], 470)
if abs(wave - 470) < 20:
refl = rdn_image[band, :, :] * np.pi * d2 / (solar_flux[band] *
cos_sun_zenith)
good_mask_image &= (refl > 0.01)
del refl
# If reflectance at 850 nm is less than 0.10, mask out these pixels
wave, band = get_closest_wave(rdn_header['wavelength'], 850)
if abs(wave - 850) < 20:
refl = rdn_image[band, :, :] * np.pi * d2 / (solar_flux[band] *
cos_sun_zenith)
good_mask_image &= (refl > 0.10)
del refl
# If reflectance at 1600 nm is less than 0.10, mask out these pixels
wave, band = get_closest_wave(rdn_header['wavelength'], 1600)
if abs(wave - 1600) < 20:
refl = rdn_image[band, :, :] * np.pi * d2 / (solar_flux[band] *
cos_sun_zenith)
good_mask_image &= (refl > 0.10)
del refl
del wave, band, solar_flux, cos_sun_zenith, d2
# Estimate water vapor columns
wvc_image = np.zeros((rdn_header['lines'], rdn_header['samples']))
for sensor_index, sensor_dict in sensors.items():
# Read water vapor column models
wvc_model_file = sensor_dict['wvc_model_file']
wvc_models = json.load(open(wvc_model_file, 'r'))
# Estimate WVC
for wvc_model in wvc_models.values():
# Find band indices
bands = []
for wave in wvc_model['wavelength']:
_, band = get_closest_wave(rdn_header['wavelength'], wave)
bands.append(band)
del band
# Calculate ratios
ratio_image = rdn_image[bands[1], :, :] / (
1e-10 + rdn_image[bands[0], :, :] * wvc_model['weight'][0] +
rdn_image[bands[2], :, :] * wvc_model['weight'][1])
# Calculate water vapor columns
wvc_image[good_mask_image] += np.interp(
ratio_image[good_mask_image], wvc_model['ratio'],
wvc_model['wvc']).astype('float32')
# Clear data
del ratio_image, bands, wvc_model
rdn_image.flush()
del rdn_image
# Average
wvc_image /= len(sensors)
# Read solar flux
wvc_image[~good_mask_image] = wvc_image[good_mask_image].mean()
logger.info(
'WVC [mm] statistics: min=%.2f, max=%.2f, avg=%.2f, sd=%.2f.' %
(wvc_image[good_mask_image].min(), wvc_image[good_mask_image].max(),
wvc_image[good_mask_image].mean(), wvc_image[good_mask_image].std()))
wvc_image[bg_mask_image] = -1000.0
del bg_mask_image, good_mask_image
# Save water vapor column
fid = open(wvc_file, 'wb')
fid.write(wvc_image.astype('float32').tostring())
fid.close()
# Write header
wvc_header = empty_envi_header()
wvc_header['description'] = 'Water vapor column [mm]'
wvc_header['file type'] = 'ENVI Standard'
wvc_header['bands'] = 1
wvc_header['lines'] = rdn_header['lines']
wvc_header['samples'] = rdn_header['samples']
wvc_header['file type'] = 'ENVI Standard'
wvc_header['interleave'] = 'bsq'
wvc_header['byte order'] = 0
wvc_header['data type'] = 4
wvc_header['data ignore value'] = -1000.0
wvc_header['map info'] = rdn_header['map info']
wvc_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(wvc_file + '.hdr', wvc_header)
del wvc_header, rdn_header
logger.info('Save the WVC image to %s.' % (wvc_file))
def atm_corr_image(flight_dict):
""" Do atmospheric corrections on the whole image.
Arguments:
flight_dict: dict
Flight dictionary.
"""
if os.path.exists(flight_dict['refl_file']):
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
return
from ENVI import read_envi_header, write_envi_header
from AtmLUT import read_binary_metadata
# Read radiance image.
rdn_header = read_envi_header(
os.path.splitext(flight_dict['merged_rdn_file'])[0] + '.hdr')
# rdn_image = np.memmap(flight_dict['merged_rdn_file'],
# dtype='float32',
# mode='r',
# shape=(rdn_header['bands'],
# rdn_header['lines'],
# rdn_header['samples']))
# Read atmospheric lookup table.
atm_lut_metadata = read_binary_metadata(
flight_dict['resampled_atm_lut_file'] + '.meta')
atm_lut_metadata['shape'] = tuple(
[int(v) for v in atm_lut_metadata['shape']])
atm_lut_WVC = np.array([float(v) for v in atm_lut_metadata['WVC']])
atm_lut_VIS = np.array([float(v) for v in atm_lut_metadata['VIS']])
atm_lut_VZA = np.array([float(v) for v in atm_lut_metadata['VZA']])
atm_lut_RAA = np.array([float(v) for v in atm_lut_metadata['RAA']])
atm_lut = np.memmap(flight_dict['resampled_atm_lut_file'],
dtype=atm_lut_metadata['dtype'],
mode='r',
shape=atm_lut_metadata['shape']
) # shape = (RHO, WVC, VIS, VZA, RAA, WAVE)
# Read VZA and RAA image.
sca_header = read_envi_header(
os.path.splitext(flight_dict['merged_sca_file'])[0] + '.hdr')
saa = float(sca_header['sun azimuth'])
sca_image = np.memmap(flight_dict['merged_sca_file'],
dtype='float32',
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# vza
vza_image = np.copy(sca_image[0, :, :])
# raa
raa_image = saa - sca_image[1, :, :]
raa_image[raa_image < 0] += 360.0
raa_image[raa_image > 180] = 360.0 - raa_image[raa_image > 180]
# clear data
sca_image.flush()
del sca_header, saa
del sca_image
# Read wvc and vis image.
wvc_header = read_envi_header(
os.path.splitext(flight_dict['wvc_file'])[0] + '.hdr')
tmp_wvc_image = np.memmap(flight_dict['wvc_file'],
mode='r',
dtype='float32',
shape=(wvc_header['lines'],
wvc_header['samples']))
wvc_image = np.copy(tmp_wvc_image)
vis_header = read_envi_header(
os.path.splitext(flight_dict['vis_file'])[0] + '.hdr')
tmp_vis_image = np.memmap(flight_dict['vis_file'],
dtype='float32',
mode='r',
shape=(vis_header['lines'],
vis_header['samples']))
vis_image = np.copy(tmp_vis_image)
tmp_wvc_image.flush()
tmp_vis_image.flush()
del wvc_header, vis_header
del tmp_vis_image, tmp_wvc_image
# Read background mask.
bg_header = read_envi_header(
os.path.splitext(flight_dict['background_mask_file'])[0] + '.hdr')
bg_mask = np.memmap(flight_dict['background_mask_file'],
dtype='bool',
mode='r',
shape=(bg_header['lines'], bg_header['samples']))
idx = ~bg_mask
max_WVC = atm_lut_WVC.max()
max_VIS = atm_lut_VIS.max()
max_VZA = atm_lut_VZA.max()
max_RAA = atm_lut_RAA.max()
wvc_image[wvc_image >= max_WVC] = max_WVC - 0.1
vis_image[vis_image >= max_VIS] = max_VIS - 0.1
vza_image[vza_image >= max_VZA] = max_VZA - 0.1
raa_image[raa_image >= max_RAA] = max_RAA - 0.1
del max_WVC, max_VIS, max_VZA, max_RAA
# remove outliers in wvc and vis.
wvc = wvc_image[idx]
avg_wvc = wvc.mean()
std_wvc = wvc.std()
index = (np.abs(wvc_image - avg_wvc) > 2 * std_wvc) & (idx)
wvc_image[index] = avg_wvc
del wvc
vis = vis_image[idx]
avg_vis = vis.mean()
std_vis = vis.std()
index = (np.abs(vis_image - avg_vis) > 2 * std_vis) & (idx)
vis_image[index] = avg_vis
del vis
del index
del idx
fid = open(flight_dict['refl_file'], 'wb')
# Do atmosphere correction.
info = 'Bands = '
for band in range(rdn_header['bands']):
if band % 20 == 0:
info += '%d, ' % (band + 1)
if (rdn_header['wavelength'][band] >= 1340.0
and rdn_header['wavelength'][band] <= 1440.0) or (
rdn_header['wavelength'][band] >= 1800.0
and rdn_header['wavelength'][band] <= 1980.0
) or rdn_header['wavelength'][band] >= 2460.0:
# fid.write(np.zeros((rdn_header['lines'], rdn_header['samples'])).astype('float32').tostring())
fid.write(
np.zeros((rdn_header['lines'], rdn_header['samples']
)).astype('int16').tostring()) # in int, by Ting
else:
# offset = rdn_header['header offset']+4*band*rdn_header['lines']*rdn_header['samples']# in bytes
offset = rdn_header[
'header offset'] + 2 * band * rdn_header['lines'] * rdn_header[
'samples'] # in bytes, int16 consists 2 bytes, by Ting
rdn_image = np.memmap(
flight_dict['merged_rdn_file'],
# dtype='float32',
dtype='int16', # in int, by Ting
mode='r',
offset=offset,
shape=(rdn_header['lines'], rdn_header['samples']))
refl = atm_corr_band(
atm_lut_WVC,
atm_lut_VIS,
atm_lut_VZA,
atm_lut_RAA,
np.copy(atm_lut[..., band]),
wvc_image,
vis_image,
vza_image,
raa_image,
rdn_image /
1000, # divided by 1000 to convert back to original rdn, by Ting.
bg_mask)
refl = refl * 10000 # reflectance times 10000 to convert to int, by Ting
# fid.write(refl.astype('float32').tostring())
fid.write(refl.astype('int16').tostring()) # save in int, by Ting
rdn_image.flush()
del refl, rdn_image
fid.close()
info += '%d, Done!' % band
logger.info(info)
# Clear data
del wvc_image, vis_image, vza_image, raa_image
atm_lut.flush()
bg_mask.flush()
del atm_lut, bg_mask
rdn_header['description'] = 'Reflectance [0-1]'
write_envi_header(
os.path.splitext(flight_dict['refl_file'])[0] + '.hdr', rdn_header)
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
def build_wvc_model(wvc_model_file, atm_lut_file, rdn_header_file, vis=40):
""" Build water vapor models.
Parameters
----------
wvc_model_file: str
Water vapor column model filename.
atm_lut_file: str
Atmosphere lookup table file.
rdn_header_file: str
Radiance header filename.
vis: float
Visibility in [km].
"""
if os.path.exists(wvc_model_file):
logger.info('Write WVC models to %s.' % wvc_model_file)
return
from AtmLUT import read_binary_metadata, get_interp_range
from ENVI import read_envi_header
from Spectra import get_closest_wave, resample_spectra
import json
# Get sensor wavelengths & FWHMs
header = read_envi_header(rdn_header_file)
sensor_waves = np.array(header['wavelength'])
sensor_fwhms = np.array(header['fwhm'])
# Read atmospheric lookup table (ALT) metadata
metadata = read_binary_metadata(atm_lut_file + '.meta')
metadata['shape'] = tuple([int(v) for v in metadata['shape']])
atm_lut_WVC = np.array([float(v) for v in metadata['WVC']])
atm_lut_VIS = np.array([float(v) for v in metadata['VIS']])
atm_lut_VZA = np.array([float(v) for v in metadata['VZA']])
atm_lut_RAA = np.array([float(v) for v in metadata['RAA']])
atm_lut_WAVE = np.array([float(v) for v in metadata['WAVE']])
# VZA index
vza_index = np.where(atm_lut_VZA == min(atm_lut_VZA))[0][-1]
# RAA index
raa_index = np.where(atm_lut_RAA == min(atm_lut_RAA))[0][-1]
# Load atmospheric lookup table
atm_lut = np.memmap(
atm_lut_file, dtype='float32', mode='r',
shape=metadata['shape']) # shape=(RHO, WVC, VIS, VZA, RAA, WAVE)
# Subtract path radiance
atm_lut_rdn = atm_lut[1, :, vza_index, raa_index, :] - atm_lut[
0, :, vza_index, raa_index, :] # shape=(WVC, VIS, WAVE)
atm_lut.flush()
del atm_lut
# VIS interpolation range
vis_dict = get_interp_range(atm_lut_VIS, vis)
# Do interpolation
interp_rdn = np.zeros((len(atm_lut_WVC), len(atm_lut_WAVE)))
for vis_index, vis_delta in vis_dict.items():
interp_rdn += atm_lut_rdn[:, vis_index, :] * vis_delta
del atm_lut_rdn, vis_index, vis_delta
# Build WVC models
model_waves = [(890, 940, 1000), (1070, 1130, 1200)]
wvc_model = dict()
for waves in model_waves:
# Get model wavelengths
wave_1, wave_2, wave_3 = waves
left_wave, left_band = get_closest_wave(sensor_waves, wave_1)
middle_wave, middle_band = get_closest_wave(sensor_waves, wave_2)
right_wave, right_band = get_closest_wave(sensor_waves, wave_3)
# Build the model
if np.abs(left_wave - wave_1) < 20 and np.abs(
middle_wave - wave_2) < 20 and np.abs(right_wave -
wave_3) < 20:
model = dict()
# Bands
model['band'] = [int(left_band), int(middle_band), int(right_band)]
# Wavelengths
model['wavelength'] = [left_wave, middle_wave, right_wave]
# Weights
left_weight = (right_wave - middle_wave) / (right_wave - left_wave)
right_weight = (middle_wave - left_wave) / (right_wave - left_wave)
model['weight'] = [left_weight, right_weight]
# Ratios & WVCs
resampled_rdn = resample_spectra(interp_rdn, atm_lut_WAVE,
sensor_waves[model['band']],
sensor_fwhms[model['band']])
ratio = resampled_rdn[:, 1] / (left_weight * resampled_rdn[:, 0] +
right_weight * resampled_rdn[:, 2])
index = np.argsort(ratio)
model['ratio'] = list(ratio[index])
model['wvc'] = list(atm_lut_WVC[index])
# Save model parameters
wvc_model['WVC_Model_%d' % wave_2] = model
# Clear data
del left_weight, right_weight, resampled_rdn, ratio, index, model
del interp_rdn
# Save WVC models to a .JSON file
with open(wvc_model_file, 'w') as fid:
json.dump(wvc_model, fid, indent=4)
logger.info('Write WVC models to %s.' % wvc_model_file)
def detect_smile_effect(sensor_dict, atm_lut_file):
""" Detect smile effect.
Parameters
----------
sensor_dict: dict
Sensor configurations.
atm_lut_file: str
Raw atmosphere lookup table filename.
"""
if os.path.exists(sensor_dict['smile_effect_at_atm_features_file']
) and os.path.exists(sensor_dict['smile_effect_file']):
logger.info(
'Write smile effect at atmosphere aborption features to %s.' %
sensor_dict['smile_effect_at_atm_features_file'])
logger.info('Write smile effect to %s.' %
sensor_dict['smile_effect_file'])
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
from Spectra import get_closest_wave
from scipy import optimize, interpolate
import json
# Read averaged radiance
header = read_envi_header(sensor_dict['avg_rdn_file'] + '.hdr')
sensor_rdn = np.memmap(sensor_dict['avg_rdn_file'],
mode='r',
dtype='float32',
shape=(header['lines'],
header['samples'])) # shape=(bands, samples)
samples = header['samples']
sensor_wave = np.array([float(v) for v in header['waves'].split(',')])
sensor_fwhm = np.array([float(v) for v in header['fwhms'].split(',')])
tmp_vza = header['VZA'].split(',')
tmp_raa = header['RAA'].split(',')
vza = []
raa = []
for i in range(samples):
try:
vza.append(float(tmp_vza[i]))
raa.append(float(tmp_raa[i]))
except:
vza.append(np.nan)
raa.append(np.nan)
logger.info('No spectrum for column: %s.' % (i + 1))
del tmp_vza, tmp_raa
del header
vza = np.array(vza)
raa = np.array(raa)
if np.all(np.isnan(vza)) or np.all(np.isnan(raa)):
raise IOError(
"Cannot detect smile effects since all columns do not have spectra."
)
nonnan_index = np.where((~np.isnan(vza)) & (~np.isnan(vza)))[0]
vza = np.interp(np.arange(samples), nonnan_index, vza[nonnan_index])
raa = np.interp(np.arange(samples), nonnan_index, raa[nonnan_index])
# Assign visibility
vis = [40] * samples
# Estimate water vapor column
logger.info('Estimate WVC from averaged radiance spectra.')
wvc_models = json.load(open(sensor_dict['wvc_model_file'], 'r'))
wvc = np.zeros(samples)
for model in wvc_models.values():
ratio = sensor_rdn[model['band'][1], :] / (
sensor_rdn[model['band'][0], :] * model['weight'][0] +
sensor_rdn[model['band'][2], :] * model['weight'][1])
wvc += np.interp(ratio, model['ratio'], model['wvc'])
del ratio
wvc /= len(wvc_models)
wvc_isnan = np.isnan(wvc) # Look for NaN values
if np.any(wvc_isnan):
logger.info('WVC could not be estimated for some columns. '
'Missing values will be interpolated.')
interpolate_values(wvc,
wvc_isnan) # Replace NaNs with interpolated values
logger.info('WVC [mm] statistics: min=%.2f, max=%.2f, avg=%.2f, sd=%.2f.' %
(wvc.min(), wvc.max(), wvc.mean(), wvc.std()))
del wvc_models, model
# Interpolate atmospheric lookup table
logger.info('Interpolate atmospheric look-up table.')
atm_lut_wave, atm_lut_rdn = interp_atm_lut(atm_lut_file, wvc, vis, vza,
raa) # shape=(samples, bands)
del wvc, vis, vza, raa
# Detect smile effects at atmospheric absorption features
logger.info('Detect smile effects at atmosphere absorption features.')
shifts = []
band_indices = []
n_features = 0
for wave, wave_range in absorption_features.items():
# Get sensor band range
sensor_wave0, sensor_band0 = get_closest_wave(sensor_wave,
wave_range[0])
sensor_wave1, sensor_band1 = get_closest_wave(sensor_wave,
wave_range[1])
center_wave, band_index = get_closest_wave(sensor_wave, wave)
# Check if continue
if abs(sensor_wave0 - wave_range[0]) > 20 or abs(sensor_wave1 -
wave_range[1]) > 20:
continue
# Get LUT band range
_, atm_lut_band0 = get_closest_wave(atm_lut_wave, sensor_wave0 - 20)
_, atm_lut_band1 = get_closest_wave(atm_lut_wave, sensor_wave1 + 20)
# Optimize
logger.info(
'Absorption feature center wavelength and range [nm] = %d: %d-%d.'
% (wave, wave_range[0], wave_range[1]))
x = []
for sample in range(samples):
p = optimize.minimize(
cost_fun, [0, 0],
method='BFGS',
args=(sensor_wave[sensor_band0:sensor_band1 + 1],
sensor_fwhm[sensor_band0:sensor_band1 + 1],
sensor_rdn[sensor_band0:sensor_band1 + 1, sample],
atm_lut_wave[atm_lut_band0:atm_lut_band1],
atm_lut_rdn[sample, atm_lut_band0:atm_lut_band1]))
x.append(p.x)
x = np.array(x)
# Do linear interpolation for invalid values
tmp_index = ~(np.abs(x[:, 0]) > 10.0)
x[:, 0] = np.interp(np.arange(samples),
np.arange(samples)[tmp_index], x[tmp_index, 0])
x[:, 1] = np.interp(np.arange(samples),
np.arange(samples)[tmp_index], x[tmp_index, 1])
del tmp_index
# Append shifts
shifts.append(x)
band_indices.append(band_index)
n_features += 1
# Clear data
del p, x, atm_lut_rdn
sensor_rdn.flush()
del sensor_rdn
# Reshape data
shifts = np.dstack(shifts).astype('float32').swapaxes(0, 1).swapaxes(1, 2)
# Write shifts to binary file
fid = open(sensor_dict['smile_effect_at_atm_features_file'], 'wb')
fid.write(shifts[0, :, :].tostring()) # Shift in wavelength
fid.write(shifts[1, :, :].tostring()) # Shift in fwhm
fid.close()
# Write header
header = empty_envi_header()
header[
'description'] = 'Smile effects detected at atmosphere absorption features'
header['file type'] = 'ENVI Standard'
header['bands'] = 2
header['lines'] = n_features
header['samples'] = samples
header['interleave'] = 'bsq'
header['byte order'] = 0
header['data type'] = 4
header['spectral center wavelengths'] = list(sensor_wave[band_indices])
header['spectral bandwiths'] = list(sensor_fwhm[band_indices])
header['band indices'] = band_indices
write_envi_header(
sensor_dict['smile_effect_at_atm_features_file'] + '.hdr', header)
del header
logger.info('Write smile effect at atmosphere absorption features to %s.' %
sensor_dict['smile_effect_at_atm_features_file'])
# Do interpolation
fid = open(sensor_dict['smile_effect_file'], 'wb')
x = np.arange(samples)
y = sensor_wave[band_indices]
x_new = x
y_new = sensor_wave
# Center wavelength
z = np.zeros((shifts.shape[1], shifts.shape[2]), dtype='float64')
for feature in range(shifts.shape[1]):
p = np.poly1d(np.polyfit(x, shifts[0, feature, :], 4))
z[feature, :] = p(x)
f = interpolate.interp2d(x, y, z, kind='cubic')
z_new = f(x_new, y_new) + np.expand_dims(sensor_wave, axis=1)
fid.write(z_new.astype('float32').tostring())
# Bandwidth
z = np.zeros((shifts.shape[1], shifts.shape[2]), dtype='float64')
for feature in range(shifts.shape[1]):
p = np.poly1d(np.polyfit(x, shifts[1, feature, :], 5))
z[feature, :] = p(x)
f = interpolate.interp2d(x, y, z, kind='cubic')
z_new = f(x_new, y_new) + np.expand_dims(sensor_fwhm, axis=1)
fid.write(z_new.astype('float32').tostring())
fid.close()
del f, x, y, z, z_new
# Write header
header = empty_envi_header()
header[
'description'] = 'Spectral Center Wavelengths and Spectral Bandwidths'
header['file type'] = 'ENVI Standard'
header['bands'] = 2
header['lines'] = len(y_new)
header['samples'] = len(x_new)
header['interleave'] = 'bsq'
header['byte order'] = 0
header['data type'] = 4
write_envi_header(sensor_dict['smile_effect_file'] + '.hdr', header)
del header, x_new, y_new
logger.info('Write smile effect to %s.' % sensor_dict['smile_effect_file'])
def atm_corr_image(flight_dict):
""" Whole-image atmospheric correction.
Parameters
----------
flight_dict: dict
Flight dictionary.
"""
if os.path.exists(flight_dict['refl_file']):
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
return
from ENVI import read_envi_header, write_envi_header
from AtmLUT import read_binary_metadata
# Read radiance image header
rdn_header = read_envi_header(
os.path.splitext(flight_dict['merged_rdn_file'])[0] + '.hdr')
# Read atmospheric lookup table
atm_lut_metadata = read_binary_metadata(
flight_dict['resampled_atm_lut_file'] + '.meta')
atm_lut_metadata['shape'] = tuple(
[int(v) for v in atm_lut_metadata['shape']])
atm_lut_WVC = np.array([float(v) for v in atm_lut_metadata['WVC']])
atm_lut_VIS = np.array([float(v) for v in atm_lut_metadata['VIS']])
atm_lut_VZA = np.array([float(v) for v in atm_lut_metadata['VZA']])
atm_lut_RAA = np.array([float(v) for v in atm_lut_metadata['RAA']])
atm_lut = np.memmap(flight_dict['resampled_atm_lut_file'],
dtype=atm_lut_metadata['dtype'],
mode='r',
shape=atm_lut_metadata['shape']
) # shape=(RHO, WVC, VIS, VZA, RAA, WAVE)
# Read scan angles (SCA) image
sca_header = read_envi_header(
os.path.splitext(flight_dict['merged_sca_file'])[0] + '.hdr')
saa = float(sca_header['sun azimuth'])
sca_image = np.memmap(flight_dict['merged_sca_file'],
dtype='float32',
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# View zenith angle (VZA)
vza_image = np.copy(sca_image[0, :, :])
# Relative azimuth angle (RAA)
raa_image = saa - sca_image[1, :, :]
raa_image[raa_image < 0] += 360.0
raa_image[raa_image > 180] = 360.0 - raa_image[raa_image > 180]
# Clear data
sca_image.flush()
del sca_header, saa
del sca_image
# Read WVC & VIS images
wvc_header = read_envi_header(
os.path.splitext(flight_dict['wvc_file'])[0] + '.hdr')
tmp_wvc_image = np.memmap(flight_dict['wvc_file'],
mode='r',
dtype='float32',
shape=(wvc_header['lines'],
wvc_header['samples']))
wvc_image = np.copy(tmp_wvc_image)
vis_header = read_envi_header(
os.path.splitext(flight_dict['vis_file'])[0] + '.hdr')
tmp_vis_image = np.memmap(flight_dict['vis_file'],
dtype='float32',
mode='r',
shape=(vis_header['lines'],
vis_header['samples']))
vis_image = np.copy(tmp_vis_image)
tmp_wvc_image.flush()
tmp_vis_image.flush()
del wvc_header, vis_header
del tmp_vis_image, tmp_wvc_image
# Read background mask
bg_header = read_envi_header(
os.path.splitext(flight_dict['background_mask_file'])[0] + '.hdr')
bg_mask = np.memmap(flight_dict['background_mask_file'],
dtype='bool',
mode='r',
shape=(bg_header['lines'], bg_header['samples']))
idx = ~bg_mask
max_WVC = atm_lut_WVC.max()
max_VIS = atm_lut_VIS.max()
max_VZA = atm_lut_VZA.max()
max_RAA = atm_lut_RAA.max()
# Enforce cutoffs in ALT parameters
wvc_image[wvc_image >= max_WVC] = max_WVC - 0.1
vis_image[vis_image >= max_VIS] = max_VIS - 0.1
vza_image[vza_image >= max_VZA] = max_VZA - 0.1
raa_image[raa_image >= max_RAA] = max_RAA - 0.1
del max_WVC, max_VIS, max_VZA, max_RAA
# Remove outliers in WVC and VIS
wvc = wvc_image[idx]
avg_wvc = wvc.mean()
std_wvc = wvc.std()
wvc_image[(np.abs(wvc_image - avg_wvc) > 2 * std_wvc)
& (~bg_mask)] = avg_wvc
del wvc
vis = vis_image[idx]
avg_vis = vis.mean()
std_vis = vis.std()
vis_image[(np.abs(vis_image - avg_vis) > 2 * std_vis)
& (~bg_mask)] = avg_vis
del vis
del idx
fid = open(flight_dict['refl_file'], 'wb')
# Do atmospheric correction
info = 'Bands = '
for band in range(rdn_header['bands']):
if band % 20 == 0:
info += '%d, ' % (band + 1)
if (rdn_header['wavelength'][band] >= 1340.0
and rdn_header['wavelength'][band] <= 1440.0) or (
rdn_header['wavelength'][band] >= 1800.0
and rdn_header['wavelength'][band] <= 1980.0
) or rdn_header['wavelength'][band] >= 2460.0:
fid.write(
np.zeros((rdn_header['lines'],
rdn_header['samples'])).astype('float32').tostring())
else:
offset = rdn_header['header offset'] + 4 * band * rdn_header[
'lines'] * rdn_header['samples'] # in bytes
rdn_image = np.memmap(flight_dict['merged_rdn_file'],
dtype='float32',
mode='r',
offset=offset,
shape=(rdn_header['lines'],
rdn_header['samples']))
refl = atm_corr_band(atm_lut_WVC, atm_lut_VIS,
atm_lut_VZA, atm_lut_RAA,
np.copy(atm_lut[...,
band]), wvc_image, vis_image,
vza_image, raa_image, rdn_image, bg_mask)
fid.write(refl.astype('float32').tostring())
rdn_image.flush()
del refl, rdn_image
fid.close()
info += '%d, Done!' % band
logger.info(info)
# Clear data
del wvc_image, vis_image, vza_image, raa_image
atm_lut.flush()
bg_mask.flush()
del atm_lut, bg_mask
rdn_header['description'] = 'Reflectance [0-1]'
write_envi_header(
os.path.splitext(flight_dict['refl_file'])[0] + '.hdr', rdn_header)
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
