def process_sca_file(raw_sca_image_file, processed_sca_image_file):
# Read raw scan angle data.
raw_sca_header = read_envi_header(
os.path.splitext(raw_sca_image_file)[0] + '.hdr')
raw_sca_image = np.memmap(raw_sca_image_file,
mode='r',
dtype='float32',
shape=(raw_sca_header['bands'],
raw_sca_header['lines'],
raw_sca_header['samples']))
# Initialize processed scan angle data.
processed_sca_image = np.memmap(processed_sca_image_file,
mode='w+',
dtype='int16',
shape=(raw_sca_header['bands'],
raw_sca_header['lines'],
raw_sca_header['samples']))
# zenith*100
angle = raw_sca_image[0, :, :] * 100
angle[angle < 0] = 9100
processed_sca_image[0, :, :] = angle.astype('int16')
del angle
# zenith*10
angle = raw_sca_image[1, :, :] * 10
angle[angle < 0] = -1
processed_sca_image[1, :, :] = angle.astype('int16')
del angle
# Clear data
processed_sca_image.flush()
raw_sca_image.flush()
del processed_sca_image, raw_sca_image
# Write header
raw_sca_header['description'] = 'SCA'
raw_sca_header['data type'] = 2
raw_sca_header['band names'] = [
'Sensor Zenith [100*deg]', 'Sensor Azimuth [10*deg]'
]
raw_sca_header['data ignore value'] = None
write_envi_header(
os.path.splitext(processed_sca_image_file)[0] + '.hdr', raw_sca_header)
def orthorectify_rdn(ortho_rdn_image_file, rdn_image_file, glt_image_file):
""" Do orthorectifications on radiance.
Arguments:
ortho_rdn_image_file: str
Orthorectified radiance filename.
rdn_image_file: str
Radiance image filename.
glt_image_file: str
Geographic look-up table image filename.
"""
if os.path.exists(ortho_rdn_image_file):
logger.info('Write the geo-rectified radiance image to %s.' %ortho_rdn_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read radiance image.
rdn_header = read_envi_header(rdn_image_file+'.hdr')
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['lines'],
rdn_header['bands'],
rdn_header['samples']))
# Read GLT image.
glt_header = read_envi_header(glt_image_file+'.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
offset=0,
shape=(glt_header['bands'],
glt_header['lines'],
glt_header['samples']))
# Get spatial indices.
I, J = np.where((glt_image[0,:,:]>=0)&(glt_image[1,:,:]>=0))
ortho_image = np.zeros((glt_header['lines'], glt_header['samples']), dtype='float32')
# Orthorectify radiance.
fid = open(ortho_rdn_image_file, 'wb')
info = 'Band (max=%d): ' %rdn_header['bands']
for band in range(rdn_header['bands']):
if band%20==0:
info += '%d, ' %(band+1)
ortho_image[:,:] = 0.0
ortho_image[I,J] = rdn_image[glt_image[0,I,J], band, glt_image[1,I,J]]
fid.write(ortho_image.tostring())
fid.close()
info += 'Done! %s' %rdn_header['bands']
logger.info(info)
del ortho_image
rdn_image.flush()
glt_image.flush()
del rdn_image, glt_image
# Write header.
ortho_rdn_header = empty_envi_header()
ortho_rdn_header['description'] = 'Geo-rectified radiance, in [mW/(cm2*um*sr)]'
ortho_rdn_header['file type'] = 'ENVI Standard'
ortho_rdn_header['samples'] = glt_header['samples']
ortho_rdn_header['lines'] = glt_header['lines']
ortho_rdn_header['bands'] = rdn_header['bands']
ortho_rdn_header['byte order'] = 0
ortho_rdn_header['header offset'] = 0
ortho_rdn_header['interleave'] = 'bsq'
ortho_rdn_header['data type'] = 4
ortho_rdn_header['wavelength'] = rdn_header['wavelength']
ortho_rdn_header['fwhm'] = rdn_header['fwhm']
ortho_rdn_header['wavelength units'] = 'nm'
ortho_rdn_header['default bands'] = rdn_header['default bands']
ortho_rdn_header['acquisition time'] = rdn_header['acquisition time']
ortho_rdn_header['map info'] = glt_header['map info']
ortho_rdn_header['coordinate system string'] = glt_header['coordinate system string']
write_envi_header(ortho_rdn_image_file+'.hdr', ortho_rdn_header)
del glt_header, rdn_header, ortho_rdn_header
logger.info('Write the geo-rectified radiance image to %s.' %ortho_rdn_image_file)
def orthorectify_sca(ortho_sca_image_file, sca_image_file, glt_image_file):
""" Do orthorectifications on SCA.
Arguments:
ortho_sca_image_file: str
Orthorectified SCA image filename.
sca_image_file: str
SCA image filename.
glt_image_file: str
Geographic look-up table image filename.
"""
if os.path.exists(ortho_sca_image_file):
logger.info('Write the geo-rectified SCA image to %s.' %ortho_sca_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read SCA image.
sca_header = read_envi_header(os.path.splitext(sca_image_file)[0]+'.hdr')
# Read GLT image.
glt_header = read_envi_header(os.path.splitext(glt_image_file)[0]+'.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
shape=(glt_header['bands'],
glt_header['lines'],
glt_header['samples']))
# Get spatial indices.
I, J = np.where((glt_image[0,:,:]>=0)&(glt_image[1,:,:]>=0))
ortho_sca_image = np.zeros((glt_header['lines'], glt_header['samples']), dtype='float32')
# Orthorectify SCA.
fid = open(ortho_sca_image_file, 'wb')
for band in range(sca_header['bands']):
ortho_sca_image[:,:] = -1000.0
offset = sca_header['header offset']++4*band*sca_header['lines']*sca_header['samples']
sca_image = np.memmap(sca_image_file,
dtype='float32',
mode='r',
offset=offset,
shape=(sca_header['lines'], sca_header['samples']))
ortho_sca_image[I,J] = sca_image[glt_image[0,I,J], glt_image[1,I,J]]
fid.write(ortho_sca_image.tostring())
sca_image.flush()
del sca_image
fid.close()
del ortho_sca_image
glt_image.flush()
del glt_image
# Write header
ortho_sca_header = empty_envi_header()
ortho_sca_header['description'] = 'Geo-rectified SCA, in [deg]'
ortho_sca_header['file type'] = 'ENVI Standard'
ortho_sca_header['samples'] = glt_header['samples']
ortho_sca_header['lines'] = glt_header['lines']
ortho_sca_header['bands'] = sca_header['bands']
ortho_sca_header['byte order'] = 0
ortho_sca_header['header offset'] = 0
ortho_sca_header['data ignore value'] = -1000.0
ortho_sca_header['interleave'] = 'bsq'
ortho_sca_header['data type'] = 4
ortho_sca_header['band names'] = sca_header['band names']
ortho_sca_header['sun zenith'] = sca_header['sun zenith']
ortho_sca_header['sun azimuth'] = sca_header['sun azimuth']
ortho_sca_header['map info'] = glt_header['map info']
ortho_sca_header['coordinate system string'] = glt_header['coordinate system string']
write_envi_header(ortho_sca_image_file+'.hdr', ortho_sca_header)
del glt_header, sca_header, ortho_sca_header
logger.info('Write the geo-rectified SCA image to %s.' %ortho_sca_image_file)
def atm_corr_image(flight_dict):
""" Whole-image atmospheric correction.
Parameters
----------
flight_dict: dict
Flight dictionary.
"""
if os.path.exists(flight_dict['refl_file']):
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
return
from ENVI import read_envi_header, write_envi_header
from AtmLUT import read_binary_metadata
# Read radiance image header
rdn_header = read_envi_header(
os.path.splitext(flight_dict['merged_rdn_file'])[0] + '.hdr')
# Read atmospheric lookup table
atm_lut_metadata = read_binary_metadata(
flight_dict['resampled_atm_lut_file'] + '.meta')
atm_lut_metadata['shape'] = tuple(
[int(v) for v in atm_lut_metadata['shape']])
atm_lut_WVC = np.array([float(v) for v in atm_lut_metadata['WVC']])
atm_lut_VIS = np.array([float(v) for v in atm_lut_metadata['VIS']])
atm_lut_VZA = np.array([float(v) for v in atm_lut_metadata['VZA']])
atm_lut_RAA = np.array([float(v) for v in atm_lut_metadata['RAA']])
atm_lut = np.memmap(flight_dict['resampled_atm_lut_file'],
dtype=atm_lut_metadata['dtype'],
mode='r',
shape=atm_lut_metadata['shape']
) # shape=(RHO, WVC, VIS, VZA, RAA, WAVE)
# Read scan angles (SCA) image
sca_header = read_envi_header(
os.path.splitext(flight_dict['merged_sca_file'])[0] + '.hdr')
saa = float(sca_header['sun azimuth'])
sca_image = np.memmap(flight_dict['merged_sca_file'],
dtype='float32',
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# View zenith angle (VZA)
vza_image = np.copy(sca_image[0, :, :])
# Relative azimuth angle (RAA)
raa_image = saa - sca_image[1, :, :]
raa_image[raa_image < 0] += 360.0
raa_image[raa_image > 180] = 360.0 - raa_image[raa_image > 180]
# Clear data
sca_image.flush()
del sca_header, saa
del sca_image
# Read WVC & VIS images
wvc_header = read_envi_header(
os.path.splitext(flight_dict['wvc_file'])[0] + '.hdr')
tmp_wvc_image = np.memmap(flight_dict['wvc_file'],
mode='r',
dtype='float32',
shape=(wvc_header['lines'],
wvc_header['samples']))
wvc_image = np.copy(tmp_wvc_image)
vis_header = read_envi_header(
os.path.splitext(flight_dict['vis_file'])[0] + '.hdr')
tmp_vis_image = np.memmap(flight_dict['vis_file'],
dtype='float32',
mode='r',
shape=(vis_header['lines'],
vis_header['samples']))
vis_image = np.copy(tmp_vis_image)
tmp_wvc_image.flush()
tmp_vis_image.flush()
del wvc_header, vis_header
del tmp_vis_image, tmp_wvc_image
# Read background mask
bg_header = read_envi_header(
os.path.splitext(flight_dict['background_mask_file'])[0] + '.hdr')
bg_mask = np.memmap(flight_dict['background_mask_file'],
dtype='bool',
mode='r',
shape=(bg_header['lines'], bg_header['samples']))
idx = ~bg_mask
max_WVC = atm_lut_WVC.max()
max_VIS = atm_lut_VIS.max()
max_VZA = atm_lut_VZA.max()
max_RAA = atm_lut_RAA.max()
# Enforce cutoffs in ALT parameters
wvc_image[wvc_image >= max_WVC] = max_WVC - 0.1
vis_image[vis_image >= max_VIS] = max_VIS - 0.1
vza_image[vza_image >= max_VZA] = max_VZA - 0.1
raa_image[raa_image >= max_RAA] = max_RAA - 0.1
del max_WVC, max_VIS, max_VZA, max_RAA
# Remove outliers in WVC and VIS
wvc = wvc_image[idx]
avg_wvc = wvc.mean()
std_wvc = wvc.std()
wvc_image[(np.abs(wvc_image - avg_wvc) > 2 * std_wvc)
& (~bg_mask)] = avg_wvc
del wvc
vis = vis_image[idx]
avg_vis = vis.mean()
std_vis = vis.std()
vis_image[(np.abs(vis_image - avg_vis) > 2 * std_vis)
& (~bg_mask)] = avg_vis
del vis
del idx
fid = open(flight_dict['refl_file'], 'wb')
# Do atmospheric correction
info = 'Bands = '
for band in range(rdn_header['bands']):
if band % 20 == 0:
info += '%d, ' % (band + 1)
if (rdn_header['wavelength'][band] >= 1340.0
and rdn_header['wavelength'][band] <= 1440.0) or (
rdn_header['wavelength'][band] >= 1800.0
and rdn_header['wavelength'][band] <= 1980.0
) or rdn_header['wavelength'][band] >= 2460.0:
fid.write(
np.zeros((rdn_header['lines'],
rdn_header['samples'])).astype('float32').tostring())
else:
offset = rdn_header['header offset'] + 4 * band * rdn_header[
'lines'] * rdn_header['samples'] # in bytes
rdn_image = np.memmap(flight_dict['merged_rdn_file'],
dtype='float32',
mode='r',
offset=offset,
shape=(rdn_header['lines'],
rdn_header['samples']))
refl = atm_corr_band(atm_lut_WVC, atm_lut_VIS,
atm_lut_VZA, atm_lut_RAA,
np.copy(atm_lut[...,
band]), wvc_image, vis_image,
vza_image, raa_image, rdn_image, bg_mask)
fid.write(refl.astype('float32').tostring())
rdn_image.flush()
del refl, rdn_image
fid.close()
info += '%d, Done!' % band
logger.info(info)
# Clear data
del wvc_image, vis_image, vza_image, raa_image
atm_lut.flush()
bg_mask.flush()
del atm_lut, bg_mask
rdn_header['description'] = 'Reflectance [0-1]'
write_envi_header(
os.path.splitext(flight_dict['refl_file'])[0] + '.hdr', rdn_header)
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
def clip_hyspex(raw_image_file, clipped_image_file, clipped_imugps_file):
""" clip the raw hyspex data based on given extent
Arguments:
raw_image_file: str
Raw hyspex filename.
l_start: int
The starting line.
l_end: int
The endding line.
clipped_image_file: str
Clipped hyspex image filename
"""
if os.path.exists(clipped_image_file):
logger.info('Write the clipped image to %s.' % clipped_image_file)
return
from ENVI import read_envi_header, write_envi_header
# Load the clipped imugps file:
clipped_imugps = np.loadtxt(clipped_imugps_file)
l_start = int(clipped_imugps[0, 0])
l_end = int(clipped_imugps[-1, 0])
# Read DN image.
dn_header = read_envi_header(os.path.splitext(raw_image_file)[0] + '.hdr')
dn_image = np.memmap(raw_image_file,
dtype='uint16',
mode='r',
offset=dn_header['header offset'],
shape=(dn_header['lines'], dn_header['bands'],
dn_header['samples']))
# read the metadata from the raw image:
lines_raw = dn_header['lines']
offset = dn_header['header offset']
fid_r = open(raw_image_file, 'rb')
fid_w = open(clipped_image_file, 'wb')
temp = fid_r.read(offset)
fid_w.write(temp)
fid_r.close()
# Set up the start line and the end line:
s = l_start - 1
total_lines = int(l_end - s)
# Adjust the file pointer position:
fid_w.seek(dn_header['header offset'], 0)
for from_line in range(s, l_end, 500):
# Determine chunck size.
to_line = min(from_line + 500, l_end)
clipped = dn_image[from_line:to_line, :, :]
# Write clipped data to the file.
fid_w.write(clipped.tostring())
# Clear temporary data.
del clipped, to_line
fid_w.close()
# Clear data.
dn_image.flush()
del dn_image
# Write header.
clipped_header = dn_header
clipped_header['clip tag'] = 1
clipped_header['starting line'] = l_start
clipped_header['ending line'] = l_end
clipped_header['lines'] = total_lines
clipped_header['original lines'] = lines_raw
write_envi_header(
os.path.splitext(clipped_image_file)[0] + '.hdr', clipped_header)
del clipped_header, dn_header
logger.info('Write the clipped image to %s.' % clipped_image_file)
def average_rdn(avg_rdn_file, rdn_image_file, sca_image_file,
pre_class_image_file):
""" Average radiance along each column.
Parameters
----------
avg_rdn_file: str
Average radiance data filename.
rdn_image_file: 3D array
Radiance image filename, in BIL format.
sca_image_file: 3D array
Scan angle image filename, in BSQ format.
pre_class_image_file: str
Pre-classification image filename.
"""
if os.path.exists(avg_rdn_file):
logger.info('Write the averaged radiance data to %s.' % avg_rdn_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read radiance image data
rdn_header = read_envi_header(os.path.splitext(rdn_image_file)[0] + '.hdr')
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['lines'], rdn_header['bands'],
rdn_header['samples']))
# Read classification map
pre_class_header = read_envi_header(
os.path.splitext(pre_class_image_file)[0] + '.hdr')
pre_class_image = np.memmap(pre_class_image_file,
dtype='uint8',
mode='r',
shape=(pre_class_header['lines'],
pre_class_header['samples']))
mask = pre_class_image == 5 # 5: land (clear)
pre_class_image.flush()
del pre_class_header, pre_class_image
# Average radiance along each column
fid = open(avg_rdn_file, 'wb')
info = 'Band (max=%d): ' % rdn_header['bands']
for band in range(rdn_header['bands']):
if band % 20 == 0:
info += '%d, ' % (band + 1)
# Build temporary mask
tmp_mask = mask & (rdn_image[:, band, :] > 0.0)
# Average radiance
rdn = np.ma.array(rdn_image[:, band, :],
mask=~tmp_mask).mean(axis=0) # shape = (samples,1)
# Interpolate bad radiance values
for bad_sample in np.where(rdn.mask)[0]:
if bad_sample == 0:
rdn[bad_sample] = rdn[bad_sample + 1]
elif bad_sample == rdn_header['samples'] - 1:
rdn[bad_sample] = rdn[bad_sample - 1]
else:
rdn[bad_sample] = (rdn[bad_sample - 1] +
rdn[bad_sample + 1]) / 2.0
# Write average radiance data to file
fid.write(rdn.data.astype('float32'))
# Clear data
del tmp_mask, rdn
fid.close()
info += str(rdn_header['bands'])
logger.info(info)
rdn_image.flush()
del rdn_image
# Read scan angles
sca_header = read_envi_header(os.path.splitext(sca_image_file)[0] + '.hdr')
sca_image = np.memmap(sca_image_file,
dtype='float32',
mode='r',
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
saa = float(sca_header['sun azimuth'])
# Average scan angles
avg_vza = np.ma.array(sca_image[0, :, :], mask=~mask).mean(axis=0)
avg_vaa = np.ma.array(sca_image[1, :, :], mask=~mask).mean(axis=0)
avg_raa = saa - avg_vaa
avg_raa[avg_raa < 0] += 360.0
avg_raa[avg_raa > 180] = 360.0 - avg_raa[avg_raa > 180]
sca_image.flush()
del sca_image
# Write header
avg_rdn_header = empty_envi_header()
avg_rdn_header['description'] = 'Averaged radiance in [mW/(cm2*um*sr)]'
avg_rdn_header['samples'] = rdn_header['samples']
avg_rdn_header['lines'] = rdn_header['bands']
avg_rdn_header['bands'] = 1
avg_rdn_header['byte order'] = 0
avg_rdn_header['header offset'] = 0
avg_rdn_header['interleave'] = 'bsq'
avg_rdn_header['data type'] = 4
avg_rdn_header['waves'] = rdn_header['wavelength']
avg_rdn_header['fwhms'] = rdn_header['fwhm']
avg_rdn_header['VZA'] = list(avg_vza)
avg_rdn_header['RAA'] = list(avg_raa)
write_envi_header(avg_rdn_file + '.hdr', avg_rdn_header)
logger.info('Write the averaged radiance data to %s.' % avg_rdn_file)
def hyspex_extract(src_img, bands, out_img):
"""
extract certain bands from hyspex images (note: images are in BSQ)
:param src_img:
:param out_img:
:param bands: list; bands to be extracted, starting from 0
:return:
"""
src_header = read_envi_header(src_img + '.hdr')
# src_image = np.memmap(src_img,
# dtype='int16',
# mode='r',
# offset=src_header['header offset'],
# shape=(src_header['lines'],
# src_header['samples'],
# src_header['bands'],))
# Create empty array for the output image:
# out = np.memmap(out_img, dtype='int16',
# mode='w+',
# shape=(src_header['lines'],
# src_header['samples'],
# len(bands))
# )
# Read the given bands:
i = 0
with open(out_img, 'wb') as outfile:
for b in bands:
print('Now reading band {}'.format(b))
src_image = np.memmap(
src_img,
dtype='int16',
mode='r',
shape=(src_header['lines'], src_header['samples']),
offset=int(src_header['lines'] * src_header['samples'] * b *
16 / 8))
# # Create empty array for the output image:
# out = np.memmap(out_img, dtype='int16',
# mode='w+',
# shape=(src_header['lines'],
# src_header['samples']),
# offset=int(src_header['lines'] * src_header['samples'] * i * 16 / 8))
# out[:, :, i] = src_image[:]
outfile.write(src_image[:])
del src_image
i = i + 1
outfile.close()
# flush memory:
# out.flush()
# src_image.flush()
# Modify the header for out_image:
tgt_header = src_header
# number of bands:
tgt_header['bands'] = len(bands)
# wavelength:
wvl = [src_header['wavelength'][b] for b in bands]
tgt_header['wavelength'] = wvl
# fwhm:
fwhm = [src_header['fwhm'][b] for b in bands]
tgt_header['fwhm'] = fwhm
write_envi_header(out_img + '.hdr', tgt_header)
def dn2rdn_Hyspex(rdn_image_file, dn_image_file, radio_cali_file,
acquisition_time):
""" Do Hyspex radiometric calibration.
Parameters
----------
rdn_image_file: str
Radiance image filename.
dn_image_file: str
Hyspex DN image filename.
radio_cali_file: str
Hyspex radiometric calibration coefficients filename.
acquisition_time: datetime object
Acquisition time.
"""
if os.path.exists(rdn_image_file):
logger.info('Write the radiance image to %s.' % rdn_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read calibration coefficients
radio_cali_header = read_envi_header(
os.path.splitext(radio_cali_file)[0] + '.hdr')
radio_cali_coeff = np.memmap(radio_cali_file,
dtype='float64',
mode='r',
shape=(radio_cali_header['bands'],
radio_cali_header['lines'],
radio_cali_header['samples']))
wavelengths = np.array(
[float(v) for v in radio_cali_header['waves'].split(',')])
fwhms = np.array([float(v) for v in radio_cali_header['fwhms'].split(',')])
# Read DN image
dn_header = read_envi_header(os.path.splitext(dn_image_file)[0] + '.hdr')
dn_image = np.memmap(dn_image_file,
dtype='uint16',
mode='r',
offset=dn_header['header offset'],
shape=(dn_header['lines'], dn_header['bands'],
dn_header['samples']))
# Get gain coefficients
gain = radio_cali_coeff[0, :, :] # shape=(bands, samples)
# Do radiometric calibration
info = 'Line (max=%d): ' % dn_header['lines']
fid = open(rdn_image_file, 'wb')
for from_line in range(0, dn_header['lines'], 500):
info += '%d, ' % (from_line + 1)
# Determine chunk size
to_line = min(from_line + 500, dn_header['lines'])
# Get offset coefficients
if radio_cali_header['bands'] == 2:
offset = radio_cali_coeff[1, :, :] # shape=(bands, samples)
else:
background = np.stack([radio_cali_coeff[1, :, :]] *
(to_line - from_line))
backgroundLast = np.stack([radio_cali_coeff[2, :, :]] *
(to_line - from_line))
factor = np.arange(from_line, to_line) / dn_header['lines']
offset = background + (
backgroundLast - background
) * factor[:, np.newaxis,
np.newaxis] # shape=(to_line-from_line, bands, samples)
del background, backgroundLast, factor
# Convert DN to radiance
rdn = (dn_image[from_line:to_line, :, :].astype('float32') -
offset) * gain # shape=(to_line-from_line, bands, samples)
# Write radiance to file
fid.write(rdn.astype('float32').tostring())
# Clear temporary data
del rdn, to_line, offset
fid.close()
info += '%d, Done!' % dn_header['lines']
logger.info(info)
# Clear data
dn_image.flush()
radio_cali_coeff.flush()
del gain, from_line
del dn_image, radio_cali_coeff
# Write header
rdn_header = empty_envi_header()
rdn_header['description'] = 'Hyspex radiance in mW/(cm2*um*sr)'
rdn_header['file type'] = 'ENVI Standard'
rdn_header['samples'] = dn_header['samples']
rdn_header['lines'] = dn_header['lines']
rdn_header['bands'] = dn_header['bands']
rdn_header['byte order'] = 0
rdn_header['header offset'] = 0
rdn_header['interleave'] = 'bil'
rdn_header['data type'] = 4
rdn_header['wavelength'] = list(wavelengths)
rdn_header['fwhm'] = list(fwhms)
rdn_header['wavelength units'] = 'nm'
rdn_header['default bands'] = dn_header['default bands']
rdn_header['acquisition time'] = acquisition_time.strftime(
'%Y-%m-%dT%H:%M:%S.%f')
write_envi_header(os.path.splitext(rdn_image_file)[0] + '.hdr', rdn_header)
del radio_cali_header, dn_header, rdn_header
logger.info('Write the radiance image to %s.' % rdn_image_file)
def calculate_igm(igm_image_file, imugps_file, sensor_model_file,
dem_image_file, boresight_options):
""" Create an input geometry (IGM) image.
Arguments:
igm_image_file: str
The IGM image filename.
imugps_file: str
The IMUGPS filename.
sensor_model_file: str
The sensor model filename.
dem_image_file: str
The DEM image filename.
boresight_options: list of boolean
Boresight offset options, true or false.
"""
if os.path.exists(igm_image_file):
logger.info('Write the IGM to %s.' % igm_image_file)
return
from ENVI import empty_envi_header, write_envi_header
from scipy import interpolate
# Read IMU and GPS data.
imugps = np.loadtxt(
imugps_file
) # ID, X, Y, Z, R, P, H, R_Offset, P_Offset, H_Offset, Grid_Convergence
# Read sensor model data.
sensor_model = np.loadtxt(sensor_model_file, skiprows=1)[:, 1:]
# Read DEM data.
ds = gdal.Open(dem_image_file, gdal.GA_ReadOnly)
dem_image = ds.GetRasterBand(1).ReadAsArray()
dem_geotransform = ds.GetGeoTransform()
dem_prj = ds.GetProjection()
ds = None
# Apply boresight offsets.
if boresight_options[0]:
imugps[:, 4] += imugps[:, 7]
if boresight_options[1]:
imugps[:, 5] += imugps[:, 8]
if boresight_options[2]:
imugps[:, 6] += imugps[:, 9]
if boresight_options[3]:
imugps[:, 3] += imugps[:, 10]
imugps[:, 6] -= imugps[:, 11] # Heading - grid convergence
# Get scan vectors.
L0 = get_scan_vectors(imugps[:, 4:7], sensor_model)
del sensor_model
# Get start and end points of ray tracing.
index = dem_image > 0.0
dem_min = dem_image[index].min()
dem_max = dem_image[index].max()
del index
xyz0, xyz1 = get_xyz0_xyz1(imugps[:, 1:4], L0, dem_min, dem_max)
# Ray-tracing.
lines = imugps.shape[0]
samples = L0.shape[1]
logger.info('Beginning ray-tracing ({} scanlines)...'.format(lines))
igm_image = ray_tracer_ufunc(xyz0, xyz1, L0, dem_image, dem_geotransform)
del dem_image, xyz0, xyz1, L0, imugps
logger.info('Ray-tracing complete.')
# Interpolate IGM.
nan_lines = []
nonnan_lines = []
for line in np.arange(lines):
# Find Nan values.
nan_flag = np.isnan(igm_image[0, line, :])
# If all columns are Nan values;
if np.all(nan_flag):
del nan_flag
nan_lines.append(line)
continue
# If some columns are Nan values;
if np.any(nan_flag):
nan_samples = np.arange(samples)[nan_flag]
nonnan_samples = np.arange(samples)[~nan_flag]
f = interpolate.interp1d(nonnan_samples,
igm_image[:, line, nonnan_samples],
axis=1,
fill_value="extrapolate")
igm_image[:, line, nan_samples] = f(nan_samples)
del nan_samples, nonnan_samples, f, nan_flag
nonnan_lines.append(line)
for nan_line in nan_lines:
index = np.argmin(np.abs(nonnan_lines - nan_line))
nonnan_line = nonnan_lines[index]
igm_image[:, nan_line, :] = igm_image[:, nonnan_line, :]
del index, nonnan_line
logger.info('IGM interpolation complete.')
# Write IGM image.
fid = open(igm_image_file, 'wb')
fid.write(igm_image.tostring())
fid.close()
del igm_image
# Write IGM header file.
igm_header = empty_envi_header()
igm_header['description'] = 'IGM (map coordinate system=%s)' % (dem_prj)
igm_header['file type'] = 'ENVI Standard'
igm_header['samples'] = samples
igm_header['lines'] = lines
igm_header['bands'] = 3
igm_header['byte order'] = 0
igm_header['header offset'] = 0
igm_header['interleave'] = 'bsq'
igm_header['data type'] = 5
igm_header['band names'] = ['IGM Map X', 'IGM Map Y', 'IGM Map Z']
igm_header_file = os.path.splitext(igm_image_file)[0] + '.hdr'
write_envi_header(igm_header_file, igm_header)
del igm_header, dem_prj
logger.info('Write the IGM to %s.' % igm_image_file)
def pre_classification(pre_class_image_file,
rdn_image_file,
sun_zenith,
distance,
background_mask_file=None):
""" Pre-classify the image.
Notes
-----
(1) The classification algorithm used here is from ATCOR.
Parameters
----------
pre_class_image_file: str
Pre-classification image filename.
rdn_image_file: str
Radiance image filename, either BIL or BSQ.
sun_zenith: float
Sun zenith angle in degrees.
distance: float
Sun-Earth distance.
background_mask_file: str
Background mask filename.
"""
if os.path.exists(pre_class_image_file):
logger.info('Write the pre-classification map to %s.' %
(pre_class_image_file))
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
from Spectra import get_closest_wave, resample_solar_flux
# Read radiance image data
rdn_header = read_envi_header(os.path.splitext(rdn_image_file)[0] + '.hdr')
if rdn_header['interleave'].lower() == 'bil':
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['lines'], rdn_header['bands'],
rdn_header['samples']))
elif rdn_header['interleave'].lower() == 'bsq':
rdn_image = np.memmap(rdn_image_file,
dtype='float32',
mode='r',
shape=(rdn_header['bands'], rdn_header['lines'],
rdn_header['samples']))
else:
logger.error('Cannot read radiance data from %s format file.' %
rdn_header['interleave'])
# Read solar flux
solar_flux = resample_solar_flux(solar_flux_file, rdn_header['wavelength'],
rdn_header['fwhm'])
cos_sun_zenith = np.cos(np.deg2rad(sun_zenith))
d2 = distance**2
# Initialize classification
pre_class_image = np.memmap(pre_class_image_file,
dtype='uint8',
mode='w+',
shape=(rdn_header['lines'],
rdn_header['samples']))
pre_class_image[:, :] = 0
# Define VNIR sensor wavelengths
blue_wave, blue_band = get_closest_wave(rdn_header['wavelength'], 470)
green_wave, green_band = get_closest_wave(rdn_header['wavelength'], 550)
red_wave, red_band = get_closest_wave(rdn_header['wavelength'], 660)
nir_wave, nir_band = get_closest_wave(rdn_header['wavelength'], 850)
# Define SWIR sensor wavelengths
cirrus_wave, cirrus_band = get_closest_wave(rdn_header['wavelength'], 1380)
swir1_wave, swir1_band = get_closest_wave(rdn_header['wavelength'], 1650)
swir2_wave, swir2_band = get_closest_wave(rdn_header['wavelength'], 2200)
# Determine the sensor type
if_vnir = abs(blue_wave - 470) < 20 and abs(green_wave - 550) < 20 and abs(
red_wave - 660) < 20 and abs(nir_wave - 850) < 20
if_swir = abs(cirrus_wave - 1380) < 20 and abs(
swir1_wave - 1650) < 20 and abs(swir2_wave - 2200) < 20
if_whole = if_vnir and if_swir
# Do classification
if if_whole:
# Calculate the reflectance at different bands
if rdn_header['interleave'].lower() == 'bil':
blue_refl = rdn_image[:, blue_band, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[:, green_band, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[:, red_band, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[:, nir_band, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
cirrus_refl = rdn_image[:, cirrus_band, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[:, swir1_band, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[:, swir2_band, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
else:
blue_refl = rdn_image[blue_band, :, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[green_band, :, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[red_band, :, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[nir_band, :, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
cirrus_refl = rdn_image[cirrus_band, :, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[swir1_band, :, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[swir2_band, :, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
rdn_image.flush()
del rdn_image
# Calculate NDSI
ndsi = (green_refl - swir1_refl) / (green_refl + swir1_refl + 1e-10)
# Water
water = (nir_refl < 0.05) & (swir1_refl < 0.03)
# Land
land = ~water
# Shadow
shadow = (water | land) & (green_refl < 0.01)
land = land & (~shadow)
water = water & (~shadow)
# Cloud over land
Tc = 0.20
cloud_over_land = land & (blue_refl > Tc) & (red_refl > 0.15) & (
nir_refl < red_refl * 2.0) & (nir_refl > red_refl * 0.8) & (
nir_refl > swir1_refl) & (ndsi < 0.7)
land = land & (~cloud_over_land)
# Cloud over water
cloud_over_water = water & (blue_refl < 0.40) & (blue_refl > 0.20) & (
green_refl < blue_refl) & (nir_refl < green_refl) & (
swir1_refl < 0.15) & (ndsi < 0.20)
water = water & (~cloud_over_water)
# Cloud shadow
cloud_shadow = (water | land) & (nir_refl < 0.12) & (
nir_refl > 0.04) & (swir1_refl < 0.20)
land = land & (~cloud_shadow)
water = water & (~cloud_shadow)
# Snow & ice
snow_ice = land & (((blue_refl > 0.22) & (ndsi > 0.60)) |
((green_refl > 0.22) & (ndsi > 0.25) &
(swir2_refl < 0.5 * green_refl)))
land = land & (~snow_ice)
# Cirrus over land & water
thin_cirrus = (cirrus_refl < 0.015) & (cirrus_refl > 0.010)
medium_cirrus = (cirrus_refl < 0.025) & (cirrus_refl >= 0.015)
thick_cirrus = (cirrus_refl < 0.040) & (cirrus_refl >= 0.025)
thin_cirrus_over_land = land & thin_cirrus
land = land & (~thin_cirrus_over_land)
medium_cirrus_over_land = land & medium_cirrus
land = land & (~medium_cirrus_over_land)
thick_cirrus_over_land = land & thick_cirrus
land = land & (~thick_cirrus_over_land)
thin_cirrus_over_water = water & thin_cirrus
water = water & (~thin_cirrus_over_water)
medium_cirrus_over_water = water & thin_cirrus
water = water & (~medium_cirrus_over_water)
thick_cirrus_over_water = water & thin_cirrus
water = water & (~thick_cirrus_over_water)
cirrus_cloud = (water
| land) & (cirrus_refl < 0.050) & (cirrus_refl > 0.040)
land = land & (~cirrus_cloud)
water = water & (~cirrus_cloud)
thick_cirrus_cloud = (water | land) & (cirrus_refl > 0.050)
land = land & (~thick_cirrus_cloud)
water = water & (~thick_cirrus_cloud)
# Haze over water
T2 = 0.04
T1 = 0.12
thin_haze_over_water = water & (nir_refl >= T1) & (nir_refl <= 0.06)
water = water & (~thin_haze_over_water)
medium_haze_over_water = water & (nir_refl >= 0.06) & (nir_refl <= T2)
water = water & (~medium_haze_over_water)
# Assign class values
pre_class_image[shadow] = 1
pre_class_image[thin_cirrus_over_water] = 2
pre_class_image[medium_cirrus_over_water] = 3
pre_class_image[thick_cirrus_over_water] = 4
pre_class_image[land] = 5
pre_class_image[snow_ice] = 7
pre_class_image[thin_cirrus_over_land] = 8
pre_class_image[medium_cirrus_over_land] = 9
pre_class_image[thick_cirrus_over_land] = 10
pre_class_image[thin_haze_over_water] = 13
pre_class_image[medium_haze_over_water] = 14
pre_class_image[cloud_over_land] = 15
pre_class_image[cloud_over_water] = 16
pre_class_image[water] = 17
pre_class_image[cirrus_cloud] = 18
pre_class_image[thick_cirrus_cloud] = 19
pre_class_image[cloud_shadow] = 22
# Clear data
del shadow, thin_cirrus_over_water, medium_cirrus_over_water
del thick_cirrus_over_water, land, snow_ice
del thin_cirrus_over_land, medium_cirrus_over_land, thick_cirrus_over_land
del thin_haze_over_water, medium_haze_over_water, cloud_over_land
del cloud_over_water, water, cirrus_cloud
del thick_cirrus_cloud, cloud_shadow
elif if_vnir:
# Calculate the reflectance at different bands
if rdn_header['interleave'].lower() == 'bil':
blue_refl = rdn_image[:, blue_band, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[:, green_band, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[:, red_band, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[:, nir_band, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
else:
blue_refl = rdn_image[blue_band, :, :] * np.pi * d2 / (
solar_flux[blue_band] * cos_sun_zenith)
green_refl = rdn_image[green_band, :, :] * np.pi * d2 / (
solar_flux[green_band] * cos_sun_zenith)
red_refl = rdn_image[red_band, :, :] * np.pi * d2 / (
solar_flux[red_band] * cos_sun_zenith)
nir_refl = rdn_image[nir_band, :, :] * np.pi * d2 / (
solar_flux[nir_band] * cos_sun_zenith)
rdn_image.flush()
del rdn_image
# Water
water = nir_refl < 0.05
# Land
land = ~water
# Shadow
shadow = (water | land) & (green_refl < 0.01)
land = land & (~shadow)
water = water & (~shadow)
# Cloud over land
Tc = 0.20
cloud_over_land = land & (blue_refl > Tc) & (red_refl > 0.15) & (
nir_refl < red_refl * 2.0) & (nir_refl > red_refl * 0.8)
land = land & (~cloud_over_land)
# Cloud over water
cloud_over_water = water & (blue_refl < 0.40) & (blue_refl > 0.20) & (
green_refl < blue_refl) & (nir_refl < green_refl)
water = water & (~cloud_over_water)
# Cloud shadow
cloud_shadow = (water | land) & (nir_refl < 0.12) & (nir_refl > 0.04)
land = land & (~cloud_shadow)
water = water & (~cloud_shadow)
# Haze over water
T2 = 0.04
T1 = 0.12
thin_haze_over_water = water & (nir_refl >= T1) & (nir_refl <= 0.06)
water = water & (~thin_haze_over_water)
medium_haze_over_water = water & (nir_refl >= 0.06) & (nir_refl <= T2)
water = water & (~medium_haze_over_water)
# Assign class values
pre_class_image[shadow] = 1
pre_class_image[land] = 5
pre_class_image[thin_haze_over_water] = 13
pre_class_image[medium_haze_over_water] = 14
pre_class_image[cloud_over_land] = 15
pre_class_image[cloud_over_water] = 16
pre_class_image[water] = 17
pre_class_image[cloud_shadow] = 22
# Clear data
del shadow, land, thin_haze_over_water
del medium_haze_over_water, cloud_over_land, cloud_over_water
del water, cloud_shadow
elif if_swir:
# Calculate the reflectance at different bands
if rdn_header['interleave'] == 'bil':
cirrus_refl = rdn_image[:, cirrus_band, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[:, swir1_band, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[:, swir2_band, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
else:
cirrus_refl = rdn_image[cirrus_band, :, :] * np.pi * d2 / (
solar_flux[cirrus_band] * cos_sun_zenith)
swir1_refl = rdn_image[swir1_band, :, :] * np.pi * d2 / (
solar_flux[swir1_band] * cos_sun_zenith)
swir2_refl = rdn_image[swir2_band, :, :] * np.pi * d2 / (
solar_flux[swir2_band] * cos_sun_zenith)
rdn_image.flush()
del rdn_image
# Water
water = swir1_refl < 0.03
# Land
land = ~water
# Cirrus over land & water
thin_cirrus = (cirrus_refl < 0.015) & (cirrus_refl > 0.010)
medium_cirrus = (cirrus_refl < 0.025) & (cirrus_refl >= 0.015)
thick_cirrus = (cirrus_refl < 0.040) & (cirrus_refl >= 0.025)
thin_cirrus_over_land = land & thin_cirrus
land = land & (~thin_cirrus_over_land)
medium_cirrus_over_land = land & medium_cirrus
land = land & (~medium_cirrus_over_land)
thick_cirrus_over_land = land & thick_cirrus
land = land & (~thick_cirrus_over_land)
thin_cirrus_over_water = water & thin_cirrus
water = water & (~thin_cirrus_over_water)
medium_cirrus_over_water = water & thin_cirrus
water = water & (~medium_cirrus_over_water)
thick_cirrus_over_water = water & thin_cirrus
water = water & (~thick_cirrus_over_water)
cirrus_cloud = (water
| land) & (cirrus_refl < 0.050) & (cirrus_refl > 0.040)
land = land & (~cirrus_cloud)
water = water & (~cirrus_cloud)
thick_cirrus_cloud = (water | land) & (cirrus_refl > 0.050)
land = land & (~thick_cirrus_cloud)
water = water & (~thick_cirrus_cloud)
# Assign class values
pre_class_image[thin_cirrus_over_water] = 2
pre_class_image[medium_cirrus_over_water] = 3
pre_class_image[thick_cirrus_over_water] = 4
pre_class_image[land] = 5
pre_class_image[thin_cirrus_over_land] = 8
pre_class_image[medium_cirrus_over_land] = 9
pre_class_image[thick_cirrus_over_land] = 10
pre_class_image[water] = 17
pre_class_image[cirrus_cloud] = 18
pre_class_image[thick_cirrus_cloud] = 19
# Clear data
del thin_cirrus_over_water, medium_cirrus_over_water, thick_cirrus_over_water,
del land, thin_cirrus_over_land, medium_cirrus_over_land, thick_cirrus_over_land
del water, cirrus_cloud, thick_cirrus_cloud
else:
logger.error(
'Cannot find appropriate wavelengths to do pre-classification.')
# Apply background mask
if background_mask_file is not None:
bg_mask_header = read_envi_header(
os.path.splitext(background_mask_file)[0] + '.hdr')
bg_mask_image = np.memmap(background_mask_file,
mode='r',
dtype='bool',
shape=(bg_mask_header['lines'],
bg_mask_header['samples']))
pre_class_image[bg_mask_image] = 0
bg_mask_image.flush()
del bg_mask_image
# Clear data
pre_class_image.flush()
del pre_class_image
# Write header
pre_class_header = empty_envi_header()
pre_class_header['description'] = 'Pre-classification'
pre_class_header['file type'] = 'ENVI Standard'
pre_class_header['samples'] = rdn_header['samples']
pre_class_header['lines'] = rdn_header['lines']
pre_class_header['classes'] = 23
pre_class_header['bands'] = 1
pre_class_header['byte order'] = 0
pre_class_header['header offset'] = 0
pre_class_header['interleave'] = 'bsq'
pre_class_header['data type'] = 1
pre_class_header['class names'] = [
'0: geocoded background', '1: shadow', '2: thin cirrus (water)',
'3: medium cirrus (water)', '4: thick cirrus (water)',
'5: land (clear)', '6: saturated', '7: snow/ice',
'8: thin cirrus (land)', '9: medium cirrus (land)',
'10: thick cirrus (land)', '11: thin haze (land)',
'12: medium haze (land)', '13: thin haze/glint (water)',
'14: med. haze/glint (water)', '15: cloud (land)', '16: cloud (water)',
'17: water', '18: cirrus cloud', '19: cirrus cloud (thick)',
'20: bright', '21: topogr. shadow', '22: cloud shadow'
]
pre_class_header['class lookup'] = [
80, 80, 80, 0, 0, 0, 0, 160, 250, 0, 200, 250, 0, 240, 250, 180, 100,
40, 200, 0, 0, 250, 250, 250, 240, 240, 170, 225, 225, 150, 210, 210,
120, 250, 250, 100, 230, 230, 80, 80, 100, 250, 130, 130, 250, 180,
180, 180, 160, 160, 240, 0, 0, 250, 180, 180, 100, 160, 160, 100, 200,
200, 200, 40, 40, 40, 50, 50, 50
]
pre_class_header['map info'] = rdn_header['map info']
pre_class_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(
os.path.splitext(pre_class_image_file)[0] + '.hdr', pre_class_header)
logger.info('Write the pre-classification map to %s.' %
(pre_class_image_file))
def merge_rdn(merged_image_file, mask_file, sensors):
""" Merge radiance images.
Arguments:
merged_image_file: str
Merged radiance image filename.
mask_file: str
Background mask filename.
sensors: dict
Sensor dictionaries.
"""
if os.path.exists(merged_image_file):
logger.info('Write the merged refletance image to %s.' %
merged_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read mask.
mask_header = read_envi_header(os.path.splitext(mask_file)[0] + '.hdr')
mask_image = np.memmap(mask_file,
mode='r',
dtype='bool',
shape=(mask_header['lines'],
mask_header['samples']))
# Get the map upper-left coordinates and pixel sizes of VNIR and SWIR images.
ulx, uly, pixel_size = float(mask_header['map info'][3]), float(
mask_header['map info'][4]), float(mask_header['map info'][5])
# Determine regular map grids.
x, y = np.meshgrid(ulx + np.arange(mask_header['samples']) * pixel_size,
uly - np.arange(mask_header['lines']) * pixel_size)
del ulx, uly, pixel_size
# Read radiance header and image.
header_dict = dict()
image_file_dict = dict()
bands_waves_fwhms = []
for sensor_index, sensor_dict in sensors.items():
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_rdn_image_file'])[0] + '.hdr')
for band in range(tmp_header['bands']):
bands_waves_fwhms.append([
'%s_%d' % (sensor_index, band), tmp_header['wavelength'][band],
tmp_header['fwhm'][band]
])
header_dict[sensor_index] = tmp_header
image_file_dict[sensor_index] = sensor_dict['ortho_rdn_image_file']
bands_waves_fwhms.sort(key=lambda x: x[1])
# Merge images.
wavelengths = []
fwhms = []
fid = open(merged_image_file, 'wb')
for v in bands_waves_fwhms:
# Determine which sensor, band to read.
sensor_index, band = v[0].split('_')
band = int(band)
wavelengths.append(v[1])
fwhms.append(v[2])
header = header_dict[sensor_index]
image_file = image_file_dict[sensor_index]
# Write image.
if ((v[1] >= 1339.0) &
(v[1] <= 1438.0)) | ((v[1] >= 1808.0) &
(v[1] <= 1978.0)) | (v[1] >= 2467.0):
# resampled_image = np.zeros(x.shape)
resampled_image = np.zeros(x.shape,
dtype='int16') # in int, by Ting
else:
offset = header[
'header offset'] + 2 * band * header['lines'] * header[
'samples'] # in bytes, int16 consists 2 bytes
rdn_image = np.memmap(
image_file,
# dtype='float32',
dtype='int16', # in int, by Ting
mode='r',
offset=offset,
shape=(header['lines'], header['samples']))
resampled_image = resample_ortho_rdn(
np.copy(rdn_image) /
1000, # divided by 1000 to convert back to original rdn
float(header_dict[sensor_index]['map info'][3]),
float(header_dict[sensor_index]['map info'][4]),
float(header_dict[sensor_index]['map info'][5]),
x,
y)
# resampled_image[mask_image] = 0.0
resampled_image = resampled_image * 1000 # times 1000 to convert to int, by Ting
resampled_image[mask_image] = 0 # in int, by Ting
rdn_image.flush()
del rdn_image
# fid.write(resampled_image.astype('float32').tostring())
fid.write(resampled_image.astype(
'int16').tostring()) # write in int ,by Ting
del resampled_image
fid.close()
del header_dict, image_file_dict, x, y
mask_image.flush()
del mask_image
# Write header.
header = empty_envi_header()
header['description'] = 'Merged radiance, in [mW/(cm2*um*sr)]'
header['file type'] = 'ENVI Standard'
header['samples'] = mask_header['samples']
header['lines'] = mask_header['lines']
header['bands'] = len(wavelengths)
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
# header['data type'] = 4
header['data type'] = 2 # in int, by Ting
header['wavelength'] = wavelengths
header['fwhm'] = fwhms
header['wavelength units'] = 'nm'
header['acquisition time'] = tmp_header['acquisition time']
header['map info'] = mask_header['map info']
header['coordinate system string'] = mask_header[
'coordinate system string']
write_envi_header(os.path.splitext(merged_image_file)[0] + '.hdr', header)
del header, tmp_header
logger.info('Write the merged refletance image to %s.' % merged_image_file)
def merge_dem_sca(background_mask_file, merged_dem_file, merged_sca_file,
sensors):
""" Merge DEM and SCA images.
Arguments:
background_mask_file: str
Background mask filename.
merged_dem_file: str
Merged DEM file.
merged_sca_file: str
Merged SCA file.
sensors: dict
Sensor dictionaries.
"""
if os.path.exists(background_mask_file) and os.path.exists(
merged_dem_file) and os.path.exists(merged_sca_file):
logger.info('Write the background mask to %s.' % background_mask_file)
logger.info('Write the merged DEM image to %s.' % merged_dem_file)
logger.info('Write the merged SCA image to %s.' % merged_sca_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
"""
Get spatial extent and pixel size.
"""
ulx, uly, lrx, lry, pixel_size = -np.inf, np.inf, np.inf, -np.inf, np.inf
for sensor_index, sensor_dict in sensors.items():
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_dem_image_file'])[0] + '.hdr')
tmp_ulx, tmp_uly = float(tmp_header['map info'][3]), float(
tmp_header['map info'][4])
tmp_pixel_size = float(tmp_header['map info'][5])
tmp_lrx, tmp_lry = tmp_ulx + tmp_header[
'samples'] * tmp_pixel_size, tmp_uly - tmp_header[
'lines'] * tmp_pixel_size
if tmp_ulx > ulx:
ulx = tmp_ulx
if tmp_uly < uly:
uly = tmp_uly
if tmp_lrx < lrx:
lrx = tmp_lrx
if tmp_lry > lry:
lry = tmp_lry
if tmp_pixel_size < pixel_size:
pixel_size = tmp_pixel_size
del tmp_ulx, tmp_uly, tmp_pixel_size, tmp_lrx, tmp_lry
# Process at 2x the smallest pixel size
pixel_size = 2 * pixel_size
# Find lower right corners of first CRS-aligned pixels at the
# upper left & lower right corners which are fully in the image
ulx, uly = np.ceil(ulx / pixel_size +
1) * pixel_size, np.floor(uly / pixel_size -
1) * pixel_size
lrx, lry = np.floor(lrx / pixel_size -
1) * pixel_size, np.ceil(lry / pixel_size +
1) * pixel_size
map_info = [
tmp_header['map info'][0], 1, 1, ulx, uly, pixel_size, pixel_size
] + tmp_header['map info'][7:]
crs = tmp_header['coordinate system string']
del tmp_header
logger.info('The spatial range and pixel size of merged images:')
logger.info('Map x = %.2f - %.2f' % (ulx, lrx))
logger.info('Map y = %.2f - %.2f' % (lry, uly))
logger.info('Pixel size = %.2f' % pixel_size)
"""
Determine regular map grids.
"""
x, y = np.meshgrid(np.arange(ulx, lrx, pixel_size),
np.arange(uly, lry, -pixel_size))
mask = np.full(x.shape, True, dtype='bool')
"""
Build a background mask.
"""
# Use DEM and SCA images to build this mask.
for sensor_index, sensor_dict in sensors.items():
# Use dem.
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_dem_image_file'])[0] + '.hdr')
tmp_image = np.memmap(sensor_dict['ortho_dem_image_file'],
dtype='float32',
mode='r',
shape=(tmp_header['lines'],
tmp_header['samples']))
tmp_ulx, tmp_uly = float(tmp_header['map info'][3]), float(
tmp_header['map info'][4])
tmp_pixel_size = float(tmp_header['map info'][5])
resampled_image = resample_ortho_dem(np.copy(tmp_image), tmp_ulx,
tmp_uly, tmp_pixel_size, x, y)
mask = mask & (resampled_image > 0.0)
# Use sca.
tmp_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_sca_image_file'])[0] + '.hdr')
tmp_image = np.memmap(sensor_dict['ortho_sca_image_file'],
dtype='float32',
mode='r',
shape=(tmp_header['bands'], tmp_header['lines'],
tmp_header['samples']))
tmp_ulx, tmp_uly = float(tmp_header['map info'][3]), float(
tmp_header['map info'][4])
tmp_pixel_size = float(tmp_header['map info'][5])
resampled_image = resample_ortho_sca(np.copy(tmp_image[0, :, :]),
tmp_ulx, tmp_uly, tmp_pixel_size,
x, y)
mask = mask & (resampled_image > 0.0)
# Clear data.
del tmp_header, tmp_ulx, tmp_uly, tmp_pixel_size, resampled_image
tmp_image.flush()
del tmp_image
mask = ~mask # 1: background pixels; 0: non-background pixels.
# Write the mask to a file.
fid = open(background_mask_file, 'wb')
fid.write(mask.tostring())
fid.close()
# Write the mask header to a file.
header = empty_envi_header()
header[
'description'] = 'Background mask (0: non-background; 1: background)'
header['file type'] = 'ENVI Standard'
header['samples'] = mask.shape[1]
header['lines'] = mask.shape[0]
header['bands'] = 1
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
header['data type'] = 1
header['map info'] = map_info
header['coordinate system string'] = crs
write_envi_header(
os.path.splitext(background_mask_file)[0] + '.hdr', header)
del header
logger.info('Write the background mask to %s.' % background_mask_file)
"""
Merge DEM.
"""
# Read the first DEM.
sensor_index = list(sensors.keys())[0]
sensor_dict = sensors[sensor_index]
raw_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_dem_image_file'])[0] + '.hdr')
raw_image = np.memmap(sensor_dict['ortho_dem_image_file'],
dtype='float32',
mode='r',
shape=(raw_header['lines'], raw_header['samples']))
resampled_image = resample_ortho_dem(np.copy(raw_image),
float(raw_header['map info'][3]),
float(raw_header['map info'][4]),
float(raw_header['map info'][5]), x,
y)
resampled_image[mask] = -1000.0
# Write the merged DEM to a file.
fid = open(merged_dem_file, 'wb')
fid.write(resampled_image.astype('float32').tostring())
fid.close()
del raw_header, sensor_index, sensor_dict, resampled_image
raw_image.flush()
del raw_image
# Write the merged DEM header to a file.
header = empty_envi_header()
header['description'] = 'Merged DEM, in [m]'
header['file type'] = 'ENVI Standard'
header['samples'] = mask.shape[1]
header['lines'] = mask.shape[0]
header['bands'] = 1
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
header['data type'] = 4
header['data ignore value'] = -1000.0
header['map info'] = map_info
header['coordinate system string'] = crs
write_envi_header(os.path.splitext(merged_dem_file)[0] + '.hdr', header)
del header
logger.info('Write the merged DEM image to %s.' % merged_dem_file)
"""
Merge SCA.
"""
# Read the first SCA.
sensor_index = list(sensors.keys())[0]
sensor_dict = sensors[sensor_index]
raw_header = read_envi_header(
os.path.splitext(sensor_dict['ortho_sca_image_file'])[0] + '.hdr')
raw_image = np.memmap(sensor_dict['ortho_sca_image_file'],
dtype='float32',
mode='r',
shape=(raw_header['bands'], raw_header['lines'],
raw_header['samples']))
# Write data.
fid = open(merged_sca_file, 'wb')
for band in range(raw_header['bands']):
resampled_image = resample_ortho_sca(np.copy(raw_image[band, :, :]),
float(raw_header['map info'][3]),
float(raw_header['map info'][4]),
float(raw_header['map info'][5]),
x, y)
resampled_image[mask] = -1000.0
fid.write(resampled_image.astype('float32').tostring())
del resampled_image
fid.close()
del sensor_index, sensor_dict
raw_image.flush()
del raw_image
# Write header.
header = empty_envi_header()
header['description'] = 'Merged SCA, in [deg]'
header['file type'] = 'ENVI Standard'
header['samples'] = mask.shape[1]
header['lines'] = mask.shape[0]
header['bands'] = 2
header['byte order'] = 0
header['header offset'] = 0
header['interleave'] = 'bsq'
header['data type'] = 4
header['band names'] = ['Sensor Zenith [deg]', 'Sensor Azimuth [deg]']
header['sun azimuth'] = raw_header['sun azimuth']
header['sun zenith'] = raw_header['sun zenith']
header['data ignore value'] = -1000.0
header['map info'] = map_info
header['coordinate system string'] = crs
write_envi_header(os.path.splitext(merged_sca_file)[0] + '.hdr', header)
del raw_header, header
logger.info('Write the merged SCA image to %s.' % merged_sca_file)
def estimate_vis(vis_file, ddv_file, atm_lut_file, rdn_file, sca_file,
background_mask_file):
""" Estimate visibility.
Arguments:
vis_file: str
Visibility map filename.
ddv_file: str
Dark dense vegetation map filename.
atm_lut_file: str
Atmospheric lookup table filename.
rdn_file: str
Radiance filename.
sca_file: str
Scan angle filename.
background_mask_file:
Background mask filename.
"""
if os.path.exists(ddv_file) and os.path.exists(vis_file):
logger.info('Write the visibility map to %s.' % vis_file)
logger.info('Write the DDV map to %s.' % ddv_file)
return
from ENVI import read_envi_header, empty_envi_header, write_envi_header
from AtmLUT import read_binary_metadata
from Spectra import get_closest_wave
from AtmCorr import atm_corr_band
# Read radiance header.
rdn_header = read_envi_header(os.path.splitext(rdn_file)[0] + '.hdr')
# Find VNIR and SWIR sensor wavelengths.
red_wave, red_band = get_closest_wave(rdn_header['wavelength'], 660)
nir_wave, nir_band = get_closest_wave(rdn_header['wavelength'], 850)
swir1_wave, swir1_band = get_closest_wave(rdn_header['wavelength'], 1650)
swir2_wave, swir2_band = get_closest_wave(rdn_header['wavelength'], 2130)
# Determine the sensor type.
if_vnir = abs(red_wave - 660) < 20 and abs(nir_wave - 850) < 20
if_swir = abs(swir1_wave - 1650) < 20 or abs(swir2_wave - 2130) < 20
if if_vnir and if_swir:
logger.info(
'Both VNIR and SWIR bands are used for estimating visibility.')
elif if_vnir:
logger.info('Only VNIR bands are used for estimating visibility.')
elif if_swir:
logger.info(
'Only SWIR bands are available, a constant visibility (23 km) is used.'
)
else:
logger.error(
'Cannot find appropriate bands for estimating visibility.')
# Read atmospheric lookup table.
atm_lut_metadata = read_binary_metadata(atm_lut_file + '.meta')
atm_lut_metadata['shape'] = tuple(
[int(v) for v in atm_lut_metadata['shape']])
atm_lut_RHO = np.array([float(v) for v in atm_lut_metadata['RHO']])
atm_lut_WVC = np.array([float(v) for v in atm_lut_metadata['WVC']])
atm_lut_VIS = np.array([float(v) for v in atm_lut_metadata['VIS']])
atm_lut_VZA = np.array([float(v) for v in atm_lut_metadata['VZA']])
atm_lut_RAA = np.array([float(v) for v in atm_lut_metadata['RAA']])
atm_lut = np.memmap(atm_lut_file,
dtype=atm_lut_metadata['dtype'],
mode='r',
shape=atm_lut_metadata['shape']
) # shape = (RHO, WVC, VIS, VZA, RAA, WAVE)
# Read radiance image.
rdn_image = np.memmap(
rdn_file,
# dtype='float32',
dtype='int16', # in int, by Ting
mode='r',
shape=(rdn_header['bands'], rdn_header['lines'],
rdn_header['samples']))
# Read VZA and RAA image.
sca_header = read_envi_header(os.path.splitext(sca_file)[0] + '.hdr')
saa = float(sca_header['sun azimuth'])
sca_image = np.memmap(sca_file,
dtype='float32',
offset=0,
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# vza
vza_image = np.copy(sca_image[0, :, :])
# raa
raa_image = saa - sca_image[1, :, :]
raa_image[raa_image < 0] += 360.0
raa_image[raa_image > 180] = 360.0 - raa_image[raa_image > 180]
raa_image[raa_image == 180] = 179.0
# Constrain RAA within bounds of ALT
raa_max = atm_lut_RAA.max()
raa_image[raa_image > raa_max] = raa_max - 5e-2
# clear data
sca_image.flush()
del sca_header, saa, sca_image
# Set visibility and water vapor column values.
metadata = read_binary_metadata(atm_lut_file + '.meta')
tmp_wvc_image = np.ones(
vza_image.shape) * atm_db_wvc_lut[metadata['atm_mode']]
tmp_vis_image = np.ones(vza_image.shape) * 23
del metadata
# Read background mask.
bg_header = read_envi_header(
os.path.splitext(background_mask_file)[0] + '.hdr')
bg_mask = np.memmap(background_mask_file,
dtype='bool',
mode='r',
shape=(bg_header['lines'], bg_header['samples']))
# Calculate NDVI.
red_refl = atm_corr_band(
atm_lut_WVC,
atm_lut_VIS,
atm_lut_VZA,
atm_lut_RAA,
atm_lut[..., red_band],
tmp_wvc_image,
tmp_vis_image,
vza_image,
raa_image,
rdn_image[red_band, :, :] /
1000, # divided by 1000 to convert to original rdn, by Ting
bg_mask)
nir_refl = atm_corr_band(atm_lut_WVC, atm_lut_VIS, atm_lut_VZA,
atm_lut_RAA, atm_lut[...,
nir_band], tmp_wvc_image,
tmp_vis_image, vza_image, raa_image,
rdn_image[nir_band, :, :] / 1000, bg_mask)
ndvi = (nir_refl - red_refl) / (nir_refl + red_refl + 1e-10)
vis_image = np.zeros((rdn_header['lines'], rdn_header['samples']))
if if_vnir and if_swir:
# Decide which SWIR band to use.
if abs(swir2_wave - 2130) < 20:
swir_wave = swir2_wave
swir_band = swir2_band
swir_refl_upper_bounds = [0.05, 0.10, 0.12]
red_swir_ratio = 0.50
else:
swir_wave = swir1_wave
swir_band = swir1_band
swir_refl_upper_bounds = [0.10, 0.15, 0.18]
red_swir_ratio = 0.25
# Calculate swir refletance.
swir_refl = atm_corr_band(
atm_lut_WVC,
atm_lut_VIS,
atm_lut_VZA,
atm_lut_RAA,
atm_lut[..., swir_band],
tmp_wvc_image,
tmp_vis_image,
vza_image,
raa_image,
rdn_image[swir_band, :, :] /
1000, # divided by 1000 to convert to original rdn, by Ting
bg_mask)
# Get DDV mask.
for swir_refl_upper_bound in swir_refl_upper_bounds:
ddv_mask = (ndvi > 0.10) & (swir_refl < swir_refl_upper_bound) & (
swir_refl > 0.01)
percentage = np.sum(ddv_mask[~bg_mask]) / np.sum(~bg_mask)
if percentage > 0.02:
logger.info(
'The SWIR wavelength %.2f is used for detecting dark dense vegetation.'
% swir_wave)
logger.info('The SWIR reflectance upper boundary is %.2f.' %
swir_refl_upper_bound)
logger.info('The number of DDV pixels is %.2f%%.' %
(percentage * 100))
break
# Estimate Visibility.
rows, columns = np.where(ddv_mask)
# Estimate red reflectance.
red_refl[rows, columns] = red_swir_ratio * swir_refl[rows, columns]
del swir_refl
for row, column in zip(rows, columns):
interp_rdn = interp_atm_lut(atm_lut_RHO, atm_lut_WVC, atm_lut_VZA,
atm_lut_RAA, atm_lut[..., red_band],
red_refl[row, column],
tmp_wvc_image[row, column],
vza_image[row,
column], raa_image[row,
column])
vis_image[row, column] = np.interp(
rdn_image[red_band, row, column] / 1000, interp_rdn[::-1],
atm_lut_VIS[::-1]
) # divided by 1000 to convert to original rdn, by Ting
rdn_image.flush()
del rdn_image
logger.info(
'Visibility [km] statistics: min=%.2f, max=%.2f, avg=%.2f, sd=%.2f.'
% (vis_image[ddv_mask].min(), vis_image[ddv_mask].max(),
vis_image[ddv_mask].mean(), vis_image[ddv_mask].std()))
# Fill gaps with average values.
vis_image[~ddv_mask] = vis_image[ddv_mask].mean()
vis_image[bg_mask] = -1000.0
# Write the visibility data.
fid = open(vis_file, 'wb')
fid.write(vis_image.astype('float32').tostring())
fid.close()
del vis_image
vis_header = empty_envi_header()
vis_header['description'] = 'Visibility [km]'
vis_header['samples'] = rdn_header['samples']
vis_header['lines'] = rdn_header['lines']
vis_header['bands'] = 1
vis_header['byte order'] = 0
vis_header['header offset'] = 0
vis_header['interleave'] = 'bsq'
vis_header['data ignore value'] = -1000.0
vis_header['data type'] = 4
vis_header['map info'] = rdn_header['map info']
vis_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(os.path.splitext(vis_file)[0] + '.hdr', vis_header)
logger.info('Write the visibility image to %s.' % vis_file)
# Write the DDV data.
fid = open(ddv_file, 'wb')
fid.write(ddv_mask.tostring())
fid.close()
del ddv_mask
ddv_header = empty_envi_header()
ddv_header[
'description'] = 'DDV mask (1: Dark dense vegetaion; 0: non-dark dense vegetation)'
ddv_header['samples'] = rdn_header['samples']
ddv_header['lines'] = rdn_header['lines']
ddv_header['bands'] = 1
ddv_header['byte order'] = 0
ddv_header['header offset'] = 0
ddv_header['interleave'] = 'bsq'
ddv_header['data type'] = 1
ddv_header['map info'] = rdn_header['map info']
ddv_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(os.path.splitext(ddv_file)[0] + '.hdr', ddv_header)
logger.info('Write the DDV mask to %s.' % ddv_file)
elif if_vnir:
pass
elif if_swir:
pass
# Clear data
del ndvi, red_refl, nir_refl, rdn_header
relative_azimuth[relative_azimuth<0] += 360.0 relative_azimuth[relative_azimuth>180] = 360.0-relative_azimuth[relative_azimuth>180] relative_azimuth = relative_azimuth*10 relative_azimuth[bg_mask_image] = -1 relative_azimuth = np.int16(relative_azimuth) abg_altitude = np.ones(relative_azimuth.shape, dtype='int16')*674 abg_altitude[bg_mask_image] = -1 import matplotlib.pyplot as plt plt.imshow(view_zenith) plt.colorbar() plt.show() plt.imshow(relative_azimuth) plt.colorbar() plt.show() fid = open(new_sca_file, 'wb') fid.write(view_zenith.tostring()) fid.write(relative_azimuth.tostring()) fid.write(abg_altitude.tostring()) fid.close() raw_sca_header['bands'] = 3 raw_sca_header['data type'] = 2 raw_sca_header['GPS long-lat-alt'] = [-93.16, 45.40, 676.65] raw_sca_header['heading[deg]'] = 273.5 raw_sca_header['DEM height[m]'] = 255.12 write_envi_header(new_sca_file+'.hdr', raw_sca_header)
def orthorectify_dem(ortho_dem_image_file, igm_image_file, glt_image_file):
""" Do orthorectifications on DEM.
Arguments:
ortho_dem_image_file: str
Orthorectified DEM image filename.
igm_image_file: str
IGM image filename.
glt_image_file: str
Geographic look-up table image filename.
"""
if os.path.exists(ortho_dem_image_file):
logger.info('Write the geo-rectified DEM image to %s.' %ortho_dem_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read IGM image (the third band is DEM).
igm_header = read_envi_header(igm_image_file+'.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
shape=(igm_header['bands'],
igm_header['lines'],
igm_header['samples']))
del igm_header
# Read GLT image.
glt_header = read_envi_header(glt_image_file+'.hdr')
glt_image = np.memmap(glt_image_file,
dtype=np.int32,
mode='r',
shape=(glt_header['bands'],
glt_header['lines'],
glt_header['samples']))
# Get spatial indices.
I, J = np.where((glt_image[0,:,:]>=0)&(glt_image[1,:,:]>=0))
ortho_dem_image = np.zeros((glt_header['lines'], glt_header['samples']), dtype='float32')
# Orthorectify DEM.
fid = open(ortho_dem_image_file, 'wb')
ortho_dem_image[:,:] = -1000.0
ortho_dem_image[I,J] = igm_image[2, glt_image[0,I,J], glt_image[1,I,J]]
fid.write(ortho_dem_image.tostring())
fid.close()
del ortho_dem_image
igm_image.flush()
glt_image.flush()
del igm_image, glt_image
# Write header.
ortho_dem_header = empty_envi_header()
ortho_dem_header['description'] = 'Geo-rectified DEM, in [m]'
ortho_dem_header['file type'] = 'ENVI Standard'
ortho_dem_header['samples'] = glt_header['samples']
ortho_dem_header['lines'] = glt_header['lines']
ortho_dem_header['bands'] = 1
ortho_dem_header['byte order'] = 0
ortho_dem_header['header offset'] = 0
ortho_dem_header['data ignore value'] = -1000.0
ortho_dem_header['interleave'] = 'bsq'
ortho_dem_header['data type'] = 4
ortho_dem_header['map info'] = glt_header['map info']
ortho_dem_header['coordinate system string'] = glt_header['coordinate system string']
write_envi_header(ortho_dem_image_file+'.hdr', ortho_dem_header)
del glt_header, ortho_dem_header
logger.info('Write the geo-rectified DEM image to %s.' %ortho_dem_image_file)
def calculate_sca(sca_image_file, imugps_file, igm_image_file, sun_angles):
""" Create a scan angle (SCA) image.
Arguments:
sca_image_file: str
Scan angle filename.
imu_gps_file: str
IMU/GPS filename.
igm_image_file: str
IGM image filename.
sun_angles: list
Sun angles [sun zenith, sun azimuth], in degrees.
"""
if os.path.exists(sca_image_file):
logger.info('Write the SCA to %s.' % sca_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read IGM data.
igm_header = read_envi_header(os.path.splitext(igm_image_file)[0] + '.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
offset=0,
shape=(igm_header['bands'], igm_header['lines'],
igm_header['samples']))
# Read GPS data.
imugps = np.loadtxt(imugps_file) # ID, X, Y, Z, R, P, H, ...
# Calculate sensor angles.
DX = igm_image[0, :, :] - np.expand_dims(imugps[:, 1], axis=1)
DY = igm_image[1, :, :] - np.expand_dims(imugps[:, 2], axis=1)
DZ = igm_image[2, :, :] - np.expand_dims(imugps[:, 3], axis=1)
igm_image.flush()
del imugps, igm_image
view_zenith = np.abs(np.arctan(np.sqrt(DX**2 + DY**2) / DZ))
index = view_zenith >= np.deg2rad(40.0)
if np.any(index):
view_zenith[index] = np.deg2rad(39.0)
del index
view_azimuth = np.arcsin(np.abs(DX) / np.sqrt(DX**2 + DY**2))
index = (DX > 0) & (DY < 0)
view_azimuth[index] = np.pi - view_azimuth[index]
index = (DX < 0) & (DY < 0)
view_azimuth[index] = np.pi + view_azimuth[index]
index = (DX < 0) & (DY > 0)
view_azimuth[index] = 2 * np.pi - view_azimuth[index]
del DX, DY, DZ, index
# Save scan angles.
fid = open(sca_image_file, 'wb')
fid.write(np.rad2deg(view_zenith).astype('float32').tostring())
fid.write(np.rad2deg(view_azimuth).astype('float32').tostring())
fid.close()
del view_zenith, view_azimuth
# Write scan angle header file.
sca_header = empty_envi_header()
sca_header['description'] = 'SCA [deg]'
sca_header['file type'] = 'ENVI Standard'
sca_header['samples'] = igm_header['samples']
sca_header['lines'] = igm_header['lines']
sca_header['bands'] = 2
sca_header['byte order'] = 0
sca_header['header offset'] = 0
sca_header['interleave'] = 'bsq'
sca_header['data type'] = 4
sca_header['band names'] = ['Sensor Zenith [deg]', 'Sensor Azimuth [deg]']
sca_header['sun zenith'] = sun_angles[0]
sca_header['sun azimuth'] = sun_angles[1]
sca_header_file = os.path.splitext(sca_image_file)[0] + '.hdr'
write_envi_header(sca_header_file, sca_header)
del sca_header
logger.info('Write the SCA to %s.' % sca_image_file)
def make_radio_cali_file_Hyspex(radio_cali_file, dn_image_file, setting_file):
""" Make a Hyspex radiometric calibration file.
Parameters
----------
radio_cali_file: str
Hyspex radiometric calibration coefficiets filename.
dn_image_file: str
Hyspex digital number (DN) image filename.
setting_file: str
Hyspex radiometric calibration setting filename.
"""
if os.path.exists(radio_cali_file):
logger.info('Write the radiometric calibration coefficients to %s.' %
radio_cali_file)
return
from ENVI import empty_envi_header, write_envi_header
from Spectra import estimate_fwhms_from_waves
from scipy import constants
# Read metadata from raw HySpex image
header = dict()
try:
fid = open(dn_image_file, 'rb')
except:
logger.error('Cannot read calibration data from %s.' % dn_image_file)
header['word'] = np.fromfile(fid, dtype=np.int8, count=8)
header['hdrSize'] = np.fromfile(fid, dtype=np.int32, count=1)[0]
header['serialNumber'] = np.fromfile(fid, dtype=np.int32, count=1)[0]
header['configFile'] = np.fromfile(fid, dtype=np.int8, count=200)
header['settingFile'] = np.fromfile(fid, dtype=np.int8, count=120)
header['scalingFactor'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['electronics'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['comsettingsElectronics'] = np.fromfile(fid,
dtype=np.uint32,
count=1)[0]
header['comportElectronics'] = np.fromfile(fid, dtype=np.int8, count=44)
header['fanSpeed'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['backTemperature'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['Pback'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['Iback'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['Dback'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['comport'] = np.fromfile(fid, dtype=np.int8, count=64)
header['detectstring'] = np.fromfile(fid, dtype=np.int8, count=200)
header['sensor'] = np.fromfile(fid, dtype=np.int8, count=176)
header['temperature_end'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['temperature_start'] = np.fromfile(fid, dtype=np.float64,
count=1)[0]
header['temperature_calibration'] = np.fromfile(fid,
dtype=np.float64,
count=1)[0]
header['framegrabber'] = np.fromfile(fid, dtype=np.int8, count=200)
header['ID'] = np.fromfile(fid, dtype=np.int8, count=200)
header['supplier'] = np.fromfile(fid, dtype=np.int8, count=200)
header['leftGain'] = np.fromfile(fid, dtype=np.int8, count=32)
header['rightGain'] = np.fromfile(fid, dtype=np.int8, count=32)
header['comment'] = np.fromfile(fid, dtype=np.int8, count=200)
header['backgroundFile'] = np.fromfile(fid, dtype=np.int8, count=200)
header['recordHD'] = np.fromfile(fid, dtype=np.int8, count=1)
header['unknownPOINTER'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['serverIndex'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['comsettings'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['numberOfBackground'] = np.fromfile(fid, dtype=np.uint32,
count=1)[0]
header['spectralSize'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['spatialSize'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['binning'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['detected'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['integrationTime'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['frameperiod'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['defaultR'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['defaultG'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['defaultB'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['bitshift'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['temperatureOffset'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['shutter'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['backgroundPresent'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['power'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['current'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['bias'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['bandwidth'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['vin'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['vref'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['sensorVin'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['sensorVref'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['coolingTemperature'] = np.fromfile(fid, dtype=np.uint32,
count=1)[0]
header['windowStart'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['windowStop'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['readoutTime'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['p'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['i'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['d'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['numberOfFrames'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['nobp'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['dw'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['EQ'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['lens'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['FOVexp'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['scanningMode'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['calibAvailable'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['numberOfAvg'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['SF'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['apertureSize'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['pixelSizeX'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['pixelSizeY'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['temperature'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['maxFramerate'] = np.fromfile(fid, dtype=np.float64, count=1)[0]
header['spectralCalibPOINTER'] = np.fromfile(fid, dtype=np.int32,
count=1)[0]
header['REPOINTER'] = np.fromfile(fid, dtype=np.int32, count=1)[0]
header['QEPOINTER'] = np.fromfile(fid, dtype=np.int32, count=1)[0]
header['backgroundPOINTER'] = np.fromfile(fid, dtype=np.int32, count=1)[0]
header['badPixelsPOINTER'] = np.fromfile(fid, dtype=np.int32, count=1)[0]
header['imageFormat'] = np.fromfile(fid, dtype=np.uint32, count=1)[0]
header['spectralVector'] = np.fromfile(fid,
dtype=np.float64,
count=header['spectralSize'])
header['QE'] = np.fromfile(fid,
dtype=np.float64,
count=header['spectralSize'])
header['RE'] = np.fromfile(fid,
dtype=np.float64,
count=header['spectralSize'] *
header['spatialSize'])
header['RE'].shape = (header['spectralSize'], header['spatialSize'])
header['background'] = np.fromfile(fid,
dtype=np.float64,
count=header['spectralSize'] *
header['spatialSize'])
header['background'].shape = (header['spectralSize'],
header['spatialSize'])
header['badPixels'] = np.fromfile(fid,
dtype=np.uint32,
count=header['nobp'])
if header['serialNumber'] >= 3000 and header['serialNumber'] <= 5000:
header['backgroundLast'] = np.fromfile(fid,
dtype=np.float64,
count=header['spectralSize'] *
header['spatialSize'])
header['backgroundLast'].shape = (header['spectralSize'],
header['spatialSize'])
fid.close()
# Convert int8 array to string
def from_int8array_to_string(int8_array):
string = ''
for int8_value in int8_array:
if int8_value == 0:
break
string += chr(int8_value)
return string
for key in header:
if header[key].dtype.name == 'int8':
header[key] = from_int8array_to_string(header[key])
# Replace QE, wavelength & RE values if setting file exists
if setting_file is not None:
setting = get_hyspex_setting(setting_file)
header['QE'] = np.array(setting['QE'])
header['spectralVector'] = np.array(setting['spectral_calib'])
header['RE'] = np.array(setting['RE']).reshape(
(setting['spectral_size'], setting['spatial_size']))
# Calculate other values
header['spectralSampling'] = np.diff(header['spectralVector'])
header['spectralSampling'] = np.append(header['spectralSampling'],
header['spectralSampling'][-1])
header['solidAngle'] = header['pixelSizeX'] * header['pixelSizeY']
header['satValue'] = np.power(2, 16.0 - header['bitshift']) - 1
header['fwhms'] = estimate_fwhms_from_waves(header['spectralVector'])
# Calculate gain & offset coefficients
fid = open(radio_cali_file, 'wb')
# Gain
h = constants.Planck # Planck constant
c = constants.c * 1e+9 # Light speed, in [nm/s]
SF = header['SF'] # Image data scaling factor, float
RE = header['RE'] # Relative efficiency, shape=(bands, samples)
QE = np.expand_dims(header['QE'],
axis=1) # Quantum efficiency, shape=(bands, 1)
center_wavelength = np.expand_dims(header['spectralVector'],
axis=1) # shape=(bands, 1)
wavelength_interval = np.expand_dims(header['spectralSampling'],
axis=1) # shape=(bands, 1)
integration_time = header['integrationTime'] # float
aperture_area = np.pi * header['apertureSize'] * header[
'apertureSize'] # float
solid_angle = header['solidAngle'] # float
gain = h * c * 1e6 / (
RE * QE * SF * integration_time * aperture_area * solid_angle *
wavelength_interval * center_wavelength
) * 100.0 # shape=(bands, samples); 100.0 to convert radiance to mW/(cm2*um*sr).
fid.write(gain.astype('float64').tostring())
del h, c, SF, RE, QE, center_wavelength, wavelength_interval, integration_time, aperture_area, solid_angle, gain
# Offset
fid.write(header['background'].tostring())
if header['serialNumber'] >= 3000 and header['serialNumber'] <= 5000:
fid.write(header['backgroundLast'].tostring())
bands = 3
else:
bands = 2
fid.close()
# Write header
radio_cali_header = empty_envi_header()
radio_cali_header = empty_envi_header()
radio_cali_header[
'description'] = 'Hyspex radiometric calibration coefficients.'
radio_cali_header['file type'] = 'ENVI Standard'
radio_cali_header['bands'] = bands
radio_cali_header['lines'] = header['spectralSize']
radio_cali_header['samples'] = header['spatialSize']
radio_cali_header['interleave'] = 'bsq'
radio_cali_header['byte order'] = 0
radio_cali_header['data type'] = 5
radio_cali_header['band names'] = [
'gain', 'background'
] if bands == 2 else ['gain', 'background', 'backgroundLast']
radio_cali_header['waves'] = list(header['spectralVector'])
radio_cali_header['fwhms'] = list(header['fwhms'])
write_envi_header(
os.path.splitext(radio_cali_file)[0] + '.hdr', radio_cali_header)
del radio_cali_header, header
logger.info('Write the radiometric calibration coefficients to %s.' %
radio_cali_file)
def build_glt(glt_image_file, igm_image_file, pixel_size, map_crs):
""" Create a geographic lookup table (GLT) image.
Notes:
(1) This code is adapted from Adam Chlus's ([email protected]) script.
(2) The GLT image consists of two bands:
Band 0: Sample Lookup:
Pixel values indicate the column number of the pixel
in the input geometry file that belongs at the given Y
location in the output image.
Band 1: Line Lookup:
Pixel values indicate the row number of the pixel
in the input geometry file that belongs at the given X
location in the output image.
(3) For more details about GLT, refer to https://www.harrisgeospatial.com/
docs/GeoreferenceFromInputGeometry.html.
Arguments:
glt_image_file: str
Geographic look-up table filename.
igm_image_file: str
Input geometry filename.
pixel_size: float
Output image pixel size.
map_crs: osr object
GLT image map coordinate system.
"""
if os.path.exists(glt_image_file):
logger.info('Write the GLT to %s.' % glt_image_file)
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
# Read IGM.
igm_header = read_envi_header(os.path.splitext(igm_image_file)[0] + '.hdr')
igm_image = np.memmap(igm_image_file,
dtype='float64',
mode='r',
offset=0,
shape=(igm_header['bands'], igm_header['lines'],
igm_header['samples']))
# Estimate output spatial extent.
X_Min = igm_image[0, :, :].min()
X_Max = igm_image[0, :, :].max()
Y_Min = igm_image[1, :, :].min()
Y_Max = igm_image[1, :, :].max()
X_Min = np.floor(X_Min / pixel_size) * pixel_size - pixel_size
X_Max = np.ceil(X_Max / pixel_size) * pixel_size + pixel_size
Y_Min = np.floor(Y_Min / pixel_size) * pixel_size - pixel_size
Y_Max = np.ceil(Y_Max / pixel_size) * pixel_size + pixel_size
igm_image.flush()
del igm_image
# Build VRT for IGM.
igm_vrt_file = os.path.splitext(igm_image_file)[0] + '.vrt'
igm_vrt = open(igm_vrt_file, 'w')
igm_vrt.write('<VRTDataset rasterxsize="%s" rasterysize="%s">\n' %
(igm_header['samples'], igm_header['lines']))
for band in range(igm_header['bands']):
igm_vrt.write('\t<VRTRasterBand dataType="%s" band="%s">\n' %
("Float64", band + 1))
igm_vrt.write('\t\t<SimpleSource>\n')
igm_vrt.write(
'\t\t\t<SourceFilename relativeToVRT="1">%s</SourceFilename>\n' %
igm_image_file.replace('&', '&'))
igm_vrt.write('\t\t\t<SourceBand>%s</SourceBand>\n' % (band + 1))
igm_vrt.write(
'\t\t\t<SourceProperties RasterXSize="%s" RasterYSize="%s" DataType="%s" BlockXSize="%s" BlockYSize="%s" />\n'
% (igm_header['samples'], igm_header['lines'], "Float64",
igm_header['samples'], 1))
igm_vrt.write(
'\t\t\t<SrcRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(igm_header['samples'], igm_header['lines']))
igm_vrt.write(
'\t\t\t<DstRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(igm_header['samples'], igm_header['lines']))
igm_vrt.write("\t\t</SimpleSource>\n")
igm_vrt.write("\t</VRTRasterBand>\n")
igm_vrt.write("</VRTDataset>\n")
igm_vrt.close()
# Make IGM index image.
index_image_file = igm_image_file + '_Index'
index_image = np.memmap(index_image_file,
dtype='int32',
mode='w+',
offset=0,
shape=(2, igm_header['lines'],
igm_header['samples']))
index_image[0, :, :], index_image[1, :, :] = np.mgrid[
0:igm_header['lines'], 0:igm_header['samples']]
index_image.flush()
del index_image
index_header = empty_envi_header()
index_header['description'] = 'IGM Image Index'
index_header['samples'] = igm_header['samples']
index_header['lines'] = igm_header['lines']
index_header['bands'] = 2
index_header['byte order'] = 0
index_header['header offset'] = 0
index_header['interleave'] = 'bsq'
index_header['data type'] = 3
index_header['band names'] = ['Image Row', 'Image Column']
index_header_file = os.path.splitext(index_image_file)[0] + '.hdr'
write_envi_header(index_header_file, index_header)
# Build VRT for IGM Index.
index_vrt_file = os.path.splitext(index_image_file)[0] + '.vrt'
index_vrt = open(index_vrt_file, 'w')
index_vrt.write('<VRTDataset rasterxsize="%s" rasterysize="%s">\n' %
(index_header['samples'], index_header['lines']))
index_vrt.write('\t<Metadata Domain = "GEOLOCATION">\n')
index_vrt.write('\t\t<mdi key="X_DATASET">%s</mdi>\n' %
igm_image_file.replace('&', '&'))
index_vrt.write('\t\t<mdi key="X_BAND">1</mdi>\n')
index_vrt.write('\t\t<mdi key="Y_DATASET">%s</mdi>\n' %
igm_image_file.replace('&', '&'))
index_vrt.write('\t\t<mdi key="Y_BAND">2</mdi>\n')
index_vrt.write('\t\t<mdi key="PIXEL_OFFSET">0</mdi>\n')
index_vrt.write('\t\t<mdi key="LINE_OFFSET">0</mdi>\n')
index_vrt.write('\t\t<mdi key="PIXEL_STEP">1</mdi>\n')
index_vrt.write('\t\t<mdi key="LINE_STEP">1</mdi>\n')
index_vrt.write('\t</Metadata>\n')
for band in range(index_header['bands']):
index_vrt.write('\t<VRTRasterBand dataType="%s" band="%s">\n' %
("Int16", band + 1))
index_vrt.write('\t\t<SimpleSource>\n')
index_vrt.write(
'\t\t\t<SourceFilename relativeToVRT="1">%s</SourceFilename>\n' %
index_image_file.replace('&', '&'))
index_vrt.write('\t\t\t<SourceBand>%s</SourceBand>\n' % (band + 1))
index_vrt.write(
'\t\t\t<SourceProperties RasterXSize="%s" RasterYSize="%s" DataType="%s" BlockXSize="%s" BlockYSize="%s" />\n'
% (index_header['samples'], index_header['lines'], "Int32",
index_header['samples'], 1))
index_vrt.write(
'\t\t\t<SrcRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(index_header['samples'], index_header['lines']))
index_vrt.write(
'\t\t\t<DstRect xOff="0" yOff="0" xSize="%s" ySize="%s" />\n' %
(index_header['samples'], index_header['lines']))
index_vrt.write("\t\t</SimpleSource>\n")
index_vrt.write("\t</VRTRasterBand>\n")
index_vrt.write("</VRTDataset>\n")
index_vrt.close()
# Build GLT.
tmp_glt_image_file = glt_image_file + '.tif'
tmp_glt_image = gdal.Warp(tmp_glt_image_file,
index_vrt_file,
outputBounds=(X_Min, Y_Min, X_Max, Y_Max),
xRes=pixel_size,
yRes=pixel_size,
resampleAlg='near',
dstSRS=map_crs.ExportToProj4(),
dstNodata=-1,
multithread=True,
geoloc=True)
del tmp_glt_image
# Convert the .tif file to ENVI format.
ds = gdal.Open(tmp_glt_image_file, gdal.GA_ReadOnly)
lines = ds.RasterYSize
samples = ds.RasterXSize
glt_image = np.memmap(glt_image_file,
dtype='int32',
mode='w+',
offset=0,
shape=(2, lines, samples))
for band in range(ds.RasterCount):
glt_image[band, :, :] = ds.GetRasterBand(band + 1).ReadAsArray()
glt_image.flush()
del glt_image
ds = None
# Write GLT header.
glt_header = empty_envi_header()
glt_header['description'] = 'GLT'
glt_header['file type'] = 'ENVI Standard'
glt_header['samples'] = samples
glt_header['lines'] = lines
glt_header['bands'] = 2
glt_header['byte order'] = 0
glt_header['header offset'] = 0
glt_header['interleave'] = 'bsq'
glt_header['data type'] = 3
glt_header['data ignore value'] = -1
glt_header['band names'] = ['Image Row', 'Image Column']
glt_header['coordinate system string'] = map_crs.ExportToWkt()
glt_header['map info'] = [
map_crs.GetAttrValue('projcs').replace(',', ''), 1, 1, X_Min, Y_Max,
pixel_size, pixel_size, ' ', ' ',
map_crs.GetAttrValue('datum'),
map_crs.GetAttrValue('unit')
]
if map_crs.GetAttrValue('PROJECTION').lower() == 'transverse_mercator':
glt_header['map info'][7] = map_crs.GetUTMZone()
if Y_Max > 0.0:
glt_header['map info'][8] = 'North'
else:
glt_header['map info'][8] = 'South'
glt_header_file = os.path.splitext(glt_image_file)[0] + '.hdr'
write_envi_header(glt_header_file, glt_header)
# Remove temporary files.
os.remove(index_image_file)
os.remove(index_vrt_file)
os.remove(index_header_file)
os.remove(igm_vrt_file)
os.remove(tmp_glt_image_file)
logger.info('Write the GLT to %s.' % glt_image_file)
def resample_rdn(resampled_rdn_image_file, raw_rdn_image_file,
smile_effect_file):
""" Resample radiance spectra.
Parameters
----------
resampled_rdn_image_file: str
Resampled radiance image filename.
raw_rdn_image_file: str
Raw radiance image filename.
smile_effect_file: str
Smile effect filename.
"""
if os.path.exists(resampled_rdn_image_file):
logger.info('Write the spectrally resampled radiance image to %s.' %
resampled_rdn_image_file)
return
from ENVI import read_envi_header, write_envi_header
from scipy import interpolate
# Read raw radiance image
raw_rdn_header = read_envi_header(
os.path.splitext(raw_rdn_image_file)[0] + '.hdr')
raw_rdn_image = np.memmap(raw_rdn_image_file,
dtype='float32',
mode='r',
shape=(raw_rdn_header['lines'],
raw_rdn_header['bands'],
raw_rdn_header['samples']))
# Read spectral center wavelengths & bandwidths
smile_effect_header = read_envi_header(
os.path.splitext(smile_effect_file)[0] + '.hdr')
smile_effect_data = np.memmap(smile_effect_file,
dtype='float32',
mode='r',
shape=(smile_effect_header['bands'],
smile_effect_header['lines'],
smile_effect_header['samples']))
# Do spectral interpolation
info = 'Line (max=%d): ' % smile_effect_header['lines']
fid = open(resampled_rdn_image_file, 'wb')
for from_line in range(0, raw_rdn_header['lines'], 500):
info += '%d, ' % (from_line + 1)
# Determine chunk size
to_line = min(from_line + 500, raw_rdn_header['lines'])
# Initialize
rdn = np.zeros((to_line - from_line, raw_rdn_header['bands'],
raw_rdn_header['samples']
)) # shape=(from_line:to_line, bands, samples)
# Do spectral interpolation
for sample in range(raw_rdn_header['samples']):
# NOTE: No NaN values may be present in the input!
f = interpolate.interp1d(smile_effect_data[0, :, sample],
raw_rdn_image[from_line:to_line, :,
sample],
kind='cubic',
fill_value='extrapolate',
axis=1)
rdn[:, :, sample] = f(raw_rdn_header['wavelength'])
del f
# Write interpolated radiance to file
fid.write(rdn.astype('float32').tostring())
del rdn, to_line
fid.close()
info += '%d, Done!' % smile_effect_header['lines']
logger.info(info)
# Clear data
del smile_effect_data
raw_rdn_image.flush()
del raw_rdn_image
# Write header
write_envi_header(
os.path.splitext(resampled_rdn_image_file)[0] + '.hdr', raw_rdn_header)
del raw_rdn_header
logger.info('Write the spectrally resampled radiance image to %s.' %
resampled_rdn_image_file)
def estimate_wvc(wvc_file, rdn_file, sensors, sun_zenith, distance,
background_mask_file):
""" Estimate water vapor column.
Parameters
----------
wvc_file: str
Water vapor column image filename.
rdn_file: str
Radiance image filename.
sensors: dict
Sensors.
mask_file: str
Mask image filename.
sun_zenith: float
Sun zenith angle.
distance: float
Earth-to-sun distance.
"""
if os.path.exists(wvc_file):
logger.info('Save the WVC image to %s.' % (wvc_file))
return
from ENVI import read_envi_header, empty_envi_header, write_envi_header
from Spectra import get_closest_wave, resample_solar_flux
import json
# Read radiance image
rdn_header = read_envi_header(rdn_file + '.hdr')
rdn_image = np.memmap(rdn_file,
dtype='float32',
mode='r',
shape=(rdn_header['bands'], rdn_header['lines'],
rdn_header['samples']))
# Read boundary mask
bg_mask_header = read_envi_header(background_mask_file + '.hdr')
bg_mask_image = np.memmap(background_mask_file,
mode='r',
dtype='bool',
shape=(bg_mask_header['lines'],
bg_mask_header['samples']))
# Mask out dark pixels
good_mask_image = np.full((rdn_header['lines'], rdn_header['samples']),
True)
solar_flux = resample_solar_flux(solar_flux_file, rdn_header['wavelength'],
rdn_header['fwhm'])
cos_sun_zenith = np.cos(np.deg2rad(sun_zenith))
d2 = distance**2
# If reflectance at 850 nm is less than 0.01, mask out these pixels
wave, band = get_closest_wave(rdn_header['wavelength'], 470)
if abs(wave - 470) < 20:
refl = rdn_image[band, :, :] * np.pi * d2 / (solar_flux[band] *
cos_sun_zenith)
good_mask_image &= (refl > 0.01)
del refl
# If reflectance at 850 nm is less than 0.10, mask out these pixels
wave, band = get_closest_wave(rdn_header['wavelength'], 850)
if abs(wave - 850) < 20:
refl = rdn_image[band, :, :] * np.pi * d2 / (solar_flux[band] *
cos_sun_zenith)
good_mask_image &= (refl > 0.10)
del refl
# If reflectance at 1600 nm is less than 0.10, mask out these pixels
wave, band = get_closest_wave(rdn_header['wavelength'], 1600)
if abs(wave - 1600) < 20:
refl = rdn_image[band, :, :] * np.pi * d2 / (solar_flux[band] *
cos_sun_zenith)
good_mask_image &= (refl > 0.10)
del refl
del wave, band, solar_flux, cos_sun_zenith, d2
# Estimate water vapor columns
wvc_image = np.zeros((rdn_header['lines'], rdn_header['samples']))
for sensor_index, sensor_dict in sensors.items():
# Read water vapor column models
wvc_model_file = sensor_dict['wvc_model_file']
wvc_models = json.load(open(wvc_model_file, 'r'))
# Estimate WVC
for wvc_model in wvc_models.values():
# Find band indices
bands = []
for wave in wvc_model['wavelength']:
_, band = get_closest_wave(rdn_header['wavelength'], wave)
bands.append(band)
del band
# Calculate ratios
ratio_image = rdn_image[bands[1], :, :] / (
1e-10 + rdn_image[bands[0], :, :] * wvc_model['weight'][0] +
rdn_image[bands[2], :, :] * wvc_model['weight'][1])
# Calculate water vapor columns
wvc_image[good_mask_image] += np.interp(
ratio_image[good_mask_image], wvc_model['ratio'],
wvc_model['wvc']).astype('float32')
# Clear data
del ratio_image, bands, wvc_model
rdn_image.flush()
del rdn_image
# Average
wvc_image /= len(sensors)
# Read solar flux
wvc_image[~good_mask_image] = wvc_image[good_mask_image].mean()
logger.info(
'WVC [mm] statistics: min=%.2f, max=%.2f, avg=%.2f, sd=%.2f.' %
(wvc_image[good_mask_image].min(), wvc_image[good_mask_image].max(),
wvc_image[good_mask_image].mean(), wvc_image[good_mask_image].std()))
wvc_image[bg_mask_image] = -1000.0
del bg_mask_image, good_mask_image
# Save water vapor column
fid = open(wvc_file, 'wb')
fid.write(wvc_image.astype('float32').tostring())
fid.close()
# Write header
wvc_header = empty_envi_header()
wvc_header['description'] = 'Water vapor column [mm]'
wvc_header['file type'] = 'ENVI Standard'
wvc_header['bands'] = 1
wvc_header['lines'] = rdn_header['lines']
wvc_header['samples'] = rdn_header['samples']
wvc_header['file type'] = 'ENVI Standard'
wvc_header['interleave'] = 'bsq'
wvc_header['byte order'] = 0
wvc_header['data type'] = 4
wvc_header['data ignore value'] = -1000.0
wvc_header['map info'] = rdn_header['map info']
wvc_header['coordinate system string'] = rdn_header[
'coordinate system string']
write_envi_header(wvc_file + '.hdr', wvc_header)
del wvc_header, rdn_header
logger.info('Save the WVC image to %s.' % (wvc_file))
def atm_corr_image(flight_dict):
""" Do atmospheric corrections on the whole image.
Arguments:
flight_dict: dict
Flight dictionary.
"""
if os.path.exists(flight_dict['refl_file']):
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
return
from ENVI import read_envi_header, write_envi_header
from AtmLUT import read_binary_metadata
# Read radiance image.
rdn_header = read_envi_header(
os.path.splitext(flight_dict['merged_rdn_file'])[0] + '.hdr')
# rdn_image = np.memmap(flight_dict['merged_rdn_file'],
# dtype='float32',
# mode='r',
# shape=(rdn_header['bands'],
# rdn_header['lines'],
# rdn_header['samples']))
# Read atmospheric lookup table.
atm_lut_metadata = read_binary_metadata(
flight_dict['resampled_atm_lut_file'] + '.meta')
atm_lut_metadata['shape'] = tuple(
[int(v) for v in atm_lut_metadata['shape']])
atm_lut_WVC = np.array([float(v) for v in atm_lut_metadata['WVC']])
atm_lut_VIS = np.array([float(v) for v in atm_lut_metadata['VIS']])
atm_lut_VZA = np.array([float(v) for v in atm_lut_metadata['VZA']])
atm_lut_RAA = np.array([float(v) for v in atm_lut_metadata['RAA']])
atm_lut = np.memmap(flight_dict['resampled_atm_lut_file'],
dtype=atm_lut_metadata['dtype'],
mode='r',
shape=atm_lut_metadata['shape']
) # shape = (RHO, WVC, VIS, VZA, RAA, WAVE)
# Read VZA and RAA image.
sca_header = read_envi_header(
os.path.splitext(flight_dict['merged_sca_file'])[0] + '.hdr')
saa = float(sca_header['sun azimuth'])
sca_image = np.memmap(flight_dict['merged_sca_file'],
dtype='float32',
shape=(sca_header['bands'], sca_header['lines'],
sca_header['samples']))
# vza
vza_image = np.copy(sca_image[0, :, :])
# raa
raa_image = saa - sca_image[1, :, :]
raa_image[raa_image < 0] += 360.0
raa_image[raa_image > 180] = 360.0 - raa_image[raa_image > 180]
# clear data
sca_image.flush()
del sca_header, saa
del sca_image
# Read wvc and vis image.
wvc_header = read_envi_header(
os.path.splitext(flight_dict['wvc_file'])[0] + '.hdr')
tmp_wvc_image = np.memmap(flight_dict['wvc_file'],
mode='r',
dtype='float32',
shape=(wvc_header['lines'],
wvc_header['samples']))
wvc_image = np.copy(tmp_wvc_image)
vis_header = read_envi_header(
os.path.splitext(flight_dict['vis_file'])[0] + '.hdr')
tmp_vis_image = np.memmap(flight_dict['vis_file'],
dtype='float32',
mode='r',
shape=(vis_header['lines'],
vis_header['samples']))
vis_image = np.copy(tmp_vis_image)
tmp_wvc_image.flush()
tmp_vis_image.flush()
del wvc_header, vis_header
del tmp_vis_image, tmp_wvc_image
# Read background mask.
bg_header = read_envi_header(
os.path.splitext(flight_dict['background_mask_file'])[0] + '.hdr')
bg_mask = np.memmap(flight_dict['background_mask_file'],
dtype='bool',
mode='r',
shape=(bg_header['lines'], bg_header['samples']))
idx = ~bg_mask
max_WVC = atm_lut_WVC.max()
max_VIS = atm_lut_VIS.max()
max_VZA = atm_lut_VZA.max()
max_RAA = atm_lut_RAA.max()
wvc_image[wvc_image >= max_WVC] = max_WVC - 0.1
vis_image[vis_image >= max_VIS] = max_VIS - 0.1
vza_image[vza_image >= max_VZA] = max_VZA - 0.1
raa_image[raa_image >= max_RAA] = max_RAA - 0.1
del max_WVC, max_VIS, max_VZA, max_RAA
# remove outliers in wvc and vis.
wvc = wvc_image[idx]
avg_wvc = wvc.mean()
std_wvc = wvc.std()
index = (np.abs(wvc_image - avg_wvc) > 2 * std_wvc) & (idx)
wvc_image[index] = avg_wvc
del wvc
vis = vis_image[idx]
avg_vis = vis.mean()
std_vis = vis.std()
index = (np.abs(vis_image - avg_vis) > 2 * std_vis) & (idx)
vis_image[index] = avg_vis
del vis
del index
del idx
fid = open(flight_dict['refl_file'], 'wb')
# Do atmosphere correction.
info = 'Bands = '
for band in range(rdn_header['bands']):
if band % 20 == 0:
info += '%d, ' % (band + 1)
if (rdn_header['wavelength'][band] >= 1340.0
and rdn_header['wavelength'][band] <= 1440.0) or (
rdn_header['wavelength'][band] >= 1800.0
and rdn_header['wavelength'][band] <= 1980.0
) or rdn_header['wavelength'][band] >= 2460.0:
# fid.write(np.zeros((rdn_header['lines'], rdn_header['samples'])).astype('float32').tostring())
fid.write(
np.zeros((rdn_header['lines'], rdn_header['samples']
)).astype('int16').tostring()) # in int, by Ting
else:
# offset = rdn_header['header offset']+4*band*rdn_header['lines']*rdn_header['samples']# in bytes
offset = rdn_header[
'header offset'] + 2 * band * rdn_header['lines'] * rdn_header[
'samples'] # in bytes, int16 consists 2 bytes, by Ting
rdn_image = np.memmap(
flight_dict['merged_rdn_file'],
# dtype='float32',
dtype='int16', # in int, by Ting
mode='r',
offset=offset,
shape=(rdn_header['lines'], rdn_header['samples']))
refl = atm_corr_band(
atm_lut_WVC,
atm_lut_VIS,
atm_lut_VZA,
atm_lut_RAA,
np.copy(atm_lut[..., band]),
wvc_image,
vis_image,
vza_image,
raa_image,
rdn_image /
1000, # divided by 1000 to convert back to original rdn, by Ting.
bg_mask)
refl = refl * 10000 # reflectance times 10000 to convert to int, by Ting
# fid.write(refl.astype('float32').tostring())
fid.write(refl.astype('int16').tostring()) # save in int, by Ting
rdn_image.flush()
del refl, rdn_image
fid.close()
info += '%d, Done!' % band
logger.info(info)
# Clear data
del wvc_image, vis_image, vza_image, raa_image
atm_lut.flush()
bg_mask.flush()
del atm_lut, bg_mask
rdn_header['description'] = 'Reflectance [0-1]'
write_envi_header(
os.path.splitext(flight_dict['refl_file'])[0] + '.hdr', rdn_header)
logger.info('Write the reflectance image to %s.' %
flight_dict['refl_file'])
def prepare_dem(dem_image_file, dem, imugps_file, fov, map_crs, pixel_size):
""" Prepare DEM data.
Parameters
----------
dem_image_file: str
Processed DEM image filename.
dem: str or float
Input DEM image filename, or user-specified elevation value [m].
imugps_file: str
IMUGPS data filename.
fov: float
Sensor field of view [deg].
map_crs: osr object
Map coordinate system.
pixel_size: float
Image pixel size [deg].
"""
if os.path.exists(dem_image_file):
logger.info('Write the DEM to %s.' % dem_image_file)
return
from ENVI import empty_envi_header, write_envi_header
# Estimate spatial extent of flight area
""" Notes:
(1) The `altitude` here is above the mean sea level, or the Earth
ellipsoid surface, rather than above the ground surface. Strictly speaking,
it should be subtracted by the average elevation before calculating `half_swath`.
(2) A buffer of 50 m is used to ensure the processed DEM image covers the whole flight area.
"""
buffer = 500
imugps = np.loadtxt(imugps_file)
altitude = imugps[:, 3].max()
half_swath = np.tan(np.deg2rad(fov / 2)) * altitude
x_min = imugps[:, 1].min() - half_swath - buffer
x_max = imugps[:, 1].max() + half_swath + buffer
y_min = imugps[:, 2].min() - half_swath - buffer
y_max = imugps[:, 2].max() + half_swath + buffer
del imugps, half_swath, altitude, buffer
# Process DEM
if type(
dem
) is str: # If `old_dem` is a file, then clip it to the flight area
# Read raw DEM
ds = gdal.Open(dem, gdal.GA_ReadOnly)
# Get image rows & columns
geotransform = ds.GetGeoTransform()
col_0 = int((x_min - geotransform[0]) / geotransform[1])
col_1 = int((x_max - geotransform[0]) / geotransform[1])
row_0 = int((y_max - geotransform[3]) / geotransform[5])
row_1 = int((y_min - geotransform[3]) / geotransform[5])
if (col_0 > ds.RasterXSize - 1) or (row_0 > ds.RasterYSize - 1) or (
col_1 < 0) or (row_1 < 0):
logger.error('The input DEM image does not cover the flight area.')
raise IOError(
'The input DEM image does not cover the flight area.')
col_0 = max(col_0, 0)
row_0 = max(row_0, 0)
col_1 = min(col_1, ds.RasterXSize - 1)
row_1 = min(row_1, ds.RasterYSize - 1)
cols = int(col_1 - col_0)
rows = int(row_1 - row_0)
# Read subset of DEM
dem_image = ds.GetRasterBand(1).ReadAsArray(col_0, row_0, cols, rows)
dem_image = dem_image.astype('float32')
# Write clipped DEM image
fid = open(dem_image_file, 'wb')
fid.write(dem_image.tostring())
fid.close()
ds = None
del dem_image
# Update geotransform
geotransform = (geotransform[0] + col_0 * geotransform[1],
geotransform[1], 0,
geotransform[3] + row_0 * geotransform[5], 0,
geotransform[5])
del col_0, col_1, row_0, row_1
elif type(dem) in [int, float]:
# Make a flat DEM
rows = int((y_max - y_min) / pixel_size)
cols = int((x_max - x_min) / pixel_size)
dem_image = np.ones((rows, cols)) * dem
# Write clipped DEM image
fid = open(dem_image_file, 'wb')
fid.write(dem_image.astype('float32').tostring())
fid.close()
del dem_image
# Update geotransform
geotransform = (x_min, pixel_size, 0, y_max, 0, -pixel_size)
else:
logger.error('Cannot process DEM due to the wrong input.')
# Write clipped DEM header
dem_header = empty_envi_header()
dem_header['description'] = 'DEM, in [m]'
dem_header['file type'] = 'ENVI Standard'
dem_header['samples'] = cols
dem_header['lines'] = rows
dem_header['bands'] = 1
dem_header['byte order'] = 0
dem_header['header offset'] = 0
dem_header['interleave'] = 'bsq'
dem_header['data type'] = 4
dem_header['coordinate system string'] = map_crs.ExportToWkt()
dem_header['map info'] = [
map_crs.GetAttrValue('projcs').replace(',', ''), 1, 1, geotransform[0],
geotransform[3], geotransform[1], geotransform[1], ' ', ' ',
map_crs.GetAttrValue('datum').replace(',', ''),
map_crs.GetAttrValue('unit')
]
if map_crs.GetAttrValue('PROJECTION').lower() == 'transverse_mercator':
dem_header['map info'][7] = map_crs.GetUTMZone()
if y_min > 0.0:
dem_header['map info'][8] = 'North'
else:
dem_header['map info'][8] = 'South'
write_envi_header(os.path.splitext(dem_image_file)[0] + '.hdr', dem_header)
del dem_header
logger.info('Write the DEM to %s.' % dem_image_file)
def detect_smile_effect(sensor_dict, atm_lut_file):
""" Detect smile effect.
Parameters
----------
sensor_dict: dict
Sensor configurations.
atm_lut_file: str
Raw atmosphere lookup table filename.
"""
if os.path.exists(sensor_dict['smile_effect_at_atm_features_file']
) and os.path.exists(sensor_dict['smile_effect_file']):
logger.info(
'Write smile effect at atmosphere aborption features to %s.' %
sensor_dict['smile_effect_at_atm_features_file'])
logger.info('Write smile effect to %s.' %
sensor_dict['smile_effect_file'])
return
from ENVI import empty_envi_header, read_envi_header, write_envi_header
from Spectra import get_closest_wave
from scipy import optimize, interpolate
import json
# Read averaged radiance
header = read_envi_header(sensor_dict['avg_rdn_file'] + '.hdr')
sensor_rdn = np.memmap(sensor_dict['avg_rdn_file'],
mode='r',
dtype='float32',
shape=(header['lines'],
header['samples'])) # shape=(bands, samples)
samples = header['samples']
sensor_wave = np.array([float(v) for v in header['waves'].split(',')])
sensor_fwhm = np.array([float(v) for v in header['fwhms'].split(',')])
tmp_vza = header['VZA'].split(',')
tmp_raa = header['RAA'].split(',')
vza = []
raa = []
for i in range(samples):
try:
vza.append(float(tmp_vza[i]))
raa.append(float(tmp_raa[i]))
except:
vza.append(np.nan)
raa.append(np.nan)
logger.info('No spectrum for column: %s.' % (i + 1))
del tmp_vza, tmp_raa
del header
vza = np.array(vza)
raa = np.array(raa)
if np.all(np.isnan(vza)) or np.all(np.isnan(raa)):
raise IOError(
"Cannot detect smile effects since all columns do not have spectra."
)
nonnan_index = np.where((~np.isnan(vza)) & (~np.isnan(vza)))[0]
vza = np.interp(np.arange(samples), nonnan_index, vza[nonnan_index])
raa = np.interp(np.arange(samples), nonnan_index, raa[nonnan_index])
# Assign visibility
vis = [40] * samples
# Estimate water vapor column
logger.info('Estimate WVC from averaged radiance spectra.')
wvc_models = json.load(open(sensor_dict['wvc_model_file'], 'r'))
wvc = np.zeros(samples)
for model in wvc_models.values():
ratio = sensor_rdn[model['band'][1], :] / (
sensor_rdn[model['band'][0], :] * model['weight'][0] +
sensor_rdn[model['band'][2], :] * model['weight'][1])
wvc += np.interp(ratio, model['ratio'], model['wvc'])
del ratio
wvc /= len(wvc_models)
wvc_isnan = np.isnan(wvc) # Look for NaN values
if np.any(wvc_isnan):
logger.info('WVC could not be estimated for some columns. '
'Missing values will be interpolated.')
interpolate_values(wvc,
wvc_isnan) # Replace NaNs with interpolated values
logger.info('WVC [mm] statistics: min=%.2f, max=%.2f, avg=%.2f, sd=%.2f.' %
(wvc.min(), wvc.max(), wvc.mean(), wvc.std()))
del wvc_models, model
# Interpolate atmospheric lookup table
logger.info('Interpolate atmospheric look-up table.')
atm_lut_wave, atm_lut_rdn = interp_atm_lut(atm_lut_file, wvc, vis, vza,
raa) # shape=(samples, bands)
del wvc, vis, vza, raa
# Detect smile effects at atmospheric absorption features
logger.info('Detect smile effects at atmosphere absorption features.')
shifts = []
band_indices = []
n_features = 0
for wave, wave_range in absorption_features.items():
# Get sensor band range
sensor_wave0, sensor_band0 = get_closest_wave(sensor_wave,
wave_range[0])
sensor_wave1, sensor_band1 = get_closest_wave(sensor_wave,
wave_range[1])
center_wave, band_index = get_closest_wave(sensor_wave, wave)
# Check if continue
if abs(sensor_wave0 - wave_range[0]) > 20 or abs(sensor_wave1 -
wave_range[1]) > 20:
continue
# Get LUT band range
_, atm_lut_band0 = get_closest_wave(atm_lut_wave, sensor_wave0 - 20)
_, atm_lut_band1 = get_closest_wave(atm_lut_wave, sensor_wave1 + 20)
# Optimize
logger.info(
'Absorption feature center wavelength and range [nm] = %d: %d-%d.'
% (wave, wave_range[0], wave_range[1]))
x = []
for sample in range(samples):
p = optimize.minimize(
cost_fun, [0, 0],
method='BFGS',
args=(sensor_wave[sensor_band0:sensor_band1 + 1],
sensor_fwhm[sensor_band0:sensor_band1 + 1],
sensor_rdn[sensor_band0:sensor_band1 + 1, sample],
atm_lut_wave[atm_lut_band0:atm_lut_band1],
atm_lut_rdn[sample, atm_lut_band0:atm_lut_band1]))
x.append(p.x)
x = np.array(x)
# Do linear interpolation for invalid values
tmp_index = ~(np.abs(x[:, 0]) > 10.0)
x[:, 0] = np.interp(np.arange(samples),
np.arange(samples)[tmp_index], x[tmp_index, 0])
x[:, 1] = np.interp(np.arange(samples),
np.arange(samples)[tmp_index], x[tmp_index, 1])
del tmp_index
# Append shifts
shifts.append(x)
band_indices.append(band_index)
n_features += 1
# Clear data
del p, x, atm_lut_rdn
sensor_rdn.flush()
del sensor_rdn
# Reshape data
shifts = np.dstack(shifts).astype('float32').swapaxes(0, 1).swapaxes(1, 2)
# Write shifts to binary file
fid = open(sensor_dict['smile_effect_at_atm_features_file'], 'wb')
fid.write(shifts[0, :, :].tostring()) # Shift in wavelength
fid.write(shifts[1, :, :].tostring()) # Shift in fwhm
fid.close()
# Write header
header = empty_envi_header()
header[
'description'] = 'Smile effects detected at atmosphere absorption features'
header['file type'] = 'ENVI Standard'
header['bands'] = 2
header['lines'] = n_features
header['samples'] = samples
header['interleave'] = 'bsq'
header['byte order'] = 0
header['data type'] = 4
header['spectral center wavelengths'] = list(sensor_wave[band_indices])
header['spectral bandwiths'] = list(sensor_fwhm[band_indices])
header['band indices'] = band_indices
write_envi_header(
sensor_dict['smile_effect_at_atm_features_file'] + '.hdr', header)
del header
logger.info('Write smile effect at atmosphere absorption features to %s.' %
sensor_dict['smile_effect_at_atm_features_file'])
# Do interpolation
fid = open(sensor_dict['smile_effect_file'], 'wb')
x = np.arange(samples)
y = sensor_wave[band_indices]
x_new = x
y_new = sensor_wave
# Center wavelength
z = np.zeros((shifts.shape[1], shifts.shape[2]), dtype='float64')
for feature in range(shifts.shape[1]):
p = np.poly1d(np.polyfit(x, shifts[0, feature, :], 4))
z[feature, :] = p(x)
f = interpolate.interp2d(x, y, z, kind='cubic')
z_new = f(x_new, y_new) + np.expand_dims(sensor_wave, axis=1)
fid.write(z_new.astype('float32').tostring())
# Bandwidth
z = np.zeros((shifts.shape[1], shifts.shape[2]), dtype='float64')
for feature in range(shifts.shape[1]):
p = np.poly1d(np.polyfit(x, shifts[1, feature, :], 5))
z[feature, :] = p(x)
f = interpolate.interp2d(x, y, z, kind='cubic')
z_new = f(x_new, y_new) + np.expand_dims(sensor_fwhm, axis=1)
fid.write(z_new.astype('float32').tostring())
fid.close()
del f, x, y, z, z_new
# Write header
header = empty_envi_header()
header[
'description'] = 'Spectral Center Wavelengths and Spectral Bandwidths'
header['file type'] = 'ENVI Standard'
header['bands'] = 2
header['lines'] = len(y_new)
header['samples'] = len(x_new)
header['interleave'] = 'bsq'
header['byte order'] = 0
header['data type'] = 4
write_envi_header(sensor_dict['smile_effect_file'] + '.hdr', header)
del header, x_new, y_new
logger.info('Write smile effect to %s.' % sensor_dict['smile_effect_file'])
def resample_rdn(resampled_rdn_image_file, raw_rdn_image_file,
smile_effect_file):
""" Resample radiance spectra.
Arguments:
resampled_rdn_image_file: str
Resampled radiance image filename.
raw_rdn_image_file: str
Raw radiance image filename.
smile_effect_file: str
Smile effect filename.
"""
if os.path.exists(resampled_rdn_image_file):
logger.info('Write the spectrally resampled radiance image to %s.' %
resampled_rdn_image_file)
return
from ENVI import read_envi_header, write_envi_header
from scipy import interpolate
# Read the old radiance image.
raw_rdn_header = read_envi_header(
os.path.splitext(raw_rdn_image_file)[0] + '.hdr')
raw_rdn_image = np.memmap(
raw_rdn_image_file,
# dtype='float32',
dtype='int16', # to int, by Ting
mode='r',
shape=(raw_rdn_header['lines'], raw_rdn_header['bands'],
raw_rdn_header['samples']))
# Read spectral center wavelengths and bandwidths.
smile_effect_header = read_envi_header(
os.path.splitext(smile_effect_file)[0] + '.hdr')
smile_effect_data = np.memmap(smile_effect_file,
dtype='float32',
mode='r',
shape=(smile_effect_header['bands'],
smile_effect_header['lines'],
smile_effect_header['samples']))
# Do spectral interpolation.
info = 'Line (max=%d): ' % smile_effect_header['lines']
fid = open(resampled_rdn_image_file, 'wb')
for from_line in range(0, raw_rdn_header['lines'], 500):
info += '%d, ' % (from_line + 1)
# Determine chunck size.
to_line = min(from_line + 500, raw_rdn_header['lines'])
# Initialize.
rdn = np.zeros((to_line - from_line, raw_rdn_header['bands'],
raw_rdn_header['samples']
)) # shape=(from_line:to_line, bands, samples)
# Do spectral interpolation.
for sample in range(raw_rdn_header['samples']):
f = interpolate.interp1d(
smile_effect_data[0, :, sample],
raw_rdn_image[from_line:to_line, :, sample] /
1000, # divided by 1000 to convert back to original rdn, by Ting
kind='cubic',
fill_value='extrapolate',
axis=1)
rdn[:, :, sample] = f(raw_rdn_header['wavelength'])
rdn[:, :,
sample] = rdn[:, :,
sample] * 1000 # times 1000 to convert to int, by Ting
del f
# Write interpolated radiance into the file.
# fid.write(rdn.astype('float32').tostring())
fid.write(rdn.astype('int16').tostring()) # write to int, by Ting
del rdn, to_line
fid.close()
info += '%d, Done!' % smile_effect_header['lines']
logger.info(info)
# Clear data.
del smile_effect_data
raw_rdn_image.flush()
del raw_rdn_image
# Write the header.
write_envi_header(
os.path.splitext(resampled_rdn_image_file)[0] + '.hdr', raw_rdn_header)
del raw_rdn_header
logger.info('Write the spectrally resampled radiance image to %s.' %
resampled_rdn_image_file)
