def main():
# Parse arguments
parser = argparse.ArgumentParser(description="Apply OE to a block of data.")
parser.add_argument('input_radiance', type=str)
parser.add_argument('input_loc', type=str)
parser.add_argument('input_obs', type=str)
parser.add_argument('working_directory', type=str)
parser.add_argument('sensor', type=str, choices=['ang', 'avcl', 'neon', 'prism'])
parser.add_argument('--copy_input_files', type=int, choices=[0,1], default=0)
parser.add_argument('--h2o', action='store_true')
parser.add_argument('--modtran_path', type=str)
parser.add_argument('--wavelength_path', type=str)
parser.add_argument('--surface_category', type=str, default="multicomponent_surface")
parser.add_argument('--aerosol_climatology_path', type=str, default=None)
parser.add_argument('--rdn_factors_path', type=str)
parser.add_argument('--surface_path', type=str)
parser.add_argument('--channelized_uncertainty_path', type=str)
parser.add_argument('--lut_config_file', type=str)
parser.add_argument('--logging_level', type=str, default="INFO")
parser.add_argument('--log_file', type=str, default=None)
parser.add_argument('--n_cores', type=int, default=-1)
parser.add_argument('--presolve', choices=[0,1], type=int, default=0)
parser.add_argument('--empirical_line', choices=[0,1], type=int, default=0)
args = parser.parse_args()
if args.copy_input_files == 1:
args.copy_input_files = True
else:
args.copy_input_files = False
if args.log_file is None:
logging.basicConfig(format='%(message)s', level=args.logging_level)
else:
logging.basicConfig(format='%(message)s', level=args.logging_level, filename=args.log_file)
lut_params = LUTConfig(args.lut_config_file)
paths = Pathnames(args)
paths.make_directories()
paths.stage_files()
# Based on the sensor type, get appropriate year/month/day info fro intial condition.
# We'll adjust for line length and UTC day overrun later
if args.sensor == 'ang':
# parse flightline ID (AVIRIS-NG assumptions)
dt = datetime.strptime(paths.fid[3:], '%Y%m%dt%H%M%S')
dayofyear = dt.timetuple().tm_yday
elif args.sensor == 'avcl':
# parse flightline ID (AVIRIS-CL assumptions)
dt = datetime.strptime('20{}t000000'.format(paths.fid[1:7]), '%Y%m%dt%H%M%S')
dayofyear = dt.timetuple().tm_yday
elif args.sensor == 'neon':
dt = datetime.strptime(paths.fid, 'NIS01_%Y%m%d_%H%M%S')
dayofyear = dt.timetuple().tm_yday
elif args.sensor == 'prism':
dt = datetime.strptime(paths.fid[3:], '%Y%m%dt%H%M%S')
dayofyear = dt.timetuple().tm_yday
h_m_s, day_increment, mean_path_km, mean_to_sensor_azimuth, mean_to_sensor_zenith, valid, \
to_sensor_azimuth_lut_grid, to_sensor_zenith_lut_grid = get_metadata_from_obs(paths.obs_working_path, lut_params)
if day_increment:
dayofyear += 1
gmtime = float(h_m_s[0] + h_m_s[1] / 60.)
# get radiance file, wavelengths
if args.wavelength_path:
chn, wl, fwhm = np.loadtxt(args.wavelength_path).T
else:
radiance_dataset = envi.open(paths.radiance_working_path + '.hdr')
wl = np.array([float(w) for w in radiance_dataset.metadata['wavelength']])
if 'fwhm' in radiance_dataset.metadata:
fwhm = np.array([float(f) for f in radiance_dataset.metadata['fwhm']])
elif 'FWHM' in radiance_dataset.metadata:
fwhm = np.array([float(f) for f in radiance_dataset.metadata['FWHM']])
else:
fwhm = np.ones(wl.shape) * (wl[1] - wl[0])
# Close out radiance dataset to avoid potential confusion
del radiance_dataset
# Convert to microns if needed
if wl[0] > 100:
wl = wl / 1000.0
fwhm = fwhm / 1000.0
# write wavelength file
wl_data = np.concatenate([np.arange(len(wl))[:, np.newaxis], wl[:, np.newaxis],
fwhm[:, np.newaxis]], axis=1)
np.savetxt(paths.wavelength_path, wl_data, delimiter=' ')
mean_latitude, mean_longitude, mean_elevation_km, elevation_lut_grid = get_metadata_from_loc(paths.loc_working_path, lut_params)
# Need a 180 - here, as this is already in MODTRAN convention
mean_altitude_km = mean_elevation_km + np.cos(np.deg2rad(180 - mean_to_sensor_zenith)) * mean_path_km
logging.info('Path (km): %f, 180 - To-sensor Zenith (deg): %f, To-sensor Azimuth (deg) : %f, Altitude: %6.2f km' %
(mean_path_km, mean_to_sensor_zenith, mean_to_sensor_azimuth, mean_altitude_km))
# Superpixel segmentation
if args.empirical_line == 1:
if not exists(paths.lbl_working_path) or not exists(paths.radiance_working_path):
logging.info('Segmenting...')
segment(spectra=(paths.radiance_working_path, paths.lbl_working_path),
flag=-9999, npca=5, segsize=SEGMENTATION_SIZE, nchunk=CHUNKSIZE)
# Extract input data per segment
for inp, outp in [(paths.radiance_working_path, paths.rdn_subs_path),
(paths.obs_working_path, paths.obs_subs_path),
(paths.loc_working_path, paths.loc_subs_path)]:
if not exists(outp):
logging.info('Extracting ' + outp)
extractions(inputfile=inp, labels=paths.lbl_working_path,
output=outp, chunksize=CHUNKSIZE, flag=-9999)
if args.presolve == 1 and (not exists(paths.h2o_subs_path + '.hdr') or not exists(paths.h2o_subs_path)):
write_modtran_template(atmosphere_type='ATM_MIDLAT_SUMMER', fid=paths.fid, altitude_km=mean_altitude_km,
dayofyear=dayofyear, latitude=mean_latitude, longitude=mean_longitude,
to_sensor_azimuth=mean_to_sensor_azimuth, to_sensor_zenith=mean_to_sensor_zenith,
gmtime=gmtime, elevation_km=mean_elevation_km,
output_file=paths.h2o_template_path, ihaze_type='AER_NONE')
# TODO: this is effectively redundant from the radiative_transfer->modtran. Either devise a way
# to port in from there, or put in utils to reduce redundancy.
xdir = {
'linux': 'linux',
'darwin': 'macos',
'windows': 'windows'
}
name = 'H2O_bound_test'
filebase = os.path.join(paths.lut_h2o_directory, name)
with open(paths.h2o_template_path, 'r') as f:
bound_test_config = json.load(f)
bound_test_config['MODTRAN'][0]['MODTRANINPUT']['NAME'] = name
bound_test_config['MODTRAN'][0]['MODTRANINPUT']['ATMOSPHERE']['H2OSTR'] = 50
with open(filebase + '.json', 'w') as fout:
fout.write(json.dumps(bound_test_config, cls=SerialEncoder, indent=4, sort_keys=True))
cwd = os.getcwd()
os.chdir(paths.lut_h2o_directory)
cmd = os.path.join(paths.modtran_path, 'bin', xdir[platform], 'mod6c_cons ' + filebase + '.json')
try:
subprocess.call(cmd, shell=True, timeout=10)
except:
pass
os.chdir(cwd)
max_water = None
with open(filebase + '.tp6') as tp6file:
for count, line in enumerate(tp6file):
if 'The water column is being set to the maximum' in line:
max_water = line.split(',')[1].strip()
max_water = float(max_water.split(' ')[0])
break
if max_water is None:
logging.error('Could not find MODTRAN H2O upper bound in file {}'.format(filebase + '.tp6'))
raise KeyError('Could not find MODTRAN H2O upper bound')
# Write the presolve connfiguration file
h2o_grid = np.linspace(0.01, max_water - 0.01, 10).round(2)
logging.info('Pre-solve H2O grid: {}'.format(h2o_grid))
logging.info('Writing H2O pre-solve configuration file.')
build_presolve_config(paths, h2o_grid, args.n_cores, args.surface_category)
# Run modtran retrieval
logging.info('Run ISOFIT initial guess')
retrieval_h2o = isofit.Isofit(paths.h2o_config_path, level='DEBUG')
retrieval_h2o.run()
# clean up unneeded storage
for to_rm in ['*r_k', '*t_k', '*tp7', '*wrn', '*psc', '*plt', '*7sc', '*acd']:
cmd = 'rm ' + join(paths.lut_h2o_directory, to_rm)
logging.info(cmd)
os.system(cmd)
# Define h2o grid, either from presolve or LUTConfig
if args.presolve == 1:
h2o = envi.open(paths.h2o_subs_path + '.hdr')
h2o_est = h2o.read_band(-1)[:].flatten()
h2o_lut_grid = np.linspace(np.percentile(h2o_est[h2o_est > lut_params.h2o_min], 5),
np.percentile(h2o_est[h2o_est > lut_params.h2o_min], 95),
lut_params.num_h2o_lut_elements)
if (np.abs(h2o_lut_grid[-1] - h2o_lut_grid[0]) < 0.03):
new_h2o_lut_grid = np.linspace(h2o_lut_grid[0] - 0.1* np.ceil(lut_params.num_h2o_lut_elements/2.),
h2o_lut_grid[0] + 0.1*np.ceil(lut_params.num_h2o_lut_elements/2.),
lut_params.num_h2o_lut_elements)
logging.warning('Warning: h2o lut grid from presolve detected as {}-{}, which is very narrow. Expanding to {}-{}. Advised to check presolve solutions thoroughly.'.format(h2o_lut_grid[0],h2o_lut_grid[-1], new_h2o_lut_grid[0], new_h2o_lut_grid[-1]))
h2o_lut_grid = new_h2o_lut_grid
else:
h2o_lut_grid = np.linspace(lut_params.default_h2o_lut_range[0], lut_params.default_h2o_lut_range[1],
lut_params.num_h2o_lut_elements)
logging.info('Full (non-aerosol) LUTs:\nElevation: {}\nTo-sensor azimuth: {}\nTo-sensor zenith: {}\nh2o-vis: {}:'.format(elevation_lut_grid, to_sensor_azimuth_lut_grid, to_sensor_zenith_lut_grid, h2o_lut_grid))
logging.info(paths.state_subs_path)
if not exists(paths.state_subs_path) or \
not exists(paths.uncert_subs_path) or \
not exists(paths.rfl_subs_path):
write_modtran_template(atmosphere_type='ATM_MIDLAT_SUMMER', fid=paths.fid, altitude_km=mean_altitude_km,
dayofyear=dayofyear, latitude=mean_latitude, longitude=mean_longitude,
to_sensor_azimuth=mean_to_sensor_azimuth, to_sensor_zenith=mean_to_sensor_zenith,
gmtime=gmtime, elevation_km=mean_elevation_km, output_file=paths.modtran_template_path)
logging.info('Writing main configuration file.')
build_main_config(paths, lut_params, h2o_lut_grid, elevation_lut_grid, to_sensor_azimuth_lut_grid,
to_sensor_zenith_lut_grid, mean_latitude, mean_longitude, dt,
args.empirical_line == 1, args.n_cores, args.surface_category)
# Run modtran retrieval
logging.info('Running ISOFIT with full LUT')
retrieval_full = isofit.Isofit(paths.modtran_config_path, level='DEBUG')
retrieval_full.run()
# clean up unneeded storage
for to_rm in ['*r_k', '*t_k', '*tp7', '*wrn', '*psc', '*plt', '*7sc', '*acd']:
cmd = 'rm ' + join(paths.lut_modtran_directory, to_rm)
logging.info(cmd)
os.system(cmd)
if not exists(paths.rfl_working_path) or not exists(paths.uncert_working_path):
# Empirical line
logging.info('Empirical line inference')
empirical_line(reference_radiance_file=paths.rdn_subs_path,
reference_reflectance_file=paths.rfl_subs_path,
reference_uncertainty_file=paths.uncert_subs_path,
reference_locations_file=paths.loc_subs_path,
segmentation_file=paths.lbl_working_path,
input_radiance_file=paths.radiance_working_path,
input_locations_file=paths.loc_working_path,
output_reflectance_file=paths.rfl_working_path,
output_uncertainty_file=paths.uncert_working_path,
isofit_config=paths.modtran_config_path)
logging.info('Done.')
def do_inverse(isofit_inv: dict, radfile: pathlib.Path,
est_refl_file: pathlib.Path, est_state_file: pathlib.Path,
atm_coef_file: pathlib.Path, post_unc_file: pathlib.Path,
overwrite: bool, use_empirical_line: bool):
if use_empirical_line:
# Segment first, then run on segmented file
SEGMENTATION_SIZE = 40
CHUNKSIZE = 256
lbl_working_path = radfile.parent / str(radfile).replace(
"toa-radiance", "segmentation")
rdn_subs_path = radfile.with_suffix("-subs")
rfl_subs_path = est_refl_file.with_suffix("-subs")
state_subs_path = est_state_file.with_suffix("-subs")
atm_subs_path = atm_coef_file.with_suffix("-subs")
unc_subs_path = post_unc_file.with_suffix("-subs")
isofit_inv["input"]["measured_radiance_file"] = str(rdn_subs_path)
isofit_inv["output"] = {
"estimated_reflectance_file": str(rfl_subs_path),
"estimated_state_file": str(state_subs_path),
"atmospheric_coefficients_file": str(atm_subs_path),
"posterior_uncertainty_file": str(unc_subs_path)
}
if not overwrite and lbl_working_path.exists(
) and rdn_subs_path.exists():
logger.info(
"Skipping segmentation and extraction because files exist.")
else:
logger.info(
"Fixing any radiance values slightly less than zero...")
rad_img = sp.open_image(str(radfile) + ".hdr")
rad_m = rad_img.open_memmap(writable=True)
nearzero = np.logical_and(rad_m < 0, rad_m > -2)
rad_m[nearzero] = 0.0001
del rad_m
del rad_img
logger.info("Segmenting...")
segment(spectra=(str(radfile), str(lbl_working_path)),
flag=-9999,
npca=5,
segsize=SEGMENTATION_SIZE,
nchunk=CHUNKSIZE)
logger.info("Extracting...")
extractions(inputfile=str(radfile),
labels=str(lbl_working_path),
output=str(rdn_subs_path),
chunksize=CHUNKSIZE,
flag=-9999)
else:
# Run Isofit directly
isofit_inv["input"]["measured_radiance_file"] = str(radfile)
isofit_inv["output"] = {
"estimated_reflectance_file": str(est_refl_file),
"estimated_state_file": str(est_state_file),
"atmospheric_coefficients_file": str(atm_coef_file),
"posterior_uncertainty_file": str(post_unc_file)
}
if not overwrite and pathlib.Path(
isofit_inv["output"]["estimated_reflectance_file"]).exists():
logger.info("Skipping inversion because output file exists.")
else:
invfile = radfile.parent / (
str(radfile).replace("toa-radiance", "inverse") + ".json")
json.dump(isofit_inv, open(invfile, "w"), indent=2)
Isofit(invfile).run()
if use_empirical_line:
if not overwrite and est_refl_file.exists():
logger.info("Skipping empirical line because output exists.")
else:
logger.info("Applying empirical line...")
empirical_line(reference_radiance_file=str(rdn_subs_path),
reference_reflectance_file=str(rfl_subs_path),
reference_uncertainty_file=str(unc_subs_path),
reference_locations_file=None,
segmentation_file=str(lbl_working_path),
input_radiance_file=str(radfile),
input_locations_file=None,
output_reflectance_file=str(est_refl_file),
output_uncertainty_file=str(post_unc_file),
isofit_config=str(invfile))
def main():
# Parse arguments
parser = argparse.ArgumentParser(
description="Apply OE to a block of data.")
parser.add_argument('input_radiance', type=str)
parser.add_argument('input_loc', type=str)
parser.add_argument('input_obs', type=str)
parser.add_argument('working_directory', type=str)
parser.add_argument('sensor', type=str, choices=['ang', 'avcl', 'neon'])
parser.add_argument('--copy_input_files',
type=int,
choices=[0, 1],
default=0)
parser.add_argument('--h2o', action='store_true')
parser.add_argument('--modtran_path', type=str)
parser.add_argument('--wavelength_path', type=str)
parser.add_argument('--aerosol_climatology_path', type=str, default=None)
parser.add_argument('--rdn_factors_path', type=str)
parser.add_argument('--surface_path', type=str)
parser.add_argument('--channelized_uncertainty_path', type=str)
parser.add_argument('--lut_config_file', type=str)
parser.add_argument('--logging_level', type=str, default="INFO")
parser.add_argument('--log_file', type=str, default=None)
args = parser.parse_args()
if args.copy_input_files == 1:
args.copy_input_files = True
else:
args.copy_input_files = False
if args.log_file is None:
logging.basicConfig(format='%(message)s', level=args.logging_level)
else:
logging.basicConfig(format='%(message)s',
level=args.logging_level,
filename=args.log_file)
lut_params = LUTConfig(args.lut_config_file)
paths = Pathnames(args)
paths.make_directories()
paths.stage_files()
# Based on the sensor type, get appropriate year/month/day info fro intial condition.
# We'll adjust for line length and UTC day overrun later
if args.sensor == 'ang':
# parse flightline ID (AVIRIS-NG assumptions)
dt = datetime.strptime(paths.fid[3:], '%Y%m%dt%H%M%S')
dayofyear = dt.timetuple().tm_yday
elif args.sensor == 'avcl':
# parse flightline ID (AVIRIS-CL assumptions)
dt = datetime.strptime('20{}t000000'.format(paths.fid[1:7]),
'%Y%m%dt%H%M%S')
dayofyear = dt.timetuple().tm_yday
elif args.sensor == 'neon':
dt = datetime.strptime(paths.fid, 'NIS01_%Y%m%d_%H%M%S')
dayofyear = dt.timetuple().tm_yday
h_m_s, day_increment, mean_path_km, mean_to_sensor_azimuth, mean_to_sensor_zenith, valid, \
to_sensor_azimuth_lut_grid, to_sensor_zenith_lut_grid = get_metadata_from_obs(paths.obs_working_path, lut_params)
if day_increment:
dayofyear += 1
gmtime = float(h_m_s[0] + h_m_s[1] / 60.)
# Superpixel segmentation
if not exists(paths.lbl_working_path) or not exists(
paths.radiance_working_path):
logging.info('Segmenting...')
segment(spectra=(paths.radiance_working_path, paths.lbl_working_path),
flag=-9999,
npca=5,
segsize=SEGMENTATION_SIZE,
nchunk=CHUNKSIZE)
# Extract input data per segment
for inp, outp in [(paths.radiance_working_path, paths.rdn_subs_path),
(paths.obs_working_path, paths.obs_subs_path),
(paths.loc_working_path, paths.loc_subs_path)]:
if not exists(outp):
logging.info('Extracting ' + outp)
extractions(inputfile=inp,
labels=paths.lbl_working_path,
output=outp,
chunksize=CHUNKSIZE,
flag=-9999)
# get radiance file, wavelengths
if args.wavelength_path:
chn, wl, fwhm = np.loadtxt(args.wavelength_path).T
else:
radiance_dataset = envi.open(paths.rdn_subs_path + '.hdr')
wl = np.array(
[float(w) for w in radiance_dataset.metadata['wavelength']])
if 'fwhm' in radiance_dataset.metadata:
fwhm = np.array(
[float(f) for f in radiance_dataset.metadata['fwhm']])
if 'FWHM' in radiance_dataset.metadata:
fwhm = np.array(
[float(f) for f in radiance_dataset.metadata['FWHM']])
else:
fwhm = np.ones(wl.shape) * (wl[1] - wl[0])
# Convert to microns if needed
if wl[0] > 100:
wl = wl / 1000.0
fwhm = fwhm / 1000.0
# write wavelength file
wl_data = np.concatenate([
np.arange(len(wl))[:, np.newaxis], wl[:, np.newaxis], fwhm[:,
np.newaxis]
],
axis=1)
np.savetxt(paths.wavelength_path, wl_data, delimiter=' ')
mean_latitude, mean_longitude, mean_elevation_km, elevation_lut_grid = get_metadata_from_loc(
paths.loc_working_path, lut_params)
# Need a 180 - here, as this is already in MODTRAN convention
mean_altitude_km = mean_elevation_km + np.cos(
np.deg2rad(180 - mean_to_sensor_zenith)) * mean_path_km
logging.info(
'Path (km): %f, 180 - To-sensor Zenith (deg): %f, To-sensor Azimuth (deg) : %f, Altitude: %6.2f km'
% (mean_path_km, mean_to_sensor_zenith, mean_to_sensor_azimuth,
mean_altitude_km))
if not exists(paths.h2o_subs_path + '.hdr') or not exists(
paths.h2o_subs_path):
write_modtran_template(atmosphere_type='ATM_MIDLAT_SUMMER',
fid=paths.fid,
altitude_km=mean_altitude_km,
dayofyear=dayofyear,
latitude=mean_latitude,
longitude=mean_longitude,
to_sensor_azimuth=mean_to_sensor_azimuth,
to_sensor_zenith=mean_to_sensor_zenith,
gmtime=gmtime,
elevation_km=mean_elevation_km,
output_file=paths.h2o_template_path,
ihaze_type='AER_NONE')
# Write the presolve connfiguration file
logging.info('Writing H2O pre-solve configuration file.')
build_presolve_config(paths, np.linspace(0.5, 5, 10))
# Run modtran retrieval
logging.info('Run ISOFIT initial guess')
retrieval_h2o = isofit.Isofit(paths.h2o_config_path, level='DEBUG')
retrieval_h2o.run()
# clean up unneeded storage
for to_rm in [
'*r_k', '*t_k', '*tp7', '*wrn', '*psc', '*plt', '*7sc', '*acd'
]:
cmd = 'rm ' + join(paths.lut_h2o_directory, to_rm)
logging.info(cmd)
os.system(cmd)
# Extract h2o grid avoiding the zero label (periphery, bad data)
# and outliers
h2o = envi.open(paths.h2o_subs_path + '.hdr')
h2o_est = h2o.read_band(-1)[:].flatten()
h2o_lut_grid = np.linspace(
np.percentile(h2o_est[h2o_est > lut_params.h2o_min], 5),
np.percentile(h2o_est[h2o_est > lut_params.h2o_min], 95),
lut_params.num_h2o_lut_elements)
logging.info(
'Full (non-aerosol) LUTs:\nElevation: {}\nTo-sensor azimuth: {}\nTo-sensor zenith: {}\nh2o-vis: {}:'
.format(elevation_lut_grid, to_sensor_azimuth_lut_grid,
to_sensor_zenith_lut_grid, h2o_lut_grid))
logging.info(paths.state_subs_path)
if not exists(paths.state_subs_path) or \
not exists(paths.uncert_subs_path) or \
not exists(paths.rfl_subs_path):
write_modtran_template(atmosphere_type='ATM_MIDLAT_SUMMER',
fid=paths.fid,
altitude_km=mean_altitude_km,
dayofyear=dayofyear,
latitude=mean_latitude,
longitude=mean_longitude,
to_sensor_azimuth=mean_to_sensor_azimuth,
to_sensor_zenith=mean_to_sensor_zenith,
gmtime=gmtime,
elevation_km=mean_elevation_km,
output_file=paths.modtran_template_path)
logging.info('Writing main configuration file.')
build_main_config(paths, lut_params, h2o_lut_grid, elevation_lut_grid,
to_sensor_azimuth_lut_grid,
to_sensor_zenith_lut_grid, mean_latitude,
mean_longitude, dt)
# Run modtran retrieval
logging.info('Running ISOFIT with full LUT')
retrieval_full = isofit.Isofit(paths.modtran_config_path,
level='DEBUG')
retrieval_full.run()
# clean up unneeded storage
for to_rm in [
'*r_k', '*t_k', '*tp7', '*wrn', '*psc', '*plt', '*7sc', '*acd'
]:
cmd = 'rm ' + join(paths.lut_modtran_directory, to_rm)
logging.info(cmd)
os.system(cmd)
if not exists(paths.rfl_working_path) or not exists(
paths.uncert_working_path):
# Empirical line
logging.info('Empirical line inference')
empirical_line(reference_radiance=paths.rdn_subs_path,
reference_reflectance=paths.rfl_subs_path,
reference_uncertainty=paths.uncert_subs_path,
reference_locations=paths.loc_subs_path,
hashfile=paths.lbl_working_path,
input_radiance=paths.radiance_working_path,
input_locations=paths.loc_working_path,
output_reflectance=paths.rfl_working_path,
output_uncertainty=paths.uncert_working_path,
isofit_config=paths.modtran_config_path)
logging.info('Done.')