def __init__(self, config_file, row_column='', level='INFO', logfile=None):
# Explicitly set the number of threads to be 1, so we more effectively
# run in parallel
os.environ["MKL_NUM_THREADS"] = "1"
# Set logging level
self.loglevel = level
self.logfile = logfile
logging.basicConfig(format='%(levelname)s:%(message)s', level=self.loglevel, filename=self.logfile)
self.rows = None
self.cols = None
self.config = None
# Load configuration file
self.config = configs.create_new_config(config_file)
self.config.get_config_errors()
# Initialize ray for parallel execution
rayargs = {'address': self.config.implementation.ip_head,
'_redis_password': self.config.implementation.redis_password,
'ignore_reinit_error': self.config.implementation.ray_ignore_reinit_error,
'_temp_dir': self.config.implementation.ray_temp_dir,
'local_mode': self.config.implementation.n_cores == 1}
# We can only set the num_cpus if running on a single-node
if self.config.implementation.ip_head is None and self.config.implementation.redis_password is None:
rayargs['num_cpus'] = self.config.implementation.n_cores
if self.config.implementation.debug_mode is False:
ray.init(**rayargs)
self.workers = None
def __init__(self, config_file, row_column='', level='INFO', logfile=None):
# Explicitly set the number of threads to be 1, so we more effectively
#run in parallel
os.environ["MKL_NUM_THREADS"] = "1"
# Set logging level
self.loglevel = level
self.logfile = logfile
logging.basicConfig(format='%(levelname)s:%(message)s', level=self.loglevel, filename=self.logfile)
self.rows = None
self.cols = None
self.config = None
self.fm = None
self.iv = None
self.io = None
self.states = None
# Load configuration file
self.config = configs.create_new_config(config_file)
self.config.get_config_errors()
# Initialize ray for parallel execution
rayargs = {'address': self.config.implementation.ip_head,
'redis_password': self.config.implementation.redis_password,
'ignore_reinit_error':True,
'local_mode': self.config.implementation.n_cores == 1}
# only specify a temporary directory if we are not connecting to
# a ray cluster
if rayargs['local_mode']:
rayargs['temp_dir'] = self.config.implementation.ray_temp_dir
# Used to run on a VPN
ray.services.get_node_ip_address = lambda: '127.0.0.1'
# We can only set the num_cpus if running on a single-node
if self.config.implementation.ip_head is None and self.config.implementation.redis_password is None:
rayargs['num_cpus'] = self.config.implementation.n_cores
ray.init(**rayargs)
if len(row_column) > 0:
ranges = row_column.split(',')
if len(ranges) == 1:
self.rows, self.cols = [int(ranges[0])], None
if len(ranges) == 2:
row_start, row_end = ranges
self.rows, self.cols = range(
int(row_start), int(row_end)), None
elif len(ranges) == 4:
row_start, row_end, col_start, col_end = ranges
line_start, line_end, samp_start, samp_end = ranges
self.rows = range(int(row_start), int(row_end))
self.cols = range(int(col_start), int(col_end))
# Build the forward model and inversion objects
self._init_nonpicklable_objects()
self.io = IO(self.config, self.fm, self.iv, self.rows, self.cols)
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm, iv, [0], [0])
# Simulate a measurement and write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[0]
def __init__(self, config_file, row_column='', level='INFO', logfile=None):
# Explicitly set the number of threads to be 1, so we more effectively
# run in parallel
os.environ["MKL_NUM_THREADS"] = "1"
# Set logging level
self.loglevel = level
self.logfile = logfile
logging.basicConfig(format='%(levelname)s:%(message)s',
level=self.loglevel,
filename=self.logfile)
self.rows = None
self.cols = None
self.config = None
# Load configuration file
self.config = configs.create_new_config(config_file)
self.config.get_config_errors()
# Initialize ray for parallel execution
rayargs = {
'address': self.config.implementation.ip_head,
'_redis_password': self.config.implementation.redis_password,
'ignore_reinit_error':
self.config.implementation.ray_ignore_reinit_error,
'local_mode': self.config.implementation.n_cores == 1
}
# only specify a temporary directory if we are not connecting to
# a ray cluster
if rayargs['local_mode']:
rayargs['_temp_dir'] = self.config.implementation.ray_temp_dir
# Used to run on a VPN
ray.services.get_node_ip_address = lambda: '127.0.0.1'
# We can only set the num_cpus if running on a single-node
if self.config.implementation.ip_head is None and self.config.implementation.redis_password is None:
rayargs['num_cpus'] = self.config.implementation.n_cores
ray.init(**rayargs)
self.workers = None
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm, iv, [0], [0])
geom = None
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[-1]
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm)
# Simulate a measurement and write result
for row in range(io.n_rows):
for col in range(io.n_cols):
id = io.get_components_at_index(row, col)
if id is not None:
states = iv.invert(id.meas, id.geom)
io.write_spectrum(row, col, states, fm, iv)
assert True
return states[0]
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm)
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row in range(io.n_rows):
for col in range(io.n_cols):
id = io.get_components_at_index(row, col)
if id is not None:
states = iv.invert(id.meas, id.geom)
io.write_spectrum(row, col, states, fm, iv)
assert True
return states[-1]
def _run_chunk(start_line: int, stop_line: int, reference_radiance_file: str,
reference_reflectance_file: str,
reference_uncertainty_file: str, reference_locations_file: str,
input_radiance_file: str, input_locations_file: str,
segmentation_file: str, isofit_config: str,
output_reflectance_file: str, output_uncertainty_file: str,
radiance_factors: np.array, nneighbors: int,
nodata_value: float, loglevel: str, logfile: str) -> None:
"""
Args:
start_line: line to start empirical line run at
stop_line: line to stop empirical line run at
reference_radiance_file: source file for radiance (interpolation built from this)
reference_reflectance_file: source file for reflectance (interpolation built from this)
reference_uncertainty_file: source file for uncertainty (interpolation built from this)
reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this)
input_radiance_file: input radiance file (interpolate over this)
input_locations_file: input location file (interpolate over this)
segmentation_file: input file noting the per-pixel segmentation used
isofit_config: path to isofit configuration JSON file
output_reflectance_file: location to write output reflectance to
output_uncertainty_file: location to write output uncertainty to
radiance_factors: radiance adjustment factors
nneighbors: number of neighbors to use for interpolation
nodata_value: nodata value of input and output
loglevel: logging level
logfile: logging file
Returns:
None
"""
logging.basicConfig(format='%(message)s', level=loglevel, filename=logfile)
# Load reference images
reference_radiance_img = envi.open(reference_radiance_file + '.hdr',
reference_radiance_file)
reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr',
reference_reflectance_file)
reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr',
reference_uncertainty_file)
reference_locations_img = envi.open(reference_locations_file + '.hdr',
reference_locations_file)
n_reference_lines, n_radiance_bands, n_reference_columns = [
int(reference_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
n_reference_uncertainty_bands = int(
reference_uncertainty_img.metadata['bands'])
# Load input images
input_radiance_img = envi.open(input_radiance_file + '.hdr',
input_radiance_file)
n_input_lines, n_input_bands, n_input_samples = [
int(input_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
input_locations_img = envi.open(input_locations_file + '.hdr',
input_locations_file)
n_location_bands = int(input_locations_img.metadata['bands'])
# Load output images
output_reflectance_img = envi.open(output_reflectance_file + '.hdr',
output_reflectance_file)
output_uncertainty_img = envi.open(output_uncertainty_file + '.hdr',
output_uncertainty_file)
n_output_reflectance_bands = int(output_reflectance_img.metadata['bands'])
n_output_uncertainty_bands = int(output_uncertainty_img.metadata['bands'])
# Load reference data
reference_locations_mm = reference_locations_img.open_memmap(
interleave='bip', writable=False)
reference_locations = np.array(reference_locations_mm[:, :, :]).reshape(
(n_reference_lines, n_location_bands))
reference_radiance_mm = reference_radiance_img.open_memmap(
interleave='bip', writable=False)
reference_radiance = np.array(reference_radiance_mm[:, :, :]).reshape(
(n_reference_lines, n_radiance_bands))
reference_reflectance_mm = reference_reflectance_img.open_memmap(
interleave='bip', writable=False)
reference_reflectance = np.array(
reference_reflectance_mm[:, :, :]).reshape(
(n_reference_lines, n_radiance_bands))
reference_uncertainty_mm = reference_uncertainty_img.open_memmap(
interleave='bip', writable=False)
reference_uncertainty = np.array(
reference_uncertainty_mm[:, :, :]).reshape(
(n_reference_lines, n_reference_uncertainty_bands))
reference_uncertainty = reference_uncertainty[:, :
n_radiance_bands].reshape(
(n_reference_lines,
n_radiance_bands))
# Load segmentation data
if segmentation_file:
segmentation_img = envi.open(segmentation_file + '.hdr',
segmentation_file)
segmentation_img = segmentation_img.read_band(0)
else:
segmentation_img = None
# Prepare instrument model, if available
if isofit_config is not None:
config = configs.create_new_config(isofit_config)
instrument = Instrument(config)
logging.info('Loading instrument')
# Make sure the instrument is configured for single-pixel noise (no averaging)
instrument.integrations = 1
else:
instrument = None
# Load radiance factors
if radiance_factors is None:
radiance_adjustment = np.ones(n_radiance_bands, )
else:
radiance_adjustment = np.loadtxt(radiance_factors)
# Load Tree
loc_scaling = np.array([1e5, 1e5, 0.1])
scaled_ref_loc = reference_locations * loc_scaling
tree = KDTree(scaled_ref_loc)
# Assume (heuristically) that, for distance purposes, 1 m vertically is
# comparable to 10 m horizontally, and that there are 100 km per latitude
# degree. This is all approximate of course. Elevation appears in the
# Third element, and the first two are latitude/longitude coordinates
# Iterate through image
hash_table = {}
for row in np.arange(start_line, stop_line):
# Load inline input data
input_radiance_mm = input_radiance_img.open_memmap(interleave='bip',
writable=False)
input_radiance = np.array(input_radiance_mm[row, :, :])
input_radiance = input_radiance * radiance_adjustment
input_locations_mm = input_locations_img.open_memmap(interleave='bip',
writable=False)
input_locations = np.array(input_locations_mm[row, :, :])
output_reflectance_row = np.zeros(input_radiance.shape) + nodata_value
output_uncertainty_row = np.zeros(input_radiance.shape) + nodata_value
nspectra, start = 0, time.time()
for col in np.arange(n_input_samples):
x = input_radiance[col, :]
if np.all(np.isclose(x, nodata_value)):
output_reflectance_row[col, :] = nodata_value
output_uncertainty_row[col, :] = nodata_value
continue
bhat = None
if segmentation_img is not None:
hash_idx = segmentation_img[row, col]
if hash_idx in hash_table:
bhat, bmarg, bcov = hash_table[hash_idx]
else:
loc = reference_locations[
np.array(hash_idx, dtype=int), :] * loc_scaling
else:
loc = input_locations[col, :] * loc_scaling
if bhat is None:
dists, nn = tree.query(loc, nneighbors)
xv = reference_radiance[nn, :]
yv = reference_reflectance[nn, :]
uv = reference_uncertainty[nn, :]
bhat = np.zeros((n_radiance_bands, 2))
bmarg = np.zeros((n_radiance_bands, 2))
bcov = np.zeros((n_radiance_bands, 2, 2))
for i in np.arange(n_radiance_bands):
use = yv[:, i] > 0
n = sum(use)
X = np.concatenate((np.ones((n, 1)), xv[use, i:i + 1]),
axis=1)
W = np.diag(np.ones(n)) # /uv[use, i])
y = yv[use, i:i + 1]
try:
bhat[i, :] = (inv(X.T @ W @ X) @ X.T @ W @ y).T
bcov[i, :, :] = inv(X.T @ W @ X)
except:
bhat[i, :] = 0
bcov[i, :, :] = 0
bmarg[i, :] = np.diag(bcov[i, :, :])
if (segmentation_img is not None) and not (hash_idx in hash_table):
hash_table[hash_idx] = bhat, bmarg, bcov
A = np.array((np.ones(n_radiance_bands), x))
output_reflectance_row[col, :] = (np.multiply(bhat.T,
A).sum(axis=0))
# Calculate uncertainties. Sy approximation rather than Seps for
# speed, for now... but we do take into account instrument
# radiometric uncertainties
if instrument is None:
output_uncertainty_row[col, :] = np.sqrt(
np.multiply(bmarg.T, A).sum(axis=0))
else:
Sy = instrument.Sy(x, geom=None)
calunc = instrument.bval[:instrument.n_chan]
output_uncertainty_row[col, :] = np.sqrt(
np.diag(Sy) + pow(calunc * x, 2)) * bhat[:, 1]
# if loglevel == 'DEBUG':
# plot_example(xv, yv, bhat)
nspectra = nspectra + 1
elapsed = float(time.time() - start)
logging.info('row {}/{}, ({}/{} local), {} spectra per second'.format(
row, n_input_lines, int(row - start_line),
int(stop_line - start_line), round(float(nspectra) / elapsed, 2)))
del input_locations_mm
del input_radiance_mm
output_reflectance_row = output_reflectance_row.transpose((1, 0))
output_uncertainty_row = output_uncertainty_row.transpose((1, 0))
shp = output_reflectance_row.shape
output_reflectance_row = output_reflectance_row.reshape(
(1, shp[0], shp[1]))
shp = output_uncertainty_row.shape
output_uncertainty_row = output_uncertainty_row.reshape(
(1, shp[0], shp[1]))
_write_bil_chunk(
output_reflectance_row, output_reflectance_file, row,
(n_input_lines, n_output_reflectance_bands, n_input_samples))
_write_bil_chunk(
output_uncertainty_row, output_uncertainty_file, row,
(n_input_lines, n_output_uncertainty_bands, n_input_samples))
def empirical_line(reference_radiance_file: str,
reference_reflectance_file: str,
reference_uncertainty_file: str,
reference_locations_file: str,
segmentation_file: str,
input_radiance_file: str,
input_locations_file: str,
output_reflectance_file: str,
output_uncertainty_file: str,
nneighbors: int = 400,
nodata_value: float = -9999.0,
level: str = 'INFO',
logfile: str = None,
radiance_factors: np.array = None,
isofit_config: str = None,
n_cores: int = -1) -> None:
"""
Perform an empirical line interpolation for reflectance and uncertainty extrapolation
Args:
reference_radiance_file: source file for radiance (interpolation built from this)
reference_reflectance_file: source file for reflectance (interpolation built from this)
reference_uncertainty_file: source file for uncertainty (interpolation built from this)
reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this)
segmentation_file: input file noting the per-pixel segmentation used
input_radiance_file: input radiance file (interpolate over this)
input_locations_file: input location file (interpolate over this)
output_reflectance_file: location to write output reflectance to
output_uncertainty_file: location to write output uncertainty to
nneighbors: number of neighbors to use for interpolation
nodata_value: nodata value of input and output
level: logging level
logfile: logging file
radiance_factors: radiance adjustment factors
isofit_config: path to isofit configuration JSON file
n_cores: number of cores to run on
Returns:
None
"""
loglevel = level
logging.basicConfig(format='%(message)s', level=loglevel, filename=logfile)
# Open input data to check that band formatting is correct
# Load reference set radiance
reference_radiance_img = envi.open(reference_radiance_file + '.hdr',
reference_radiance_file)
n_reference_lines, n_radiance_bands, n_reference_columns = [
int(reference_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_reference_columns != 1:
raise IndexError("Reference data should be a single-column list")
# Load reference set reflectance
reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr',
reference_reflectance_file)
nrefr, nbr, srefr = [
int(reference_reflectance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefr != n_reference_lines or nbr != n_radiance_bands or srefr != n_reference_columns:
raise IndexError("Reference file dimension mismatch (reflectance)")
# Load reference set uncertainty, assuming reflectance uncertainty is
# recoreded in the first n_radiance_bands channels of data
reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr',
reference_uncertainty_file)
nrefu, ns, srefu = [
int(reference_uncertainty_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefu != n_reference_lines or ns < n_radiance_bands or srefu != n_reference_columns:
raise IndexError("Reference file dimension mismatch (uncertainty)")
# Load reference set locations
reference_locations_img = envi.open(reference_locations_file + '.hdr',
reference_locations_file)
nrefl, lb, ls = [
int(reference_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefl != n_reference_lines or lb != 3:
raise IndexError("Reference file dimension mismatch (locations)")
input_radiance_img = envi.open(input_radiance_file + '.hdr',
input_radiance_file)
n_input_lines, n_input_bands, n_input_samples = [
int(input_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_radiance_bands != n_input_bands:
msg = 'Number of channels mismatch: input (%i) vs. reference (%i)'
raise IndexError(msg % (nbr, n_radiance_bands))
input_locations_img = envi.open(input_locations_file + '.hdr',
input_locations_file)
nll, nlb, nls = [
int(input_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nll != n_input_lines or nlb != 3 or nls != n_input_samples:
raise IndexError('Input location dimension mismatch')
# Create output files
output_metadata = input_radiance_img.metadata
output_metadata['interleave'] = 'bil'
output_reflectance_img = envi.create_image(output_reflectance_file +
'.hdr',
ext='',
metadata=output_metadata,
force=True)
output_uncertainty_img = envi.create_image(output_uncertainty_file +
'.hdr',
ext='',
metadata=output_metadata,
force=True)
# Now cleanup inputs and outputs, we'll write dynamically above
del output_reflectance_img, output_uncertainty_img
del reference_reflectance_img, reference_uncertainty_img, reference_locations_img, input_radiance_img, input_locations_img
# Initialize ray cluster
start_time = time.time()
if isofit_config is not None:
iconfig = configs.create_new_config(isofit_config)
else:
# If none, create a temporary config to get default ray parameters
iconfig = configs.Config({})
if n_cores == -1:
n_cores = iconfig.implementation.n_cores
rayargs = {
'ignore_reinit_error': iconfig.implementation.ray_ignore_reinit_error,
'local_mode': n_cores == 1,
"address": iconfig.implementation.ip_head,
"_redis_password": iconfig.implementation.redis_password
}
# only specify a temporary directory if we are not connecting to
# a ray cluster
if rayargs['local_mode']:
rayargs['_temp_dir'] = iconfig.implementation.ray_temp_dir
# Used to run on a VPN
ray.services.get_node_ip_address = lambda: '127.0.0.1'
# We can only set the num_cpus if running on a single-node
if iconfig.implementation.ip_head is None and iconfig.implementation.redis_password is None:
rayargs['num_cpus'] = n_cores
ray.init(**rayargs)
atexit.register(ray.shutdown)
n_ray_cores = ray.available_resources()["CPU"]
n_cores = min(n_ray_cores, n_input_lines)
logging.info(
'Beginning empirical line inversions using {} cores'.format(n_cores))
# Break data into sections
line_sections = np.linspace(0,
n_input_lines,
num=int(n_cores + 1),
dtype=int)
start_time = time.time()
# Run the pool (or run serially)
results = []
for l in range(len(line_sections) - 1):
args = (line_sections[l], line_sections[l + 1],
reference_radiance_file, reference_reflectance_file,
reference_uncertainty_file, reference_locations_file,
input_radiance_file, input_locations_file, segmentation_file,
isofit_config, output_reflectance_file,
output_uncertainty_file, radiance_factors, nneighbors,
nodata_value, level, logfile)
results.append(_run_chunk.remote(*args))
_ = ray.get(results)
total_time = time.time() - start_time
logging.info(
'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core'
.format(total_time, line_sections[-1] * n_input_samples / total_time,
line_sections[-1] * n_input_samples / total_time / n_cores))
def main():
# Parse arguments
parser = argparse.ArgumentParser(description="built luts for emulation.")
parser.add_argument('-ip_head', type=str)
parser.add_argument('-redis_password', type=str)
parser.add_argument('-n_cores', type=int, default=1)
parser.add_argument('-train', type=int, default=1, choices=[0,1])
parser.add_argument('-cleanup', type=int, default=0, choices=[0,1])
args = parser.parse_args()
args.train = args.train == 1
args.cleanup = args.cleanup == 1
dayofyear = 200
if args.train:
to_solar_zenith_lut = [0, 12.5, 25, 37.5, 50]
to_solar_azimuth_lut = [180]
to_sensor_azimuth_lut = [180]
to_sensor_zenith_lut = [140, 160, 180]
altitude_km_lut = [2, 4, 7, 10, 15, 25]
elevation_km_lut = [0, 0.75, 1.5, 2.25, 4.5]
h2o_lut_grid = np.round(np.linspace(0.1,5,num=5),3).tolist() + [0.6125]
h2o_lut_grid.sort()
aerfrac_2_lut_grid = np.round(np.linspace(0.01,1,num=5),3).tolist() + [0.5]
aerfrac_2_lut_grid.sort()
else:
# HOLDOUT SET
to_solar_zenith_lut = [6, 18, 30,45]
to_solar_azimuth_lut = [60, 300]
to_sensor_azimuth_lut = [180]
to_sensor_zenith_lut = [145, 155, 165, 175]
altitude_km_lut = [3, 5.5, 8.5, 12.5, 17.5]
elevation_km_lut = [0.325, 1.025, 1.875, 2.575, 4.2]
h2o_lut_grid = np.round(np.linspace(0.5,4.5,num=4),3)
aerfrac_2_lut_grid = np.round(np.linspace(0.125,0.9,num=4),3)
n_lut_build = np.product([len(to_solar_zenith_lut),
len(to_solar_azimuth_lut),
len(to_sensor_zenith_lut),
len(to_sensor_azimuth_lut),
len(altitude_km_lut),
len(elevation_km_lut),
len(h2o_lut_grid),
len(aerfrac_2_lut_grid)])
print('Num LUTs to build: {}'.format(n_lut_build))
print('Expected MODTRAN runtime: {} hrs'.format(n_lut_build*1.5))
print('Expected MODTRAN runtime: {} days'.format(n_lut_build*1.5/24))
print('Expected MODTRAN runtime per (40-core) node: {} days'.format(n_lut_build*1.5/24/40))
# Create wavelength file
wl = np.arange(0.350, 2.550, 0.0005)
wl_file_contents = np.zeros((len(wl),3))
wl_file_contents[:,0] = np.arange(len(wl),dtype=int)
wl_file_contents[:,1] = wl
wl_file_contents[:,2] = 0.0005
np.savetxt('support/hifidelity_wavelengths.txt',wl_file_contents,fmt='%.5f')
# Initialize ray for parallel execution
rayargs = {'address': args.ip_head,
'redis_password': args.redis_password,
'local_mode': args.n_cores == 1}
if args.n_cores < 40:
rayargs['num_cpus'] = args.n_cores
ray.init(**rayargs)
print(ray.cluster_resources())
template_dir = 'templates'
if os.path.isdir(template_dir) is False:
os.mkdir(template_dir)
modtran_template_file = os.path.join(template_dir,'modtran_template.json')
if args.training:
isofit_config_file = os.path.join(template_dir,'isofit_template_v2.json')
else:
isofit_config_file = os.path.join(template_dir,'isofit_template_holdout.json')
write_modtran_template(atmosphere_type='ATM_MIDLAT_SUMMER', fid=os.path.splitext(modtran_template_file)[0],
altitude_km=altitude_km_lut[0],
dayofyear=dayofyear, latitude=to_solar_azimuth_lut[0], longitude=to_solar_zenith_lut[0],
to_sensor_azimuth=to_sensor_azimuth_lut[0], to_sensor_zenith=to_sensor_zenith_lut[0],
gmtime=0, elevation_km=elevation_km_lut[0], output_file=modtran_template_file)
# Make sure H2O grid is fully valid
with open(modtran_template_file, 'r') as f:
modtran_config = json.load(f)
modtran_config['MODTRAN'][0]['MODTRANINPUT']['GEOMETRY']['IPARM'] = 12
modtran_config['MODTRAN'][0]['MODTRANINPUT']['ATMOSPHERE']['H2OOPT'] = '+'
modtran_config['MODTRAN'][0]['MODTRANINPUT']['AEROSOLS']['VIS'] = 100
with open(modtran_template_file, 'w') as fout:
fout.write(json.dumps(modtran_config, cls=SerialEncoder, indent=4, sort_keys=True))
paths = Paths(os.path.join('.',os.path.basename(modtran_template_file)), args.training)
build_main_config(paths, isofit_config_file, to_solar_azimuth_lut, to_solar_zenith_lut,
aerfrac_2_lut_grid, h2o_lut_grid, elevation_km_lut, altitude_km_lut, to_sensor_azimuth_lut,
to_sensor_zenith_lut, n_cores=args.n_cores)
config = configs.create_new_config(isofit_config_file)
isofit_modtran = ModtranRT(config.forward_model.radiative_transfer.radiative_transfer_engines[0],
config)
isofit_sixs = SixSRT(config.forward_model.radiative_transfer.radiative_transfer_engines[1],
config)
# cleanup
if args.cleanup:
for to_rm in ['*r_k', '*t_k', '*tp7', '*wrn', '*psc', '*plt', '*7sc', '*acd']:
cmd = 'find {os.path.join(paths.lut_modtran_directory)} -name "{to_rm}"')
print(cmd)
os.system(cmd)
def main():
# Parse arguments
parser = argparse.ArgumentParser(description="built luts for emulation.")
parser.add_argument('-config_file', type=str, default='templates/isofit_template.json')
parser.add_argument('-keys', type=str, default=['transm', 'rhoatm', 'sphalb'], nargs='+')
parser.add_argument('-munge_dir', type=str, default='munged')
args = parser.parse_args()
np.random.seed(13)
for key_ind, key in enumerate(args.keys):
munge_file = os.path.join(args.munge_dir, key + '.npz')
if os.path.isfile(munge_file) is False:
config = configs.create_new_config(args.config_file)
# Note - this goes way faster if you comment out the Vector Interpolater build section in each of these
isofit_modtran = ModtranRT(config.forward_model.radiative_transfer.radiative_transfer_engines[0],
config, build_lut = False)
isofit_sixs = SixSRT(config.forward_model.radiative_transfer.radiative_transfer_engines[1],
config, build_lut = False)
sixs_results = get_obj_res(isofit_sixs, key, resample=False)
modtran_results = get_obj_res(isofit_modtran, key)
if os.path.isdir(os.path.dirname(munge_file) is False):
os.mkdir(os.path.dirname(munge_file))
for fn in isofit_modtran.files:
mod_output = isofit_modtran.load_rt(fn)
sol_irr = mod_output['sol']
if np.all(np.isfinite(sol_irr)):
break
np.savez(munge_file, modtran_results=modtran_results, sixs_results=sixs_results, sol_irr=sol_irr)
modtran_results = None
sixs_results = None
for key_ind, key in enumerate(args.keys):
munge_file = os.path.join(args.munge_dir, key + '.npz')
npzf = np.load(munge_file)
dim1 = int(np.product(np.array(npzf['modtran_results'].shape)[:-1]))
dim2 = npzf['modtran_results'].shape[-1]
if modtran_results is None:
modtran_results = np.zeros((dim1,dim2*len(args.keys)))
modtran_results[:,dim2*key_ind:dim2*(key_ind+1)] = npzf['modtran_results']
dim1 = int(np.product(np.array(npzf['sixs_results'].shape)[:-1]))
dim2 = npzf['sixs_results'].shape[-1]
if sixs_results is None:
sixs_results = np.zeros((dim1,dim2*len(args.keys)))
sixs_results[:,dim2*key_ind:dim2*(key_ind+1)] = npzf['sixs_results']
sol_irr = npzf['sol_irr']
config = configs.create_new_config(args.config_file)
isofit_modtran = ModtranRT(config.forward_model.radiative_transfer.radiative_transfer_engines[0],
config, build_lut=False)
isofit_sixs = SixSRT(config.forward_model.radiative_transfer.radiative_transfer_engines[1],
config, build_lut=False)
sixs_names = isofit_sixs.lut_names
modtran_names = isofit_modtran.lut_names
if 'elev' in sixs_names:
sixs_names[sixs_names.index('elev')] = 'GNDALT'
if 'viewzen' in sixs_names:
sixs_names[sixs_names.index('viewzen')] = 'OBSZEN'
if 'viewaz' in sixs_names:
sixs_names[sixs_names.index('viewaz')] = 'TRUEAZ'
if 'alt' in sixs_names:
sixs_names[sixs_names.index('alt')] = 'H1ALT'
if 'AOT550' in sixs_names:
sixs_names[sixs_names.index('AOT550')] = 'AERFRAC_2'
reorder_sixs = [sixs_names.index(x) for x in modtran_names]
points = isofit_modtran.points.copy()
points_sixs = isofit_sixs.points.copy()[:,reorder_sixs]
if 'OBSZEN' in modtran_names:
print('adjusting')
points_sixs[:, modtran_names.index('OBSZEN')] = 180 - points_sixs[:, modtran_names.index('OBSZEN')]
ind = np.lexsort(tuple([points[:,x] for x in range(points.shape[-1])]))
points = points[ind,:]
modtran_results = modtran_results[ind,:]
ind_sixs = np.lexsort(tuple([points_sixs[:,x] for x in range(points_sixs.shape[-1])]))
points_sixs = points_sixs[ind_sixs,:]
sixs_results = sixs_results[ind_sixs,:]
good_data = np.all(np.isnan(modtran_results) == False,axis=1)
good_data[np.any(np.isnan(sixs_results),axis=1)] = False
modtran_results = modtran_results[good_data,:]
sixs_results = sixs_results[good_data,:]
points = points[good_data,...]
points_sixs = points_sixs[good_data,...]
print(sixs_results.shape)
print(modtran_results.shape)
tmp = isofit_sixs.load_rt(isofit_sixs.files[0])
np.savez(os.path.join(args.munge_dir, 'combined_training_data.npz'), sixs_results=sixs_results, modtran_results=modtran_results,
points=points, points_sixs=points_sixs, keys=args.keys, point_names=modtran_names, modtran_wavelengths=isofit_modtran.wl,
sixs_wavelengths=isofit_sixs.grid,
sol_irr=sol_irr)
def _run_chunk(start_line: int, stop_line: int, reference_radiance_file: str,
reference_atm_file: str, reference_locations_file: str,
input_radiance_file: str, input_locations_file: str,
segmentation_file: str, isofit_config: dict,
output_reflectance_file: str, output_uncertainty_file: str,
radiance_factors: np.array, nneighbors: int,
nodata_value: float) -> None:
"""
Args:
start_line: line to start empirical line run at
stop_line: line to stop empirical line run at
reference_radiance_file: source file for radiance (interpolation built from this)
reference_atm_file: source file for atmosphere coefficients (interpolation built from this)
reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this)
input_radiance_file: input radiance file (interpolate over this)
input_locations_file: input location file (interpolate over this)
segmentation_file: input file noting the per-pixel segmentation used
isofit_config: dictionary-stype isofit configuration
output_reflectance_file: location to write output reflectance to
output_uncertainty_file: location to write output uncertainty to
radiance_factors: radiance adjustment factors
nneighbors: number of neighbors to use for interpolation
nodata_value: nodata value of input and output
Returns:
None
"""
# Load reference images
reference_radiance_img = envi.open(envi_header(reference_radiance_file),
reference_radiance_file)
reference_atm_img = envi.open(envi_header(reference_atm_file),
reference_atm_file)
reference_locations_img = envi.open(envi_header(reference_locations_file),
reference_locations_file)
n_reference_lines, n_radiance_bands, n_reference_columns = [
int(reference_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
# Load input images
input_radiance_img = envi.open(envi_header(input_radiance_file),
input_radiance_file)
n_input_lines, n_input_bands, n_input_samples = [
int(input_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
wl = np.array(
[float(w) for w in input_radiance_img.metadata['wavelength']])
input_locations_img = envi.open(envi_header(input_locations_file),
input_locations_file)
n_location_bands = int(input_locations_img.metadata['bands'])
# Load output images
output_reflectance_img = envi.open(envi_header(output_reflectance_file),
output_reflectance_file)
output_uncertainty_img = envi.open(envi_header(output_uncertainty_file),
output_uncertainty_file)
n_output_reflectance_bands = int(output_reflectance_img.metadata['bands'])
n_output_uncertainty_bands = int(output_uncertainty_img.metadata['bands'])
# Load reference data
reference_locations_mm = reference_locations_img.open_memmap(
interleave='source', writable=False)
reference_locations = np.array(reference_locations_mm[:, :, :]).reshape(
(n_reference_lines, n_location_bands))
reference_radiance_mm = reference_radiance_img.open_memmap(
interleave='source', writable=False)
reference_radiance = np.array(reference_radiance_mm[:, :, :]).reshape(
(n_reference_lines, n_radiance_bands))
reference_atm_mm = reference_atm_img.open_memmap(interleave='source',
writable=False)
reference_atm = np.array(reference_atm_mm[:, :, :]).reshape(
(n_reference_lines, n_radiance_bands * 5))
rhoatm = reference_atm[:, :n_radiance_bands]
sphalb = reference_atm[:, n_radiance_bands:(n_radiance_bands * 2)]
transm = reference_atm[:, (n_radiance_bands * 2):(n_radiance_bands * 3)]
solirr = reference_atm[:, (n_radiance_bands * 3):(n_radiance_bands * 4)]
coszen = reference_atm[:, (n_radiance_bands * 4):(n_radiance_bands * 5)]
# Load segmentation data
if segmentation_file:
segmentation_img = envi.open(envi_header(segmentation_file),
segmentation_file)
segmentation_img = segmentation_img.read_band(0)
else:
segmentation_img = None
# Prepare instrument model, if available
if isofit_config is not None:
config = configs.create_new_config(isofit_config)
instrument = Instrument(config)
logging.info('Loading instrument')
else:
instrument = None
# Load radiance factors
if radiance_factors is None:
radiance_adjustment = np.ones(n_radiance_bands, )
else:
radiance_adjustment = np.loadtxt(radiance_factors)
# PCA coefficients
rdn_pca = PCA(n_components=2)
reference_pca = rdn_pca.fit_transform(reference_radiance *
radiance_adjustment)
# Create the tree to find nearest neighbor segments.
# Assume (heuristically) that, for distance purposes, 1 m vertically is
# comparable to 10 m horizontally, and that there are 100 km per latitude
# degree. This is all approximate of course. Elevation appears in the
# Third element, and the first two are latitude/longitude coordinates
# The fourth and fifth elements are "spectral distance" determined by the
# top principal component coefficients
loc_scaling = np.array([1e5, 1e5, 10, 100, 100])
scaled_ref_loc = np.concatenate(
(reference_locations, reference_pca), axis=1) * loc_scaling
tree = KDTree(scaled_ref_loc)
# Fit GP parameters on transmissivity of an H2O feature, in the
# first 400 datapoints
use = np.arange(min(len(rhoatm), 400))
h2oband = np.argmin(abs(wl - 940))
scale = (500, 500, 500, 500, 500)
bounds = ((100, 2000), (100, 2000), (100, 2000), (100, 2000), (100, 2000))
kernel = RBF(length_scale=scale, length_scale_bounds=bounds) +\
WhiteKernel(noise_level=0.01, noise_level_bounds=(1e-10, 0.1))
gp = GaussianProcessRegressor(kernel=kernel, alpha=0.0, normalize_y=True)
gp = gp.fit(scaled_ref_loc[use, :], transm[use, h2oband])
kernel = gp.kernel_
# Iterate through image. Each segment has its own GP, stored in a
# hash table indexed by location in the segmentation map
hash_table = {}
for row in np.arange(start_line, stop_line):
# Load inline input data
input_radiance_mm = input_radiance_img.open_memmap(interleave='source',
writable=False)
input_radiance = np.array(input_radiance_mm[row, :, :])
if input_radiance_img.metadata['interleave'] == 'bil':
input_radiance = input_radiance.transpose((1, 0))
input_radiance = input_radiance * radiance_adjustment
input_locations_mm = input_locations_img.open_memmap(
interleave='source', writable=False)
input_locations = np.array(input_locations_mm[row, :, :])
if input_locations_img.metadata['interleave'] == 'bil':
input_locations = input_locations.transpose((1, 0))
output_reflectance_row = np.zeros(input_radiance.shape) + nodata_value
output_uncertainty_row = np.zeros(input_radiance.shape) + nodata_value
nspectra, start = 0, time.time()
for col in np.arange(n_input_samples):
# Get radiance, pca coordinates, physical location for this datum
my_rdn = input_radiance[col, :]
my_pca = rdn_pca.transform(my_rdn[np.newaxis, :])
my_loc = np.r_[input_locations[col, :], my_pca[0, :]] * loc_scaling
if np.all(np.isclose(my_rdn, nodata_value)):
output_reflectance_row[col, :] = nodata_value
output_uncertainty_row[col, :] = nodata_value
continue
# Retrieve or build the GP
gp_rhoatm, gp_sphalb, gp_transm, irr = None, None, None, None
hash_idx = segmentation_img[row, col]
if hash_idx in hash_table:
gp_rhoatm, gp_sphalb, gp_transm, irr = hash_table[hash_idx]
else:
# There is no GP for this segment, so we build one from
# the atmospheric coefficients from closest neighbors
dists, nn = tree.query(my_loc, nneighbors)
neighbor_rhoatm = rhoatm[nn, :]
neighbor_transm = transm[nn, :]
neighbor_sphalb = sphalb[nn, :]
neighbor_coszen = coszen[nn, :]
neighbor_solirr = solirr[nn, :]
neighbor_locs = scaled_ref_loc[nn, :]
# Create a new GP using the optimized parameters as a fixed kernel
gp_rhoatm = GaussianProcessRegressor(kernel=kernel,
alpha=0.0,
normalize_y=True,
optimizer=None)
gp_rhoatm.fit(neighbor_locs, neighbor_rhoatm)
gp_sphalb = GaussianProcessRegressor(kernel=kernel,
alpha=0.0,
normalize_y=True,
optimizer=None)
gp_sphalb.fit(neighbor_locs, neighbor_sphalb)
gp_transm = GaussianProcessRegressor(kernel=kernel,
alpha=0.0,
normalize_y=True,
optimizer=None)
gp_transm.fit(neighbor_locs, neighbor_transm)
irr = solirr[1, :] * coszen[1, :]
irr[irr < 1e-8] = 1e-8
hash_table[hash_idx] = (gp_rhoatm, gp_sphalb, gp_transm, irr)
my_rhoatm = gp_rhoatm.predict(my_loc[np.newaxis, :])
my_sphalb = gp_sphalb.predict(my_loc[np.newaxis, :])
my_transm = gp_transm.predict(my_loc[np.newaxis, :])
my_rho = (my_rdn * np.pi) / irr
my_rfl = 1.0 / (my_transm / (my_rho - my_rhoatm) + my_sphalb)
output_reflectance_row[col, :] = my_rfl
# Calculate uncertainties. Sy approximation rather than Seps for
# speed, for now... but we do take into account instrument
# radiometric uncertainties
#output_uncertainty_row[col, :] = np.zeros()
#if instrument is None:
#else:
# Sy = instrument.Sy(x, geom=None)
# calunc = instrument.bval[:instrument.n_chan]
# output_uncertainty_row[col, :] = np.sqrt(
# np.diag(Sy) + pow(calunc * x, 2)) * bhat[:, 1]
# if loglevel == 'DEBUG':
# plot_example(xv, yv, bhat)
nspectra = nspectra + 1
elapsed = float(time.time() - start)
logging.info('row {}/{}, ({}/{} local), {} spectra per second'.format(
row, n_input_lines, int(row - start_line),
int(stop_line - start_line), round(float(nspectra) / elapsed, 2)))
del input_locations_mm
del input_radiance_mm
output_reflectance_row = output_reflectance_row.transpose((1, 0))
output_uncertainty_row = output_uncertainty_row.transpose((1, 0))
shp = output_reflectance_row.shape
output_reflectance_row = output_reflectance_row.reshape(
(1, shp[0], shp[1]))
shp = output_uncertainty_row.shape
output_uncertainty_row = output_uncertainty_row.reshape(
(1, shp[0], shp[1]))
_write_bil_chunk(
output_reflectance_row, output_reflectance_file, row,
(n_input_lines, n_output_reflectance_bands, n_input_samples))
_write_bil_chunk(
output_uncertainty_row, output_uncertainty_file, row,
(n_input_lines, n_output_uncertainty_bands, n_input_samples))
