def __init__(self, config: configs.Config, loglevel: str, logfile: str, worker_id: int = None, total_workers: int = None):
"""
Worker class to help run a subset of spectra.
Args:
config: isofit configuration
loglevel: output logging level
logfile: output logging file
worker_id: worker ID for logging reference
total_workers: the total number of workers running, for logging reference
"""
logging.basicConfig(format='%(levelname)s:%(message)s', level=loglevel, filename=logfile)
self.config = config
self.fm = ForwardModel(self.config)
if self.config.implementation.mode == 'mcmc_inversion':
self.iv = MCMCInversion(self.config, self.fm)
elif self.config.implementation.mode in ['inversion', 'simulation']:
self.iv = Inversion(self.config, self.fm)
else:
# This should never be reached due to configuration checking
raise AttributeError('Config implementation mode node valid')
self.io = IO(self.config, self.fm)
self.approximate_total_spectra = None
if total_workers is not None:
self.approximate_total_spectra = self.io.n_cols * self.io.n_rows / total_workers
self.worker_id = worker_id
self.completed_spectra = 0
def _init_nonpicklable_objects(self) -> None:
""" Internal function to initialize objects that cannot be pickled
"""
self.fm = ForwardModel(self.config)
if self.config.implementation.mode == 'mcmc_inversion':
self.iv = MCMCInversion(self.config, self.fm)
elif self.config.implementation.mode in ['inversion', 'simulation']:
self.iv = Inversion(self.config, self.fm)
else:
# This should never be reached due to configuration checking
raise AttributeError('Config implementation mode node valid')
def __init__(self, config: configs.Config, loglevel: str, logfile: str):
logging.basicConfig(format='%(levelname)s:%(message)s', level=loglevel, filename=logfile)
self.config = config
self.fm = ForwardModel(self.config)
if self.config.implementation.mode == 'mcmc_inversion':
self.iv = MCMCInversion(self.config, self.fm)
elif self.config.implementation.mode in ['inversion', 'simulation']:
self.iv = Inversion(self.config, self.fm)
else:
# This should never be reached due to configuration checking
raise AttributeError('Config implementation mode node valid')
self.io = IO(self.config, self.fm)
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
config = load_config('config_forward.json')
fm = ForwardModel(config['forward_model'])
iv = Inversion(config['inversion'], fm)
io = IO(config, fm, iv, [0], [0])
# Simulate a measurement and write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[0]
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm, iv, [0], [0])
# Simulate a measurement and write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[0]
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
config = load_config('config_inversion.json')
fm = ForwardModel(config['forward_model'])
iv = Inversion(config['inversion'], fm)
io = IO(config, fm, iv, [0], [0])
geom = None
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[-1]
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm, iv, [0], [0])
geom = None
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[-1]
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm)
# Simulate a measurement and write result
for row in range(io.n_rows):
for col in range(io.n_cols):
id = io.get_components_at_index(row, col)
if id is not None:
states = iv.invert(id.meas, id.geom)
io.write_spectrum(row, col, states, fm, iv)
assert True
return states[0]
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm)
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row in range(io.n_rows):
for col in range(io.n_cols):
id = io.get_components_at_index(row, col)
if id is not None:
states = iv.invert(id.meas, id.geom)
io.write_spectrum(row, col, states, fm, iv)
assert True
return states[-1]
def run(self, row_column = None):
"""
Iterate over spectra, reading and writing through the IO
object to handle formatting, buffering, and deferred write-to-file.
Attempts to avoid reading the entire file into memory, or hitting
the physical disk too often.
row_column: The user can specify
* a single number, in which case it is interpreted as a row
* a comma-separated pair, in which case it is interpreted as a
row/column tuple (i.e. a single spectrum)
* a comma-separated quartet, in which case it is interpreted as
a row, column range in the order (line_start, line_end, sample_start,
sample_end) all values are inclusive.
If none of the above, the whole cube will be analyzed.
"""
logging.info("Building first forward model, will generate any necessary LUTs")
fm = ForwardModel(self.config)
if row_column is not None:
ranges = row_column.split(',')
if len(ranges) == 1:
self.rows, self.cols = [int(ranges[0])], None
if len(ranges) == 2:
row_start, row_end = ranges
self.rows, self.cols = range(
int(row_start), int(row_end)), None
elif len(ranges) == 4:
row_start, row_end, col_start, col_end = ranges
self.rows = range(int(row_start), int(row_end) + 1)
self.cols = range(int(col_start), int(col_end) + 1)
else:
io = IO(self.config, fm)
self.rows = range(io.n_rows)
self.cols = range(io.n_cols)
del io, fm
index_pairs = np.vstack([x.flatten(order='f') for x in np.meshgrid(self.rows, self.cols)]).T
n_iter = index_pairs.shape[0]
if self.config.implementation.n_cores is None:
n_workers = multiprocessing.cpu_count()
else:
n_workers = self.config.implementation.n_cores
# Max out the number of workers based on the number of tasks
n_workers = min(n_workers, n_iter)
if self.workers is None:
remote_worker = ray.remote(Worker)
self.workers = ray.util.ActorPool([remote_worker.remote(self.config, self.loglevel, self.logfile, n, n_workers)
for n in range(n_workers)])
start_time = time.time()
n_tasks = min(n_workers * self.config.implementation.task_inflation_factor, n_iter)
logging.info(f'Beginning {n_iter} inversions in {n_tasks} chunks using {n_workers} cores')
# Divide up spectra to run into chunks
index_sets = np.linspace(0, n_iter, num=n_tasks, dtype=int)
if len(index_sets) == 1:
indices_to_run = [index_pairs[0:1,:]]
else:
indices_to_run = [index_pairs[index_sets[l]:index_sets[l + 1], :]
for l in range(len(index_sets) - 1)]
res = list(self.workers.map_unordered(lambda a, b: a.run_set_of_spectra.remote(b),
indices_to_run))
total_time = time.time() - start_time
logging.info(f'Inversions complete. {round(total_time,2)}s total, {round(n_iter/total_time,4)} spectra/s, '
f'{round(n_iter/total_time/n_workers,4)} spectra/s/core')
def build_output(self, states: List, input_data: InputData,
fm: ForwardModel, iv: Inversion):
"""
Build the output to be written to disk as a dictionary
Args:
states: results states from inversion. In the MCMC case, these are interpreted as samples from the
posterior, otherwise they are a gradient descent trajectory (with the last spectrum being the converged
solution).
input_data: an InputData object
fm: the forward model used to solve the inversion
iv: the inversion object
"""
if len(states) == 0:
# Write a bad data flag
atm_bad = np.zeros(len(fm.instrument.n_chan) * 5) * -9999.0
state_bad = np.zeros(len(fm.statevec)) * -9999.0
data_bad = np.zeros(fm.instrument.n_chan) * -9999.0
to_write = {
'estimated_state_file': state_bad,
'estimated_reflectance_file': data_bad,
'estimated_emission_file': data_bad,
'modeled_radiance_file': data_bad,
'apparent_reflectance_file': data_bad,
'path_radiance_file': data_bad,
'simulated_measurement_file': data_bad,
'algebraic_inverse_file': data_bad,
'atmospheric_coefficients_file': atm_bad,
'radiometry_correction_file': data_bad,
'spectral_calibration_file': data_bad,
'posterior_uncertainty_file': state_bad
}
else:
to_write = {}
meas = input_data.meas
geom = input_data.geom
reference_reflectance = input_data.reference_reflectance
if self.config.implementation.mode == 'inversion_mcmc':
state_est = states.mean(axis=0)
else:
state_est = states[-1, :]
############ Start with all of the 'independent' calculations
if 'estimated_state_file' in self.output_datasets:
to_write['estimated_state_file'] = state_est
if 'path_radiance_file' in self.output_datasets:
path_est = fm.calc_meas(state_est,
geom,
rfl=np.zeros(self.meas_wl.shape))
to_write['path_radiance_file'] = np.column_stack(
(fm.instrument.wl_init, path_est))
if 'spectral_calibration_file' in self.output_datasets:
# Spectral calibration
wl, fwhm = fm.calibration(state_est)
cal = np.column_stack(
[np.arange(0, len(wl)), wl / 1000.0, fwhm / 1000.0])
to_write['spectral_calibration_file'] = cal
if 'posterior_uncertainty_file' in self.output_datasets:
S_hat, K, G = iv.calc_posterior(state_est, geom, meas)
to_write['posterior_uncertainty_file'] = np.sqrt(
np.diag(S_hat))
############ Now proceed to the calcs where they may be some overlap
if any(item in
['estimated_emission_file', 'apparent_reflectance_file']
for item in self.output_datasets):
Ls_est = fm.calc_Ls(state_est, geom)
if any(item in [
'estimated_reflectance_file', 'apparent_reflectance_file',
'modeled_radiance_file', 'simulated_measurement_file'
] for item in self.output_datasets):
lamb_est = fm.calc_lamb(state_est, geom)
if any(item in
['modeled_radiance_file', 'simulated_measurement_file']
for item in self.output_datasets):
meas_est = fm.calc_meas(state_est, geom, rfl=lamb_est)
if 'modeled_radiance_file' in self.output_datasets:
to_write['modeled_radiance_file'] = np.column_stack(
(fm.instrument.wl_init, meas_est))
if 'simulated_measurement_file' in self.output_datasets:
meas_sim = fm.instrument.simulate_measurement(
meas_est, geom)
to_write['simulated_measurement_file'] = np.column_stack(
(self.meas_wl, meas_sim))
if 'estimated_emission_file' in self.output_datasets:
to_write['estimated_emission_file'] = np.column_stack(
(self.meas_wl, Ls_est))
if 'estimated_reflectance_file' in self.output_datasets:
to_write['estimated_reflectance_file'] = np.column_stack(
(self.meas_wl, lamb_est))
if 'apparent_reflectance_file' in self.output_datasets:
# Upward emission & glint and apparent reflectance
apparent_rfl_est = lamb_est + Ls_est
to_write['apparent_reflectance_file'] = np.column_stack(
(self.meas_wl, apparent_rfl_est))
x_surface, x_RT, x_instrument = fm.unpack(state_est)
if any(item in
['algebraic_inverse_file', 'atmospheric_coefficients_file']
for item in self.output_datasets):
rfl_alg_opt, Ls, coeffs = invert_algebraic(
fm.surface, fm.RT, fm.instrument, x_surface, x_RT,
x_instrument, meas, geom)
if 'algebraic_inverse_file' in self.output_datasets:
to_write['algebraic_inverse_file'] = np.column_stack(
(self.meas_wl, rfl_alg_opt))
if 'atmospheric_coefficients_file' in self.output_datasets:
rhoatm, sphalb, transm, solar_irr, coszen, transup = coeffs
atm = np.column_stack(
list(coeffs[:4]) +
[np.ones((len(self.meas_wl), 1)) * coszen])
atm = atm.T.reshape((len(self.meas_wl) * 5, ))
to_write['atmospheric_coefficients_file'] = atm
if 'radiometry_correction_file' in self.output_datasets:
factors = np.ones(len(self.meas_wl))
if 'reference_reflectance_file' in self.input_datasets:
meas_est = fm.calc_meas(state_est,
geom,
rfl=reference_reflectance)
factors = meas_est / meas
else:
logging.warning('No reflectance reference')
to_write['radiometry_correction_file'] = factors
return to_write
surface_file,
1,
"multicomponent_surface",
engine,
observables)
full_config = Config({"forward_model":{"instrument":instrument_config,
"surface":surface_config,
"radiative_transfer": rtm_config}})
#Start up ray
if ray.is_initialized():
ray.shutdown()
ray.init(num_cpus = 48)
# Run intial forward model
fm = ForwardModel(full_config)
inversion_settings = {"implementation": {"mode": "inversion",
"inversion": {
"windows": windows}}}
inverse_config = Config(inversion_settings)
iv = Inversion(inverse_config, fm)
# Refine water vapor LUT
water = []
window =5
for line in range(10,990,100):
print(line)
obs_mean = observables.get_chunk(2,990,
line-window,line+window).mean(axis =(0,1))
loc_mean = location.get_chunk(2,990,