def _run_set_of_spectra(self, index_start: int, index_stop: int) -> None:
"""Internal function to run a chunk of spectra
Args:
index_start: spectral index to start execution at
index_stop: spectral index to stop execution at
"""
logging.basicConfig(format='%(levelname)s:%(message)s', level=self.loglevel, filename=self.logfile)
self._init_nonpicklable_objects()
io = IO(self.config, self.fm, self.iv, self.rows, self.cols)
for index in range(index_start, index_stop):
success, row, col, meas, geom = io.get_components_at_index(
index)
# Only run through the inversion if we got some data
if success:
if meas is not None and all(meas < -49.0):
# Bad data flags
self.states = []
else:
# The inversion returns a list of states, which are
# intepreted either as samples from the posterior (MCMC case)
# or as a gradient descent trajectory (standard case). For
# a trajectory, the last spectrum is the converged solution.
self.states = self.iv.invert(meas, geom)
# Write the spectra to disk
io.write_spectrum(row, col, self.states, meas,
geom, flush_immediately=True)
if (index - index_start) % 100 == 0:
logging.info(
'Core at start index {} completed inversion {}/{}'.format(index_start, index-index_start,
index_stop-index_start))
class Worker(object):
def __init__(self, config: configs.Config, loglevel: str, logfile: str):
logging.basicConfig(format='%(levelname)s:%(message)s', level=loglevel, filename=logfile)
self.config = config
self.fm = ForwardModel(self.config)
if self.config.implementation.mode == 'mcmc_inversion':
self.iv = MCMCInversion(self.config, self.fm)
elif self.config.implementation.mode in ['inversion', 'simulation']:
self.iv = Inversion(self.config, self.fm)
else:
# This should never be reached due to configuration checking
raise AttributeError('Config implementation mode node valid')
self.io = IO(self.config, self.fm)
def run_set_of_spectra(self, indices: np.array):
for index in range(0, indices.shape[0]):
logging.debug("Read chunk of spectra")
row, col = indices[index,0], indices[index,1]
input_data = self.io.get_components_at_index(row, col)
if input_data is not None:
logging.debug("Run model")
# The inversion returns a list of states, which are
# intepreted either as samples from the posterior (MCMC case)
# or as a gradient descent trajectory (standard case). For
# a trajectory, the last spectrum is the converged solution.
states = self.iv.invert(input_data.meas, input_data.geom)
logging.debug("Write chunk of spectra")
# Write the spectra to disk
try:
self.io.write_spectrum(row, col, states, self.fm, self.iv)
except ValueError as err:
logging.info(
f"""
Encountered the following ValueError in (row,col) ({row},{col}).
Results for this pixel will be all zeros.
"""
)
logging.error(err)
if index % 100 == 0:
logging.info(f'Core at start location ({row},{col}) completed {index}/{indices.shape[0]}')
self.io.flush_buffers()
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
config = load_config('config_forward.json')
fm = ForwardModel(config['forward_model'])
iv = Inversion(config['inversion'], fm)
io = IO(config, fm, iv, [0], [0])
# Simulate a measurement and write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[0]
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm, iv, [0], [0])
# Simulate a measurement and write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[0]
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
config = load_config('config_inversion.json')
fm = ForwardModel(config['forward_model'])
iv = Inversion(config['inversion'], fm)
io = IO(config, fm, iv, [0], [0])
geom = None
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[-1]
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm, iv, [0], [0])
geom = None
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row, col, meas, geom, configs in io:
states = iv.invert(meas, geom)
io.write_spectrum(row, col, states, meas, geom)
assert True
return states[-1]
def run_forward():
"""Simulate the remote measurement of a spectrally uniform surface."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm)
# Simulate a measurement and write result
for row in range(io.n_rows):
for col in range(io.n_cols):
id = io.get_components_at_index(row, col)
if id is not None:
states = iv.invert(id.meas, id.geom)
io.write_spectrum(row, col, states, fm, iv)
assert True
return states[0]
def run_inverse():
"""Invert the remote measurement."""
# Configure the surface/atmosphere/instrument model
testdir, fname = os.path.split(os.path.abspath(__file__))
datadir = os.path.join(testdir, 'data')
config = create_new_config(os.path.join(datadir, 'config_forward.json'))
fm = ForwardModel(config)
iv = Inversion(config, fm)
io = IO(config, fm)
# Get our measurement from the simulation results, and invert.
# Calculate uncertainties at the solution state, write result
for row in range(io.n_rows):
for col in range(io.n_cols):
id = io.get_components_at_index(row, col)
if id is not None:
states = iv.invert(id.meas, id.geom)
io.write_spectrum(row, col, states, fm, iv)
assert True
return states[-1]
class Worker(object):
def __init__(self, config: configs.Config, loglevel: str, logfile: str, worker_id: int = None, total_workers: int = None):
"""
Worker class to help run a subset of spectra.
Args:
config: isofit configuration
loglevel: output logging level
logfile: output logging file
worker_id: worker ID for logging reference
total_workers: the total number of workers running, for logging reference
"""
logging.basicConfig(format='%(levelname)s:%(message)s', level=loglevel, filename=logfile)
self.config = config
self.fm = ForwardModel(self.config)
if self.config.implementation.mode == 'mcmc_inversion':
self.iv = MCMCInversion(self.config, self.fm)
elif self.config.implementation.mode in ['inversion', 'simulation']:
self.iv = Inversion(self.config, self.fm)
else:
# This should never be reached due to configuration checking
raise AttributeError('Config implementation mode node valid')
self.io = IO(self.config, self.fm)
self.approximate_total_spectra = None
if total_workers is not None:
self.approximate_total_spectra = self.io.n_cols * self.io.n_rows / total_workers
self.worker_id = worker_id
self.completed_spectra = 0
def run_set_of_spectra(self, indices: np.array):
for index in range(0, indices.shape[0]):
logging.debug("Read chunk of spectra")
row, col = indices[index,0], indices[index,1]
input_data = self.io.get_components_at_index(row, col)
self.completed_spectra += 1
if input_data is not None:
logging.debug("Run model")
# The inversion returns a list of states, which are
# intepreted either as samples from the posterior (MCMC case)
# or as a gradient descent trajectory (standard case). For
# a trajectory, the last spectrum is the converged solution.
states = self.iv.invert(input_data.meas, input_data.geom)
logging.debug("Write chunk of spectra")
# Write the spectra to disk
try:
self.io.write_spectrum(row, col, states, self.fm, self.iv)
except ValueError as err:
logging.info(
f"""
Encountered the following ValueError in (row,col) ({row},{col}).
Results for this pixel will be all zeros.
"""
)
logging.error(err)
if index % 100 == 0:
if self.worker_id is not None and self.approximate_total_spectra is not None:
percent = np.round(self.completed_spectra / self.approximate_total_spectra * 100,2)
logging.info(f'Worker {self.worker_id} completed {self.completed_spectra}/~{self.approximate_total_spectra}:: {percent}% complete')
else:
logging.info(f'Worker at start location ({row},{col}) completed {index}/{indices.shape[0]}')
self.io.flush_buffers()