def main(infile):
inhdr = infile + ".hdr"
outhdr = os.path.basename(infile)[0:-11] + "_glint900metric.hdr"
img = envi.open(inhdr, infile)
wl = s.array([float(w) for w in img.metadata['wavelength']])
if (wl[0] < 100):
wl = wl * 1000
fwhm = s.array([float(w) for w in img.metadata['fwhm']])
b900 = s.argmin(abs(wl - 900.0))
## b1050 = s.argmin(abs(wl-1050.0))
b900imgs = img.read_bands([b900 - 1, b900, b900 + 1])
## b1050imgs = img.read_bands([b1050-1, b1050, b1050+1])
glintm900 = np.median(b900imgs, axis=2).reshape(
(b900imgs.shape[0], 1, b900imgs.shape[1]))
## glintm1050 = np.median(b1050imgs, axis=2).reshape((b1050imgs.shape[0], 1, b1050imgs.shape[1]))
# make output glint metric file and open memmap
metadata = img.metadata.copy()
metadata['bands'] = '%i' % 1
metadata['interleave'] = 'bil'
metadata['data type'] = '4'
out = envi.create_image(outhdr, metadata, ext='', force=True)
outmm = out.open_memmap(interleave='source', writable=True)
outmm[:, :, :] = glintm900
del outmm, out, glintm900, img
def main():
# Read in CSV file and convert to BIP ENVI file
csv_path = sys.argv[1]
output_path = csv_path.replace(".csv", "")
output_hdr_path = output_path + ".hdr"
spectra_df = pd.read_csv(csv_path, dtype="float32")
spectra_df = spectra_df.fillna(-9999)
lines, bands = spectra_df.shape
wavelengths = spectra_df.keys().values
hdr = {
"lines": str(lines),
"samples": "1",
"bands": str(bands),
"header offset": "0",
"file type": "ENVI Standard",
"data type": "4",
"interleave": "bip",
"byte order": "0",
"data ignore value": "-9999",
"wavelength": wavelengths
}
out_file = envi.create_image(output_hdr_path, hdr, ext='', force=True)
output_mm = out_file.open_memmap(interleave='source', writable=True)
# Iterate through dataframe and write to output memmap
for index, row in spectra_df.iterrows():
output_mm[index, 0, :] = row.values
# Write to disk
del output_mm
def remap(inputfile, labels, outputfile, flag, chunksize):
"""."""
ref_file = inputfile
lbl_file = labels
out_file = outputfile
nchunk = chunksize
ref_img = envi.open(ref_file + '.hdr', ref_file)
ref_meta = ref_img.metadata
ref_mm = ref_img.open_memmap(interleave='source', writable=False)
ref = np.array(ref_mm[:, :])
lbl_img = envi.open(lbl_file + '.hdr', lbl_file)
lbl_meta = lbl_img.metadata
labels = lbl_img.read_band(0)
nl = int(lbl_meta['lines'])
ns = int(lbl_meta['samples'])
nb = int(ref_meta['bands'])
out_meta = dict([(k, v) for k, v in ref_meta.items()])
out_meta["samples"] = ns
out_meta["bands"] = nb
out_meta["lines"] = nl
out_meta['data type'] = ref_meta['data type']
out_meta["interleave"] = "bil"
out_img = envi.create_image(out_file + '.hdr',
metadata=out_meta,
ext='',
force=True)
out_mm = out_img.open_memmap(interleave='source', writable=True)
# Iterate through image "chunks," restoring as we go
for lstart in np.arange(0, nl, nchunk):
print(lstart)
del out_mm
out_mm = out_img.open_memmap(interleave='source', writable=True)
# Which labels will we extract? ignore zero index
lend = min(lstart + nchunk, nl)
lbl = labels[lstart:lend, :]
out = flag * np.ones((lbl.shape[0], nb, lbl.shape[1]))
for row in range(lbl.shape[0]):
for col in range(lbl.shape[1]):
out[row, :, col] = np.squeeze(ref[int(lbl[row, col]), :])
out_mm[lstart:lend, :, :] = out
def extractions(inputfile,
labels,
output,
chunksize,
flag,
n_cores: int = 1,
ray_address: str = None,
ray_redis_password: str = None,
ray_temp_dir: str = None,
ray_ip_head=None,
logfile: str = None,
loglevel: str = 'INFO'):
"""..."""
in_file = inputfile
lbl_file = labels
out_file = output
nchunk = chunksize
dtm = {'4': np.float32, '5': np.float64}
# Open input data, get dimensions
in_img = envi.open(in_file + '.hdr', in_file)
meta = in_img.metadata
nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
img_mm = in_img.open_memmap(interleave='bip', writable=False)
lbl_img = envi.open(lbl_file + '.hdr', lbl_file)
labels = lbl_img.read_band(0)
un_labels = np.unique(labels).tolist()
if 0 not in un_labels:
un_labels.insert(0, 0)
nout = len(un_labels)
# Start up a ray instance for parallel work
rayargs = {
'ignore_reinit_error': True,
'local_mode': n_cores == 1,
"address": ray_address,
"_redis_password": ray_redis_password
}
if rayargs['local_mode']:
rayargs['_temp_dir'] = ray_temp_dir
# Used to run on a VPN
ray.services.get_node_ip_address = lambda: '127.0.0.1'
# We can only set the num_cpus if running on a single-node
if ray_ip_head is None and ray_redis_password is None:
rayargs['num_cpus'] = n_cores
ray.init(**rayargs)
atexit.register(ray.shutdown)
labelid = ray.put(labels)
jobs = []
for lstart in np.arange(0, nl, nchunk):
lend = min(lstart + nchunk, nl)
jobs.append(
extract_chunk.remote(lstart,
lend,
in_file,
labelid,
flag,
logfile=logfile,
loglevel=loglevel))
# Collect results
rreturn = [ray.get(jid) for jid in jobs]
# Iterate through image "chunks," segmenting as we go
out = np.zeros((nout, nb))
for idx, ret in rreturn:
if ret is not None:
for _output_index in range(len(idx)):
out[un_labels.index(idx[_output_index]),
...] = ret[_output_index, ...]
del rreturn
ray.shutdown()
out[np.logical_not(np.isfinite(out))] = flag
meta["lines"] = str(nout)
meta["bands"] = str(nb)
meta["samples"] = '1'
meta["interleave"] = "bil"
out_img = envi.create_image(out_file + '.hdr',
metadata=meta,
ext='',
force=True)
out_mm = np.memmap(out_file,
dtype=dtm[meta['data type']],
mode='w+',
shape=(nout, 1, nb))
if dtm[meta['data type']] == np.float32:
out_mm[:, 0, :] = np.array(out, np.float32)
else:
out_mm[:, 0, :] = np.array(out, np.float64)
#Working
startrow = 0
endrow = lat
startcol = 0
endcol = long
import itertools
p = Pool(cpuNode)
p.starmap(
GGGl,
itertools.product(range(startrow, endrow, maxchuckSize),
range(startcol, endcol, maxchuckSize), [maxchuckSize],
[{
"YAM": True
}]))
p.close()
# In[72]:
#Writing output
meta = MSAVI.metadata
meta["band names"] = outputparams
meta['bands'] = str(len(meta["band names"]))
meta['data type'] = '4'
#del meta['description']
img = envi.create_image(outputName, meta, force=True)
mm = img.open_memmap(writable=True)
mm[:, :, :] = fp[:, :, :]
def interpolate_atmosphere(reference_radiance_file: str,
reference_atm_file: str,
reference_locations_file: str,
segmentation_file: str,
input_radiance_file: str,
input_locations_file: str,
output_reflectance_file: str,
output_uncertainty_file: str,
nneighbors: int = 15,
nodata_value: float = -9999.0,
level: str = 'INFO',
radiance_factors: np.array = None,
isofit_config: dict = None,
n_cores: int = -1) -> None:
"""
Perform a Gaussian process interpolation of atmospheric parameters. It relies on precalculated
atmospheric coefficients at a subset of spatial locations stored in a file. The file has
each coefficient defined for every radiance channel, appearing in the order: (1) atmospheric
path reflectance; (2) spherical sky albedo; (3) total diffuse and direct transmittance of the
two-part downwelling and upwelling path; (4) extraterrestrial solar irradiance; (5) cosine of solar
zenith angle.
Args:
reference_radiance_file: source file for radiance (interpolation built from this)
reference_atm_file: source file for atmospheric coefficients (interpolation from this)
reference_locations_file: source file for file locations (lon, lat, elev), (interpolation from this)
segmentation_file: input file noting the per-pixel segmentation used
input_radiance_file: input radiance file (interpolate over this)
input_locations_file: input location file (interpolate over this)
output_reflectance_file: location to write output reflectance
output_uncertainty_file: location to write output uncertainty
nneighbors: number of neighbors to use for interpolation
nodata_value: nodata value of input and output
level: logging level
radiance_factors: radiance adjustment factors
isofit_config: dictionary-stype isofit configuration
n_cores: number of cores to run on
Returns:
None
"""
loglevel = level
logging.basicConfig(format='%(message)s', level=loglevel)
# Open input data to check that band formatting is correct
# Load reference set radiance
reference_radiance_img = envi.open(envi_header(reference_radiance_file),
reference_radiance_file)
n_reference_lines, n_radiance_bands, n_reference_columns = [
int(reference_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_reference_columns != 1:
raise IndexError("Reference data should be a single-column list")
# Load reference set atmospheric coefficients
reference_atm_img = envi.open(envi_header(reference_atm_file),
reference_atm_file)
nrefa, nba, srefa = [
int(reference_atm_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefa != n_reference_lines or srefa != n_reference_columns:
raise IndexError("Reference file dimension mismatch (atmosphere)")
if nba != (n_radiance_bands * 5):
raise IndexError(
"Reference atmosphere file has incorrect dimensioning")
# Load reference set locations
reference_locations_img = envi.open(envi_header(reference_locations_file),
reference_locations_file)
nrefl, lb, ls = [
int(reference_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefl != n_reference_lines or lb != 3:
raise IndexError("Reference file dimension mismatch (locations)")
input_radiance_img = envi.open(envi_header(input_radiance_file),
input_radiance_file)
n_input_lines, n_input_bands, n_input_samples = [
int(input_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_radiance_bands != n_input_bands:
msg = 'Number of channels mismatch: input (%i) vs. reference (%i)'
raise IndexError(msg % (n_input_bands, n_radiance_bands))
input_locations_img = envi.open(envi_header(input_locations_file),
input_locations_file)
nll, nlb, nls = [
int(input_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nll != n_input_lines or nlb != 3 or nls != n_input_samples:
raise IndexError('Input location dimension mismatch')
# Create output files
output_metadata = input_radiance_img.metadata
output_metadata['interleave'] = 'bil'
output_reflectance_img = envi.create_image(
envi_header(output_reflectance_file),
ext='',
metadata=output_metadata,
force=True)
output_uncertainty_img = envi.create_image(
envi_header(output_uncertainty_file),
ext='',
metadata=output_metadata,
force=True)
# Now cleanup inputs and outputs, we'll write dynamically above
del output_reflectance_img, output_uncertainty_img
del reference_atm_img, reference_locations_img, input_radiance_img, input_locations_img
# Determine the number of cores to use
if n_cores == -1:
n_cores = multiprocessing.cpu_count()
n_cores = min(n_cores, n_input_lines)
# Break data into sections
line_sections = np.linspace(0, n_input_lines, num=n_cores + 1, dtype=int)
# Set up our pool
pool = multiprocessing.Pool(processes=n_cores)
start_time = time.time()
logging.info(
'Beginning atmospheric interpolation inversions using {} cores'.format(
n_cores))
# Run the pool (or run serially)
results = []
for l in range(len(line_sections) - 1):
args = (
line_sections[l],
line_sections[l + 1],
reference_radiance_file,
reference_atm_file,
reference_locations_file,
input_radiance_file,
input_locations_file,
segmentation_file,
isofit_config,
output_reflectance_file,
output_uncertainty_file,
radiance_factors,
nneighbors,
nodata_value,
)
if n_cores != 1:
results.append(pool.apply_async(_run_chunk, args))
else:
_run_chunk(*args)
pool.close()
pool.join()
total_time = time.time() - start_time
logging.info(
'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core'
.format(total_time, line_sections[-1] * n_input_samples / total_time,
line_sections[-1] * n_input_samples / total_time / n_cores))
def main():
description = 'Spectroscopic Surface & Atmosphere Fitting'
parser = argparse.ArgumentParser()
parser.add_argument('config_file')
parser.add_argument('--row_column', default='')
parser.add_argument('--profile', default='')
args = parser.parse_args()
# Setup
config = json.load(open(args.config_file, 'r'))
configdir, f = split(abspath(args.config_file))
config = expand_all_paths(config, configdir)
fm = ForwardModel(config['forward_model'])
iv = Inversion(config['inversion'], fm)
out = Output(config, iv)
simulation_mode = (not ('input' in config)) or \
(not ('measured_radiance_file' in config['input']))
text_mode = simulation_mode or \
config['input']['measured_radiance_file'].endswith('txt')
# Do we apply a radiance correction?
if (not simulation_mode) and \
'radiometry_correction_file' in config['input']:
radiance_correction_file = config['input'][
'radiometry_correction_file']
radiance_correction, wl = spectrumLoad(radiance_correction_file)
else:
radiance_correction = None
if text_mode:
# ------------------------------- Text mode
# build geometry object
if 'input' in config:
obs, loc, glt = None, None, None
if 'glt_file' in config['input']:
glt = s.loadtxt(config['input']['glt_file'])
if 'obs_file' in config['input']:
obs = s.loadtxt(config['input']['obs_file'])
if 'loc_file' in config['input']:
loc = s.loadtxt(config['input']['loc_file'])
geom = Geometry(obs=obs, glt=glt, loc=loc)
else:
geom = None
if simulation_mode:
# Get our measurement from instrument data or the initial state vector
state_est = fm.init_val.copy()
rdn_est = fm.calc_rdn(state_est, geom)
rdn_meas = rdn_est.copy()
rdn_sim = fm.instrument.simulate_measurement(rdn_meas, geom)
rfl_est, rdn_est, path_est, S_hat, K, G =\
iv.forward_uncertainty(state_est, rdn_meas, geom)
else:
# Invert instrument measurement
rdn_meas, wl = spectrumLoad(
config['input']['measured_radiance_file'])
if radiance_correction is not None:
rdn_meas = rdn_meas * radiance_correction
rdn_sim = None
if len(args.profile) > 0:
cProfile.runctx('iv.invert(rdn_meas, geom, None)', globals(),
locals())
sys.exit(0)
else:
state_est = iv.invert(rdn_meas, geom, out=out)
rfl_est, rdn_est, path_est, S_hat, K, G =\
iv.forward_uncertainty(state_est, rdn_meas, geom)
out.write_spectrum(state_est, rfl_est, rdn_est, path_est, rdn_meas,
rdn_sim, geom)
else:
# ------------------------------ Binary mode
meas_file = config['input']['measured_radiance_file']
meas_hdr = meas_file + '.hdr'
meas = envi.open(meas_hdr, meas_file)
nl, nb, ns = [
int(meas.metadata[n]) for n in ('lines', 'bands', 'samples')
]
# Do we apply a flatfield correction?
if 'flatfield_correction_file' in config['input']:
ffile = config['input']['flatfield_correction_file']
fcor = envi.open(ffile + '.hdr', ffile)
fcor_mm = fcor.open_memmap(interleave='source', writable=False)
flatfield = s.array(fcor_mm[0, :, :])
else:
flatfield = None
if 'obs_file' in config['input']:
obs_file = config['input']['obs_file']
obs_hdr = obs_file + '.hdr'
obs = envi.open(obs_hdr, obs_file)
if int(obs.metadata['bands']) != 11 and \
int(obs.metadata['bands']) != 10:
raise ValueError('Expected 10 or 11 bands in OBS file')
if int(obs.metadata['lines']) != nl or \
int(obs.metadata['samples']) != ns:
raise ValueError('obs file dimensions do not match radiance')
else:
obs_file, obs_hdr = None, None
if 'glt_file' in config['input']:
glt_file = config['input']['glt_file']
glt_hdr = glt_file + '.hdr'
glt = envi.open(glt_hdr, glt_file)
if int(glt.metadata['bands']) != 2:
raise ValueError('Expected two bands in GLT file')
if int(glt.metadata['lines']) != nl or \
int(glt.metadata['samples']) != ns:
raise ValueError('GLT file dimensions do not match radiance')
else:
glt_file, glt_hdr = None, None
if 'loc_file' in config['input']:
loc_file = config['input']['loc_file']
loc_hdr = loc_file + '.hdr'
loc = envi.open(loc_hdr, loc_file)
if int(loc.metadata['bands']) != 3:
raise ValueError('Expected three bands in LOC file')
if int(loc.metadata['lines']) != nl or \
int(loc.metadata['samples']) != ns:
raise ValueError('loc file dimensions do not match radiance')
else:
loc_file, loc_hdr = None, None
rfl_file = config['output']['estimated_reflectance_file']
rfl_hdr = rfl_file + '.hdr'
rfl_meta = meas.metadata.copy()
if not os.path.exists(rfl_hdr):
rfl = envi.create_image(rfl_hdr, rfl_meta, ext='', force=True)
del rfl
rfl = envi.open(rfl_hdr, rfl_file)
state_file = config['output']['estimated_state_file']
state_hdr = state_file + '.hdr'
state_meta = meas.metadata.copy()
state_meta['bands'] = len(fm.statevec)
state_meta['band names'] = fm.statevec[:]
if not os.path.exists(state_hdr):
state = envi.create_image(state_hdr,
state_meta,
ext='',
force=True)
del state
state = envi.open(state_hdr, state_file)
mdl_file = config['output']['modeled_radiance_file']
mdl_hdr = mdl_file + '.hdr'
mdl_meta = meas.metadata.copy()
if not os.path.exists(mdl_hdr):
mdl = envi.create_image(mdl_hdr, mdl_meta, ext='', force=True)
del mdl
mdl = envi.open(mdl_hdr, mdl_file)
path_file = config['output']['path_radiance_file']
path_hdr = path_file + '.hdr'
path_meta = meas.metadata.copy()
if not os.path.exists(path_hdr):
path = envi.create_image(path_hdr, path_meta, ext='', force=True)
del path
path = envi.open(path_hdr, path_file)
post_file = config['output']['posterior_uncertainty_file']
post_hdr = post_file + '.hdr'
post_meta = state_meta.copy()
if not os.path.exists(post_hdr):
post = envi.create_image(post_hdr, post_meta, ext='', force=True)
del post
post = envi.open(post_hdr, post_file)
if 'component_file' in config['output']:
comp_file = config['output']['component_file']
comp_hdr = comp_file + '.hdr'
comp_meta = state_meta.copy()
comp_meta['bands'] = 1
comp_meta['band names'] = '{Surface Model Component}'
if not os.path.exists(comp_hdr):
comp = envi.create_image(comp_hdr,
comp_meta,
ext='',
force=True)
del comp
comp = envi.open(comp_hdr, comp_file)
else:
comp = None
meas_mm, obs_mm, state_mm, rfl_mm, path_mm, mdl_mm = \
None, None, None, None, None, None
# Did the user specify a row,column tuple, or a row?
if len(args.row_column) < 1:
lines, samps = range(nl), range(ns)
else:
# Restrict the range of the retrieval if overridden.
ranges = args.row_column.split(',')
if len(ranges) == 1:
lines, samps = [int(ranges[0])], range(ns)
if len(ranges) == 2:
line_start, line_end = ranges
lines, samps = range(int(line_start), int(line_end)), range(ns)
elif len(ranges) == 4:
line_start, line_end, samp_start, samp_end = ranges
lines = range(int(line_start), int(line_end))
samps = range(int(samp_start), int(samp_end))
# analyze the image one frame at a time
for i in lines:
# Flush cache every once in a while
print('line %i/%i' % (i, nl))
if meas_mm is None or i % 100 == 0:
# Refresh Radiance buffer
del meas
meas = envi.open(meas_hdr, meas_file)
meas_mm = meas.open_memmap(interleave='source', writable=False)
# Refresh OBS buffer
if obs_hdr is not None:
del obs
obs = envi.open(obs_hdr, obs_file)
obs_mm = obs.open_memmap(interleave='source',
writable=False)
# Refresh GLT buffer
if glt_hdr is not None:
del glt
glt = envi.open(glt_hdr, glt_file)
glt_mm = glt.open_memmap(interleave='source',
writable=False)
# Refresh LOC buffer
if loc_hdr is not None:
del loc
loc = envi.open(loc_hdr, loc_file)
loc_mm = loc.open_memmap(interleave='source',
writable=False)
# Refresh Output buffers
del rfl
rfl = envi.open(rfl_hdr, rfl_file)
rfl_mm = rfl.open_memmap(interleave='source', writable=True)
del state
state = envi.open(state_hdr, state_file)
state_mm = state.open_memmap(interleave='source',
writable=True)
del path
path = envi.open(path_hdr, path_file)
path_mm = path.open_memmap(interleave='source', writable=True)
del mdl
mdl = envi.open(mdl_hdr, mdl_file)
mdl_mm = mdl.open_memmap(interleave='source', writable=True)
del post
post = envi.open(post_hdr, post_file)
post_mm = post.open_memmap(interleave='source', writable=True)
if comp is not None:
del comp
comp = envi.open(comp_hdr, comp_file)
comp_mm = comp.open_memmap(interleave='source',
writable=True)
# translate to BIP
meas_frame = s.array(meas_mm[i, :, :]).T
if obs_hdr is not None:
obs_frame = s.array(obs_mm[i, :, :])
if glt_hdr is not None:
glt_frame = s.array(glt_mm[i, :, :])
if loc_hdr is not None:
loc_frame = s.array(loc_mm[i, :, :])
init = None
nl = int(rfl_meta['lines'])
ns = int(rfl_meta['samples'])
nb = int(rfl_meta['bands'])
nsv = int(state_meta['bands'])
if comp is not None:
comp_frame = s.zeros((1, ns))
post_frame = s.zeros((nsv, ns), dtype=s.float32)
rfl_frame = s.zeros((nb, ns), dtype=s.float32)
state_frame = s.zeros((nsv, ns), dtype=s.float32)
path_frame = s.zeros((nb, ns), dtype=s.float32)
mdl_frame = s.zeros((nb, ns), dtype=s.float32)
for j in samps:
try:
# Use AVIRIS-C convention?
if loc_hdr is not None and "t01p00r" in loc_file:
loc_frame[:, 2] = loc_frame[:, 2] / \
1000.0 # translate to km
rdn_meas = meas_frame[j, :]
# Bad data test
if all(rdn_meas < -49.0):
raise OOBError()
if obs_hdr is not None:
obs_spectrum = obs_frame[j, :]
else:
obs_spectrum = None
if glt_hdr is not None:
pc = abs(glt_frame[j, 0]) - 1
if pc < 0:
raise OOBError()
glt_spectrum = glt_frame[j, :]
else:
pc = j
glt_spectrum = None
if loc_hdr is not None:
loc_spectrum = loc_frame[j, :]
else:
loc_spectrum = None
if flatfield is not None:
rdn_meas = rdn_meas * flatfield[:, pc]
if radiance_correction is not None:
rdn_meas = rdn_meas * radiance_correction
geom = Geometry(obs_spectrum,
glt_spectrum,
loc_spectrum,
pushbroom_column=pc)
# Inversion
if len(args.profile) > 0:
cProfile.runctx('iv.invert(rdn_meas, geom, None)',
globals(), locals())
sys.exit(0)
else:
state_est = iv.invert(rdn_meas, geom, None, init=init)
rfl_est, rdn_est, path_est, S_hat, K, G =\
iv.forward_uncertainty(state_est, rdn_meas, geom)
#init = state_est.copy()
# write spectrum
state_surf = state_est[iv.fm.surface_inds]
post_frame[:, j] = s.sqrt(s.diag(S_hat))
rfl_frame[:, j] = rfl_est
state_frame[:, j] = state_est
path_frame[:, j] = path_est
mdl_frame[:, j] = rdn_est
if comp is not None:
comp_frame[:, j] = iv.fm.surface.component(
state_surf, geom)
except OOBError:
post_frame[:, j] = -9999 * s.ones((nsv))
rfl_frame[:, j] = -9999 * s.ones((nb))
state_frame[:, j] = -9999 * s.ones((nsv))
path_frame[:, j] = -9999 * s.ones((nb))
mdl_frame[:, j] = -9999 * s.ones((nb))
if comp is not None:
comp_frame[:, j] = -9999
post_mm[i, :, :] = post_frame.copy()
rfl_mm[i, :, :] = rfl_frame.copy()
state_mm[i, :, :] = state_frame.copy()
path_mm[i, :, :] = path_frame.copy()
mdl_mm[i, :, :] = mdl_frame.copy()
if comp is not None:
comp_mm[i, :, :] = comp_frame.copy()
del post_mm
del rfl_mm
del state_mm
del path_mm
del mdl_mm
if comp is not None:
del comp_mm
def main():
# parse arguments
desc = "Break up images into smaller chunks"
parser = argparse.ArgumentParser(description=desc)
parser.add_argument('input', type=str, metavar='INPUT',
help='input reflectance file (ENVI header present)')
args = parser.parse_args()
s.set_printoptions(precision=4)
s.seterr(all='ignore')
scenario = args.scenario
# Set up input and output file paths
in_file = args.input
in_hdr = find_header(in_file)
suffix = ['a', 'b', 'c', 'd']
img = envi.open(in_hdr, in_file)
nl = int(float(img.metadata['lines'])/2.0)
ns = int(float(img.metadata['samples'])/2.0)
rowbounds = [[0, nl], [0, nl], \
[nl, img.metadata['lines']], [nl, img.metadata['lines']]]
colbounds = [[0, ns], [ns, img.metadata['samples']], \
[0, ns], [ns, img.metadata['samples']]]
for j in [0,1,2,3]:
out_file = os.path.splitext(in_file)[0] + suffix[j]
out_file = os.path.splitext(in_file)[0] + suffix[j] + '.hdr'
img = envi.open(in_hdr, in_file)
envi.read_subregion((0,1), (0,img.shape[1]), use_memmap=False)
inmm = img.open_memmap(interleave='source', writable=False)
## wl = s.array([float(w) for w in img.metadata['wavelength']])
## if (wl[0] < 100):
## wl = wl * 1000
## fwhm = s.array([float(w) for w in img.metadata['fwhm']])
# set up metadata and constants
const = constants(wl, args.mode, scenario, args.root_dir+'/data/')
# make output Rb file and open memmap
metadata = img.metadata.copy()
metadata['bands'] = '%i' % sum(const.use_out)
metadata['interleave'] = 'bil'
metadata['wavelength'] = ['%7.2f'%w for w in wl[const.use_out]]
metadata['data type'] = '2'
out = envi.create_image(out_hdr, metadata, ext='',force=True)
outmm = out.open_memmap(interleave='source', writable=True)
# iterate over rows
if args.rows is None:
start, fin = 0, nl
else:
start, fin = [int(f) for f in args.rows.split(',')]
for i in range(start, fin):
# Flush cache
print 'line %i/%i' % (i,nl)
if args.depth_file is not None:
del bathy
bathy= spectral.io.envi.open(args.depth_file+'.hdr', args.depth_file)
bathymm = bathy.open_memmap(interleave='source', writable=False)
bath = s.array(bathymm[i,:,:])
if bathy.metadata['interleave'] == 'bil':
bath = bath.T
else:
bath = None
del img
img = spectral.io.envi.open(in_hdr, in_file)
inmm = img.open_memmap(interleave='source', writable=False)
Rrs = s.array(inmm[i,:,:])
if img.metadata['interleave'] == 'bil':
Rrs = Rrs.T
if int(img.metadata['data type']) != 4:
Rrs = Rrs / s.float32(10000)
del out_Rb
out_Rb = spectral.io.envi.open(out_Rb_hdr, out_Rb_file)
outmm_Rb = out_Rb.open_memmap(interleave='source', writable=True)
Rb_frame = s.zeros(outmm_Rb.shape[1:])
del out_Kd
out_Kd = spectral.io.envi.open(out_Kd_hdr, out_Kd_file)
outmm_Kd = out_Kd.open_memmap(interleave='source', writable=True)
Kd_frame = s.zeros(outmm_Kd.shape[1:])
del out_bb
out_bb = spectral.io.envi.open(out_bb_hdr, out_bb_file)
outmm_bb = out_bb.open_memmap(interleave='source', writable=True)
bb_frame = s.zeros(outmm_bb.shape[1:])
del out_dep
out_dep = spectral.io.envi.open(out_dep_hdr, out_dep_file)
outmm_dep = out_dep.open_memmap(interleave='source', writable=True)
dep_frame = s.zeros((outmm_Rb.shape[1],1))
del out_mix
out_mix = spectral.io.envi.open(out_mix_hdr, out_mix_file)
outmm_mix = out_mix.open_memmap(interleave='source', writable=True)
mix_frame = s.zeros(outmm_mix.shape[1:])
# iterate over columns
if args.columns is None:
colstart, colfin = 0, img.ncols
else:
colstart, colfin = [int(f) for f in args.columns.split(',')]
for col in range(colstart, colfin):
if col%100==0:
print 'column '+str(col)
# check for land and bad data flags
if Rrs[col,s.argmin(abs(wl-1000))] > 0.05 or all(Rrs[col,:] <= 0):
continue
# convert to Rrs
Rrs_raw = Rrs[col,:] / s.pi
# subtract glint
b900 = s.argmin(abs(wl-900.0))
glint = max(0.0001, s.median(Rrs_raw[(b900-2):(b900+3)]))
Rrs_raw = Rrs_raw - glint
if all(Rrs_raw < 0):
continue # out of bounds data
# optimize
arg = (wl, Rrs_raw, const)
state = init_state(wl, Rrs_raw, const)
if args.mode != 'aop':
print("Only AOP retrievals have pure C support")
state, r = leastsq(err, state, args=arg, xtol=1e-6, ftol=1e-6)
elif bath is None:
state, r = c_leastsq_aop(err,state,args=arg,libaop=libaop)
else:
b = bath[col,-1]
state, r = c_leastsq_aop(err,state,args=arg,libaop=libaop,bath=b)
Rrs_model, Kd, bb, R_b, R_std = forward_rrs(wl, Rrs_raw, state, const,
est_sd=True)
# plot output
if args.plot:
print('plotting output')
err(state, wl, Rrs_raw, const, errbar=None, plot=True)
# Transform some parameters out of log space, and Chl-a to mg/m^3
# via the Lee relation
water_state = state[:const.nwater]
depth_state = state[const.nwater]
mix_state = state[(const.nwater+1):]
state_phys = s.r_[water_state, depth_state, mix_state]
if const.mode=='iop':
state_phys[0] = s.exp(s.log(state_phys[0]/0.06)/0.65)
# text dump
if args.textout is not None:
print('writing to '+args.textout)
with open(args.textout,'w') as f:
f.write('#Depth: %6.2f m\n'%depth_state)
f.write('#wl,Rrs.,model,Rb,Rb_sigma_factor,Kd,bb\n')
for i in s.where(const.use_out)[0]:
f.write('%8.6f,%8.6f,%8.6f,%8.6f,%8.6f,%8.6f,%8.6f\n'%\
(wl[i],Rrs_raw[i],Rrs_model[i],R_b[i],R_std[i],Kd[i],bb[i]))
# write output files
dep_frame[col,0] = state_phys[const.nbb+const.nKd]
bb_frame[col,:] = bb[const.use_out]
Kd_frame[col,:] = Kd[const.use_out]
Rb_frame[col,:] = R_b[const.use_out]
mix_frame[col,:] = mix_state
outmm_Rb[i,:,:] = Rb_frame
outmm_Kd[i,:,:] = Kd_frame
outmm_bb[i,:,:] = bb_frame
outmm_mix[i,:,:] = mix_frame
outmm_dep[i,:,:] = dep_frame.T
if __name__ == "__main__":
main()
def segment(spectra: tuple,
nodata_value: float,
npca: int,
segsize: int,
nchunk: int,
n_cores: int = 1,
ray_address: str = None,
ray_redis_password: str = None,
ray_temp_dir=None,
ray_ip_head=None,
logfile=None,
loglevel='INFO'):
"""
Segment an image using SLIC on a PCA.
Args:
spectra: tuple of filepaths of image to segment and (optionally) output label file
nodata_value: data to ignore in radiance image
npca: number of pca components to use
segsize: mean segmentation size
nchunk: size of each image chunk
n_cores: number of cores to use
ray_address: ray address to connect to (for multinode implementation)
ray_redis_password: ray password to use (for multinode implementation)
ray_temp_dir: ray temp directory to reference
ray_ip_head: ray ip head to reference (for multinode use)
logfile: logging file to output to
loglevel: logging level to use
"""
logging.basicConfig(format='%(levelname)s:%(message)s',
level=loglevel,
filename=logfile)
in_file = spectra[0]
if len(spectra) > 1 and type(spectra) is tuple:
lbl_file = spectra[1]
else:
lbl_file = spectra + '_lbl'
# Open input data, get dimensions
in_img = envi.open(in_file + '.hdr', in_file)
meta = in_img.metadata
nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
# Start up a ray instance for parallel work
rayargs = {
'ignore_reinit_error': True,
'local_mode': n_cores == 1,
"address": ray_address,
"redis_password": ray_redis_password
}
if rayargs['local_mode']:
rayargs['temp_dir'] = ray_temp_dir
# Used to run on a VPN
ray.services.get_node_ip_address = lambda: '127.0.0.1'
# We can only set the num_cpus if running on a single-node
if ray_ip_head is None and ray_redis_password is None:
rayargs['num_cpus'] = n_cores
ray.init(**rayargs)
atexit.register(ray.shutdown)
# Iterate through image "chunks," segmenting as we go
next_label = 1
all_labels = np.zeros((nl, ns), dtype=np.int64)
jobs = []
for lstart in np.arange(0, nl, nchunk):
# Extract data
lend = min(lstart + nchunk, nl)
jobs.append(
segment_chunk.remote(lstart,
lend,
in_file,
nodata_value,
npca,
segsize,
logfile=logfile,
loglevel=loglevel))
# Collect results, making sure each chunk is distinct (note, label indexing is arbitrary,
# so not attempt at sorting is made)
rreturn = [ray.get(jid) for jid in jobs]
for lstart, lend, ret in rreturn:
if ret is not None:
logging.debug(f'Collecting chunk: {lstart}')
chunk_label = ret.copy()
chunk_label[chunk_label != 0] += np.max(all_labels)
all_labels[lstart:lend, ...] = chunk_label
del rreturn
ray.shutdown()
# Final file I/O
logging.debug('Writing output')
lbl_meta = {
"samples": str(ns),
"lines": str(nl),
"bands": "1",
"header offset": "0",
"file type": "ENVI Standard",
"data type": "4",
"interleave": "bil"
}
lbl_img = envi.create_image(lbl_file + '.hdr',
lbl_meta,
ext='',
force=True)
lbl_mm = lbl_img.open_memmap(interleave='source', writable=True)
lbl_mm[:, :] = np.array(all_labels, dtype=np.float32).reshape((nl, 1, ns))
del lbl_mm
def empirical_line(reference_radiance_file: str,
reference_reflectance_file: str,
reference_uncertainty_file: str,
reference_locations_file: str,
segmentation_file: str,
input_radiance_file: str,
input_locations_file: str,
output_reflectance_file: str,
output_uncertainty_file: str,
nneighbors: int = 400,
nodata_value: float = -9999.0,
level: str = 'INFO',
logfile: str = None,
radiance_factors: np.array = None,
isofit_config: str = None,
n_cores: int = -1) -> None:
"""
Perform an empirical line interpolation for reflectance and uncertainty extrapolation
Args:
reference_radiance_file: source file for radiance (interpolation built from this)
reference_reflectance_file: source file for reflectance (interpolation built from this)
reference_uncertainty_file: source file for uncertainty (interpolation built from this)
reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this)
segmentation_file: input file noting the per-pixel segmentation used
input_radiance_file: input radiance file (interpolate over this)
input_locations_file: input location file (interpolate over this)
output_reflectance_file: location to write output reflectance to
output_uncertainty_file: location to write output uncertainty to
nneighbors: number of neighbors to use for interpolation
nodata_value: nodata value of input and output
level: logging level
logfile: logging file
radiance_factors: radiance adjustment factors
isofit_config: path to isofit configuration JSON file
n_cores: number of cores to run on
Returns:
None
"""
loglevel = level
logging.basicConfig(format='%(message)s', level=loglevel, filename=logfile)
# Open input data to check that band formatting is correct
# Load reference set radiance
reference_radiance_img = envi.open(reference_radiance_file + '.hdr',
reference_radiance_file)
n_reference_lines, n_radiance_bands, n_reference_columns = [
int(reference_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_reference_columns != 1:
raise IndexError("Reference data should be a single-column list")
# Load reference set reflectance
reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr',
reference_reflectance_file)
nrefr, nbr, srefr = [
int(reference_reflectance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefr != n_reference_lines or nbr != n_radiance_bands or srefr != n_reference_columns:
raise IndexError("Reference file dimension mismatch (reflectance)")
# Load reference set uncertainty, assuming reflectance uncertainty is
# recoreded in the first n_radiance_bands channels of data
reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr',
reference_uncertainty_file)
nrefu, ns, srefu = [
int(reference_uncertainty_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefu != n_reference_lines or ns < n_radiance_bands or srefu != n_reference_columns:
raise IndexError("Reference file dimension mismatch (uncertainty)")
# Load reference set locations
reference_locations_img = envi.open(reference_locations_file + '.hdr',
reference_locations_file)
nrefl, lb, ls = [
int(reference_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefl != n_reference_lines or lb != 3:
raise IndexError("Reference file dimension mismatch (locations)")
input_radiance_img = envi.open(input_radiance_file + '.hdr',
input_radiance_file)
n_input_lines, n_input_bands, n_input_samples = [
int(input_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_radiance_bands != n_input_bands:
msg = 'Number of channels mismatch: input (%i) vs. reference (%i)'
raise IndexError(msg % (nbr, n_radiance_bands))
input_locations_img = envi.open(input_locations_file + '.hdr',
input_locations_file)
nll, nlb, nls = [
int(input_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nll != n_input_lines or nlb != 3 or nls != n_input_samples:
raise IndexError('Input location dimension mismatch')
# Create output files
output_metadata = input_radiance_img.metadata
output_metadata['interleave'] = 'bil'
output_reflectance_img = envi.create_image(output_reflectance_file +
'.hdr',
ext='',
metadata=output_metadata,
force=True)
output_uncertainty_img = envi.create_image(output_uncertainty_file +
'.hdr',
ext='',
metadata=output_metadata,
force=True)
# Now cleanup inputs and outputs, we'll write dynamically above
del output_reflectance_img, output_uncertainty_img
del reference_reflectance_img, reference_uncertainty_img, reference_locations_img, input_radiance_img, input_locations_img
# Initialize ray cluster
start_time = time.time()
if isofit_config is not None:
iconfig = configs.create_new_config(isofit_config)
else:
# If none, create a temporary config to get default ray parameters
iconfig = configs.Config({})
if n_cores == -1:
n_cores = iconfig.implementation.n_cores
rayargs = {
'ignore_reinit_error': iconfig.implementation.ray_ignore_reinit_error,
'local_mode': n_cores == 1,
"address": iconfig.implementation.ip_head,
"_redis_password": iconfig.implementation.redis_password
}
# only specify a temporary directory if we are not connecting to
# a ray cluster
if rayargs['local_mode']:
rayargs['_temp_dir'] = iconfig.implementation.ray_temp_dir
# Used to run on a VPN
ray.services.get_node_ip_address = lambda: '127.0.0.1'
# We can only set the num_cpus if running on a single-node
if iconfig.implementation.ip_head is None and iconfig.implementation.redis_password is None:
rayargs['num_cpus'] = n_cores
ray.init(**rayargs)
atexit.register(ray.shutdown)
n_ray_cores = ray.available_resources()["CPU"]
n_cores = min(n_ray_cores, n_input_lines)
logging.info(
'Beginning empirical line inversions using {} cores'.format(n_cores))
# Break data into sections
line_sections = np.linspace(0,
n_input_lines,
num=int(n_cores + 1),
dtype=int)
start_time = time.time()
# Run the pool (or run serially)
results = []
for l in range(len(line_sections) - 1):
args = (line_sections[l], line_sections[l + 1],
reference_radiance_file, reference_reflectance_file,
reference_uncertainty_file, reference_locations_file,
input_radiance_file, input_locations_file, segmentation_file,
isofit_config, output_reflectance_file,
output_uncertainty_file, radiance_factors, nneighbors,
nodata_value, level, logfile)
results.append(_run_chunk.remote(*args))
_ = ray.get(results)
total_time = time.time() - start_time
logging.info(
'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core'
.format(total_time, line_sections[-1] * n_input_samples / total_time,
line_sections[-1] * n_input_samples / total_time / n_cores))
def segment(spectra, flag, npca, segsize, nchunk):
"""."""
in_file = spectra[0]
if len(spectra) > 1:
lbl_file = spectra[1]
else:
lbl_file = spectra + '_lbl'
# Open input data, get dimensions
in_img = envi.open(in_file + '.hdr', in_file)
meta = in_img.metadata
nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
img_mm = in_img.open_memmap(interleave='source', writable=False)
if meta['interleave'] != 'bil':
raise ValueError('I need BIL interleave.')
# Iterate through image "chunks," segmenting as we go
next_label = 1
all_labels = s.zeros((nl, ns))
for lstart in s.arange(0, nl, nchunk):
del img_mm
print(lstart)
# Extract data
lend = min(lstart + nchunk, nl)
img_mm = in_img.open_memmap(interleave='source', writable=False)
x = s.array(img_mm[lstart:lend, :, :]).transpose((0, 2, 1))
nc = x.shape[0]
x = x.reshape((nc * ns, nb))
# Excluding bad locations, calculate top PCA coefficients
use = s.all(abs(x - flag) > 1e-6, axis=1)
mu = x[use, :].mean(axis=0)
C = s.cov(x[use, :], rowvar=False)
[v, d] = eigh(C)
# Determine segmentation compactness scaling based on eigenvalues
cmpct = norm(s.sqrt(v[-npca:]))
# Project, redimension as an image with "npca" channels, and segment
x_pca = (x - mu) @ d[:, -npca:]
x_pca[use < 1, :] = 0.0
x_pca = x_pca.reshape([nc, ns, npca])
valid = use.reshape([nc, ns, 1])
seg_in_chunk = int(sum(use) / float(segsize))
labels = slic(x_pca,
n_segments=seg_in_chunk,
compactness=cmpct,
max_iter=10,
sigma=0,
multichannel=True,
enforce_connectivity=True,
min_size_factor=0.5,
max_size_factor=3)
# Reindex the subscene labels and place them into the larger scene
labels = labels.reshape([nc * ns])
labels[s.logical_not(use)] = 0
labels[use] = labels[use] + next_label
next_label = max(labels) + 1
labels = labels.reshape([nc, ns])
all_labels[lstart:lend, :] = labels
# Reindex
labels_sorted = s.sort(s.unique(all_labels))
lbl = s.zeros((nl, ns))
for i, val in enumerate(labels_sorted):
lbl[all_labels == val] = i
# Final file I/O
del img_mm
lbl_meta = {
"samples": str(ns),
"lines": str(nl),
"bands": "1",
"header offset": "0",
"file type": "ENVI Standard",
"data type": "4",
"interleave": "bil"
}
lbl_img = envi.create_image(lbl_file + '.hdr',
lbl_meta,
ext='',
force=True)
lbl_mm = lbl_img.open_memmap(interleave='source', writable=True)
lbl_mm[:, :] = s.array(lbl, dtype=s.float32).reshape((nl, 1, ns))
del lbl_mm
def extractions(inputfile, labels, output, chunksize, flag, n_cores: int = 1, ray_address: str = None, ray_redis_password: str = None, ray_temp_dir: str = None, ray_ip_head = None, logfile: str = None, loglevel: str = 'INFO'):
"""..."""
in_file = inputfile
lbl_file = labels
out_file = output
nchunk = chunksize
dtm = {
'4': np.float32,
'5': np.float64
}
# Open input data, get dimensions
in_img = envi.open(envi_header(in_file), in_file)
meta = in_img.metadata
nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
img_mm = in_img.open_memmap(interleave='bip', writable=False)
lbl_img = envi.open(envi_header(lbl_file), lbl_file)
labels = lbl_img.read_band(0)
un_labels = np.unique(labels).tolist()
if 0 not in un_labels:
un_labels.insert(0,0)
nout = len(un_labels)
# Start up a ray instance for parallel work
rayargs = {'ignore_reinit_error': True,
'local_mode': n_cores == 1,
"address": ray_address,
'_temp_dir': ray_temp_dir,
'_redis_password': ray_redis_password}
# We can only set the num_cpus if running on a single-node
if ray_ip_head is None and ray_redis_password is None:
rayargs['num_cpus'] = n_cores
ray.init(**rayargs)
atexit.register(ray.shutdown)
labelid = ray.put(labels)
jobs = []
for lstart in np.arange(0, nl, nchunk):
lend = min(lstart+nchunk, nl)
jobs.append(extract_chunk.remote(lstart, lend, in_file, labelid, flag, logfile=logfile, loglevel=loglevel))
# Collect results
rreturn = [ray.get(jid) for jid in jobs]
## Iterate through image "chunks," segmenting as we go
out = np.zeros((nout, nb, 1))
for idx, ret in rreturn:
if ret is not None:
out[idx, :, 0] = ret
del rreturn
ray.shutdown()
meta["lines"] = str(nout)
meta["bands"] = str(nb)
meta["samples"] = '1'
meta["interleave"] = "bil"
out_img = envi.create_image(envi_header(out_file), metadata=meta, ext='', force=True)
del out_img
if dtm[meta['data type']] == np.float32:
type = 'float32'
else:
type = 'float64'
write_bil_chunk(out, out_file, 0, out.shape, dtype=type)
def segment(spectra: tuple,
nodata_value: float,
npca: int,
segsize: int,
nchunk: int,
n_cores: int = 1,
ray_address: str = None,
ray_redis_password: str = None,
ray_temp_dir=None,
ray_ip_head=None,
logfile=None,
loglevel='INFO'):
"""
Segment an image using SLIC on a PCA.
Args:
spectra: tuple of filepaths of image to segment and (optionally) output label file
nodata_value: data to ignore in radiance image
npca: number of pca components to use
segsize: mean segmentation size
nchunk: size of each image chunk
n_cores: number of cores to use
ray_address: ray address to connect to (for multinode implementation)
ray_redis_password: ray password to use (for multinode implementation)
ray_temp_dir: ray temp directory to reference
ray_ip_head: ray ip head to reference (for multinode use)
logfile: logging file to output to
loglevel: logging level to use
"""
logging.basicConfig(format='%(levelname)s:%(message)s',
level=loglevel,
filename=logfile)
in_file = spectra[0]
if len(spectra) > 1 and type(spectra) is tuple:
lbl_file = spectra[1]
else:
lbl_file = spectra + '_lbl'
# Open input data, get dimensions
in_img = envi.open(envi_header(in_file), in_file)
meta = in_img.metadata
nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
# Start up a ray instance for parallel work
rayargs = {
'ignore_reinit_error': True,
'local_mode': n_cores == 1,
"address": ray_address,
'_temp_dir': ray_temp_dir,
"_redis_password": ray_redis_password
}
# We can only set the num_cpus if running on a single-node
if ray_ip_head is None and ray_redis_password is None:
rayargs['num_cpus'] = n_cores
ray.init(**rayargs)
atexit.register(ray.shutdown)
# Iterate through image "chunks," segmenting as we go
all_labels = np.zeros((nl, ns), dtype=np.int64)
jobs = []
# Enforce a minimum chunk size to prevent singularities downstream
# This could eventually be made a user-tunable parameter but this
# value should work in all cases
min_lines_per_chunk = 10
for lstart in np.arange(0, nl - min_lines_per_chunk, nchunk):
# Extend any chunk that falls within a small margin of the
# end of the flightline
lend = min(lstart + nchunk, nl)
if lend > (nl - min_lines_per_chunk):
lend = nl
# Extract data
jobs.append(
segment_chunk.remote(lstart,
lend,
in_file,
nodata_value,
npca,
segsize,
logfile=logfile,
loglevel=loglevel))
# Collect results, making sure each chunk is distinct, and enforce an order
next_label = 1
rreturn = [ray.get(jid) for jid in jobs]
for lstart, lend, ret in rreturn:
if ret is not None:
logging.debug(f'Collecting chunk: {lstart}')
chunk_label = ret.copy()
unique_chunk_labels = np.unique(chunk_label[chunk_label != 0])
ordered_chunk_labels = np.zeros(chunk_label.shape)
for lbl in unique_chunk_labels:
ordered_chunk_labels[chunk_label == lbl] = next_label
next_label += 1
all_labels[lstart:lend, ...] = ordered_chunk_labels
del rreturn
ray.shutdown()
# Final file I/O
logging.debug('Writing output')
lbl_meta = {
"samples": str(ns),
"lines": str(nl),
"bands": "1",
"header offset": "0",
"file type": "ENVI Standard",
"data type": "4",
"interleave": "bil"
}
lbl_img = envi.create_image(envi_header(lbl_file),
lbl_meta,
ext='',
force=True)
lbl_mm = lbl_img.open_memmap(interleave='source', writable=True)
lbl_mm[:, :] = np.array(all_labels, dtype=np.float32).reshape((nl, 1, ns))
del lbl_mm
def empirical_line(reference_radiance,
reference_reflectance,
reference_uncertainty,
reference_locations,
hashfile,
input_radiance,
input_locations,
output_reflectance,
output_uncertainty,
nneighbors=15,
flag=-9999.0,
skip=0,
level='INFO',
radiance_factors=None,
isofit_config=None):
"""..."""
def plot_example(xv, yv, b):
"""Plot for debugging purposes."""
matplotlib.rcParams['font.family'] = "serif"
matplotlib.rcParams['font.sans-serif'] = "Times"
matplotlib.rcParams["legend.edgecolor"] = "None"
matplotlib.rcParams["axes.spines.top"] = False
matplotlib.rcParams["axes.spines.bottom"] = True
matplotlib.rcParams["axes.spines.left"] = True
matplotlib.rcParams["axes.spines.right"] = False
matplotlib.rcParams['axes.grid'] = True
matplotlib.rcParams['axes.grid.axis'] = 'both'
matplotlib.rcParams['axes.grid.which'] = 'major'
matplotlib.rcParams['legend.edgecolor'] = '1.0'
plt.plot(xv[:, 113], yv[:, 113], 'ko')
plt.plot(xv[:, 113], xv[:, 113] * b[113, 1] + b[113, 0], 'k')
#plt.plot(x[113], x[113]*b[113, 1] + b[113, 0], 'ro')
plt.grid(True)
plt.xlabel('Radiance, $\mu{W }nm^{-1} sr^{-1} cm^{-2}$')
plt.ylabel('Reflectance')
plt.show(block=True)
plt.savefig('empirical_line.pdf')
eps = 1e-6
k = nneighbors
loglevel = level
# Open input data, get dimensions
logging.basicConfig(format='%(message)s', level=loglevel)
ref_rdn_file = reference_radiance
ref_rfl_file = reference_reflectance
ref_unc_file = reference_uncertainty
ref_loc_file = reference_locations
inp_hash_file = hashfile
inp_rdn_file = input_radiance
inp_loc_file = input_locations
out_rfl_file = output_reflectance
out_unc_file = output_uncertainty
# Load reference set radiance
ref_rdn_img = envi.open(ref_rdn_file + '.hdr', ref_rdn_file)
meta = ref_rdn_img.metadata
nref, nb, sref = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
if sref != 1:
raise IndexError("Reference data should be a single-column list")
ref_rdn_mm = ref_rdn_img.open_memmap(interleave='source', writable=False)
ref_rdn = s.array(ref_rdn_mm[:, :, :]).reshape((nref, nb))
# Load reference set reflectance
ref_rfl_img = envi.open(ref_rfl_file + '.hdr', ref_rfl_file)
meta = ref_rfl_img.metadata
nrefr, nbr, srefr = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
if nrefr != nref or nbr != nb or srefr != sref:
raise IndexError("Reference file dimension mismatch (reflectance)")
ref_rfl_mm = ref_rfl_img.open_memmap(interleave='source', writable=False)
ref_rfl = s.array(ref_rfl_mm[:, :, :]).reshape((nref, nb))
# Load reference set uncertainty, assuming reflectance uncertainty is
# recoreded in the first nbr channels of data
ref_unc_img = envi.open(ref_unc_file + '.hdr', ref_unc_file)
meta = ref_unc_img.metadata
nrefu, ns, srefu = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
if nrefu != nref or ns < nb or srefu != sref:
raise IndexError("Reference file dimension mismatch (uncertainty)")
ref_unc_mm = ref_unc_img.open_memmap(interleave='source', writable=False)
ref_unc = s.array(ref_unc_mm[:, :, :]).reshape((nref, ns))
ref_unc = ref_unc[:, :nbr].reshape((nref, nbr))
# Load reference set locations
ref_loc_img = envi.open(ref_loc_file + '.hdr', ref_loc_file)
meta = ref_loc_img.metadata
nrefl, lb, ls = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
if nrefl != nref or lb != 3:
raise IndexError("Reference file dimension mismatch (locations)")
ref_loc_mm = ref_loc_img.open_memmap(interleave='source', writable=False)
ref_loc = s.array(ref_loc_mm[:, :, :]).reshape((nref, lb))
# Prepare radiance adjustment
if radiance_factors is None:
rdn_factors = s.ones(nb, )
else:
rdn_factors = s.loadtxt(radiance_factors)
# Prepare instrument model, if available
if isofit_config is not None:
config = load_config(isofit_config)
instrument = Instrument(config['forward_model']['instrument'])
logging.info('Loading instrument')
else:
instrument = None
# Assume (heuristically) that, for distance purposes, 1 m vertically is
# comparable to 10 m horizontally, and that there are 100 km per latitude
# degree. This is all approximate of course. Elevation appears in the
# Third element, and the first two are latitude/longitude coordinates
loc_scaling = s.array([1e5, 1e5, 0.1])
scaled_ref_loc = ref_loc * loc_scaling
tree = KDTree(scaled_ref_loc)
inp_rdn_img = envi.open(inp_rdn_file + '.hdr', inp_rdn_file)
inp_rdn_meta = inp_rdn_img.metadata
nl, nb, ns = [int(inp_rdn_meta[n]) for n in ('lines', 'bands', 'samples')]
if nb != nbr:
msg = 'Number of channels mismatch: input (%i) vs. reference (%i)'
raise IndexError(msg % (nbr, nb))
inp_rdn_mm = inp_rdn_img.open_memmap(interleave='source', writable=False)
inp_loc_img = envi.open(inp_loc_file + '.hdr', inp_loc_file)
inp_loc_meta = inp_loc_img.metadata
nll, nlb, nls = [
int(inp_loc_meta[n]) for n in ('lines', 'bands', 'samples')
]
if nll != nl or nlb != 3 or nls != ns:
raise IndexError('Input location dimension mismatch')
inp_loc_mm = inp_loc_img.open_memmap(interleave='source', writable=False)
inp_loc = s.array(inp_loc_mm).reshape((nl, nlb, ns))
if inp_hash_file:
inp_hash_img = envi.open(inp_hash_file + '.hdr', inp_hash_file)
hash_img = inp_hash_img.read_band(0)
else:
hash_img = None
out_rfl_img = envi.create_image(out_rfl_file + '.hdr',
ext='',
metadata=inp_rdn_img.metadata,
force=True)
out_rfl_mm = out_rfl_img.open_memmap(interleave='source', writable=True)
out_unc_img = envi.create_image(out_unc_file + '.hdr',
ext='',
metadata=inp_rdn_img.metadata,
force=True)
out_unc_mm = out_unc_img.open_memmap(interleave='source', writable=True)
# Iterate through image
hash_table = {}
for row in s.arange(nl):
del inp_loc_mm
del inp_rdn_mm
del out_rfl_mm
del out_unc_mm
# Extract data
inp_rdn_mm = inp_rdn_img.open_memmap(interleave='source',
writable=False)
inp_rdn = s.array(inp_rdn_mm[row, :, :])
if inp_rdn_meta['interleave'] == 'bil':
inp_rdn = inp_rdn.transpose((1, 0))
inp_rdn = inp_rdn * rdn_factors
inp_loc_mm = inp_loc_img.open_memmap(interleave='source',
writable=False)
inp_loc = s.array(inp_loc_mm[row, :, :])
if inp_loc_meta['interleave'] == 'bil':
inp_loc = inp_loc.transpose((1, 0))
out_rfl = s.zeros(inp_rdn.shape)
out_unc = s.zeros(inp_rdn.shape)
nspectra, start = 0, time.time()
for col in s.arange(ns):
x = inp_rdn[col, :]
if s.all(abs(x - flag) < eps):
out_rfl[col, :] = flag
out_unc[col, :] = flag
continue
bhat = None
if hash_img is not None:
hash_idx = hash_img[row, col]
if hash_idx in hash_table:
bhat, bmarg, bcov = hash_table[hash_idx]
else:
loc = ref_loc[
s.array(hash_idx, dtype=int), :] * loc_scaling
else:
loc = inp_loc[col, :] * loc_scaling
if bhat is None:
dists, nn = tree.query(loc, k)
xv = ref_rdn[nn, :]
yv = ref_rfl[nn, :]
uv = ref_unc[nn, :]
bhat = s.zeros((nb, 2))
bmarg = s.zeros((nb, 2))
bcov = s.zeros((nb, 2, 2))
for i in s.arange(nb):
use = yv[:, i] > 0
n = sum(use)
X = s.concatenate((s.ones((n, 1)), xv[use, i:i + 1]),
axis=1)
W = s.diag(s.ones(n)) # /uv[use, i])
y = yv[use, i:i + 1]
bhat[i, :] = (inv(X.T @ W @ X) @ X.T @ W @ y).T
bcov[i, :, :] = inv(X.T @ W @ X)
bmarg[i, :] = s.diag(bcov[i, :, :])
if (hash_img is not None) and not (hash_idx in hash_table):
hash_table[hash_idx] = bhat, bmarg, bcov
A = s.array((s.ones(nb), x))
out_rfl[col, :] = (s.multiply(bhat.T, A).sum(axis=0))
# Calculate uncertainties. Sy approximation rather than Seps for
# speed, for now... but we do take into account instrument
# radiometric uncertainties
if instrument is None:
out_unc[col, :] = s.sqrt(s.multiply(bmarg.T, A).sum(axis=0))
else:
Sy = instrument.Sy(x, geom=None)
calunc = instrument.bval[:instrument.n_chan]
out_unc[col, :] = s.sqrt(s.diag(Sy) +
pow(calunc * x, 2)) * bhat[:, 1]
if loglevel == 'DEBUG':
plot_example(xv, yv, bhat, x, out_rfl[col, :])
nspectra = nspectra + 1
elapsed = float(time.time() - start)
logging.info('row %i/%i, %5.1f spectra per second' %
(row, nl, float(nspectra) / elapsed))
out_rfl_mm = out_rfl_img.open_memmap(interleave='source',
writable=True)
if inp_rdn_meta['interleave'] == 'bil':
out_rfl = out_rfl.transpose((1, 0))
out_rfl_mm[row, :, :] = out_rfl
out_unc_mm = out_unc_img.open_memmap(interleave='source',
writable=True)
if inp_rdn_meta['interleave'] == 'bil':
out_unc = out_unc.transpose((1, 0))
out_unc_mm[row, :, :] = out_unc
def main(in_file):
# Set up input and output file paths
in_hdr = find_header(in_file)
##########################
## MODIFIY THIS TO GET A BETTER FILENAME
##########################
out_glint_file = in_file.split('_', 1)[0] + '_glint'
out_glint_hdr = in_file.split('_', 1)[0] + '_glint.hdr'
img = envi.open(in_hdr, in_file)
inmm = img.open_memmap(interleave='source', writable=False)
wl = s.array([float(w) for w in img.metadata['wavelength']])
if (wl[0] < 100):
wl = wl * 1000
fwhm = s.array([float(w) for w in img.metadata['fwhm']])
if (wl[0] < 100):
fwhm = fwhm * 1000
# set up metadata and constants
# make output glint file and open memmap
nl = int(img.metadata['lines'])
metadata_glint = img.metadata.copy()
metadata_glint['bands'] = u'1'
metadata_glint['data type'] = u'4'
metadata_glint['band names'] = ['Glint at 900nm']
metadata_glint['interleave'] = 'bsq'
metadata_glint[
'description'] = ' make_glint.py from input ATREM reflectance '
try:
del metadata_glint['wavelength']
del metadata_glint['wavelength units']
del metadata_glint['fwhm']
del metadata_glint['raw starting band']
del metadata_glint['raw starting sample']
del metadata_glint['raw starting line']
del metadata_glint['line averaging']
except:
pass
out_glint = envi.create_image(out_glint_hdr,
metadata_glint,
ext='',
force=True)
outmm_glint = out_glint.open_memmap(interleave='source', writable=True)
# iterate over rows
start, fin = 0, nl
for i in range(start, fin):
Rw = s.array(inmm[i, :, :])
if img.metadata['interleave'] == 'bil':
Rw = Rw.T
## if the ENVI data type is unsigned 16-bit integer, just change it to be signed
## so that numpy doesn't interpret them as insanely high numbers (e.g., 65535)
if int(img.metadata['data type']) == 12:
Rw.dtype = 'int16'
if int(img.metadata['data type']) != 4:
Rw = Rw / s.float32(10000)
# iterate over columns
colstart, colfin = 0, img.ncols
glint_frame = s.zeros((outmm_glint.shape[2], 1))
for col in range(colstart, colfin):
# check for land and bad data flags
## if Rw[col,s.argmin(abs(wl-1000))] > 0.05 or all(Rw[col,:] <= 0):
if Rw[col, s.argmin(abs(wl - 1000))] > 0.10 or all(
Rw[col, :] <= 0):
continue
# convert to Rrs
Rrs_raw = Rw[col, :] / s.pi
# subtract glint
b900 = s.argmin(abs(wl - 900.0))
glint = max(0.0001, s.median(Rrs_raw[(b900 - 2):(b900 + 3)]))
if all(Rrs_raw < 0):
continue # out of bounds data
# write output files
glint_state = glint
glint_frame[col, 0] = glint_state
outmm_glint[0, i, :] = glint_frame.reshape(img.ncols)
if ((i % 500) == 0):
print 'line ' + str(i + 1)
del outmm_glint, out_glint, inmm, img
def main():
# Parse command line
description = 'Spectroscopic Surface & Atmosphere Fitting'
parser = argparse.ArgumentParser()
parser.add_argument('config_file')
parser.add_argument('--level', default='INFO')
parser.add_argument('--ip_head',
default=None,
help='ray-specific argument')
parser.add_argument('--redis_password',
default=None,
help='ray-specific argument')
parser.add_argument('--ray_temp_dir',
default=None,
help='ray-specific argument')
parser.add_argument(
'--n_cores',
type=int,
default=-1,
help="number of cores to run on. -1 for all, 1 for debug mode")
args = parser.parse_args()
logging.basicConfig(format='%(message)s', level=args.level)
# Load a parallel Pool
rayargs = {
'address': args.ip_head,
'redis_password': args.redis_password,
'local_mode': args.n_cores == 1
}
if args.n_cores != -1:
rayargs['num_cpus'] = args.n_cores
if args.ray_temp_dir is not None:
rayargs['temp_dir'] = args.ray_temp_dir
ray.init(**rayargs)
# Load the configuration file.
config = json.load(open(args.config_file, 'r'))
logging.info('Loading library')
lib = Library(config['library'])
# Get image and wavelengths
logging.info('Opening input data')
reflectance_input_header = str(config['input_reflectance'] + '.hdr')
uncertainty_input_header = str(config['input_uncertainty'] + '.hdr')
depth_output_header = str(config['output_depths'] + '.hdr')
posterior_output_header = str(config['output_posterior'] + '.hdr')
likelihood_output_header = str(config['output_likelihood'] + '.hdr')
corr_output_header = str(config['output_corr'] + '.hdr')
reflectance_ds = envi.open(reflectance_input_header)
meta = reflectance_ds.metadata.copy()
uncertainty_ds = envi.open(uncertainty_input_header)
# Now that the input images are available, resample wavelengths
if 'wavelength' in meta:
wl = np.array([float(w) for w in meta['wavelength']])
if all(wl < 100): wl = wl * 1000.0
else:
wl = lib.wl.copy()
if 'fwhm' in meta:
fwhm = np.array([float(f) for f in meta['fwhm']])
if all(fwhm < 0.1): fwhm = fwhm * 1000.0
else:
fwhm = np.ones(wl.shape) * (wl[1] - wl[0])
lib.resample(wl, fwhm)
# Create output images
meta['bands'] = lib.nchan()
if 'wavelength' in meta: del meta['wavelength']
if 'fwhm' in meta: del meta['fwhm']
meta['band names'] = lib.groups()
meta['data type'] = 4
meta['interleave'] = 'bip'
depth_ds = envi.create_image(depth_output_header, meta, force=True, ext="")
posterior_ds = envi.create_image(posterior_output_header,
meta,
force=True,
ext="")
likelihood_ds = envi.create_image(likelihood_output_header,
meta,
force=True,
ext="")
corr_ds = envi.create_image(corr_output_header, meta, force=True, ext="")
ids = [
run_one_row.remote(r, lib, reflectance_input_header,
uncertainty_input_header, depth_output_header,
posterior_output_header, likelihood_output_header,
corr_output_header)
for r in range(reflectance_ds.shape[0])
]
ret = [ray.get(id) for id in ids]
inmm = img.open_memmap(interleave='source', writable=False)
wl = s.array([float(w) for w in img.metadata['wavelength']])
if (wl[0] < 100):
wl = wl * 1000
fwhm = s.array([float(w) for w in img.metadata['fwhm']])
# set up metadata and constants
const = constants(wl, args.mode, scenario, args.root_dir+'/data/')
# make output Rb file and open memmap
metadata = img.metadata.copy()
metadata['bands'] = '%i' % sum(const.use_out)
metadata['interleave'] = 'bil'
metadata['wavelength'] = ['%7.2f'%w for w in wl[const.use_out]]
metadata['data type'] = '2'
out = envi.create_image(out_hdr, metadata, ext='',force=True)
outmm = out.open_memmap(interleave='source', writable=True)
for i in range(start, fin):
# Flush cache
print 'line %i/%i' % (i,nl)
if args.depth_file is not None:
del bathy
bathy= spectral.io.envi.open(args.depth_file+'.hdr', args.depth_file)
bathymm = bathy.open_memmap(interleave='source', writable=False)
bath = s.array(bathymm[i,:,:])
if bathy.metadata['interleave'] == 'bil':
bath = bath.T
def extractions(inputfile, labels, output, chunksize, flag):
"""..."""
in_file = inputfile
lbl_file = labels
out_file = output
nchunk = chunksize
dtm = {'4': np.float32, '5': np.float64}
# Open input data, get dimensions
in_img = envi.open(in_file + '.hdr', in_file)
meta = in_img.metadata
nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
img_mm = in_img.open_memmap(interleave='bip', writable=False)
lbl_img = envi.open(lbl_file + '.hdr', lbl_file)
labels = lbl_img.read_band(0)
nout = len(np.unique(labels))
# Iterate through image "chunks," segmenting as we go
out = np.zeros((nout, nb))
counts = np.zeros((nout))
for lstart in np.arange(0, nl, nchunk):
del img_mm
img_mm = in_img.open_memmap(interleave='bip', writable=False)
# Which labels will we extract? ignore zero index
lend = min(lstart + nchunk, nl)
active = np.unique(labels[lstart:lend, :])
active = active[active >= 1]
# Handle labels extending outside our chunk by expanding margins
active_area = np.zeros(labels.shape)
lstart_adjust, lend_adjust = lstart, lend
for i in active:
active_area[labels == i] = True
active_locs = np.where(active_area)
lstart_adjust = min(active_locs[0])
lend_adjust = max(active_locs[0]) + 1
chunk_inp = np.array(img_mm[lstart_adjust:lend_adjust, :, :])
chunk_lbl = np.array(labels[lstart_adjust:lend_adjust, :])
for i in active:
idx = int(i)
out[idx, :] = 0
locs = np.where(chunk_lbl == i)
for row, col in zip(locs[0], locs[1]):
out[idx, :] = out[idx, :] + np.squeeze(chunk_inp[row, col, :])
counts[idx] = len(locs[0])
out = np.array((out.T / counts[np.newaxis, :]).T, dtype=np.float32)
out[np.logical_not(np.isfinite(out))] = flag
meta["lines"] = str(nout)
meta["bands"] = str(nb)
meta["samples"] = '1'
meta["interleave"] = "bil"
out_img = envi.create_image(out_file + '.hdr',
metadata=meta,
ext='',
force=True)
out_mm = np.memmap(out_file,
dtype=dtm[meta['data type']],
mode='w+',
shape=(nout, 1, nb))
if dtm[meta['data type']] == np.float32:
out_mm[:, 0, :] = np.array(out, np.float32)
else:
out_mm[:, 0, :] = np.array(out, np.float64)
def createimg(hdrf,metadata,ext='',force=True):
from spectral.io.envi import create_image
return create_image(hdrf, metadata, ext=ext, force=force)
def __init__(self, fname, write=False, n_rows=None, n_cols=None, n_bands=None,
interleave=None, dtype=np.float32, wavelengths=None, fwhm=None,
band_names=None, bad_bands='[]', zrange='{0.0, 1.0}', flag=-9999.0,
ztitles='{Wavelength (nm), Magnitude}', map_info='{}'):
"""."""
self.frames = OrderedDict()
self.write = write
self.fname = os.path.abspath(fname)
self.wl = wavelengths
self.band_names = band_names
self.fwhm = fwhm
self.flag = flag
self.n_rows = n_rows
self.n_cols = n_cols
self.n_bands = n_bands
if self.fname.endswith('.txt'):
# The .txt suffix implies a space-separated ASCII text file of
# one or more data columns. This is cheap to load and store, so
# we do not defer read/write operations.
logging.debug('Inferred ASCII file format for %s' % self.fname)
self.format = 'ASCII'
if not self.write:
self.data, self.wl = load_spectrum(self.fname)
self.n_rows, self.n_cols, self.map_info = 1, 1, '{}'
if self.wl is not None:
self.n_bands = len(self.wl)
else:
self.n_bands = None
self.meta = {}
elif self.fname.endswith('.mat'):
# The .mat suffix implies a matlab-style file, i.e. a dictionary
# of 2D arrays and other matlab-like objects. This is typically
# only used for specific output products associated with single
# spectrum retrievals; there is no read option.
logging.debug('Inferred MATLAB file format for %s' % self.fname)
self.format = 'MATLAB'
if not self.write:
logging.error('Unsupported MATLAB file in input block')
raise IOError('MATLAB format in input block not supported')
else:
# Otherwise we assume it is an ENVI-format file, which is
# basically just a binary data cube with a detached human-
# readable ASCII header describing dimensions, interleave, and
# metadata. We buffer this data in self.frames, reading and
# writing individual rows of the cube on-demand.
logging.debug('Inferred ENVI file format for %s' % self.fname)
self.format = 'ENVI'
if not self.write:
# If we are an input file, the header must preexist.
if not os.path.exists(self.fname+'.hdr'):
logging.error('Could not find %s' % (self.fname+'.hdr'))
raise IOError('Could not find %s' % (self.fname+'.hdr'))
# open file and copy metadata
self.file = envi.open(self.fname + '.hdr', fname)
self.meta = self.file.metadata.copy()
self.n_rows = int(self.meta['lines'])
self.n_cols = int(self.meta['samples'])
self.n_bands = int(self.meta['bands'])
if 'data ignore value' in self.meta:
self.flag = float(self.meta['data ignore value'])
else:
self.flag = -9999.0
else:
# If we are an output file, we may have to build the header
# from scratch. Hopefully the caller has supplied the
# necessary metadata details.
meta = {
'lines': n_rows,
'samples': n_cols,
'bands': n_bands,
'byte order': 0,
'header offset': 0,
'map info': map_info,
'file_type': 'ENVI Standard',
'sensor type': 'unknown',
'interleave': interleave,
'data type': typemap[dtype],
'wavelength units': 'nm',
'z plot range': zrange,
'z plot titles': ztitles,
'fwhm': fwhm,
'bbl': bad_bands,
'band names': band_names,
'wavelength': self.wl
}
for k, v in meta.items():
if v is None:
logging.error('Must specify %s' % (k))
raise IOError('Must specify %s' % (k))
if os.path.isfile(fname+'.hdr') is False:
self.file = envi.create_image(fname+'.hdr', meta, ext='',
force=True)
else:
self.file = envi.open(fname+'.hdr')
self.open_map_with_retries()
def empirical_line(reference_radiance_file: str,
reference_reflectance_file: str,
reference_uncertainty_file: str,
reference_locations_file: str,
segmentation_file: str,
input_radiance_file: str,
input_locations_file: str,
output_reflectance_file: str,
output_uncertainty_file: str,
nneighbors: int = 15,
nodata_value: float = -9999.0,
level: str = 'INFO',
radiance_factors: np.array = None,
isofit_config: dict = None,
n_cores: int = -1) -> None:
"""
Perform an empirical line interpolation for reflectance and uncertainty extrapolation
Args:
reference_radiance_file: source file for radiance (interpolation built from this)
reference_reflectance_file: source file for reflectance (interpolation built from this)
reference_uncertainty_file: source file for uncertainty (interpolation built from this)
reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this)
segmentation_file: input file noting the per-pixel segmentation used
input_radiance_file: input radiance file (interpolate over this)
input_locations_file: input location file (interpolate over this)
output_reflectance_file: location to write output reflectance to
output_uncertainty_file: location to write output uncertainty to
nneighbors: number of neighbors to use for interpolation
nodata_value: nodata value of input and output
level: logging level
radiance_factors: radiance adjustment factors
isofit_config: dictionary-stype isofit configuration
n_cores: number of cores to run on
Returns:
None
"""
loglevel = level
logging.basicConfig(format='%(message)s', level=loglevel)
# Open input data to check that band formatting is correct
# Load reference set radiance
reference_radiance_img = envi.open(reference_radiance_file + '.hdr',
reference_radiance_file)
n_reference_lines, n_radiance_bands, n_reference_columns = [
int(reference_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_reference_columns != 1:
raise IndexError("Reference data should be a single-column list")
# Load reference set reflectance
reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr',
reference_reflectance_file)
nrefr, nbr, srefr = [
int(reference_reflectance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefr != n_reference_lines or nbr != n_radiance_bands or srefr != n_reference_columns:
raise IndexError("Reference file dimension mismatch (reflectance)")
# Load reference set uncertainty, assuming reflectance uncertainty is
# recoreded in the first n_radiance_bands channels of data
reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr',
reference_uncertainty_file)
nrefu, ns, srefu = [
int(reference_uncertainty_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefu != n_reference_lines or ns < n_radiance_bands or srefu != n_reference_columns:
raise IndexError("Reference file dimension mismatch (uncertainty)")
# Load reference set locations
reference_locations_img = envi.open(reference_locations_file + '.hdr',
reference_locations_file)
nrefl, lb, ls = [
int(reference_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nrefl != n_reference_lines or lb != 3:
raise IndexError("Reference file dimension mismatch (locations)")
input_radiance_img = envi.open(input_radiance_file + '.hdr',
input_radiance_file)
n_input_lines, n_input_bands, n_input_samples = [
int(input_radiance_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if n_radiance_bands != n_input_bands:
msg = 'Number of channels mismatch: input (%i) vs. reference (%i)'
raise IndexError(msg % (nbr, n_radiance_bands))
input_locations_img = envi.open(input_locations_file + '.hdr',
input_locations_file)
nll, nlb, nls = [
int(input_locations_img.metadata[n])
for n in ('lines', 'bands', 'samples')
]
if nll != n_input_lines or nlb != 3 or nls != n_input_samples:
raise IndexError('Input location dimension mismatch')
# Create output files
output_metadata = input_radiance_img.metadata
output_metadata['interleave'] = 'bil'
output_reflectance_img = envi.create_image(output_reflectance_file +
'.hdr',
ext='',
metadata=output_metadata,
force=True)
output_uncertainty_img = envi.create_image(output_uncertainty_file +
'.hdr',
ext='',
metadata=output_metadata,
force=True)
# Now cleanup inputs and outputs, we'll write dynamically above
del output_reflectance_img, output_uncertainty_img
del reference_reflectance_img, reference_uncertainty_img, reference_locations_img, input_radiance_img, input_locations_img
# Determine the number of cores to use
if n_cores == -1:
n_cores = multiprocessing.cpu_count()
n_cores = min(n_cores, n_input_lines)
# Break data into sections
line_sections = np.linspace(0, n_input_lines, num=n_cores + 1, dtype=int)
# Set up our pool
pool = multiprocessing.Pool(processes=n_cores)
start_time = time.time()
logging.info(
'Beginning empirical line inversions using {} cores'.format(n_cores))
# Run the pool (or run serially)
results = []
for l in range(len(line_sections) - 1):
args = (
line_sections[l],
line_sections[l + 1],
reference_radiance_file,
reference_reflectance_file,
reference_uncertainty_file,
reference_locations_file,
input_radiance_file,
input_locations_file,
segmentation_file,
isofit_config,
output_reflectance_file,
output_uncertainty_file,
radiance_factors,
nneighbors,
nodata_value,
)
if n_cores != 1:
results.append(pool.apply_async(_run_chunk, args))
else:
_run_chunk(*args)
pool.close()
pool.join()
total_time = time.time() - start_time
logging.info(
'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core'
.format(total_time, line_sections[-1] * n_input_samples / total_time,
line_sections[-1] * n_input_samples / total_time / n_cores))