def __init__(self, config, instrument):
TabularRT.__init__(self, config, instrument)
self.modtran_dir = self.find_basedir(config)
self.filtpath = os.path.join(self.lut_dir, 'wavelengths.flt')
self.template = deepcopy(json_load_ascii(
config['modtran_template_file'])['MODTRAN'])
# Insert aerosol templates, if specified
if 'aerosol_template_file' in config:
self.template[0]['MODTRANINPUT']['AEROSOLS'] = \
deepcopy(json_load_ascii(config['aerosol_template_file']))
# Insert aerosol data, if specified
if 'aerosol_model_file' in config:
aer_data = s.loadtxt(config['aerosol_model_file'])
self.aer_wl = aer_data[:, 0]
aer_data = aer_data[:, 1:].T
self.naer = int(len(aer_data)/3)
aer_absc, aer_extc, aer_asym = [], [], []
for i in range(self.naer):
aer_extc.append(aer_data[i*3])
aer_absc.append(aer_data[i*3+1])
aer_asym.append(aer_data[i*3+2])
self.aer_absc = s.array(aer_absc)
self.aer_extc = s.array(aer_extc)
self.aer_asym = s.array(aer_asym)
# Build the lookup table
self.build_lut(instrument)
def main():
parser = argparse.ArgumentParser(description="Create an instrument model")
parser.add_argument('config', type=str, metavar='INPUT')
args = parser.parse_args(sys.argv[1:])
config = json_load_ascii(args.config, shell_replace=True)
configdir, configfile = split(abspath(args.config))
infile = expand_path(configdir, config['input_radiance_file'])
outfile = expand_path(configdir, config['output_model_file'])
flatfile = expand_path(configdir, config['output_flatfield_file'])
uniformity_thresh = float(config['uniformity_threshold'])
infile_hdr = infile + '.hdr'
img = spectral.io.envi.open(infile_hdr, infile)
inmm = img.open_memmap(interleave='source', writable=False)
if img.interleave != 1:
raise ValueError('I need BIL interleave')
X = s.array(inmm[:, :, :], dtype=s.float32)
nr, nb, nc = X.shape
FF, Xhoriz, Xhorizp, use_ff = flat_field(X, uniformity_thresh)
s.array(FF, dtype=s.float32).tofile(flatfile)
with open(flatfile+'.hdr', 'w') as fout:
fout.write(hdr_template.format(lines=nb, samples=nc))
C, Xvert, Xvertp, use_C = column_covariances(X, uniformity_thresh)
cshape = (C.shape[0], C.shape[1]**2)
out = s.array(C, dtype=s.float32).reshape(cshape)
mdict = {'columns': out.shape[0], 'bands': out.shape[1],
'covariances': out, 'Xvert': Xvert, 'Xhoriz': Xhoriz,
'Xvertp': Xvertp, 'Xhorizp': Xhorizp, 'use_ff': use_ff,
'use_C': use_C}
savemat(outfile, mdict)
def main():
desc = "Add noise to a radiance spectrum or image"
parser = argparse.ArgumentParser(description=desc)
parser.add_argument('config', type=str, metavar='INPUT')
args = parser.parse_args(sys.argv[1:])
config = json_load_ascii(args.config, shell_replace=True)
configdir, configfile = split(abspath(args.config))
infile = expand_path(configdir, config['input_radiance_file'])
outfile = expand_path(configdir, config['output_radiance_file'])
instrument = Instrument(config['instrument_model'])
geom = Geometry()
if infile.endswith('txt'):
rdn, wl = spectrumLoad(infile)
Sy = instrument.Sy(rdn, geom)
rdn_noise = rdn + multivariate_normal(zeros(rdn.shape), Sy)
with open(outfile, 'w') as fout:
for w, r in zip(wl, rdn_noise):
fout.write('%8.5f %8.5f' % (w, r))
else:
raise ValueError('image cubes not yet implemented')
def modtran_driver(self, overrides):
"""Write a MODTRAN 6.0 input file"""
param = deepcopy(self.template)
# Insert basic aerosol template, if needed
for aer_key in ['VIS', 'AERTYPE', 'AOT550', 'AOD550', 'EXT550']:
if aer_key in overrides.keys():
aerosol_template = deepcopy(
json_load_ascii(self.aerosol_template))
param[0]['MODTRANINPUT']['AEROSOLS'] = aerosol_template
# Other overrides
for key, val in overrides.items():
recursive_replace(param, key, val)
if key == 'AERTYPE':
custom_aerosol = self.get_aerosol(val)
wl, absc, extc, asym = [list(q) for q in custom_aerosol]
try:
del param[0]['MODTRANINPUT']['AEROSOLS']['IHAZE']
except KeyError:
pass
nwl = len(wl)
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'NARSPC'] = nwl
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'VARSPC'] = wl
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'EXTC'] = extc
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'ABSC'] = absc
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'ASYM'] = asym
elif key == 'EXT550' or key == 'AOT550' or key == 'AOD550':
# MODTRAN 6.0 convention treats negative visibility as AOT550
recursive_replace(param, 'VIS', -val)
elif key == 'FILTNM':
param[0]['MODTRANINPUT']['SPECTRAL']['FILTNM'] = val
elif key in [
'ITYPE', 'H1ALT', 'IDAY', 'IPARM', 'PARM1', 'PARM2',
'GMTIME', 'TRUEAZ', 'OBSZEN'
]:
param[0]['MODTRANINPUT']['GEOMETRY'][key] = val
return json.dumps({"MODTRAN": param})
def modtran_driver(self, overrides):
"""Write a MODTRAN 6.0 input file"""
param = deepcopy(json_load_ascii(self.modtran_template)['MODTRAN'])
# Basic aerosol template
if 'VIS' in overrides.keys() or 'AERTYPE' in overrides.keys():
aerosol_template = deepcopy(json_load_ascii(self.aerosol_template))
param[0]['MODTRANINPUT']['AEROSOLS'] = aerosol_template
# Other overrides
for key, val in overrides.items():
recursive_replace(param, key, val)
if key == 'AERTYPE':
wl, absc, extc, asym = [list(q) for q in self.get_aerosol(val)]
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'NARSPC'] = len(wl)
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'VARSPC'] = wl
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'EXTC'] = extc
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'ABSC'] = absc
param[0]['MODTRANINPUT']['AEROSOLS']['IREGSPC'][0][
'ASYM'] = asym
elif key == 'EXT550' or key == 'AOT550' or key == 'AOD550':
vis = self.ext550_to_vis(val)
recursive_replace(param, 'VIS', vis)
elif key == 'FILTNM':
param[0]['MODTRANINPUT']['SPECTRAL']['FILTNM'] = val
elif key in [
'ITYPE', 'H1ALT', 'IDAY', 'IPARM', 'PARM1', 'PARM2',
'GMTIME', 'TRUEAZ', 'OBSZEN'
]:
param[0]['MODTRANINPUT']['GEOMETRY'][key] = val
return json.dumps({"MODTRAN": param})
def __init__(self, config, instrument):
TabularRT.__init__(self, config, instrument)
self.modtran_dir = self.find_basedir(config)
self.modtran_template_file = config['modtran_template_file']
self.template = deepcopy(
json_load_ascii(self.modtran_template_file)['MODTRAN'])
self.filtpath = os.path.join(self.lut_dir, 'wavelengths.flt')
if 'aerosol_model_file' in config:
self.aerosol_model_file = config['aerosol_model_file']
self.aerosol_template = config['aerosol_template_file']
self.build_aerosol_model()
# Build the lookup table
self.build_lut(instrument)
def main():
parser = argparse.ArgumentParser(description="Create a surface model")
parser.add_argument('config', type=str, metavar='INPUT')
args = parser.parse_args()
config = json_load_ascii(args.config, shell_replace=True)
configdir, configfile = split(abspath(args.config))
# Determine top level parameters
for q in ['output_model_file', 'sources', 'normalize', 'wavelength_file']:
if q not in config:
raise ValueError('Missing parameter: %s' % q)
wavelength_file = expand_path(configdir, config['wavelength_file'])
normalize = config['normalize']
reference_windows = config['reference_windows']
outfile = expand_path(configdir, config['output_model_file'])
if 'mixtures' in config:
mixtures = config['mixtures']
else:
mixtures = 0
# load wavelengths file
q = s.loadtxt(wavelength_file)
if q.shape[1] > 2:
q = q[:, 1:]
if q[0, 0] < 100:
q = q * 1000.0
wl = q[:, 0]
nchan = len(wl)
# build global reference windows
refwl = []
for wi, window in enumerate(reference_windows):
active_wl = aand(wl >= window[0], wl < window[1])
refwl.extend(wl[active_wl])
normind = s.array([s.argmin(abs(wl - w)) for w in refwl])
refwl = s.array(refwl, dtype=float)
# create basic model template
model = {
'normalize': normalize,
'wl': wl,
'means': [],
'covs': [],
'refwl': refwl
}
for si, source_config in enumerate(config['sources']):
# Determine source parameters
for q in [
'input_spectrum_files', 'windows', 'n_components', 'windows'
]:
if q not in source_config:
raise ValueError('Source %i is missing a parameter: %s' %
(si, q))
infiles = [
expand_path(configdir, fi)
for fi in source_config['input_spectrum_files']
]
ncomp = int(source_config['n_components'])
windows = source_config['windows']
# load spectra
spectra = []
for infile in infiles:
hdrfile = infile + '.hdr'
rfl = envi.open(hdrfile, infile)
nl, nb, ns = [
int(rfl.metadata[n]) for n in ('lines', 'bands', 'samples')
]
swl = s.array([float(f) for f in rfl.metadata['wavelength']])
rfl_mm = rfl.open_memmap(interleave='source', writable=True)
if rfl.metadata['interleave'] == 'bip':
x = s.array(rfl_mm[:, :, :])
if rfl.metadata['interleave'] == 'bil':
x = s.array(rfl_mm[:, :, :]).transpose((0, 2, 1))
x = x.reshape(nl * ns, nb)
swl = s.array([float(f) for f in rfl.metadata['wavelength']])
# import spectra and resample
for x1 in x:
p = interp1d(swl,
x1,
kind='linear',
bounds_error=False,
fill_value='extrapolate')
spectra.append(p(wl))
# calculate mixtures, if needed
n = float(len(spectra))
nmix = int(n * mixtures)
for mi in range(nmix):
s1, m1 = spectra[int(s.rand() * n)], s.rand()
s2, m2 = spectra[int(s.rand() * n)], 1.0 - m1
spectra.append(m1 * s1 + m2 * s2)
spectra = s.array(spectra)
use = s.all(s.isfinite(spectra), axis=1)
spectra = spectra[use, :]
# accumulate total list of window indices
window_idx = -s.ones((nchan), dtype=int)
for wi, win in enumerate(windows):
active_wl = aand(wl >= win['interval'][0], wl < win['interval'][1])
window_idx[active_wl] = wi
# Two step model. First step is k-means initialization
kmeans = KMeans(init='k-means++', n_clusters=ncomp, n_init=10)
kmeans.fit(spectra)
Z = kmeans.predict(spectra)
for ci in range(ncomp):
m = s.mean(spectra[Z == ci, :], axis=0)
C = s.cov(spectra[Z == ci, :], rowvar=False)
for i in range(nchan):
window = windows[window_idx[i]]
if window['correlation'] == 'EM':
C[i, i] = C[i, i] + float(window['regularizer'])
elif window['correlation'] == 'decorrelated':
ci = C[i, i]
C[:, i] = 0
C[i, :] = 0
C[i, i] = ci + float(window['regularizer'])
else:
raise ValueError('I do not recognize the source ' +
window['correlation'])
# Normalize the component spectrum if desired
if normalize == 'Euclidean':
z = s.sqrt(s.sum(pow(m[normind], 2)))
elif normalize == 'RMS':
z = s.sqrt(s.mean(pow(m[normind], 2)))
elif normalize == 'None':
z = 1.0
else:
raise ValueError('Unrecognized normalization: %s\n' %
normalize)
m = m / z
C = C / (z**2)
model['means'].append(m)
model['covs'].append(C)
model['means'] = s.array(model['means'])
model['covs'] = s.array(model['covs'])
savemat(outfile, model)
print('saving results to ' + outfile)