def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
pattern = params["grouping_pattern"]
patt_name = params['grouping_output_name']
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [
os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension
]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#identify groups using SpectrumRegex and create SpectrumGroup objects
patt_groups = []
grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
print(" Grouped spectrums: {} groups".format(len(patt_groups)))
#create output directories and save data
save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs,
patt_name, patt_groups)
def test(testsdir, verbose = False):
statuses = []
#load the spectrums
indir = os.path.join(testsdir, "input")
infs = [os.path.join(indir, "test_asdspec1_njc.txt"),
os.path.join(indir, "test_asdspec2_njc.txt")]
exdir = os.path.join(testsdir, "expect")
exfs = [os.path.join(exdir, "test_asdspec1_jc.txt"),
os.path.join(exdir, "test_asdspec2_jc.txt")]
ins = [SdalReader().read_spectrum(f) for f in infs]
exs = [SdalReader().read_spectrum(f) for f in exfs]
#test 1
jumpwaves = [1000, 1800]
stablezone = 0
jcorr = JumpCorrector(jumpwavelengths = jumpwaves,
stablezone = stablezone)
jcs = [jcorr.correct(s) for s in ins]
for (e, j) in zip(exs, jcs):
statuses.append(np.allclose(e.data, j.data, atol = 0.0001))
#status message
if all(statuses):
print("{}: PASSED".format(__file__))
else:
print("{}: FAILED".format(__file__))
#plot for ecosis sdal talk
jumpwaves = [1000, 1800]
stablezone = 0
jcorr = JumpCorrector(jumpwavelengths = jumpwaves,
stablezone = stablezone)
jcs = [jcorr.correct(s) for s in ins]
fig1, axes1 = plt.subplots(nrows = 1, ncols = 2)
axes1[0].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
axes1[1].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
axes1[1].plot(jcs[0].wavelengths, jcs[0].reflectances, 'g')
title = "L: Input, R: Corrected, Jump wavelengths = 1000, 1800"
fig1.suptitle(title)
fig1.show()
fig1.savefig("ecosis-talk-jump-correction.png")
def test(testsdir, verbose=False):
statuses = []
#load the spectrums
indir = os.path.join(testsdir, "input")
infs = [
os.path.join(indir, "test_asdspec1_njc.txt"),
os.path.join(indir, "test_asdspec2_njc.txt")
]
exdir = os.path.join(testsdir, "expect")
exfs = [
os.path.join(exdir, "test_asdspec1_jc.txt"),
os.path.join(exdir, "test_asdspec2_jc.txt")
]
ins = [SdalReader().read_spectrum(f) for f in infs]
exs = [SdalReader().read_spectrum(f) for f in exfs]
#test 1
jumpwaves = [1000, 1800]
stablezone = 0
jcorr = JumpCorrector(jumpwavelengths=jumpwaves, stablezone=stablezone)
jcs = [jcorr.correct(s) for s in ins]
for (e, j) in zip(exs, jcs):
statuses.append(np.allclose(e.data, j.data, atol=0.0001))
#status message
if all(statuses):
print("{}: PASSED".format(__file__))
else:
print("{}: FAILED".format(__file__))
#plot for ecosis sdal talk
jumpwaves = [1000, 1800]
stablezone = 0
jcorr = JumpCorrector(jumpwavelengths=jumpwaves, stablezone=stablezone)
jcs = [jcorr.correct(s) for s in ins]
fig1, axes1 = plt.subplots(nrows=1, ncols=2)
axes1[0].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
axes1[1].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
axes1[1].plot(jcs[0].wavelengths, jcs[0].reflectances, 'g')
title = "L: Input, R: Corrected, Jump wavelengths = 1000, 1800"
fig1.suptitle(title)
fig1.show()
fig1.savefig("ecosis-talk-jump-correction.png")
#get the files into benas, emnis and vauls
tdir = "/home/prabu/mycode/sdal/tests_data/test_OutlierDetector"
idir = os.path.join(tdir, "input")
ifiles = os.listdir(idir)
benafs = [os.path.join(idir, f) for f in ifiles if re.search(r'bena', f)]
emnifs = [os.path.join(idir, f) for f in ifiles if re.search(r'emni', f)]
vaulfs = [os.path.join(idir, f) for f in ifiles if re.search(r'vaul', f)]
#read in the spectra
benas = [SdalReader().read_spectrum(f) for f in benafs]
emnis = [SdalReader().read_spectrum(f) for f in emnifs]
vauls = [SdalReader().read_spectrum(f) for f in vaulfs]
specs = [benas, emnis, vauls]
#jump correct
jumpcorr = JumpCorrector(jumpwavelengths = [1000, 1800], stablezone = 0)
benajcs = [jumpcorr.correct(s) for s in benas]
emnijcs = [jumpcorr.correct(s) for s in emnis]
vauljcs = [jumpcorr.correct(s) for s in vauls]
specjcs = [benajcs, emnijcs, vauljcs]
#compute outliers and inliers
medians = []
stds = []
thresh = 3.0
od = OutlierDetector(zthresh = thresh)
for e in specjcs:
(inliers, outliers) = od.detect(e)
medians.append(od.median)
stds.append(od.std)
def process(params):
# params.print_params()
#get the project params and verify
project = params.get_params("project")
if project:
verify_project(project)
else:
print("--project is required")
sys.exit(0)
#get the resampling params
resampling = params.get_params("resampling")
#get the jumpcorrection params and verify
jumpcorrection = params.get_params("jumpcorrection")
if jumpcorrection:
verify_jumpcorrection(jumpcorrection)
#get the groupings and verify them
groupings = {grp:params.get_params(grp) for grp in params.get_groups()}
verify_groupings(params.default_group, params.get_groups(), groupings)
tags = ["raw", params.default_group]
specs = defaultdict(list)
#specs["raw"] created
#get the filenames
allfiles = os.listdir(project["indir"])
extfiles = []
for f in allfiles:
ext = get_directory_filename_extension(f)[2]
if ext == project["fileext"]:
extfiles.append(os.path.join(project["indir"], f))
#read the raw spectrums
uniquifier = WaveUniquifier()
rawspecs = [SdalReader().read_spectrum(f) for f in extfiles]
uniqspecs = [uniquifier.uniquify(s) for s in rawspecs]
specs["raw"] = uniqspecs
#specs["preproc"] created
#do the pre-processing
prepspecs = specs["raw"]
if resampling:
resampler = WaveResampler(rstype = resampling["type"],
wavestart = resampling["range"][0],
wavestop = resampling["range"][1],
spacing = resampling["spacing"])
rsspecs = [resampler.resample(s) for s in prepspecs]
prepspecs = rsspecs
if jumpcorrection:
corrector = JumpCorrector(jumpcorrection["wavelengths"],
jumpcorrection["stablezone"])
jcspecs = [corrector.correct(s) for s in prepspecs]
prepspecs = jcspecs
#detect the references
refdet = ReferenceDetector(context = "gveg")
nonrefs = []
refs = []
for s in prepspecs:
if refdet.is_reference(s):
refs.append(s)
else:
nonrefs.append(s)
specs[params.default_group] = nonrefs
#specs[group_tag] created
#do the grouping
for t in groupings:
tags.append(t)
itag = groupings[t]["intag"]
patt = groupings[t]["pattern"]
regex = SpectrumRegex()
tgrps = regex.make_groups(specs[itag], patt)
for tg in tgrps:
sg = SpectrumGroup(spectrums = tgrps[tg])
ms = sg.mean_spectrum()
ms.idstr = tg
specs[t].append(ms)
# subsets = {grp:params.get_params(grp) for grp in params.get_subsets()}
# print(subsets)
# for t in subsets:
# itag = subsets[t]["intag"]
# otag = subsets[t]["outtag"]
# wavestart = subsets[t]["range"][0]
# wavestop = subsets[t]["range"][1]
# for s in specs[itag]:
# subspec = s.wavelength_subset(wavestart, wavestop)
# subspec.idstr = subspec.idstr + otag
# print("idstr = {}".format(subspec.idstr))
# specs[otag].append(subspec)
#create outputs
prjdir = os.path.join(project["outdir"], project["name"])
os.mkdir(prjdir)
for t in specs:
tdir = os.path.join(prjdir, t)
os.mkdir(tdir)
tgrpfn = "___{}___.csv".format(t)
for s in specs[t]:
s.write_csv(odir = tdir)
sg = SpectrumGroup(spectrums = specs[t])
sg.write_csv(tdir, tgrpfn)
#get the files into benas, emnis and vauls
tdir = "/home/prabu/mycode/sdal/tests_data/test_OutlierDetector"
idir = os.path.join(tdir, "input")
ifiles = os.listdir(idir)
benafs = [os.path.join(idir, f) for f in ifiles if re.search(r'bena', f)]
emnifs = [os.path.join(idir, f) for f in ifiles if re.search(r'emni', f)]
vaulfs = [os.path.join(idir, f) for f in ifiles if re.search(r'vaul', f)]
#read in the spectra
benas = [SdalReader().read_spectrum(f) for f in benafs]
emnis = [SdalReader().read_spectrum(f) for f in emnifs]
vauls = [SdalReader().read_spectrum(f) for f in vaulfs]
specs = [benas, emnis, vauls]
#jump correct
jumpcorr = JumpCorrector(jumpwavelengths=[1000, 1800], stablezone=0)
benajcs = [jumpcorr.correct(s) for s in benas]
emnijcs = [jumpcorr.correct(s) for s in emnis]
vauljcs = [jumpcorr.correct(s) for s in vauls]
specjcs = [benajcs, emnijcs, vauljcs]
#compute outliers and inliers
medians = []
stds = []
thresh = 3.0
od = OutlierDetector(zthresh=thresh)
for e in specjcs:
(inliers, outliers) = od.detect(e)
medians.append(od.median)
stds.append(od.std)
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#create SpectrumGroup object for raw_target_specs
raw_target_sg = SpectrumGroup()
raw_target_sg.group(raw_target_specs, "raw_target")
raw_target_sg.save_dataframe(input_dir)
raw_target_sg.save_plots(input_dir)
#create SpectrumGroup object for raw_white_specs
raw_white_sg = SpectrumGroup()
raw_white_sg.group(raw_white_specs, "raw_white")
raw_white_sg.save_dataframe(input_dir)
raw_white_sg.save_plots(input_dir)
#create SpectrumGroup object for proc_target_specs
proc_target_sg = SpectrumGroup()
proc_target_sg.group(proc_target_specs, "proc_target")
proc_target_sg.save_dataframe(input_dir)
proc_target_sg.save_plots(input_dir)
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
pattern = params["grouping_pattern"]
patt_name = params['grouping_output_name']
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#identify groups using SpectrumRegex and create SpectrumGroup objects
patt_groups = []
grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
print(" Grouped spectrums: {} groups".format(len(patt_groups)))
#create output directories and save data
save_data(input_dir,
raw_white_specs,
raw_target_specs,
proc_target_specs,
patt_name,
patt_groups)
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [
os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension
]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#create SpectrumGroup object for raw_target_specs
raw_target_sg = SpectrumGroup()
raw_target_sg.group(raw_target_specs, "raw_target")
raw_target_sg.save_dataframe(input_dir)
raw_target_sg.save_plots(input_dir)
#create SpectrumGroup object for raw_white_specs
raw_white_sg = SpectrumGroup()
raw_white_sg.group(raw_white_specs, "raw_white")
raw_white_sg.save_dataframe(input_dir)
raw_white_sg.save_plots(input_dir)
#create SpectrumGroup object for proc_target_specs
proc_target_sg = SpectrumGroup()
proc_target_sg.group(proc_target_specs, "proc_target")
proc_target_sg.save_dataframe(input_dir)
proc_target_sg.save_plots(input_dir)