def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
pattern = params["grouping_pattern"]
patt_name = params['grouping_output_name']
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [
os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension
]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#identify groups using SpectrumRegex and create SpectrumGroup objects
patt_groups = []
grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
print(" Grouped spectrums: {} groups".format(len(patt_groups)))
#create output directories and save data
save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs,
patt_name, patt_groups)
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
pattern = params["grouping_pattern"]
patt_name = params['grouping_output_name']
#read the spectrums
extension = 'sed'
spec_files = [
os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) == extension
]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#uniquify wavelengths, monotone them and interpolate to 1nm
ohr = OverlapHandler(rstype='cubic')
rs_specs = []
for s in raw_specs:
if ohr.is_1nm(s):
rs_specs.append(s)
else:
rs_specs.append(ohr.process_overlap(s))
print(" Uniquified and monotoned spectrums")
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
rs_white_specs, rs_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, rs) in zip(raw_specs, rs_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(rs)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
rs_target_specs.append(rs)
else:
raw_white_specs.append(raw)
rs_white_specs.append(rs)
print(" # white spectra: {}".format(len(rs_white_specs)))
print(" # target spectra: {}".format(len(rs_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in rs_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#identify groups using SpectrumRegex and create SpectrumGroup objects
grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
print(" Grouped spectrums: {} groups".format(len(patt_groups)))
#create output directories and save data
save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs,
patt_name, patt_groups)
def test(adir, ext):
#get the files in the data directory
ext_files = [os.path.join(adir, f) for f in os.listdir(adir)
if get_extension(f) == ext]
#read in the spectra
spectrums = [SdalReader().read_spectrum(f) for f in ext_files]
print("# of {} spectrums: {}".format(ext, len(spectrums)))
#separate the data into green vegetation and non-green vegetation
green_veg_spectrums = []
nongreen_veg_spectrums = []
ndvis = []
reflectance_ranges = []
gvd = GreenVegDetector()
for s in spectrums:
status, ndvi, reflectance_range = gvd.is_green_vegetation(s)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
green_veg_spectrums.append(s)
else:
nongreen_veg_spectrums.append(s)
print("# green vegetation spectra: {}".format(len(green_veg_spectrums)))
print("# non green vegetation spectra: {}".format(len(nongreen_veg_spectrums)))
#plotting the green vegetation spectrums
fig1 = plt.figure(figsize = (18, 5))
ax1 = fig1.add_axes([0.1, 0.1, 0.7, 0.7])
for s in green_veg_spectrums:
ax1.plot(s.wavelengths, s.reflectances, color = 'green')
ax1.axvline(x = gvd.ndvi_nir_range[0], ymin = 0.0, ymax = 1.0,
color = 'cyan')
ax1.axvline(x = gvd.ndvi_nir_range[1], ymin = 0.0, ymax = 1.0,
color = 'cyan')
ax1.axvline(x = gvd.ndvi_red_range[0], ymin = 0.0, ymax = 1.0,
color = 'yellow')
ax1.axvline(x = gvd.ndvi_red_range[1], ymin = 0.0, ymax = 1.0,
color = 'yellow')
ax1.set_xlim(left = 250, right = 2600)
ax1.set_ylim(bottom = -0.1, top = 1.1)
ax1.set_xlabel("Wavelengths", fontsize = 12)
ax1.set_ylabel("Refletances", fontsize = 12)
ax1.set_title("{} Green vegetation spectra".format(ext), fontsize = 15)
# fig1.draw()
#plotting the non-green vegetation spectrums
fig2 = plt.figure(figsize = (18, 5))
ax2 = fig2.add_axes([0.1, 0.1, 0.7, 0.7])
for s in nongreen_veg_spectrums:
ax2.plot(s.wavelengths, s.reflectances, color = 'red')
ax2.axvline(x = gvd.ndvi_nir_range[0], ymin = 0.0, ymax = 1.0,
color = 'cyan')
ax2.axvline(x = gvd.ndvi_nir_range[1], ymin = 0.0, ymax = 1.0,
color = 'cyan')
ax2.axvline(x = gvd.ndvi_red_range[0], ymin = 0.0, ymax = 1.0,
color = 'yellow')
ax2.axvline(x = gvd.ndvi_red_range[1], ymin = 0.0, ymax = 1.0,
color = 'yellow')
ax2.set_xlim(left = 250, right = 2600)
ax2.set_ylim(bottom = -0.1, top = 1.1)
ax2.set_xlabel("Wavelengths", fontsize = 12)
ax2.set_ylabel("Refletances", fontsize = 12)
ax2.set_title("{} Non vegetation spectra".format(ext), fontsize = 15)
# fig2.draw()
#plotting ndvi and reflectance range
fig3, axs3 = plt.subplots(1, 2, figsize = (16, 5))
#plotting the ndvi values
axs3[0].scatter(range(len(ndvis)), ndvis, color = 'black', alpha = 0.8)
axs3[0].axhline(y = gvd.ndvi_thresh, xmin = 0.0, xmax = 1.0, color = 'red')
axs3[0].set_xlim(left = 0, right = len(ndvis))
axs3[0].set_ylim(bottom = -1.0, top = 1.0)
axs3[0].set_xlabel("Spectrum index", fontsize = 12)
axs3[0].set_ylabel("NDVI", fontsize = 12)
axs3[0].set_title("{} spectra NDVIs".format(ext), fontsize = 15)
#plotting the reflectance range values
axs3[1].scatter(range(len(reflectance_ranges)),
reflectance_ranges,
color = 'black', alpha = 0.8)
axs3[1].axhline(y = gvd.reflectance_range_thresh,
xmin = 0.0, xmax = 1.0, color = 'red')
axs3[1].set_xlim(left = 0, right = len(reflectance_ranges))
axs3[1].set_ylim(bottom = -0.1, top = 1.0)
axs3[1].set_xlabel("Spectrum index", fontsize = 12)
axs3[1].set_ylabel("Reflectance range", fontsize = 12)
axs3[1].set_title("{} spectra reflectance ranges".format(ext), fontsize = 15)
plt.draw()
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
pattern = params["grouping_pattern"]
patt_name = params['grouping_output_name']
#read the spectrums
extension = 'sig'
spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) == extension]
# for f in spec_files:
# print(f)
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0:
return
# print("Data directory: {}".format(input_dir))
# print(" Read {} {} files".format(len(raw_specs), extension))
# for s in raw_specs:
# print(s.idstr)
#uniquify wavelengths and interpolate to 1nm
ohr = OverlapHandler(rstype = 'linear')
rs_specs = []
for s in raw_specs:
if ohr.is_1nm(s):
rs_specs.append(s)
else:
rs_specs.append(ohr.process_overlap(s))
print(" Uniquified and monotoned spectrums")
# for w in rs_specs[0].wavelengths:
# print(w)
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
rs_white_specs, rs_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, rs) in zip(raw_specs, rs_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(rs)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
rs_target_specs.append(rs)
else:
raw_white_specs.append(raw)
rs_white_specs.append(rs)
print(" # white spectra: {}".format(len(rs_white_specs)))
print(" # target spectra: {}".format(len(rs_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in rs_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#identify groups using SpectrumRegex and create SpectrumGroup objects
grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
# for k in grps_dict:
# print(k, len(grps_dict[k]))
# SpectrumGroup().group(grps_dict[k], k)
patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
print(" Grouped spectrums: {} groups".format(len(patt_groups)))
#create output directories and save data
save_data(input_dir,
raw_white_specs,
raw_target_specs,
proc_target_specs,
patt_name,
patt_groups)
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#create SpectrumGroup object for raw_target_specs
raw_target_sg = SpectrumGroup()
raw_target_sg.group(raw_target_specs, "raw_target")
raw_target_sg.save_dataframe(input_dir)
raw_target_sg.save_plots(input_dir)
#create SpectrumGroup object for raw_white_specs
raw_white_sg = SpectrumGroup()
raw_white_sg.group(raw_white_specs, "raw_white")
raw_white_sg.save_dataframe(input_dir)
raw_white_sg.save_plots(input_dir)
#create SpectrumGroup object for proc_target_specs
proc_target_sg = SpectrumGroup()
proc_target_sg.group(proc_target_specs, "proc_target")
proc_target_sg.save_dataframe(input_dir)
proc_target_sg.save_plots(input_dir)
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
pattern = params["grouping_pattern"]
patt_name = params['grouping_output_name']
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#identify groups using SpectrumRegex and create SpectrumGroup objects
patt_groups = []
grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
print(" Grouped spectrums: {} groups".format(len(patt_groups)))
#create output directories and save data
save_data(input_dir,
raw_white_specs,
raw_target_specs,
proc_target_specs,
patt_name,
patt_groups)
def process_directory(params):
#parameters for processing
input_dir = params["input_directory"]
ws = params["subset_wavelength_range"][0]
we = params["subset_wavelength_range"][1]
#read the spectrums
extension = {'asd': 1, 'ASD': 1}
spec_files = [
os.path.join(input_dir, f) for f in os.listdir(input_dir)
if get_extension(f) in extension
]
raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
if len(raw_specs) == 0: return
print("Data directory: {}".format(input_dir))
print(" Read {} {} files".format(len(raw_specs), extension))
#perform jump correction
jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
params["jumpcorrection_stablezone"])
jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
print(" Jump corrected {} spectrums(s)".format(len(jc_specs)))
#separate into green vegetation and non-green vegetation spectra
gvd = GreenVegDetector()
raw_white_specs, raw_target_specs = [], []
jc_white_specs, jc_target_specs = [], []
ndvis, reflectance_ranges = [], []
for (raw, jc) in zip(raw_specs, jc_specs):
(status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
ndvis.append(ndvi)
reflectance_ranges.append(reflectance_range)
if status:
raw_target_specs.append(raw)
jc_target_specs.append(jc)
else:
raw_white_specs.append(raw)
jc_white_specs.append(jc)
print(" # white spectra: {}".format(len(jc_white_specs)))
print(" # target spectra: {}".format(len(jc_target_specs)))
#subset to desired wavelength range
#do this for the jc_target_specs only
proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
print(" Subsetted spectra to range: {}, {}".format(ws, we))
#create SpectrumGroup object for raw_target_specs
raw_target_sg = SpectrumGroup()
raw_target_sg.group(raw_target_specs, "raw_target")
raw_target_sg.save_dataframe(input_dir)
raw_target_sg.save_plots(input_dir)
#create SpectrumGroup object for raw_white_specs
raw_white_sg = SpectrumGroup()
raw_white_sg.group(raw_white_specs, "raw_white")
raw_white_sg.save_dataframe(input_dir)
raw_white_sg.save_plots(input_dir)
#create SpectrumGroup object for proc_target_specs
proc_target_sg = SpectrumGroup()
proc_target_sg.group(proc_target_specs, "proc_target")
proc_target_sg.save_dataframe(input_dir)
proc_target_sg.save_plots(input_dir)