def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] pattern = params["grouping_pattern"] patt_name = params['grouping_output_name'] #read the spectrums extension = {'asd': 1, 'ASD': 1} spec_files = [ os.path.join(input_dir, f) for f in os.listdir(input_dir) if get_extension(f) in extension ] raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return print("Data directory: {}".format(input_dir)) print(" Read {} {} files".format(len(raw_specs), extension)) #perform jump correction jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"], params["jumpcorrection_stablezone"]) jc_specs = [jumpcorrector.correct(s) for s in raw_specs] print(" Jump corrected {} spectrums(s)".format(len(jc_specs))) #separate into green vegetation and non-green vegetation spectra gvd = GreenVegDetector() raw_white_specs, raw_target_specs = [], [] jc_white_specs, jc_target_specs = [], [] ndvis, reflectance_ranges = [], [] for (raw, jc) in zip(raw_specs, jc_specs): (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc) ndvis.append(ndvi) reflectance_ranges.append(reflectance_range) if status: raw_target_specs.append(raw) jc_target_specs.append(jc) else: raw_white_specs.append(raw) jc_white_specs.append(jc) print(" # white spectra: {}".format(len(jc_white_specs))) print(" # target spectra: {}".format(len(jc_target_specs))) #subset to desired wavelength range #do this for the jc_target_specs only proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs] print(" Subsetted spectra to range: {}, {}".format(ws, we)) #identify groups using SpectrumRegex and create SpectrumGroup objects patt_groups = [] grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern) patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict] print(" Grouped spectrums: {} groups".format(len(patt_groups))) #create output directories and save data save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs, patt_name, patt_groups)
def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] pattern = params["grouping_pattern"] patt_name = params['grouping_output_name'] #read the spectrums extension = {'asd': 1, 'ASD': 1} spec_files = [os.path.join(input_dir, f) for f in sorted(os.listdir(input_dir)) if get_extension(f) in extension] raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return
def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] pattern = params["grouping_pattern"] patt_name = params['grouping_output_name'] #read the spectrums extension = {'asd': 1, 'ASD': 1} spec_files = [ os.path.join(input_dir, f) for f in sorted(os.listdir(input_dir)) if get_extension(f) in extension ] raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return
def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] pattern = params["grouping_pattern"] patt_name = params['grouping_output_name'] #read the spectrums extension = 'sed' spec_files = [ os.path.join(input_dir, f) for f in os.listdir(input_dir) if get_extension(f) == extension ] raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return print("Data directory: {}".format(input_dir)) print(" Read {} {} files".format(len(raw_specs), extension)) #uniquify wavelengths, monotone them and interpolate to 1nm ohr = OverlapHandler(rstype='cubic') rs_specs = [] for s in raw_specs: if ohr.is_1nm(s): rs_specs.append(s) else: rs_specs.append(ohr.process_overlap(s)) print(" Uniquified and monotoned spectrums") #separate into green vegetation and non-green vegetation spectra gvd = GreenVegDetector() raw_white_specs, raw_target_specs = [], [] rs_white_specs, rs_target_specs = [], [] ndvis, reflectance_ranges = [], [] for (raw, rs) in zip(raw_specs, rs_specs): (status, ndvi, reflectance_range) = gvd.is_green_vegetation(rs) ndvis.append(ndvi) reflectance_ranges.append(reflectance_range) if status: raw_target_specs.append(raw) rs_target_specs.append(rs) else: raw_white_specs.append(raw) rs_white_specs.append(rs) print(" # white spectra: {}".format(len(rs_white_specs))) print(" # target spectra: {}".format(len(rs_target_specs))) #subset to desired wavelength range #do this for the jc_target_specs only proc_target_specs = [s.wavelength_subset(ws, we) for s in rs_target_specs] print(" Subsetted spectra to range: {}, {}".format(ws, we)) #identify groups using SpectrumRegex and create SpectrumGroup objects grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern) patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict] print(" Grouped spectrums: {} groups".format(len(patt_groups))) #create output directories and save data save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs, patt_name, patt_groups)
def test(adir, ext): #get the files in the data directory ext_files = [os.path.join(adir, f) for f in os.listdir(adir) if get_extension(f) == ext] #read in the spectra spectrums = [SdalReader().read_spectrum(f) for f in ext_files] print("# of {} spectrums: {}".format(ext, len(spectrums))) #separate the data into green vegetation and non-green vegetation green_veg_spectrums = [] nongreen_veg_spectrums = [] ndvis = [] reflectance_ranges = [] gvd = GreenVegDetector() for s in spectrums: status, ndvi, reflectance_range = gvd.is_green_vegetation(s) ndvis.append(ndvi) reflectance_ranges.append(reflectance_range) if status: green_veg_spectrums.append(s) else: nongreen_veg_spectrums.append(s) print("# green vegetation spectra: {}".format(len(green_veg_spectrums))) print("# non green vegetation spectra: {}".format(len(nongreen_veg_spectrums))) #plotting the green vegetation spectrums fig1 = plt.figure(figsize = (18, 5)) ax1 = fig1.add_axes([0.1, 0.1, 0.7, 0.7]) for s in green_veg_spectrums: ax1.plot(s.wavelengths, s.reflectances, color = 'green') ax1.axvline(x = gvd.ndvi_nir_range[0], ymin = 0.0, ymax = 1.0, color = 'cyan') ax1.axvline(x = gvd.ndvi_nir_range[1], ymin = 0.0, ymax = 1.0, color = 'cyan') ax1.axvline(x = gvd.ndvi_red_range[0], ymin = 0.0, ymax = 1.0, color = 'yellow') ax1.axvline(x = gvd.ndvi_red_range[1], ymin = 0.0, ymax = 1.0, color = 'yellow') ax1.set_xlim(left = 250, right = 2600) ax1.set_ylim(bottom = -0.1, top = 1.1) ax1.set_xlabel("Wavelengths", fontsize = 12) ax1.set_ylabel("Refletances", fontsize = 12) ax1.set_title("{} Green vegetation spectra".format(ext), fontsize = 15) # fig1.draw() #plotting the non-green vegetation spectrums fig2 = plt.figure(figsize = (18, 5)) ax2 = fig2.add_axes([0.1, 0.1, 0.7, 0.7]) for s in nongreen_veg_spectrums: ax2.plot(s.wavelengths, s.reflectances, color = 'red') ax2.axvline(x = gvd.ndvi_nir_range[0], ymin = 0.0, ymax = 1.0, color = 'cyan') ax2.axvline(x = gvd.ndvi_nir_range[1], ymin = 0.0, ymax = 1.0, color = 'cyan') ax2.axvline(x = gvd.ndvi_red_range[0], ymin = 0.0, ymax = 1.0, color = 'yellow') ax2.axvline(x = gvd.ndvi_red_range[1], ymin = 0.0, ymax = 1.0, color = 'yellow') ax2.set_xlim(left = 250, right = 2600) ax2.set_ylim(bottom = -0.1, top = 1.1) ax2.set_xlabel("Wavelengths", fontsize = 12) ax2.set_ylabel("Refletances", fontsize = 12) ax2.set_title("{} Non vegetation spectra".format(ext), fontsize = 15) # fig2.draw() #plotting ndvi and reflectance range fig3, axs3 = plt.subplots(1, 2, figsize = (16, 5)) #plotting the ndvi values axs3[0].scatter(range(len(ndvis)), ndvis, color = 'black', alpha = 0.8) axs3[0].axhline(y = gvd.ndvi_thresh, xmin = 0.0, xmax = 1.0, color = 'red') axs3[0].set_xlim(left = 0, right = len(ndvis)) axs3[0].set_ylim(bottom = -1.0, top = 1.0) axs3[0].set_xlabel("Spectrum index", fontsize = 12) axs3[0].set_ylabel("NDVI", fontsize = 12) axs3[0].set_title("{} spectra NDVIs".format(ext), fontsize = 15) #plotting the reflectance range values axs3[1].scatter(range(len(reflectance_ranges)), reflectance_ranges, color = 'black', alpha = 0.8) axs3[1].axhline(y = gvd.reflectance_range_thresh, xmin = 0.0, xmax = 1.0, color = 'red') axs3[1].set_xlim(left = 0, right = len(reflectance_ranges)) axs3[1].set_ylim(bottom = -0.1, top = 1.0) axs3[1].set_xlabel("Spectrum index", fontsize = 12) axs3[1].set_ylabel("Reflectance range", fontsize = 12) axs3[1].set_title("{} spectra reflectance ranges".format(ext), fontsize = 15) plt.draw()
# -*- coding: utf-8 -*- """ Created on Mon Aug 3 13:18:15 2015 @author: pravindran """ from __future__ import print_function import os, os.path from SdalReader import SdalReader from sdal_plot_spectrums import plot_spectrums from helpers import get_extension if __name__ == "__main__": print("asd_trek.py") data_dir = "/home/pravindran/mycode/sdal_data/tests_data/asd_mixed_test/" files = [os.path.join(data_dir, f) for f in os.listdir(data_dir)] print("\n".join(files)) specs = [SdalReader().read_spectrum(f) for f in files if get_extension(f).lower() == "asd"] fig = plot_spectrums(specs, spectrums_color="cyan", title="Spectra from ASD Trek + FieldSpec") fig.savefig(os.path.join(data_dir, "plots.pdf"))
def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] pattern = params["grouping_pattern"] patt_name = params['grouping_output_name'] #read the spectrums extension = 'sig' spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) if get_extension(f) == extension] # for f in spec_files: # print(f) raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return # print("Data directory: {}".format(input_dir)) # print(" Read {} {} files".format(len(raw_specs), extension)) # for s in raw_specs: # print(s.idstr) #uniquify wavelengths and interpolate to 1nm ohr = OverlapHandler(rstype = 'linear') rs_specs = [] for s in raw_specs: if ohr.is_1nm(s): rs_specs.append(s) else: rs_specs.append(ohr.process_overlap(s)) print(" Uniquified and monotoned spectrums") # for w in rs_specs[0].wavelengths: # print(w) #separate into green vegetation and non-green vegetation spectra gvd = GreenVegDetector() raw_white_specs, raw_target_specs = [], [] rs_white_specs, rs_target_specs = [], [] ndvis, reflectance_ranges = [], [] for (raw, rs) in zip(raw_specs, rs_specs): (status, ndvi, reflectance_range) = gvd.is_green_vegetation(rs) ndvis.append(ndvi) reflectance_ranges.append(reflectance_range) if status: raw_target_specs.append(raw) rs_target_specs.append(rs) else: raw_white_specs.append(raw) rs_white_specs.append(rs) print(" # white spectra: {}".format(len(rs_white_specs))) print(" # target spectra: {}".format(len(rs_target_specs))) #subset to desired wavelength range #do this for the jc_target_specs only proc_target_specs = [s.wavelength_subset(ws, we) for s in rs_target_specs] print(" Subsetted spectra to range: {}, {}".format(ws, we)) #identify groups using SpectrumRegex and create SpectrumGroup objects grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern) # for k in grps_dict: # print(k, len(grps_dict[k])) # SpectrumGroup().group(grps_dict[k], k) patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict] print(" Grouped spectrums: {} groups".format(len(patt_groups))) #create output directories and save data save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs, patt_name, patt_groups)
def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] #read the spectrums extension = {'asd': 1, 'ASD': 1} spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) if get_extension(f) in extension] raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return print("Data directory: {}".format(input_dir)) print(" Read {} {} files".format(len(raw_specs), extension)) #perform jump correction jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"], params["jumpcorrection_stablezone"]) jc_specs = [jumpcorrector.correct(s) for s in raw_specs] print(" Jump corrected {} spectrums(s)".format(len(jc_specs))) #separate into green vegetation and non-green vegetation spectra gvd = GreenVegDetector() raw_white_specs, raw_target_specs = [], [] jc_white_specs, jc_target_specs = [], [] ndvis, reflectance_ranges = [], [] for (raw, jc) in zip(raw_specs, jc_specs): (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc) ndvis.append(ndvi) reflectance_ranges.append(reflectance_range) if status: raw_target_specs.append(raw) jc_target_specs.append(jc) else: raw_white_specs.append(raw) jc_white_specs.append(jc) print(" # white spectra: {}".format(len(jc_white_specs))) print(" # target spectra: {}".format(len(jc_target_specs))) #subset to desired wavelength range #do this for the jc_target_specs only proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs] print(" Subsetted spectra to range: {}, {}".format(ws, we)) #create SpectrumGroup object for raw_target_specs raw_target_sg = SpectrumGroup() raw_target_sg.group(raw_target_specs, "raw_target") raw_target_sg.save_dataframe(input_dir) raw_target_sg.save_plots(input_dir) #create SpectrumGroup object for raw_white_specs raw_white_sg = SpectrumGroup() raw_white_sg.group(raw_white_specs, "raw_white") raw_white_sg.save_dataframe(input_dir) raw_white_sg.save_plots(input_dir) #create SpectrumGroup object for proc_target_specs proc_target_sg = SpectrumGroup() proc_target_sg.group(proc_target_specs, "proc_target") proc_target_sg.save_dataframe(input_dir) proc_target_sg.save_plots(input_dir)
def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] pattern = params["grouping_pattern"] patt_name = params['grouping_output_name'] #read the spectrums extension = {'asd': 1, 'ASD': 1} spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) if get_extension(f) in extension] raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return print("Data directory: {}".format(input_dir)) print(" Read {} {} files".format(len(raw_specs), extension)) #perform jump correction jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"], params["jumpcorrection_stablezone"]) jc_specs = [jumpcorrector.correct(s) for s in raw_specs] print(" Jump corrected {} spectrums(s)".format(len(jc_specs))) #separate into green vegetation and non-green vegetation spectra gvd = GreenVegDetector() raw_white_specs, raw_target_specs = [], [] jc_white_specs, jc_target_specs = [], [] ndvis, reflectance_ranges = [], [] for (raw, jc) in zip(raw_specs, jc_specs): (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc) ndvis.append(ndvi) reflectance_ranges.append(reflectance_range) if status: raw_target_specs.append(raw) jc_target_specs.append(jc) else: raw_white_specs.append(raw) jc_white_specs.append(jc) print(" # white spectra: {}".format(len(jc_white_specs))) print(" # target spectra: {}".format(len(jc_target_specs))) #subset to desired wavelength range #do this for the jc_target_specs only proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs] print(" Subsetted spectra to range: {}, {}".format(ws, we)) #identify groups using SpectrumRegex and create SpectrumGroup objects patt_groups = [] grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern) patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict] print(" Grouped spectrums: {} groups".format(len(patt_groups))) #create output directories and save data save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs, patt_name, patt_groups)
def process_directory(params): #parameters for processing input_dir = params["input_directory"] ws = params["subset_wavelength_range"][0] we = params["subset_wavelength_range"][1] #read the spectrums extension = {'asd': 1, 'ASD': 1} spec_files = [ os.path.join(input_dir, f) for f in os.listdir(input_dir) if get_extension(f) in extension ] raw_specs = [SdalReader().read_spectrum(f) for f in spec_files] if len(raw_specs) == 0: return print("Data directory: {}".format(input_dir)) print(" Read {} {} files".format(len(raw_specs), extension)) #perform jump correction jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"], params["jumpcorrection_stablezone"]) jc_specs = [jumpcorrector.correct(s) for s in raw_specs] print(" Jump corrected {} spectrums(s)".format(len(jc_specs))) #separate into green vegetation and non-green vegetation spectra gvd = GreenVegDetector() raw_white_specs, raw_target_specs = [], [] jc_white_specs, jc_target_specs = [], [] ndvis, reflectance_ranges = [], [] for (raw, jc) in zip(raw_specs, jc_specs): (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc) ndvis.append(ndvi) reflectance_ranges.append(reflectance_range) if status: raw_target_specs.append(raw) jc_target_specs.append(jc) else: raw_white_specs.append(raw) jc_white_specs.append(jc) print(" # white spectra: {}".format(len(jc_white_specs))) print(" # target spectra: {}".format(len(jc_target_specs))) #subset to desired wavelength range #do this for the jc_target_specs only proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs] print(" Subsetted spectra to range: {}, {}".format(ws, we)) #create SpectrumGroup object for raw_target_specs raw_target_sg = SpectrumGroup() raw_target_sg.group(raw_target_specs, "raw_target") raw_target_sg.save_dataframe(input_dir) raw_target_sg.save_plots(input_dir) #create SpectrumGroup object for raw_white_specs raw_white_sg = SpectrumGroup() raw_white_sg.group(raw_white_specs, "raw_white") raw_white_sg.save_dataframe(input_dir) raw_white_sg.save_plots(input_dir) #create SpectrumGroup object for proc_target_specs proc_target_sg = SpectrumGroup() proc_target_sg.group(proc_target_specs, "proc_target") proc_target_sg.save_dataframe(input_dir) proc_target_sg.save_plots(input_dir)
# -*- coding: utf-8 -*- """ Created on Mon Aug 3 13:18:15 2015 @author: pravindran """ from __future__ import print_function import os, os.path from SdalReader import SdalReader from sdal_plot_spectrums import plot_spectrums from helpers import get_extension if __name__ == "__main__": print("asd_trek.py") data_dir = '/home/pravindran/mycode/sdal_data/tests_data/asd_mixed_test/' files = [os.path.join(data_dir, f) for f in os.listdir(data_dir)] print("\n".join(files)) specs = [ SdalReader().read_spectrum(f) for f in files if get_extension(f).lower() == 'asd' ] fig = plot_spectrums(specs, spectrums_color='cyan', title="Spectra from ASD Trek + FieldSpec") fig.savefig(os.path.join(data_dir, "plots.pdf"))