Exemplo n.º 1
0
def process_directory(params):

    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    pattern = params["grouping_pattern"]
    patt_name = params['grouping_output_name']

    #read the spectrums
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [
        os.path.join(input_dir, f) for f in os.listdir(input_dir)
        if get_extension(f) in extension
    ]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))

    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))

    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

    #identify groups using SpectrumRegex and create SpectrumGroup objects
    patt_groups = []
    grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
    patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
    print("   Grouped spectrums: {} groups".format(len(patt_groups)))

    #create output directories and save data
    save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs,
              patt_name, patt_groups)
def process_directory(params):

    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    pattern = params["grouping_pattern"]
    patt_name = params['grouping_output_name']

    #read the spectrums
    extension = 'sed'
    spec_files = [
        os.path.join(input_dir, f) for f in os.listdir(input_dir)
        if get_extension(f) == extension
    ]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))

    #uniquify wavelengths, monotone them and interpolate to 1nm
    ohr = OverlapHandler(rstype='cubic')
    rs_specs = []
    for s in raw_specs:
        if ohr.is_1nm(s):
            rs_specs.append(s)
        else:
            rs_specs.append(ohr.process_overlap(s))
    print("  Uniquified and monotoned spectrums")

    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    rs_white_specs, rs_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, rs) in zip(raw_specs, rs_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(rs)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            rs_target_specs.append(rs)
        else:
            raw_white_specs.append(raw)
            rs_white_specs.append(rs)
    print("   # white spectra: {}".format(len(rs_white_specs)))
    print("   # target spectra: {}".format(len(rs_target_specs)))

    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in rs_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

    #identify groups using SpectrumRegex and create SpectrumGroup objects
    grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
    patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
    print("   Grouped spectrums: {} groups".format(len(patt_groups)))

    #create output directories and save data
    save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs,
              patt_name, patt_groups)
Exemplo n.º 3
0
def test(adir, ext):
    #get the files in the data directory
    ext_files = [os.path.join(adir, f) for f in os.listdir(adir) 
                                        if get_extension(f) == ext]

    #read in the spectra 
    spectrums = [SdalReader().read_spectrum(f) for f in ext_files]
    print("# of {} spectrums: {}".format(ext, len(spectrums)))

    #separate the data into green vegetation and non-green vegetation
    green_veg_spectrums = []
    nongreen_veg_spectrums = []
    ndvis = []
    reflectance_ranges = []
    gvd = GreenVegDetector()
    for s in spectrums:
        status, ndvi, reflectance_range = gvd.is_green_vegetation(s) 
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            green_veg_spectrums.append(s)
        else:
            nongreen_veg_spectrums.append(s)    
    print("# green vegetation spectra: {}".format(len(green_veg_spectrums)))
    print("# non green vegetation spectra: {}".format(len(nongreen_veg_spectrums)))

    #plotting the green vegetation spectrums
    fig1 = plt.figure(figsize = (18, 5))
    ax1 = fig1.add_axes([0.1, 0.1, 0.7, 0.7])

    for s in green_veg_spectrums:
        ax1.plot(s.wavelengths, s.reflectances, color = 'green')
    ax1.axvline(x = gvd.ndvi_nir_range[0], ymin = 0.0, ymax = 1.0, 
                color = 'cyan')
    ax1.axvline(x = gvd.ndvi_nir_range[1], ymin = 0.0, ymax = 1.0, 
                color = 'cyan')
    ax1.axvline(x = gvd.ndvi_red_range[0], ymin = 0.0, ymax = 1.0, 
                color = 'yellow')
    ax1.axvline(x = gvd.ndvi_red_range[1], ymin = 0.0, ymax = 1.0, 
                color = 'yellow')

    ax1.set_xlim(left = 250, right = 2600)
    ax1.set_ylim(bottom = -0.1, top = 1.1)
    ax1.set_xlabel("Wavelengths", fontsize = 12)
    ax1.set_ylabel("Refletances", fontsize = 12)
    ax1.set_title("{} Green vegetation spectra".format(ext), fontsize = 15)
#    fig1.draw()

    #plotting the non-green vegetation spectrums
    fig2 = plt.figure(figsize = (18, 5))
    ax2 = fig2.add_axes([0.1, 0.1, 0.7, 0.7])

    for s in nongreen_veg_spectrums:
        ax2.plot(s.wavelengths, s.reflectances, color = 'red')
    ax2.axvline(x = gvd.ndvi_nir_range[0], ymin = 0.0, ymax = 1.0, 
                color = 'cyan')
    ax2.axvline(x = gvd.ndvi_nir_range[1], ymin = 0.0, ymax = 1.0, 
                color = 'cyan')
    ax2.axvline(x = gvd.ndvi_red_range[0], ymin = 0.0, ymax = 1.0, 
                color = 'yellow')
    ax2.axvline(x = gvd.ndvi_red_range[1], ymin = 0.0, ymax = 1.0, 
                color = 'yellow')

    ax2.set_xlim(left = 250, right = 2600)
    ax2.set_ylim(bottom = -0.1, top = 1.1)
    ax2.set_xlabel("Wavelengths", fontsize = 12)
    ax2.set_ylabel("Refletances", fontsize = 12)
    ax2.set_title("{} Non vegetation spectra".format(ext), fontsize = 15)
#    fig2.draw()

    #plotting ndvi and reflectance range
    fig3, axs3 = plt.subplots(1, 2, figsize = (16, 5))

    #plotting the ndvi values
    axs3[0].scatter(range(len(ndvis)), ndvis, color = 'black', alpha = 0.8)
    axs3[0].axhline(y = gvd.ndvi_thresh, xmin = 0.0, xmax = 1.0, color = 'red')
    axs3[0].set_xlim(left = 0, right = len(ndvis))
    axs3[0].set_ylim(bottom = -1.0, top = 1.0)
    axs3[0].set_xlabel("Spectrum index", fontsize = 12)
    axs3[0].set_ylabel("NDVI", fontsize = 12)
    axs3[0].set_title("{} spectra NDVIs".format(ext), fontsize = 15)


    #plotting the reflectance range values
    axs3[1].scatter(range(len(reflectance_ranges)), 
                    reflectance_ranges, 
                    color = 'black', alpha = 0.8)
    axs3[1].axhline(y = gvd.reflectance_range_thresh, 
                    xmin = 0.0, xmax = 1.0, color = 'red')
    axs3[1].set_xlim(left = 0, right = len(reflectance_ranges))
    axs3[1].set_ylim(bottom = -0.1, top = 1.0)
    axs3[1].set_xlabel("Spectrum index", fontsize = 12)
    axs3[1].set_ylabel("Reflectance range", fontsize = 12)
    axs3[1].set_title("{} spectra reflectance ranges".format(ext), fontsize = 15)
    
    plt.draw()
def process_directory(params):
    
    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    pattern = params["grouping_pattern"]
    patt_name = params['grouping_output_name']
    
    #read the spectrums    
    extension = 'sig'
    spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) 
                                             if get_extension(f) == extension]
#    for f in spec_files:
#        print(f)
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: 
        return    
#    print("Data directory: {}".format(input_dir))
#    print("  Read {} {} files".format(len(raw_specs), extension))
#    for s in raw_specs:
#        print(s.idstr)
    
        
    #uniquify wavelengths and interpolate to 1nm
    ohr = OverlapHandler(rstype = 'linear')
    rs_specs = []
    for s in raw_specs:
        if ohr.is_1nm(s):
            rs_specs.append(s)
        else:
            rs_specs.append(ohr.process_overlap(s))
    print("  Uniquified and monotoned spectrums")
    
#    for w in rs_specs[0].wavelengths:
#        print(w)
    
    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    rs_white_specs, rs_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, rs) in zip(raw_specs, rs_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(rs)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            rs_target_specs.append(rs)
        else:
            raw_white_specs.append(raw)
            rs_white_specs.append(rs)
    print("   # white spectra: {}".format(len(rs_white_specs)))
    print("   # target spectra: {}".format(len(rs_target_specs)))

    
    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in rs_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

        
    #identify groups using SpectrumRegex and create SpectrumGroup objects
    grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
#    for k in grps_dict:
#        print(k, len(grps_dict[k]))
#        SpectrumGroup().group(grps_dict[k], k)
    patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
    print("   Grouped spectrums: {} groups".format(len(patt_groups)))    
    
    #create output directories and save data    
    save_data(input_dir, 
              raw_white_specs, 
              raw_target_specs, 
              proc_target_specs,
              patt_name,
              patt_groups)
def process_directory(params):
    
    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    
    #read the spectrums    
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) 
                                             if get_extension(f) in extension]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return    
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))
    
    
    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))
    
    
    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    
    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

    
    #create SpectrumGroup object for raw_target_specs
    raw_target_sg = SpectrumGroup()
    raw_target_sg.group(raw_target_specs, "raw_target")
    raw_target_sg.save_dataframe(input_dir)
    raw_target_sg.save_plots(input_dir)

    #create SpectrumGroup object for raw_white_specs
    raw_white_sg = SpectrumGroup()
    raw_white_sg.group(raw_white_specs, "raw_white")
    raw_white_sg.save_dataframe(input_dir)
    raw_white_sg.save_plots(input_dir)

        
    #create SpectrumGroup object for proc_target_specs
    proc_target_sg = SpectrumGroup()
    proc_target_sg.group(proc_target_specs, "proc_target")
    proc_target_sg.save_dataframe(input_dir)
    proc_target_sg.save_plots(input_dir)
def process_directory(params):
    
    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    pattern = params["grouping_pattern"]
    patt_name = params['grouping_output_name']
    
    #read the spectrums    
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) 
                                             if get_extension(f) in extension]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return    
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))
    
    
    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))
    
    
    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    
    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

        
    #identify groups using SpectrumRegex and create SpectrumGroup objects
    patt_groups = []
    grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
    patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
    print("   Grouped spectrums: {} groups".format(len(patt_groups)))    
    
    #create output directories and save data    
    save_data(input_dir, 
              raw_white_specs, 
              raw_target_specs, 
              proc_target_specs,
              patt_name,
              patt_groups)
Exemplo n.º 7
0
def process_directory(params):

    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]

    #read the spectrums
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [
        os.path.join(input_dir, f) for f in os.listdir(input_dir)
        if get_extension(f) in extension
    ]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))

    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))

    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

    #create SpectrumGroup object for raw_target_specs
    raw_target_sg = SpectrumGroup()
    raw_target_sg.group(raw_target_specs, "raw_target")
    raw_target_sg.save_dataframe(input_dir)
    raw_target_sg.save_plots(input_dir)

    #create SpectrumGroup object for raw_white_specs
    raw_white_sg = SpectrumGroup()
    raw_white_sg.group(raw_white_specs, "raw_white")
    raw_white_sg.save_dataframe(input_dir)
    raw_white_sg.save_plots(input_dir)

    #create SpectrumGroup object for proc_target_specs
    proc_target_sg = SpectrumGroup()
    proc_target_sg.group(proc_target_specs, "proc_target")
    proc_target_sg.save_dataframe(input_dir)
    proc_target_sg.save_plots(input_dir)