コード例 #1
0
def process_directory(params):

    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    pattern = params["grouping_pattern"]
    patt_name = params['grouping_output_name']

    #read the spectrums
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [
        os.path.join(input_dir, f) for f in os.listdir(input_dir)
        if get_extension(f) in extension
    ]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))

    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))

    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

    #identify groups using SpectrumRegex and create SpectrumGroup objects
    patt_groups = []
    grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
    patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
    print("   Grouped spectrums: {} groups".format(len(patt_groups)))

    #create output directories and save data
    save_data(input_dir, raw_white_specs, raw_target_specs, proc_target_specs,
              patt_name, patt_groups)
コード例 #2
0
def test(testsdir, verbose = False):
    statuses = []

    
    #load the spectrums    
    indir = os.path.join(testsdir, "input")
    infs = [os.path.join(indir, "test_asdspec1_njc.txt"),
            os.path.join(indir, "test_asdspec2_njc.txt")]
    exdir = os.path.join(testsdir, "expect")
    exfs = [os.path.join(exdir, "test_asdspec1_jc.txt"),
            os.path.join(exdir, "test_asdspec2_jc.txt")]
    ins = [SdalReader().read_spectrum(f) for f in infs]
    exs = [SdalReader().read_spectrum(f) for f in exfs]

        
    #test 1    
    jumpwaves = [1000, 1800]    
    stablezone = 0
    jcorr = JumpCorrector(jumpwavelengths = jumpwaves, 
                          stablezone = stablezone)
    jcs = [jcorr.correct(s) for s in ins]
    for (e, j) in zip(exs, jcs):
        statuses.append(np.allclose(e.data, j.data, atol = 0.0001))
    
        
    #status message
    if all(statuses):
        print("{}: PASSED".format(__file__))
    else:
        print("{}: FAILED".format(__file__))
        

    #plot for ecosis sdal talk
    jumpwaves = [1000, 1800]
    stablezone = 0
    jcorr = JumpCorrector(jumpwavelengths = jumpwaves, 
                          stablezone = stablezone)
    jcs = [jcorr.correct(s) for s in ins]
    fig1, axes1 = plt.subplots(nrows = 1, ncols = 2)
    axes1[0].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
    axes1[1].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
    axes1[1].plot(jcs[0].wavelengths, jcs[0].reflectances, 'g')
    title = "L: Input, R: Corrected, Jump wavelengths = 1000, 1800"
    fig1.suptitle(title)
    fig1.show()
    fig1.savefig("ecosis-talk-jump-correction.png")
コード例 #3
0
def test(testsdir, verbose=False):
    statuses = []

    #load the spectrums
    indir = os.path.join(testsdir, "input")
    infs = [
        os.path.join(indir, "test_asdspec1_njc.txt"),
        os.path.join(indir, "test_asdspec2_njc.txt")
    ]
    exdir = os.path.join(testsdir, "expect")
    exfs = [
        os.path.join(exdir, "test_asdspec1_jc.txt"),
        os.path.join(exdir, "test_asdspec2_jc.txt")
    ]
    ins = [SdalReader().read_spectrum(f) for f in infs]
    exs = [SdalReader().read_spectrum(f) for f in exfs]

    #test 1
    jumpwaves = [1000, 1800]
    stablezone = 0
    jcorr = JumpCorrector(jumpwavelengths=jumpwaves, stablezone=stablezone)
    jcs = [jcorr.correct(s) for s in ins]
    for (e, j) in zip(exs, jcs):
        statuses.append(np.allclose(e.data, j.data, atol=0.0001))

    #status message
    if all(statuses):
        print("{}: PASSED".format(__file__))
    else:
        print("{}: FAILED".format(__file__))

    #plot for ecosis sdal talk
    jumpwaves = [1000, 1800]
    stablezone = 0
    jcorr = JumpCorrector(jumpwavelengths=jumpwaves, stablezone=stablezone)
    jcs = [jcorr.correct(s) for s in ins]
    fig1, axes1 = plt.subplots(nrows=1, ncols=2)
    axes1[0].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
    axes1[1].plot(ins[0].wavelengths, ins[0].reflectances, 'r')
    axes1[1].plot(jcs[0].wavelengths, jcs[0].reflectances, 'g')
    title = "L: Input, R: Corrected, Jump wavelengths = 1000, 1800"
    fig1.suptitle(title)
    fig1.show()
    fig1.savefig("ecosis-talk-jump-correction.png")
コード例 #4
0
 #get the files into benas, emnis and vauls
 tdir = "/home/prabu/mycode/sdal/tests_data/test_OutlierDetector"
 idir = os.path.join(tdir, "input")
 ifiles = os.listdir(idir)
 benafs = [os.path.join(idir, f) for f in ifiles if re.search(r'bena', f)]
 emnifs = [os.path.join(idir, f) for f in ifiles if re.search(r'emni', f)]
 vaulfs = [os.path.join(idir, f) for f in ifiles if re.search(r'vaul', f)]
 
 #read in the spectra
 benas = [SdalReader().read_spectrum(f) for f in benafs] 
 emnis = [SdalReader().read_spectrum(f) for f in emnifs]
 vauls = [SdalReader().read_spectrum(f) for f in vaulfs]
 specs = [benas, emnis, vauls]
 
 #jump correct
 jumpcorr = JumpCorrector(jumpwavelengths = [1000, 1800], stablezone = 0)
 benajcs = [jumpcorr.correct(s) for s in benas]
 emnijcs = [jumpcorr.correct(s) for s in emnis]
 vauljcs = [jumpcorr.correct(s) for s in vauls]
 specjcs = [benajcs, emnijcs, vauljcs]
 
 #compute outliers and inliers
 medians = []
 stds = []
 thresh = 3.0
 od = OutlierDetector(zthresh = thresh)
 for e in specjcs:
     (inliers, outliers) = od.detect(e)
     medians.append(od.median)
     stds.append(od.std)
     
コード例 #5
0
def process(params):
#    params.print_params()
    #get the project params and verify
    project = params.get_params("project")
    if project:
        verify_project(project)
    else:
        print("--project is required")
        sys.exit(0)
    #get the resampling params
    resampling = params.get_params("resampling")
    #get the jumpcorrection params and verify
    jumpcorrection = params.get_params("jumpcorrection")
    if jumpcorrection:
        verify_jumpcorrection(jumpcorrection)
    
    #get the groupings and verify them
    groupings = {grp:params.get_params(grp) for grp in params.get_groups()}
    verify_groupings(params.default_group, params.get_groups(), groupings)

    tags = ["raw", params.default_group]
    specs = defaultdict(list)
    #specs["raw"] created
    #get the filenames
    allfiles = os.listdir(project["indir"])
    extfiles = []
    for f in allfiles:
        ext = get_directory_filename_extension(f)[2]
        if ext == project["fileext"]:
            extfiles.append(os.path.join(project["indir"], f))
    #read the raw spectrums
    uniquifier = WaveUniquifier()
    rawspecs = [SdalReader().read_spectrum(f) for f in extfiles]
    uniqspecs = [uniquifier.uniquify(s) for s in rawspecs]
    specs["raw"] = uniqspecs
    
    #specs["preproc"] created
    #do the pre-processing
    prepspecs = specs["raw"]
    if resampling:
        resampler = WaveResampler(rstype = resampling["type"],
                                  wavestart = resampling["range"][0],
                                  wavestop = resampling["range"][1],
                                  spacing = resampling["spacing"])
        rsspecs = [resampler.resample(s) for s in prepspecs]
        prepspecs = rsspecs
    if jumpcorrection:
        corrector = JumpCorrector(jumpcorrection["wavelengths"],
                                  jumpcorrection["stablezone"])
        jcspecs = [corrector.correct(s) for s in prepspecs]
        prepspecs = jcspecs
    #detect the references
    refdet = ReferenceDetector(context = "gveg")
    nonrefs = []
    refs = []
    for s in prepspecs:
        if refdet.is_reference(s):
            refs.append(s)
        else:
            nonrefs.append(s)
    specs[params.default_group] = nonrefs
    
    #specs[group_tag] created
    #do the grouping 
    for t in groupings:
        tags.append(t)
        itag = groupings[t]["intag"]
        patt = groupings[t]["pattern"]
        regex = SpectrumRegex()
        tgrps = regex.make_groups(specs[itag], patt)
        for tg in tgrps:
            sg = SpectrumGroup(spectrums = tgrps[tg])
            ms = sg.mean_spectrum()
            ms.idstr = tg
            specs[t].append(ms)

#    subsets = {grp:params.get_params(grp) for grp in params.get_subsets()}
#    print(subsets)
#    for t in subsets:
#        itag = subsets[t]["intag"]
#        otag = subsets[t]["outtag"]
#        wavestart = subsets[t]["range"][0]
#        wavestop = subsets[t]["range"][1]
#        for s in specs[itag]:
#            subspec = s.wavelength_subset(wavestart, wavestop)
#            subspec.idstr = subspec.idstr + otag
#            print("idstr = {}".format(subspec.idstr))
#            specs[otag].append(subspec)
                           
    #create outputs
    prjdir = os.path.join(project["outdir"], project["name"])
    os.mkdir(prjdir)
    for t in specs:
        tdir = os.path.join(prjdir, t)
        os.mkdir(tdir)
        tgrpfn = "___{}___.csv".format(t)                       
        for s in specs[t]:
            s.write_csv(odir = tdir)
        sg = SpectrumGroup(spectrums = specs[t])
        sg.write_csv(tdir, tgrpfn)
コード例 #6
0
    #get the files into benas, emnis and vauls
    tdir = "/home/prabu/mycode/sdal/tests_data/test_OutlierDetector"
    idir = os.path.join(tdir, "input")
    ifiles = os.listdir(idir)
    benafs = [os.path.join(idir, f) for f in ifiles if re.search(r'bena', f)]
    emnifs = [os.path.join(idir, f) for f in ifiles if re.search(r'emni', f)]
    vaulfs = [os.path.join(idir, f) for f in ifiles if re.search(r'vaul', f)]

    #read in the spectra
    benas = [SdalReader().read_spectrum(f) for f in benafs]
    emnis = [SdalReader().read_spectrum(f) for f in emnifs]
    vauls = [SdalReader().read_spectrum(f) for f in vaulfs]
    specs = [benas, emnis, vauls]

    #jump correct
    jumpcorr = JumpCorrector(jumpwavelengths=[1000, 1800], stablezone=0)
    benajcs = [jumpcorr.correct(s) for s in benas]
    emnijcs = [jumpcorr.correct(s) for s in emnis]
    vauljcs = [jumpcorr.correct(s) for s in vauls]
    specjcs = [benajcs, emnijcs, vauljcs]

    #compute outliers and inliers
    medians = []
    stds = []
    thresh = 3.0
    od = OutlierDetector(zthresh=thresh)
    for e in specjcs:
        (inliers, outliers) = od.detect(e)
        medians.append(od.median)
        stds.append(od.std)
コード例 #7
0
def process_directory(params):
    
    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    
    #read the spectrums    
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) 
                                             if get_extension(f) in extension]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return    
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))
    
    
    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))
    
    
    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    
    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

    
    #create SpectrumGroup object for raw_target_specs
    raw_target_sg = SpectrumGroup()
    raw_target_sg.group(raw_target_specs, "raw_target")
    raw_target_sg.save_dataframe(input_dir)
    raw_target_sg.save_plots(input_dir)

    #create SpectrumGroup object for raw_white_specs
    raw_white_sg = SpectrumGroup()
    raw_white_sg.group(raw_white_specs, "raw_white")
    raw_white_sg.save_dataframe(input_dir)
    raw_white_sg.save_plots(input_dir)

        
    #create SpectrumGroup object for proc_target_specs
    proc_target_sg = SpectrumGroup()
    proc_target_sg.group(proc_target_specs, "proc_target")
    proc_target_sg.save_dataframe(input_dir)
    proc_target_sg.save_plots(input_dir)
コード例 #8
0
def process_directory(params):
    
    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]
    pattern = params["grouping_pattern"]
    patt_name = params['grouping_output_name']
    
    #read the spectrums    
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) 
                                             if get_extension(f) in extension]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return    
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))
    
    
    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))
    
    
    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    
    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

        
    #identify groups using SpectrumRegex and create SpectrumGroup objects
    patt_groups = []
    grps_dict = SpectrumRegex().make_groups(proc_target_specs, pattern)
    patt_groups = [SpectrumGroup().group(grps_dict[k], k) for k in grps_dict]
    print("   Grouped spectrums: {} groups".format(len(patt_groups)))    
    
    #create output directories and save data    
    save_data(input_dir, 
              raw_white_specs, 
              raw_target_specs, 
              proc_target_specs,
              patt_name,
              patt_groups)
コード例 #9
0
def process_directory(params):

    #parameters for processing
    input_dir = params["input_directory"]
    ws = params["subset_wavelength_range"][0]
    we = params["subset_wavelength_range"][1]

    #read the spectrums
    extension = {'asd': 1, 'ASD': 1}
    spec_files = [
        os.path.join(input_dir, f) for f in os.listdir(input_dir)
        if get_extension(f) in extension
    ]
    raw_specs = [SdalReader().read_spectrum(f) for f in spec_files]
    if len(raw_specs) == 0: return
    print("Data directory: {}".format(input_dir))
    print("  Read {} {} files".format(len(raw_specs), extension))

    #perform jump correction
    jumpcorrector = JumpCorrector(params["jumpcorrection_wavelengths"],
                                  params["jumpcorrection_stablezone"])
    jc_specs = [jumpcorrector.correct(s) for s in raw_specs]
    print("   Jump corrected {} spectrums(s)".format(len(jc_specs)))

    #separate into green vegetation and non-green vegetation spectra
    gvd = GreenVegDetector()
    raw_white_specs, raw_target_specs = [], []
    jc_white_specs, jc_target_specs = [], []
    ndvis, reflectance_ranges = [], []
    for (raw, jc) in zip(raw_specs, jc_specs):
        (status, ndvi, reflectance_range) = gvd.is_green_vegetation(jc)
        ndvis.append(ndvi)
        reflectance_ranges.append(reflectance_range)
        if status:
            raw_target_specs.append(raw)
            jc_target_specs.append(jc)
        else:
            raw_white_specs.append(raw)
            jc_white_specs.append(jc)
    print("   # white spectra: {}".format(len(jc_white_specs)))
    print("   # target spectra: {}".format(len(jc_target_specs)))

    #subset to desired wavelength range
    #do this for the jc_target_specs only
    proc_target_specs = [s.wavelength_subset(ws, we) for s in jc_target_specs]
    print("   Subsetted spectra to range: {}, {}".format(ws, we))

    #create SpectrumGroup object for raw_target_specs
    raw_target_sg = SpectrumGroup()
    raw_target_sg.group(raw_target_specs, "raw_target")
    raw_target_sg.save_dataframe(input_dir)
    raw_target_sg.save_plots(input_dir)

    #create SpectrumGroup object for raw_white_specs
    raw_white_sg = SpectrumGroup()
    raw_white_sg.group(raw_white_specs, "raw_white")
    raw_white_sg.save_dataframe(input_dir)
    raw_white_sg.save_plots(input_dir)

    #create SpectrumGroup object for proc_target_specs
    proc_target_sg = SpectrumGroup()
    proc_target_sg.group(proc_target_specs, "proc_target")
    proc_target_sg.save_dataframe(input_dir)
    proc_target_sg.save_plots(input_dir)