def process(self, spectrum_list): proc_list = [] for s in spectrum_list: wr = WaveResampler(rstype = self._interpolation, wavestart = float(math.ceil(s.wavelengths[0])), wavestop = float(math.floor(s.wavelengths[-1])), spacing = 1.0) proc_list.append(wr.resample(s)) return proc_list
def process(self, spectrum_list): proc_list = [] for s in spectrum_list: wr = WaveResampler(rstype=self._interpolation, wavestart=float(math.ceil(s.wavelengths[0])), wavestop=float(math.floor(s.wavelengths[-1])), spacing=1.0) proc_list.append(wr.resample(s)) return proc_list
def process(params): # params.print_params() #get the project params and verify project = params.get_params("project") if project: verify_project(project) else: print("--project is required") sys.exit(0) #get the resampling params resampling = params.get_params("resampling") #get the jumpcorrection params and verify jumpcorrection = params.get_params("jumpcorrection") if jumpcorrection: verify_jumpcorrection(jumpcorrection) #get the groupings and verify them groupings = {grp:params.get_params(grp) for grp in params.get_groups()} verify_groupings(params.default_group, params.get_groups(), groupings) tags = ["raw", params.default_group] specs = defaultdict(list) #specs["raw"] created #get the filenames allfiles = os.listdir(project["indir"]) extfiles = [] for f in allfiles: ext = get_directory_filename_extension(f)[2] if ext == project["fileext"]: extfiles.append(os.path.join(project["indir"], f)) #read the raw spectrums uniquifier = WaveUniquifier() rawspecs = [SdalReader().read_spectrum(f) for f in extfiles] uniqspecs = [uniquifier.uniquify(s) for s in rawspecs] specs["raw"] = uniqspecs #specs["preproc"] created #do the pre-processing prepspecs = specs["raw"] if resampling: resampler = WaveResampler(rstype = resampling["type"], wavestart = resampling["range"][0], wavestop = resampling["range"][1], spacing = resampling["spacing"]) rsspecs = [resampler.resample(s) for s in prepspecs] prepspecs = rsspecs if jumpcorrection: corrector = JumpCorrector(jumpcorrection["wavelengths"], jumpcorrection["stablezone"]) jcspecs = [corrector.correct(s) for s in prepspecs] prepspecs = jcspecs #detect the references refdet = ReferenceDetector(context = "gveg") nonrefs = [] refs = [] for s in prepspecs: if refdet.is_reference(s): refs.append(s) else: nonrefs.append(s) specs[params.default_group] = nonrefs #specs[group_tag] created #do the grouping for t in groupings: tags.append(t) itag = groupings[t]["intag"] patt = groupings[t]["pattern"] regex = SpectrumRegex() tgrps = regex.make_groups(specs[itag], patt) for tg in tgrps: sg = SpectrumGroup(spectrums = tgrps[tg]) ms = sg.mean_spectrum() ms.idstr = tg specs[t].append(ms) # subsets = {grp:params.get_params(grp) for grp in params.get_subsets()} # print(subsets) # for t in subsets: # itag = subsets[t]["intag"] # otag = subsets[t]["outtag"] # wavestart = subsets[t]["range"][0] # wavestop = subsets[t]["range"][1] # for s in specs[itag]: # subspec = s.wavelength_subset(wavestart, wavestop) # subspec.idstr = subspec.idstr + otag # print("idstr = {}".format(subspec.idstr)) # specs[otag].append(subspec) #create outputs prjdir = os.path.join(project["outdir"], project["name"]) os.mkdir(prjdir) for t in specs: tdir = os.path.join(prjdir, t) os.mkdir(tdir) tgrpfn = "___{}___.csv".format(t) for s in specs[t]: s.write_csv(odir = tdir) sg = SpectrumGroup(spectrums = specs[t]) sg.write_csv(tdir, tgrpfn)
def process_overlap(self, spec): #find the forward difference for the wavelengths diffs = np.diff(spec.wavelengths) #find where the wavelength differences are negative idxs = np.nonzero(diffs <= -0.05)[0] idxs = idxs + 1 idxs = np.hstack((np.array([0]), idxs, np.size(spec.wavelengths))) #create pieces os spectrums with increasing wavelengths pcs = [] data = spec.data uniquifier = WaveUniquifier() for k in range(1, len(idxs)): i1 = idxs[k - 1] i2 = idxs[k] s = Spectrum(data = data[i1:i2, :], idstr = spec.idstr, company = spec.company, instrument = spec.instrument) pcs.append(uniquifier.uniquify(s)) #resample the pieces into 1 nm wavelengths rspcs = [] resampler = WaveResampler() for s in pcs: wr = s.wavelength_range() start = math.ceil(wr[0]) stop = math.floor(wr[1]) resampler = WaveResampler(rstype = self._rstype, wavestart = start, wavestop = stop, spacing = 1.0) rspcs.append(resampler.resample(s)) # print(rspcs[-1].wavelengths[0], rspcs[-1].wavelengths[-1]) # print("------------------------") #chop and stitch if len(rspcs) > 1: #find the wavelengths to chop at critwaves = [rspcs[0].wavelengths[0]] for i in range(1, len(rspcs)): #find the overlapping indices lstart, lstop, rstart, rstop = -1, -1, -1, -1 rstart = 0 lstart = closest_array_index(rspcs[i].wavelengths[0], rspcs[i - 1].wavelengths) lstop = len(rspcs[i - 1].wavelengths) rstop = closest_array_index(rspcs[i - 1].wavelengths[-1], rspcs[i].wavelengths) + 1 lrefls = rspcs[i - 1].reflectances[lstart:lstop] rrefls = rspcs[i].reflectances[rstart:rstop] lwaves = rspcs[i - 1].wavelengths[lstart:lstop] critwaves.append(lwaves[np.argmin(np.abs(lrefls - rrefls))]) critwaves.append(rspcs[-1].wavelengths[-1]) # print("critwaves = {}".format(critwaves)) subdms = [] for i in range(len(rspcs)): start = closest_array_index(critwaves[i], rspcs[i].wavelengths) stop = closest_array_index(critwaves[i + 1], rspcs[i].wavelengths) + 1 subdms.append(rspcs[i].data[start:stop, :]) # print(rspcs[i].data[start:stop, :]) # print("========================") return uniquifier.uniquify(Spectrum(data = np.vstack(tuple(subdms)), idstr = spec.idstr, company = spec.company, instrument = spec.instrument)) else: return rspcs[0]
def test(testsdir, verbose = False): idir = os.path.join(testsdir, "input") edir = os.path.join(testsdir, "expect") statuses = [] #TEST 1: #resampling sig data #--create filenames ifs = ["input_sig_test1.sig", "input_sig_test2.sig", "input_sig_test3.sig"] ifs = [os.path.join(idir, f) for f in ifs] efs = ["expect_sig_test1.sig", "expect_sig_test2.sig", "expect_sig_test3.sig"] efs = [os.path.join(edir, f) for f in efs] #--load the spectrums ins = [SdalReader().read_spectrum(f) for f in ifs] uniquifier = WaveUniquifier() ins = [uniquifier.uniquify(s) for s in ins] exs = [SdalReader().read_spectrum(f) for f in efs] #--resample the input spectrums wavestart = 350.0 wavestop = 2500.0 resampler = WaveResampler(rstype = "linear", wavestart = wavestart, wavestop = wavestop, spacing = 1.0) inrs = [resampler.resample(s) for s in ins] #--subset input resampled and expect spectrums subexs = [s.wavelength_subset(wavestart, wavestop) for s in exs] #--do the comparisons testcount = 0 for (si, se) in zip(inrs, subexs): testcount = testcount + 1 statuses.append(np.allclose(si.data, se.data, atol = 0.001)) if not all(statuses): print("Failed TEST 1-{}".format(testcount)) #TEST 2: #resampling sed data #--create filenames ifs = ["input_sed_test1.sed", "input_sed_test2.sed", "input_sed_test3.sed"] ifs = [os.path.join(idir, f) for f in ifs] efs = ["expect_sed_test1.sed", "expect_sed_test2.sed", "expect_sed_test3.sed"] efs = [os.path.join(edir, f) for f in efs] #--load the spectrums ins = [SdalReader().read_spectrum(f) for f in ifs] uniquifier = WaveUniquifier() ins = [uniquifier.uniquify(s) for s in ins] exs = [SdalReader().read_spectrum(f) for f in efs] #--resample the input spectrums wavestart = 350.0 wavestop = 2500.0 resampler = WaveResampler(rstype = "cubic", wavestart = wavestart, wavestop = wavestop, spacing = 1.0) inrs = [resampler.resample(s) for s in ins] #--subset input resampled and expect spectrums subexs = [s.wavelength_subset(wavestart, wavestop) for s in exs] #--do the comparisons testcount = 0 for (si, se) in zip(inrs, subexs): testcount = testcount + 1 statuses.append(np.allclose(si.data, se.data, atol = 0.01)) if not all(statuses): print("Failed TEST 2-{}".format(testcount)) #TEST : 3 #resampling sig data #--create filenames ifs = ["input_sig_test1.sig", "input_sig_test2.sig", "input_sig_test3.sig"] ifs = [os.path.join(idir, f) for f in ifs] #--load the spectrums ins = [SdalReader().read_spectrum(f) for f in ifs] uniquifier = WaveUniquifier() ins = [uniquifier.uniquify(s) for s in ins] #--resample the input spectrums resampler = WaveResampler(rstype = "linear", wavestart = 338.0, wavestop = 2500.0, spacing = 2.0) inrs = [resampler.resample(s) for s in ins] #status message if all(statuses): print("{}: PASSED".format(__file__)) else: print("{}: FAILED".format(__file__))
def test(testsdir, verbose=False): idir = os.path.join(testsdir, "input") edir = os.path.join(testsdir, "expect") statuses = [] #TEST 1: #resampling sig data #--create filenames ifs = ["input_sig_test1.sig", "input_sig_test2.sig", "input_sig_test3.sig"] ifs = [os.path.join(idir, f) for f in ifs] efs = [ "expect_sig_test1.sig", "expect_sig_test2.sig", "expect_sig_test3.sig" ] efs = [os.path.join(edir, f) for f in efs] #--load the spectrums ins = [SdalReader().read_spectrum(f) for f in ifs] uniquifier = WaveUniquifier() ins = [uniquifier.uniquify(s) for s in ins] exs = [SdalReader().read_spectrum(f) for f in efs] #--resample the input spectrums wavestart = 350.0 wavestop = 2500.0 resampler = WaveResampler(rstype="linear", wavestart=wavestart, wavestop=wavestop, spacing=1.0) inrs = [resampler.resample(s) for s in ins] #--subset input resampled and expect spectrums subexs = [s.wavelength_subset(wavestart, wavestop) for s in exs] #--do the comparisons testcount = 0 for (si, se) in zip(inrs, subexs): testcount = testcount + 1 statuses.append(np.allclose(si.data, se.data, atol=0.001)) if not all(statuses): print("Failed TEST 1-{}".format(testcount)) #TEST 2: #resampling sed data #--create filenames ifs = ["input_sed_test1.sed", "input_sed_test2.sed", "input_sed_test3.sed"] ifs = [os.path.join(idir, f) for f in ifs] efs = [ "expect_sed_test1.sed", "expect_sed_test2.sed", "expect_sed_test3.sed" ] efs = [os.path.join(edir, f) for f in efs] #--load the spectrums ins = [SdalReader().read_spectrum(f) for f in ifs] uniquifier = WaveUniquifier() ins = [uniquifier.uniquify(s) for s in ins] exs = [SdalReader().read_spectrum(f) for f in efs] #--resample the input spectrums wavestart = 350.0 wavestop = 2500.0 resampler = WaveResampler(rstype="cubic", wavestart=wavestart, wavestop=wavestop, spacing=1.0) inrs = [resampler.resample(s) for s in ins] #--subset input resampled and expect spectrums subexs = [s.wavelength_subset(wavestart, wavestop) for s in exs] #--do the comparisons testcount = 0 for (si, se) in zip(inrs, subexs): testcount = testcount + 1 statuses.append(np.allclose(si.data, se.data, atol=0.01)) if not all(statuses): print("Failed TEST 2-{}".format(testcount)) #TEST : 3 #resampling sig data #--create filenames ifs = ["input_sig_test1.sig", "input_sig_test2.sig", "input_sig_test3.sig"] ifs = [os.path.join(idir, f) for f in ifs] #--load the spectrums ins = [SdalReader().read_spectrum(f) for f in ifs] uniquifier = WaveUniquifier() ins = [uniquifier.uniquify(s) for s in ins] #--resample the input spectrums resampler = WaveResampler(rstype="linear", wavestart=338.0, wavestop=2500.0, spacing=2.0) inrs = [resampler.resample(s) for s in ins] #status message if all(statuses): print("{}: PASSED".format(__file__)) else: print("{}: FAILED".format(__file__))