def dofit(self): print("Press Enter to input order") order = raw_input("Order: ") order = int(order) lines = scipy.empty(len(self.lines)) wave = scipy.empty(len(self.ids)) for i in range(lines.size): lines[i] = self.lines[i] wave[i] = self.ids[i] fit = scipy.empty((lines.size, 2)) fit[:, 0] = lines.copy() fit[:, 1] = wave.copy() self.solution = sf.lsqfit(fit, 'chebyshev', order) fit[:, 0] = wave.copy() fit[:, 1] = lines.copy() self.inverse = sf.lsqfit(fit, 'chebyshev', order) self.wave = sf.genfunc(self.points, 0., self.solution) pylab.close() self.__call__() for i in self.markers: wave = sf.genfunc(i, 0., self.solution) verts = self.markers[i].get_verts() bottom = verts[0][1] top = verts[1][1] wave = wave[0] mark = mpl.patches.Polygon([(wave, bottom), (wave, top)]) self.markers[wave] = mark self.ax.add_patch(mark) del self.markers[i] pylab.draw()
def combine_xyw(coords, xsoln, ysoln, wsoln, xord, yord): x = coords[1].flatten() y = coords[0].flatten() newy = ysoln.flatten() from scipy import random k = random.random(x.size) args = k.argsort() x = x[args[:y.size / 10]] newy = newy[args[:y.size / 10]] y = y[args[:y.size / 10]] newx = sf.genfunc(x, y, xsoln['back']) wave = sf.genfunc(newx, 0., wsoln) data = scipy.empty((x.size, 3)) data[:, 0] = wave.copy() data[:, 1] = y.copy() data[:, 2] = x.copy() output_to_ccdx = sf.lsqfit(data, 'chebyshev', xord, yord) data[:, 2] = newy.copy() output_to_ccdy = sf.lsqfit(data, 'chebyshev', xord, yord) data[:, 0] = x.copy() data[:, 1] = y.copy() data[:, 2] = wave.copy() ccdx_ycor_to_wave = sf.lsqfit(data, 'chebyshev', xord, yord) return { 'sky2x': output_to_ccdx, 'sky2y': output_to_ccdy, 'ccd2wave': ccdx_ycor_to_wave }
def getWsoln(sky, x, wsolution, wmodel): def refine(p, x, d, sky, model): pars = {'coeff': numpy.atleast_2d(p).T, 'type': 'polynomial'} w = sf.genfunc(x, 0., pars) c = (w > wmodel['blue']) & (w < wmodel['red']) mod = interpolate.splev(w[c], model) mod /= mod.mean() return (d[c] - mod) / abs(sky[c] + mod)**0.5 # For `by hand' fitting if wsolution is None: import id_spec wsolution = id_spec.id_spec(sky, wmodel) x0 = numpy.arange(sky.size) w = sf.genfunc(x0, 0., wsolution) wsolution = sf.lsqfit(numpy.array([x, w]).T, 'polynomial', 3) else: import pylab tmpD = sky.copy() tmpD /= sky.mean() sky /= sky.mean()**2 T = FFTFilter(tmpD, 100) Dlines = spf.get_lines(x, T, nstd=7.) Dwlines = sf.genfunc(Dlines, 0., wsolution) w = sf.genfunc(x, 0., wsolution) c = (w > wmodel['blue']) & (w < wmodel['red']) mod = interpolate.splev(w[c], wmodel['model']) Slines = spf.get_lines(w[c], mod, nstd=7.) matches = [] for j in range(Dwlines.size): diff = abs(Dwlines[j] - Slines) if diff.min() < 5. * wmodel['scale']: matches.append([Dlines[j], Slines[diff.argmin()]]) wsolution = sf.lsqfit(numpy.asarray(matches), 'polynomial', 3) start = wsolution['coeff'].flatten() coeff, ier = optimize.leastsq(refine, start, (x, tmpD, sky, wmodel['model']), epsfcn=1e-5, maxfev=10000, ftol=1e-13, xtol=1e-13) wsolution = {'coeff': numpy.atleast_2d(coeff).T, 'type': 'polynomial'} w = sf.genfunc(x, 0., wsolution) c = (w > wmodel['blue']) & (w < wmodel['red']) mod = interpolate.splev(w[c], wmodel['model']) mod /= mod.mean() pylab.plot(w[c], mod) pylab.plot(w[c], tmpD[c]) pylab.show() w = sf.genfunc(x, 0., wsolution) rwsoln = sf.lsqfit(numpy.array([w, x]).T, 'polynomial', 3) return wsolution, rwsoln
def wave_arclines(arc, arcmodel, sky, solution): from scipy import interpolate STD_LINES = scipy.sort(arcmodel['lines']) SKYLINES = [5577.338, 6300.304, 6363.78, 6553.617] #,6912.62] x = scipy.arange(arc.size).astype(scipy.float32) lines = get_lines(x, arc) skycoords = get_lines(x, sky) arccoords = lines.copy() scale = arcmodel['scale'] if scale > 1.5: SKYLINES.insert(1, 5891.) order = solution['coeff'].size - 1 fit = solution w = sf.genfunc(lines, 0., fit) matches = [] for line in STD_LINES: diff = abs(w - line) if diff.min() < 5. * scale: pos = diff.argmin() matches.append([lines[pos], line]) print matches, order fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order) order = 1 for i in range(7): matched = [m for m in matches] if skycoords.size == 0: offset = 0. break skyline = sf.genfunc(skycoords, 0., fit) offsets = [] for line in SKYLINES[:i * 2 + 2]: diff = abs(line - skyline) if diff.min() < 5. * scale: offsets.append(line - skyline[diff.argmin()]) if len(offsets) == 0: offset = 0. break offset = scipy.median(scipy.asarray(offsets)) for line in SKYLINES[:i * 2 + 2]: diff = abs(line - skyline) if diff.min() < 5. * scale: pos = diff.argmin() matched.append([skycoords[pos], line - offset]) fit = sf.lsqfit(scipy.asarray(matched), 'polynomial', order) fit['coeff'][0] += offset return fit
def new_xsoln(data, xord=3, yord=3, tol=2.): data = data.copy() height, width = data.shape indx = height / 4 mid = data[indx] xvals = scipy.arange(mid.size) middle_lines = get_lines(xvals, mid, False) ref = [] for row in range(height): ref.append(get_lines(xvals, data[row], False)) lines = [] for l in middle_lines: """ off is used to track the systematic shift of lines as we move up and down the slit. This is necessary for strongly curved lines that might be substantially shifted wrt the central row. """ off = 0. for row in range(indx, height): # The top of the slit if len(ref[row]) == 0: continue p = ref[row] + off diff = abs(l - p) if diff.min() < tol: pos = diff.argmin() lines.append([ref[row][pos], row, l]) off = l - ref[row][pos] off = 0. for row in range(indx - 1, -1, -1): # The bottom of the slit if len(ref[row]) == 0: continue p = ref[row] + off diff = abs(l - p) if diff.min() < tol: pos = diff.argmin() lines.append([ref[row][pos], row, l]) off = l - ref[row][pos] lines = scipy.asarray(lines) soln = sf.lsqfit(lines, 'chebyshev', xord, yord) tmp = lines[:, 0].copy() lines[:, 0] = lines[:, 2].copy() lines[:, 2] = tmp.copy() soln2 = sf.lsqfit(lines, 'chebyshev', xord, yord) return {'back': soln, 'forw': soln2}
def matchlines(peaks, solution, linefile, tol=30., order=3, offset=False): wave = special_functions.genfunc(peaks, 0., solution) try: lines = getlines(linefile) except: lines = linefile gooddata = [] goodlines = [] for i in range(wave.size): delta = 1e9 match = None for j in lines: if abs(j - wave[i]) < delta: delta = abs(j - wave[i]) match = j else: break if delta < tol: gooddata.append(peaks[i]) goodlines.append(match) fitdata = scipy.empty((len(gooddata), 2)) fitdata[:, 0] = scipy.asarray(gooddata) fitdata[:, 1] = scipy.asarray(goodlines) if offset: a = special_functions.genfunc(fitdata[:, 0], 0., solution) return fitdata[:, 1] - a return special_functions.lsqfit(fitdata, 'chebyshev', order)
def update_xsoln(data, soln, xord=3, yord=3, skip=False): data = data.copy() height, width = data.shape if skip: slice = data.mean(1) indx = slice.argsort()[slice.size / 4] mid = data[indx] badrow = scipy.where(slice > slice[indx] + 5. * (slice[indx])**0.5)[0] if badrow.size == 0: badrow = scipy.array([-1]) else: indx = height / 2 mid = data[indx] badrow = scipy.array([-1]) xvals = scipy.arange(mid.size) middle_lines = get_lines(xvals, mid, False) straight = sf.genfunc(middle_lines, indx, soln['back']) ref = [] lines = [] for row in range(height): if abs(row - badrow).min() == 0: continue current = get_lines(xvals, data[row], False) if current.size == 0: continue guess = sf.genfunc(current, row, soln['back']) for l in straight: diff = abs(l - guess) if diff.min() < 2: pos = diff.argmin() lines.append([current[pos], row, l]) lines = scipy.asarray(lines) newsoln = sf.lsqfit(lines, 'chebyshev', xord, yord) tmp = lines[:, 0].copy() lines[:, 0] = lines[:, 2].copy() lines[:, 2] = tmp.copy() del tmp newsoln2 = sf.lsqfit(lines, 'chebyshev', xord, yord) return {'back': newsoln, 'forw': newsoln2}
def doublematch(peaks, w, linefile, skyin, tol=30., order=3, shift=0., logfile=None): lines = scipy.asarray(skyin) lines.sort() p = peaks[0] wave = special_functions.genfunc(p, 0., w) goodmatches = [] for i in range(wave.size): delta = 1e9 match = None for j in lines: if abs(j - wave[i]) < delta: delta = abs(j - wave[i] + shift) match = j else: break if delta < tol: goodmatches.append([p[i], match]) p = peaks[1] wave = special_functions.genfunc(p, 0., w) lines = scipy.asarray(linefile) # lines = getlines(linefile) lines.sort() for i in range(wave.size): delta = 1e9 match = None for arc in lines: if abs(arc - wave[i]) < delta: delta = abs(arc - wave[i] + shift) match = arc else: break if delta < tol: goodmatches.append([p[i], match]) fitdata = scipy.asarray(goodmatches) fit = special_functions.lsqfit(fitdata, 'chebyshev', order) match = special_functions.genfunc(fitdata[:, 0], 0., fit) diff = match - fitdata[:, 1] error = stats.stats.std(diff) if logfile is not None: logentry = "\tWavelength solution error: %5.3f angstroms from %d lines\n" % ( error, diff.size) for i in range(diff.size): logentry += "\t\t%7.2f %7.2f %5.2f\n" % (fitdata[i, 1], match[i], diff[i]) keeplog(logfile, logentry) return fit, error
def wave_skylines(sky, solution): STD_LINES = [ 5197.928, 5200.286, 5202.977, 5460.735, 5577.345, 5867.5522, 5915.308, 5932.864, 6257.970, 6300.320, 6363.810, 6533.040, 6553.610, 6863.971, 6912.620, 6923.210, 6939.520, 7303.716, 7329.148, 7340.885, 7358.659, 7392.198, 7586.093, 7808.467, 7821.510, 7841.266, 7993.332, 8310.719, 8344.613, 8399.160, 8415.231, 8430.170, 8791.186, 8885.830, 8943.395, 8988.384, 9038.059, 9337.854, 9375.977, 9419.746, 9439.670, 9458.524 ] x = scipy.arange(sky.size).astype(scipy.float32) lines = get_lines(x, sky) scale = solution['coeff'][1] order = solution['coeff'].size - 1 if scale > 1.5: STD_LINES.insert(6, 5891.) w = sf.genfunc(lines, 0., solution) matches = [] for i in range(w.size): diff = abs(w[i] - STD_LINES) if diff.min() < 5. * scale: matches.append([lines[i], STD_LINES[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order) w = sf.genfunc(lines, 0., fit) matches = [] for i in range(w.size): diff = abs(w[i] - STD_LINES) if diff.min() < 5. * scale: matches.append([lines[i], STD_LINES[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order) lines = get_lines(x, sky, nstd=7.) w = sf.genfunc(lines, 0., fit) matches = [] for i in range(w.size): diff = abs(w[i] - STD_LINES) if diff.min() < 3. * scale: matches.append([lines[i], STD_LINES[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order) return fit
def combine_xy(offset, coords, xsoln, ysoln, xord, yord): x = coords[1].flatten() y = coords[0].flatten() newy = ysoln['back'].flatten() newx = sf.genfunc(x, y - offset, xsoln['back']) data = scipy.empty((x.size, 3)) data[:, 0] = x.copy() data[:, 1] = y.copy() data[:, 2] = newx.copy() soln = sf.lsqfit(data, 'chebyshev', xord, yord) tmp = data[:, 0].copy() data[:, 0] = data[:, 2].copy() data[:, 2] = tmp.copy() soln2 = sf.lsqfit(data, 'chebyshev', xord, yord) return {'back': soln, 'forw': soln2}
def combine_xw(coords, xsoln, wsoln, xord, yord): x = coords[1].flatten() y = coords[0].flatten() newx = sf.genfunc(x, y, xsoln['back']) wave = sf.genfunc(newx, 0., wsoln) data = scipy.empty((x.size, 3)) data[:, 0] = x.copy() data[:, 1] = y.copy() data[:, 2] = wave.copy() soln = sf.lsqfit(data, 'chebyshev', xord, yord) tmp = data[:, 0].copy() data[:, 0] = data[:, 2].copy() data[:, 2] = tmp.copy() soln2 = sf.lsqfit(data, 'chebyshev', xord, yord) return {'back': soln, 'forw': soln2}
def medsubtract(image,outname): data = pyfits.open(image)[0].data.copy() if data.ndim==3: data = data[0].copy() tmp = data.copy() tmp[numpy.isnan(tmp)] = 0. tmp -= numpy.sort(tmp,0)[tmp.shape[0]/5] trace = tmp.sum(1) peak = trace.argmax() center = numpy.empty(data.shape[1]) w = center.copy() for i in range(1+center.size/100): b = i*100 e = b+100 if e>center.size: e = center.size if b==e: continue center[b:e] = tmp[:,b:e].sum(1).argmax() bg = center.copy() x = numpy.arange(data.shape[0]) for i in range(center.size): d = tmp[:,i].copy() peak = center[i] if numpy.isnan(d[peak]): center[i] = peak continue fit = numpy.array([0.,d[peak],peak,1.]) cond = ~numpy.isnan(d) input = numpy.empty((d[cond].size,2)) input[:,0] = x[cond].copy() input[:,1] = d[cond].copy() fit,chi = sf.ngaussfit(input,fit) center[i] = fit[2] w[i] = fit[3] fit = sf.lsqfit(ndimage.median_filter(center,17),'polynomial',5) centroid = sf.genfunc(numpy.arange(bg.size),0.,fit) w = numpy.median(w) for i in range(bg.size): d = data[:,i].copy() d[centroid[i]-w*4:centroid[i]+w*4] = numpy.nan data[:,i] -= stats.nanmedian(d) hdu = pyfits.open(image)[0] hdu.data = data.copy() hdu.writeto(outname,clobber=True)
def ycor(img): """ Follow star traces along the x direction to correct y-distortion """ xvals = numpy.arange(img.shape[1]) yvals = numpy.arange(img.shape[0]) mid = spf.get_lines(numpy.arange(img.shape[0]), img[:, yvals.size / 2], False) ref = [] for col in range(xvals.size): ref.append(spf.get_lines(yvals, img[:, col], False)) soln = [] for l in mid: off = 0. dy = [] for col in xrange(xvals.size / 2, xvals.max()): if len(ref[col]) == 0: continue p = ref[col] + off diff = abs(l - p) if diff.min() < 1.5: pos = diff.argmin() dy.append([col, ref[col][pos] - l]) off = l - ref[col][pos] off = 0. for col in xrange(xvals.size / 2 - 1, -1, -1): if len(ref[col]) == 0: continue p = ref[col] + off diff = abs(l - p) if diff.min() < 1.5: pos = diff.argmin() dy.append([col, ref[col][pos] - l]) off = l - ref[col][pos] dy = numpy.asarray(dy) fit = sf.lsqfit(dy, 'chebyshev', 3) soln.append(sf.genfunc(xvals, 0., fit)) soln = numpy.asarray(soln) soln = numpy.median(soln, 0) y = numpy.indices(img.shape).astype(numpy.float32)[0] + soln return y, y - soln * 2
def do_fit(self, fullFit=False): p = self.peaks[self.selected == 1] fitdata = scipy.empty((self.xdata.size, 2)) fitdata[:, 0] = scipy.arange(fitdata.shape[0]) fitdata[:, 1] = self.spec.get_xdata() fitdata = fitdata[(fitdata[:, 0] > p[0] - 20) & (fitdata[:, 0] < p[-1] + 20)] ord = int(raw_input('Enter order of fit: ')) fit = sf.lsqfit(fitdata, 'polynomial', ord) self.soln = fit if fullFit == True: from scipy import interpolate, optimize spec = self.spec.get_ydata() xvals = numpy.arange(spec.size).astype(numpy.float32) def opt(p): p = numpy.array(p) n = p[-1] coeff = numpy.atleast_2d(p[:-1]).T m = {'coeff': coeff, 'type': 'polynomial'} w = sf.genfunc(xvals, 0., m) mod = n * interpolate.splev(w, self.skymodel) return (spec - mod) / abs(spec)**0.5 pars = fit['coeff'].flatten().tolist() pars.append(1.) coeff, ier = optimize.leastsq(opt, pars, maxfev=10000, epsfcn=1e-5) fit['coeff'] = numpy.atleast_2d(coeff[:-1]).T self.soln = fit print "pixel fit complete" else: print "Fit Complete" xdata = sf.genfunc(scipy.arange(self.xdata.size), 0., fit) # self.sky.set_xdata(xdata) self.data.set_xdata(xdata) for i in range(self.datapeaks.size): x = xdata[self.datapeaks[i]] l = self.datalines[i] l.set_xdata([x, x]) plt.draw()
def startrace(stars,orders): from scipy import ndimage if stars.shape[0]>1500: WIDTH = 6 fw = 9 else: WIDTH = 4 fw = 7 nrows = stars.shape[0] ncols = stars.shape[1] # This probably isn't necessary.... background = ndimage.percentile_filter(stars,10.,(37,1)) background[numpy.isnan(background)] = 0. background = ndimage.median_filter(background,7) smooth = ndimage.gaussian_filter1d(stars-background,1.5,0) stars -= background start = stars[:,int(ncols/2)].copy() # Find the traces down the middle of the image peaks = ndimage.maximum_filter(start,fw) peaks2 = ndimage.maximum_filter(start,3) avg,sig = clip(start,3.) thresh = sig*5. all = numpy.where((peaks==start)&(peaks>thresh)&(peaks==peaks2))[0] all = all.tolist() yvals = numpy.arange(nrows)*1. xvals = numpy.arange(ncols)*1. wideorders = [] solutions = [] # Loop through the orders, following the traces in each order for i in range(len(orders)): start,end = orders[i] while all[0]<start: del all[0] ordermatches = numpy.empty((0,3)) while all[0]<end: peak = all[0] col = int(ncols/2) amp = smooth[peak,col] line = peak matches = [] # Go towards the blue side of the image while col>0: line = int(line) if line<WIDTH or line+WIDTH>=nrows or stars[line-1:line+2,col].max()<amp/20.: break s1 = slice(line-WIDTH,line+WIDTH+1) line = (stars[s1,col]*yvals[s1]).sum()/stars[s1,col].sum() matches.append([col,line,peak]) col -= 1 col = int(ncols/2)+1 line = peak # Go towards the red side of the image while col<ncols: line = int(line) if line<WIDTH or line+WIDTH>=nrows: break s1 = slice(line-WIDTH,line+WIDTH+1) # The red side has some defects.... if numpy.isnan(stars[s1,col]).any(): if i==0: break fitData = numpy.array(matches)[:,:2] fit = lsqfit(fitData,'chebyshev',3) linePos = genfunc(xvals,0.,fit).astype(numpy.int32) while col<ncols-WIDTH and numpy.isnan(stars[s1,col]).any(): col += 1 s1 = slice(linePos[col]-WIDTH,linePos[col]+WIDTH+1) if col>=ncols-WIDTH: break line = int((stars[s1,col]*yvals[s1]).sum()/stars[s1,col].sum()) if stars[line-1:line+2,col].max()<amp/20.: break matches.append([col,line,peak]) col += 1 # Filter garbage; trace should be within ~2pix of the model matches = numpy.array(matches) soln = lsqfit(matches[:,:2],'cheybshev',3) resid = matches[:,1]-genfunc(matches[:,0],0.,soln) good = abs(resid)<2. ordermatches = numpy.concatenate((ordermatches,matches[good])) del all[0] matches = ordermatches ord1,ord2 = 3,3 ymap = lsqfit(matches,"chebyshev",ord1,ord2) # Clip bad points resid = matches[:,2]-genfunc(matches[:,0],matches[:,1],ymap) m,s = clip(resid,3.5) good = abs(resid-m)<3.5*s matches = matches[good] ymap = lsqfit(matches,"chebyshev",ord1,ord2) tmp = matches[:,1].copy() matches[:,1] = matches[:,2].copy() matches[:,2] = tmp.copy() ytrue = lsqfit(matches,"chebyshev",ord1,ord2) solutions.append([ytrue,ymap]) lower = int(genfunc(xvals,xvals*0.+start-3,ytrue).min()) upper = int(genfunc(xvals,xvals*0+end+3,ytrue).max())+1 if lower<0: lower = 0 if upper>stars.shape[1]: upper = stars.shape[1] wideorders.append([lower,upper]) return solutions,wideorders
def solve(d,orders): path = os.path.split(__file__)[0] lines = {} lines['cuar'] = numpy.loadtxt(path+"/data/cuar.lines") lines['hgne'] = numpy.loadtxt(path+"/data/hgne.lines") lines['xe'] = numpy.loadtxt(path+"/data/xe.lines") #startsoln = numpy.load(path+"/data/test_wavesol.dat", # allow_pickle=True) startsoln = numpy.load(path+"/data/esi_wavesolution.dat", allow_pickle=True) #arclist = d.keys() # Under python 3 this does not produce a list # and so arclist[0] below fails arclist = list(d) arclist.sort() soln = [] if d[arclist[0]].shape[1]>3000: xvals = numpy.arange(4096.) cuslice = slice(3860,3940) fw1 = 75. fw2 = 9 else: xvals = numpy.arange(1.,4096.,2.) cuslice = slice(1930,1970) fw1 = 37. fw2 = 5 """ Do a temporary kludge. In some cases, the finding of the orders fails, and only 9 orders are found. In this case, we need to skip the first of the orders """ if len(orders) == 9: ordstart = 1 dord = 1 else: ordstart = 0 dord = 0 for i in range(ordstart, 10): solution = startsoln[i] start,end = orders[(i-dord)] peaks = {} trace = {} fitD = {} WIDTH = 4 import pylab for arc in arclist: data = numpy.nanmedian(d[arc][start:end],axis=0) data[scipy.isnan(data)] = 0. if i==0 and arc=='cuar': data[cuslice] = numpy.median(data) trace[arc] = data.copy() bak = ndimage.percentile_filter(data,50.,int(fw1)) bak = getContinuum(bak,40.) data -= bak fitD[arc] = data/d[arc][start:end].std(0) p = ndimage.maximum_filter(data,fw2) std = clip(scipy.trim_zeros(data),3.)[1] nsig = ndimage.uniform_filter((data>7.*std)*1.,3) peak = scipy.where((nsig==1)&(p>10.*std)&(p==data))[0] peaks[arc] = [] for p in peak: if p-WIDTH<0 or p+WIDTH+1>xvals.size: continue x = xvals[p-WIDTH:p+WIDTH+1].copy()#-xvals[p] f = data[p-WIDTH:p+WIDTH+1].copy() fitdata = scipy.array([x,f]).T fit = scipy.array([0.,f.max(),xvals[p],1.]) fit,chi = sf.ngaussfit(fitdata,fit,weight=1) peaks[arc].append(fit[2])#+xvals[p]) for converge in range(15): wave = 10**sf.genfunc(xvals,0.,solution) refit = [] corrA = {} err = wave[int(wave.size/2)]-wave[int(wave.size/2-1)] for arc in arclist: corr = [] p = 10.**sf.genfunc(peaks[arc],0.,solution) for k in range(p.size): cent = p[k] diff = cent-lines[arc] corr.append(diff[abs(diff).argmin()]) corr = numpy.array(corr) if corr.size<4: continue m,s = clip(corr) corr = numpy.median(corr[abs(corr-m)<5.*s]) print(corr) corrA[arc] = corr # corr = m #for arc in arclist: p = 10.**sf.genfunc(peaks[arc],0.,solution) for k in range(p.size): pos = peaks[arc][k] cent = p[k] diff = abs(cent-lines[arc]-corr) if diff.min()<2.*err: refit.append([pos,lines[arc][diff.argmin()]]) refit = scipy.asarray(refit) solution = sf.lsqfit(refit,'polynomial',3) refit = [] err = solution['coeff'][1] for arc in arclist: data = trace[arc] for pos in peaks[arc]: cent = sf.genfunc(pos,0.,solution) delta = 1e9 match = None for j in lines[arc]: diff = abs(cent-j) if diff<delta and diff<1.*err: delta = diff match = j if match is not None: refit.append([pos,match]) refit = scipy.asarray(refit) refit[:,1] = numpy.log10(refit[:,1]) solution = sf.lsqfit(refit,'chebyshev',3) #refit[:,0],refit[:,1] = refit[:,1].copy(),refit[:,0].copy() #refit = numpy.array([refit[:,1],refit[:,0]]).T #refit = refit[:,::-1] solution2 = sf.lsqfit(refit[:,::-1],'chebyshev',3) #soln.append([solution,solution2]) w = 10**sf.genfunc(xvals,0.,solution) if (w==wave).all() or converge>8: print("Order %d converged in %d iterations"%(i,converge)) soln.append([solution,solution2]) break for arc in arclist: pylab.plot(w,trace[arc]) pylab.plot(w,fitD[arc]) pp = 10**sf.genfunc(peaks[arc],0.,solution) for p in pp: pylab.axvline(p,c='k') for j in 10**refit[:,1]: if j>w[0] and j<w[-1]: pylab.axvline(j) pylab.show() break return soln
def jointSolve(d,orders): import pylab path = __path__[0] lines = {} lines['cuar'] = numpy.loadtxt(path+"/data/cuar.lines") lines['hgne'] = numpy.loadtxt(path+"/data/hgne.lines") lines['xe'] = numpy.loadtxt(path+"/data/xe.lines") startsoln = numpy.load(path+"/data/esi_wavesolution.dat") startsoln = numpy.load(path+'/data/shao_wave.dat') startsoln = [i for i,j in startsoln] alldata = d['arc'] arclist = d.keys() arclist.remove('arc') soln = [] if alldata.shape[1]>3000: xvals = numpy.arange(4096.) resolve = False cuslice = slice(3860,3940) fw1 = 75. fw2 = 9 WIDTH = 4 else: xvals = numpy.arange(2048.) resolve = True cuslice = slice(1930,1970) fw1 = 37. fw2 = 7 WIDTH = 3 for i in range(10): solution = startsoln[i] start,end = orders[i] if resolve==True: tmp = numpy.arange(0.5,4096.,2.) w = sf.genfunc(tmp,0.,solution) solution = sf.lsqfit(numpy.array([xvals,w]).T,'chebyshev',3) data = numpy.nanmedian(alldata[start:end],axis=0) data[numpy.isnan(data)] = 0. if i==0: data[cuslice] = numpy.median(data) bak = ndimage.percentile_filter(data,50.,int(fw1)) data -= bak peaks = [] p = ndimage.maximum_filter(data,fw2) std = clip(numpy.trim_zeros(data),3.)[1] peak = numpy.where((p>30.*std)&(p==data))[0] for p in peak: if p-WIDTH<0 or p+WIDTH+1>xvals.size: continue x = xvals[p-WIDTH:p+WIDTH+1].copy() f = data[p-WIDTH:p+WIDTH+1].copy() fitdata = numpy.array([x,f]).T fit = numpy.array([0.,f.max(),xvals[p],1.]) fit,chi = sf.ngaussfit(fitdata,fit,weight=1) peaks.append(fit[2]) for converge in range(10): wave = 10**sf.genfunc(xvals,0.,solution) refit = [] corr = [] err = wave[wave.size/2]-wave[wave.size/2-1] p = 10.**sf.genfunc(peaks,0.,solution) for arc in arclist: for k in range(p.size): if i==0 and p[k]>4344.: continue cent = p[k] diff = cent-lines[arc] corr.append(diff[abs(diff).argmin()]) corr = numpy.array(corr) corr = corr[abs(corr)<5*err] m,s = clip(corr) corr = numpy.median(corr[abs(corr-m)<5.*s]) for arc in arclist: for k in range(p.size): if i==0 and p[k]>4344.: continue pos = peaks[k] cent = p[k] diff = abs(cent-lines[arc]-corr) if diff.min()<2.*err: refit.append([pos,lines[arc][diff.argmin()]]) refit = numpy.asarray(refit) solution = sf.lsqfit(refit,'polynomial',3) refit = [] err = solution['coeff'][1] p = sf.genfunc(peaks,0.,solution) for k in range(p.size): delta = 1e9 match = None for arc in arclist: for j in lines[arc]: if i==0 and j>4344.: continue diff = abs(p[k]-j) if diff<delta and diff<1.*err: delta = diff match = j if match is not None: refit.append([peaks[k],match]) refit = numpy.asarray(refit) refit[:,1] = numpy.log10(refit[:,1]) solution = sf.lsqfit(refit,'chebyshev',3) solution2 = sf.lsqfit(refit[:,::-1],'chebyshev',3) g = 10**sf.genfunc(peaks,0.,solution) g2 = 10**sf.genfunc(peak,0.,solution) w = 10**sf.genfunc(xvals,0.,solution) if (w==wave).all(): print("Order %d converged in %d iterations"%(i,converge)) soln.append([solution,solution2]) break pylab.plot(w,data) for arc in arclist: for j in lines[arc]: if j>w[0] and j<w[-1]: pylab.axvline(j,c='b') for j in 10**refit[:,1]: if j>w[0] and j<w[-1]: pylab.axvline(j,c='r') for j in g: pylab.axvline(j,c='g') for j in g2: pylab.axvline(j,c='c') pylab.show() break return soln
def bgsub(dir, inname, out_prefix, cal_prefix): # Where things begin and end.... blue = [1500, 1400, 1300, 1200, 1100, 900, 600, 200, 0, 0, 0] red = [3000, 3400, 3700, -1, -1, -1, -1, -1, -1, -1] readvar = 7.3 bias = pyfits.open(cal_prefix + "_bias.fits")[0].data.astype(scipy.float32) bpm = pyfits.open(cal_prefix + "_bpm.fits")[0].data.astype(scipy.float32) flat = pyfits.open(cal_prefix + "_norm.fits")[0].data.astype(scipy.float32) orders, y_soln, wideorders = numpy.load(cal_prefix + "_ycor.dat") fullsoln = numpy.load(cal_prefix + "_full.dat") try: masks = numpy.loadtxt(cal_prefix + "_bpmOrders.dat") print('BPM opened') except: print("Making BPM") mask = numpy.where(bpm == 1., 1., 0.) mask = getOrders(mask, y_soln, orders, wideorders) for i in range(len(mask)): mask[i][0] = numpy.where(mask[i][0] < 0.7, numpy.nan, 1.) f = open(cal_prefix + "_bpmOrders.dat", 'wb') cPickle.dump(mask, f, 2) f.close() hdu = pyfits.open(dir + inname) data = hdu[0].data.copy() data = biastrim(data, bias, bpm) / flat try: back = pyfits.open(out_prefix + "_bg.fits")[0].data.copy() except: strt = straighten(data, y_soln, orders, wideorders) back = scipy.zeros(strt.shape) for indx in range(len(orders)): i, j = orders[indx] bg = stats.stats.nanmedian(strt[i:j], axis=0) back[i:j] += bg back = curve(back, y_soln, orders, wideorders) pyfits.PrimaryHDU(back).writeto(out_prefix + "_bg.fits") try: strt = pyfits.open(out_prefix + "_strt.fits")[0].data.copy() except: bgsub = data - back sub = bgsub.copy() omap = sub * 0. for loop in range(3): message('CR iteration %d ...' % (loop + 1)) map = crFind(sub, data + readvar) if map.sum() == 0: message(' no new CRs found.\n') break else: message(' %d pixels flagged.\n' % (map.sum())) inmask = sub.flatten() cond = numpy.where(map.ravel() == 1)[0] inmask[cond] = numpy.where(cond % 2 == 0, 1e5, -1e5).astype(inmask.dtype) med5 = ndimage.median_filter(inmask.reshape(sub.shape), 7) med5 *= map sub = (1. - map) * sub + med5 oCRS = bgsub - sub omap += map pyfits.PrimaryHDU(oCRS).writeto('blah.fits', clobber=True) omap = omap > 0 sub = numpy.where(numpy.isfinite(bgsub), bgsub, 0.) ccdBG = numpy.where(numpy.isfinite(back), back, 0.) orig = numpy.where(numpy.isfinite(data), data, 0.) readvar = 2.7**2 data -= crmask strt = straighten(data, y_soln, orders) pyfits.PrimaryHDU(strt).writeto(out_prefix + "_strt.fits") hdulist = pyfits.HDUList([pyfits.PrimaryHDU()]) hdulist[0].header = hdu[0].header.copy() varhdu = pyfits.HDUList([pyfits.PrimaryHDU()]) varhdu[0].header = hdu[0].header.copy() for indx in range(len(orders)): i, j = orders[indx] cut = strt[i:j].copy() cut[scipy.isinf(cut)] = scipy.nan bg = stats.stats.nanmedian(cut, axis=0) B = mask[i:j] tmp = (cut - bg) * B T = tmp[3:-3, blue[indx]:red[indx]] T = T[numpy.isfinite(T)] avg, std = clip2(T) slice = stats.stats.nanmedian(tmp[:, blue[indx]:red[indx]], axis=1) slice[:3] = 0. slice[-3:] = 0. avg, std = clip2(slice[3:-3], 3.5, 0.05) good = numpy.where(slice < 2. * std, 1., 0.) good = ndimage.maximum_filter(good, 5) good = ndimage.minimum_filter(good, 15) good = good == 1 xvals = scipy.arange(good.size).astype(scipy.float32) fitdata = scipy.empty((slice[good].size, 2)) fitdata[:, 0] = xvals[good].copy() bgsub = cut.copy() for k in range(cut.shape[1]): fitdata[:, 1] = cut[:, k][good].copy() if fitdata[:, 1][scipy.isfinite(fitdata[:, 1])].size < 4: continue avg, std = clip(fitdata[:, 1]) keep = scipy.where((abs(fitdata[:, 1] - avg) < 4. * std) & scipy.isfinite(fitdata[:, 1]))[0] fit = fitdata[keep] if fit[:, 1][scipy.isfinite(fit[:, 1])].size < 4: continue fit = sf.lsqfit(fit, 'chebyshev', 1) bg = sf.genfunc(xvals, 0, fit) bgsub[:, k] -= bg solution = wave_soln[indx][0] invsoln = wave_soln[indx][1] vals = scipy.arange(cut.shape[1]).astype(scipy.float64) wave = sf.genfunc(vals, 0., solution) delta = (wave[wave.size - 1] - wave[0]) / (wave.size - 1) outc = scipy.arange(wave[0], wave[0] + 2, delta) outc = outc[:cut.shape[1]].copy() pix = sf.genfunc(outc, 0, invsoln) coords = spectools.array_coords(cut.shape).astype(numpy.float64) coords[1] *= 0. coords[1] += pix bgsub[scipy.isnan(bgsub)] = 0. out = ndimage.map_coordinates(bgsub, coords, output=scipy.float64, order=5) hdu = pyfits.ImageHDU(out.copy()) hdu.header.update('CTYPE1', 'WAVE-LOG') hdu.header.update('CRVAL1', wave[0]) hdu.header.update('CRPIX1', 1) hdu.header.update('CDELT1', delta) hdu.header.update('CTYPE2', 'LINEAR') hdu.header.update('CRVAL2', 1) hdu.header.update('CRPIX2', 1) hdu.header.update('CDELT2', 1) hdu.header.update('CD1_1', delta) hdu.header.update('CD2_2', 1) hdu.header.update('CD1_2', 0) hdu.header.update('CD2_1', 0) #hdu.header.update('PC1_1',1) #hdu.header.update('PC1_2',0) #hdu.header.update('PC2_1',0) #hdu.header.update('PC2_2',1) hdu.header.update('DC-FLAG', 1) hdu.header.update('WFITTYPE', 'LOG_LINEAR') hdu.header.update('DISPAXIS', 1) hdulist.append(hdu) cut[scipy.isnan(cut)] = 0. out = ndimage.map_coordinates(cut, coords, output=scipy.float64, order=5) hdu = pyfits.ImageHDU(out.copy() + 5.3) hdu.header.update('CTYPE1', 'WAVE-LOG') hdu.header.update('CRVAL1', wave[0]) hdu.header.update('CRPIX1', 1) hdu.header.update('CD1_1', delta) hdu.header.update('CRVAL2', 1) hdu.header.update('CRPIX2', 1) hdu.header.update('CD2_2', 1) hdu.header.update('DC-FLAG', 1) hdu.header.update('WFITTYPE', 'LOG_LINEAR') hdu.header.update('DISPAXIS', 1) varhdu.append(hdu) hdulist.verify('fix') hdulist.writeto(out_prefix + "_2d.fits") varhdu.verify('fix') varhdu.writeto(out_prefix + "_var.fits")
def skysub(x, y, z, scale): """ skysub(x,y,z,scale) Routine to determine the 2d background from data. (x,y) are the coordinates of the data, usually in the *corrected* frame. Inputs: x - 1d array describing x-coordinate, usually wavelength y - 1d array describing y-coordinate, usually corrected spatial position z - data each position (x,y) scale - approximate output scale (for knot placement). It is not, in general, possible to calculate this from x because the input coordinates are not on a regular grid. Outputs: 2d spline model of the background """ height = int(y.max() - y.min()) width = int(x.max() - x.min()) npoints = x.size midpt = y.mean() """ Very wide slits need special attention. Here we fit a first order correction to the slit and subtract it away before doing the high pixel rejection (the problem is if there is a small gradient across a wide slit, the top and bottom pixels may differ significantly, but these pixels may be close in *wavelength* and so locally (on the CCD) low pixels will be rejected in the smoothing """ if height > WIDE: zbak = z.copy() args = y.argsort() revargs = args.argsort() ymodel = ndimage.percentile_filter(z[args], 30., size=height)[revargs] fit = special_functions.lsqfit(ymodel, 'polynomial', 1) if fit['coeff'][1] * float(ymodel.size) / fit['coeff'][0] < 0.05: pass else: ymodel = special_functions.genfunc(scipy.arange(ymodel.size), 0, fit) ymodel -= ymodel.mean() z -= ymodel # Filter locally (in wavelength space) high points args = x.argsort() revargs = args.argsort() smooth = ndimage.percentile_filter(z[args], 35., size=height)[revargs] diff = z - smooth # We assume poisson statistics.... var = scipy.sqrt(scipy.fabs(z)) sigma = diff / var args = y.argsort() revargs = args.argsort() t = ndimage.median_filter(sigma[args], 9) t = ndimage.gaussian_filter(t, width)[revargs] # Source detection/rejection # Reject yvalues > 1. sigma, and weight remaining pixels w = (1.0 - t) / abs(z) skycond = ((w > 0.) & (z > 0)) x = x[skycond] y = y[skycond] z = z[skycond] # Reject residual high pixels (and very low pixels too!) args = x.argsort() revargs = args.argsort() smooth = ndimage.median_filter(z[args], height / 4.)[revargs] diff = z - smooth var = scipy.sqrt(smooth) cond = abs(diff) < 4. * var x = x[cond] y = y[cond] z = z[cond] kx = 3 ky = 1 # If the slit is long, return to original data and increase the order # of the y-fit. if height > WIDE: z = zbak[skycond] z = z[cond].astype(scipy.float64) if height > WIDE * 1.5: ky = 3 cond = z > 0. x = x[cond] y = y[cond] z = z[cond] w = 1. / z if x.size < 5. * width: kx = 1 ky = 1 # Create knots... innertx = scipy.arange(x.min() + scale / 2., x.max() - scale / 2., scale) tx = scipy.zeros(innertx.size + kx * 2 + 2) tx[0:kx + 1] = x.min() tx[kx + 1:innertx.size + kx + 1] = innertx.copy() tx[innertx.size + kx + 1:] = x.max() ty = scipy.zeros(ky * 2 + 2) ty[0:ky + 1] = y.min() ty[ky + 1:] = y.max() # ...and fit. bgfit = interpolate.bisplrep(x, y, z, w, tx=tx, ty=ty, kx=kx, ky=ky, task=-1, nxest=tx.size, nyest=ty.size, s=0) return bgfit
def skysub(x, y, z, scale): """ skysub(x,y,z,scale) Routine to determine the 2d background from data. (x,y) are the coordinates of the data, usually in the *corrected* frame. Inputs: x - 1d array describing x-coordinate, usually wavelength y - 1d array describing y-coordinate, usually corrected spatial position z - data each position (x,y) scale - approximate output scale (for knot placement). It is not, in general, possible to calculate this from x because the input coordinates are not on a regular grid. Outputs: 2d spline model of the background """ cond = (scipy.isfinite(z)) & (z > 0.) x = x[cond] y = y[cond] z = z[cond] x0 = x.copy() y0 = y.copy() z0 = z.copy() height = int(y.max() - y.min()) width = int(x.max() - x.min()) npoints = x.size midpt = y.mean() """ Very wide slits need special attention. Here we fit a first order correction to the slit and subtract it away before doing the high pixel rejection (the problem is if there is a small gradient across a wide slit, the top and bottom pixels may differ significantly, but these pixels may be close in *wavelength* and so locally (on the CCD) low pixels will be rejected in the smoothing """ if height > WIDE: zbak = z.copy() args = y.argsort() revargs = args.argsort() ymodel = ndimage.percentile_filter(z[args], 30., size=height)[revargs] fit = special_functions.lsqfit(ymodel, 'polynomial', 1) if fit['coeff'][1] * float(ymodel.size) / fit['coeff'][0] < 0.05: pass else: ymodel = special_functions.genfunc(scipy.arange(ymodel.size), 0, fit) ymodel -= ymodel.mean() z -= ymodel # Filter locally (in wavelength space) high points args = x.argsort() revargs = args.argsort() smooth = ndimage.percentile_filter(z[args], 35., size=height)[revargs] diff = z - smooth # We assume poisson statistics.... var = scipy.sqrt(scipy.fabs(z)) sigma = diff / var args = y.argsort() revargs = args.argsort() t = ndimage.median_filter(sigma[args], 9) t = ndimage.gaussian_filter(t, width) #[revargs] # Source detection/rejection # Reject yvalues > 1. sigma, and weight remaining pixels w = (1.0 - t) / abs(z[args]) if AGGRESSIVE: g = scipy.where(w <= 0, 0, 1) g = ndimage.maximum_filter(g, width * 3) g = ndimage.minimum_filter(g, width * 7) s = sigma[args].copy() b = ndimage.minimum_filter(g, width * 5) xi = scipy.arange(t.size) fitdata = scipy.empty((xi[g == 1].size, 2)) fitdata[:, 0] = xi[g == 1].copy() fitdata[:, 1] = t[g == 1].copy() fit = special_functions.lsqfit(fitdata, 'polynomial', 3) fit = special_functions.genfunc(xi, 0., fit) diff = (t - fit)[b == 1] s = diff.std() while (abs(t - fit)[(g == 1) & (b == 0)] > 2.5 * s).any(): g = b.copy() b = ndimage.minimum_filter(g, width * 5) fitdata = scipy.empty((xi[g == 1].size, 2)) fitdata[:, 0] = xi[g == 1].copy() fitdata[:, 1] = t[g == 1].copy() fit = special_functions.lsqfit(fitdata, 'polynomial', 3) fit = special_functions.genfunc(xi, 0., fit) diff = (t - fit)[b == 1] s = diff.std() w *= g skycond = ((w > 0.) & (z > 0)) x = x[skycond] y = y[skycond] z = z[skycond] # Reject residual high pixels (and very low pixels too!) args = x.argsort() revargs = args.argsort() smooth = ndimage.median_filter(z[args], height / 4.)[revargs] diff = z - smooth var = scipy.sqrt(smooth) cond = abs(diff) < 4. * var x = x[cond] y = y[cond] z = z[cond] kx = 3 ky = 1 # If the slit is long, return to original data and increase the order # of the y-fit. if height > WIDE: z = zbak[skycond] z = z[cond].astype(scipy.float64) # if height>WIDE*1.5: # ky = 3 cond = z > 0. x = x[cond] y = y[cond] z = z[cond] w = 1. / z if x.size < 5. * width: kx = 1 ky = 1 # Create knots... innertx = scipy.arange(x.min() + scale / 2., x.max() - scale / 2., 3. * scale / 4.) """ tx = scipy.zeros(innertx.size+kx*2+2) tx[0:kx+1] = x.min() tx[kx+1:innertx.size+kx+1] = innertx.copy() tx[innertx.size+kx+1:] = x.max() """ tx = scipy.linspace(x.min(), x.max(), innertx.size) xsort = scipy.sort(x) tmp = [x.min()] num = [] cnt = 0 j = 1 for i in range(xsort.size): while xsort[i] > tx[j]: if cnt > 0: if len(num) == 0 or cnt > 1 or num[-1] > 1: tmp.append(tx[j]) num.append(cnt) cnt = 0 j += 1 cnt += 1 tmp.append(x.max()) tx = scipy.asarray(tmp) ty = scipy.zeros(ky * 2 + 2) ty[0:ky + 1] = y.min() ty[ky + 1:] = y.max() #del innertx # ...and fit. bgfit = interpolate.bisplrep(x, y, z, w, tx=tx, ty=ty, kx=kx, ky=ky, task=-1, nxest=tx.size, nyest=ty.size) del x, y, z, w, tx, ty return bgfit
def arcmatch(curve,sci,arc,yforw,widemodel,finemodel,goodmodel,linemodel,disp,mswave,extra,logfile): """ Do not alter input data! """ sci = sci.copy() nsci = sci.shape[0] """ Parse extra arguments """ linefile = extra[0] cutoff = extra[1] if len(extra)==3: blue = extra[2] else: blue = 3000. height = arc.shape[0] width = arc.shape[1] """ Straighten science data """ scimodel = scipy.zeros((nsci,height,width)) for k in range(nsci): scimodel[k] = spectools.resampley(sci[k],yforw,curve) """ Skip the bottom and top of the slit, which may have artifacts """ i = 4 j = height-4 xdim = 2 ydim = 2 avg = scipy.median(scipy.median(arc)) m,std = clipped_std(arc,4.) """ The 'fiducial' arc spectrum is taken from the middle of the slit """ arcmid = stats.stats.nanmedian(arc[height/2-3:height/2+4],0) """ First we straighten the lines out. """ coords = spectools.array_coords((j-i,width)) coords[0] += 4. fitdata = arc[i:j,height:-1*height].flatten() xdata = coords[1,:,height:-1*height].flatten() ydata = coords[0,:,height:-1*height].flatten() """ Only use every third row (but at least 8 rows) for the solution to save time """ nskip = (j-i)/8 if nskip>3: nskip = 3 cond = ydata%nskip==0 fitdata = fitdata[cond] ydata = ydata[cond] xdata = xdata[cond] """ Only use pixels around the peaks (another time saver) """ thresh = scipy.median(arcmid) thresh += 5.*thresh**0.5 mask = ndimage.maximum_filter(arcmid,15) mask = fitdata>thresh fitdata = fitdata[mask] ydata = ydata[mask] xdata = xdata[mask] p = scipy.zeros((xdim+1,ydim+1)) p[1,0] = 1. p = {'coeff':p,'type':"chebyshev"} xback = arcfitfunc2(p,xdata,ydata,fitdata,arcmid) """ xdata, ydata, yorig, and newxdata hold original/transformed CCD coordinates. We don't need to do a fit to *all* points, so we skip ~every third. """ xdata = coords[1].flatten() ydata = coords[0].flatten() yorig = yforw[i:j].flatten()-curve cond = (ydata+xdata)%2==0 xdata = xdata[cond] ydata = ydata[cond] yorig = yorig[cond] newxdata = special_functions.genfunc(xdata,ydata,xback) tmp = scipy.zeros((xdata.size,3)) tmp[:,0] = newxdata.copy() tmp[:,1] = ydata.copy() tmp[:,2] = xdata.copy() xforw = special_functions.lsqfit(tmp,"chebyshev",xdim,ydim) """ Sky background model isn't needed if there aren't expected to be any sky lines. """ if cutoff>5200: bgmodel = scipy.zeros((nsci,sci.shape[2])) for k in range(nsci): tmp = spectools.resample1d(scimodel[k],xforw,"x",-999) scimodel[k] = tmp.copy() tmp.sort(axis=0) tmp[tmp==-999] = scipy.nan bg = stats.stats.nanmedian(tmp,axis=0) bg = scipy.tile(bg,(height,1)) tmp[scipy.isnan(tmp)] = bg[scipy.isnan(tmp)] tmp[scipy.isnan(tmp)] = 0. bgmodel[k] = tmp[tmp.shape[0]/4,:] del tmp,bg newarc = spectools.resample1d(arc,xforw,"x",-999) newarc[newarc==-999] = scipy.nan arcmid = stats.stats.nanmedian(newarc,axis=0) arcmid[scipy.isnan(arcmid)] = 0. """ We don't want to do the cross-correlation with all pixels; there are reflections and other badness (like higher order lines). We set the beginning of the 'useable' correlation spectrum to 150 pixels before the part of the arc with substantial flux. We set the final pixel to be 100 pixels after the last substantial peak. Here, 'substantial' means within 20% of the maximum; this in part assumes that the final peak with 20% of the maximum will be the 5460 line, so we are in affect hard-coding that the correlation be carried out to ~ 5500 angstroms. """ mask = scipy.where(arcmid>avg+4.*std)[0] first = mask[0]-150 mask = ndimage.maximum_filter(arcmid,9) mask = scipy.where(mask==arcmid,mask,0) last = scipy.where(mask>0.20*arcmid.max())[0][-1]+100 if first<0.: first = 0. if last>width: last = width """ Set reasonable starting and ending wavelengths for the correlation routines. """ minwave = mswave-disp*width*1.1 maxwave = mswave+disp*width*1.1 if minwave<2000: minwave = 2000. if maxwave>8000: maxwave = 8000. """ Due a quick fit to define initial guesses...first the starting wavelength. I've used both a correlation and a chi-square-like function to determine the start; It's still not clear which is most robust! The correlation also uses a broadened model of the arcs to make it more robust wrt deviations from a linear solution. """ broad = 10. fudge = disp/200. nsteps = 19 xmodel = arcmid[first:last].copy() xmodel = ndimage.gaussian_filter(xmodel,broad/disp) p0 = 0. p1 = disp offset = 0. max = 1e29 for l in range(nsteps): try_disp = disp + ((l-nsteps/2)*fudge) skyfit_x = scipy.arange(minwave,maxwave,try_disp) fitmodel = interpolate.splev(skyfit_x,widemodel) tratio = xmodel.max()/fitmodel.max() fitmodel *= tratio fitmodel += scipy.median(xmodel) chi2,off = push(xmodel,fitmodel) if chi2<max: p1 = try_disp p0 = (off-first)*try_disp+minwave offset = off max = chi2 firstwave = minwave+offset*p1 """ We start with a second order fit: lambda = p0 + p1*x + p2*x*x x = 0 is *not* neccessarily the first pix; it is the first *good* pixel. p2 = 0. should be a reasonable first guess. """ p0 = blue first += (p0-firstwave)/p1 print first, p0, p1 if first<0: p0 = p0-first*p1 first = 0 last = first+(5650.-p0)/p1 if last>width: last = width first = int(first) last = int(last) scale = last """ Create a normalized model of the arc data for the first refinement. """ fitdata = arcmid[first:last].copy() fitx = scipy.arange(fitdata.size).astype(scipy.float64) arclines = findlines(fitx,fitdata,20) fitmodel = scipy.zeros(3*len(arclines)+1) index = 1 for iter in range(len(arclines)): fitmodel[index] = 1. fitmodel[index+1] = arclines[iter] fitmodel[index+2] = broad/disp index += 3 arcmodel = special_functions.ngauss(fitx,fitmodel) """ We begin with a broadened arc-spectrum for the initial fit. """ xdim = 2 p = scipy.zeros((xdim+1,1)) p[0,0] = p0 p[1,0] = p1 p = {'coeff':p,'type':"chebyshev"} fitdata = arcmid[first:last].copy() skycoeff = myoptimize(p,fitx,arcmodel,scale,linemodel) debug(fitx,arcmodel,linemodel,skycoeff) skycoeff = matchlines(arclines,p,linefile,order=1,tol=10.*disp) skycoeff = matchlines(arclines,skycoeff,linefile,order=2,tol=10.*disp) skycoeff = matchlines(arclines,skycoeff,linefile,tol=10.*disp) """ 3rd order fit """ xdim = 3 p = scipy.zeros((xdim+1,1)) p[0:skycoeff['coeff'].size,0] = skycoeff['coeff'][:,0].copy() skycoeff = {'coeff':p,'type':"chebyshev"} skycoeff = myoptimize(skycoeff,fitx,arcmodel,scale,linemodel) skycoeff = matchlines(arclines,skycoeff,linefile,order=2) skycoeff = matchlines(arclines,skycoeff,linefile) skycoeff = matchlines(arclines,skycoeff,linefile) skycoeff = matchlines(arclines,skycoeff,linefile,tol=10.) """ debug() is useful here to probe the quality of the line fitting """ # debug(fitx,arcmodel,linemodel,skycoeff) # debug(fitx,fitdata,finemodel,skycoeff) xvals = scipy.arange(width)-first w = special_functions.genfunc(xvals,0,skycoeff) cond = (w>=blue)&(w<=cutoff) # 'good' pixels cond1 = scipy.where(cond)[0] # index of first good pixel fitx = scipy.arange(width).astype(scipy.float64) fitx = fitx[cond]-first fitdata = arcmid[cond] skycoeff = myoptimize(skycoeff,fitx,fitdata,scale,finemodel) arclines = findlines(fitx,fitdata,10) skycoeff = matchlines(arclines,skycoeff,linefile,tol=5.*disp) xdim = 5 ydim = 2 wave = special_functions.genfunc(newxdata-first,0.,skycoeff) sky2x = [] sky2y = [] ccd2wave = [] """ We can only use the arcs if the cutoff is too blue. """ if cutoff<5200: revmodel = scipy.zeros((wave.size,3)) revmodel[:,0] = wave.copy() revmodel[:,1] = ydata.copy() revmodel[:,2] = xdata.copy() sky2x.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim)) revmodel[:,2] = yorig.copy() sky2y.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim)) revmodel[:,0] = xdata.copy() revmodel[:,1] = ydata.copy() revmodel[:,2] = wave.copy() ccd2wave.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim)) for k in range(1,nsci): sky2x.append(sky2x[0]) sky2y.append(sky2y[0]) ccd2wave.append(ccd2wave[0]) return sky2x,sky2y,ccd2wave """ Try to use the 5577 line to refine the wavelength solution. """ wlen = wave.copy() xvals = scipy.arange(width)-first for k in range(nsci): peaks = findlines(xvals,bgmodel[k],5.) w = special_functions.genfunc(peaks,0.,skycoeff) delta = 5577.345-w offset = delta[abs(delta).argmin()] if abs(offset)>3.*disp: print "Warning: Substantial offset found!" wave = wlen+offset revmodel = scipy.zeros((wave.size,3)) revmodel[:,0] = wave.copy() revmodel[:,1] = ydata.copy() revmodel[:,2] = xdata.copy() sky2x.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim)) revmodel[:,2] = yorig.copy() sky2y.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim)) revmodel[:,0] = xdata.copy() revmodel[:,1] = ydata.copy() revmodel[:,2] = wave.copy() ccd2wave.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim)) return sky2x,sky2y,ccd2wave
def arcwave(sky, arc, arcmodel, skymodel, scale, order): from scipy import ndimage, stats, interpolate, optimize sky = sky.copy() arc = arc.copy() sky = scipy.median(sky, 0) x = scipy.arange(sky.size) x_orig = x.copy() wave = scipy.arange(3000., 10000., scale) arc_wide = ndimage.gaussian_filter(arc, 5) m = interpolate.splev(wave, arcmodel['norm']) a = arc.copy() aw = arc_wide.copy() arc = a[:a.size / 2.] x = x_orig[:a.size / 2.] arclines = get_lines(x, arc) fit = scipy.zeros(3 * arclines.size + 1) index = 1 for i in range(arclines.size): fit[index] = 1. fit[index + 1] = arclines[i] fit[index + 2] = 15. * scale index += 3 arc_wide = sf.ngauss(x, fit) """ Do an approximate chi-square between the sky model and the data over a range of offsets using a broadened data and sky model. """ max = 0. mid = 0 delta = scale / 10. s = scipy.arange(scale - delta, scale + delta, delta / 10.) for stmp in s: wtmp = scipy.arange(2000., 10000., stmp) m = interpolate.splev(wtmp, arcmodel['norm']) conv = scipy.empty(m.size - arc_wide.size + 1) for i in range(conv.size): tmp = m[i:i + arc_wide.size].copy() if tmp.max() < 0.1: conv[i] = 0. continue conv[i] = (tmp * arc_wide).sum() conv[i] = 1. / ((tmp - arc_wide)**2).sum() curr = conv.max() if curr > max: mid = conv.argmax() scale = stmp max = conv.max() wave = wtmp.copy() """ Refine the starting wavelength position using the 'true' (ie narrow) model of the sky. Searches for a minimum around the minimum found in the previous optimization. """ m = interpolate.splev(wave, arcmodel['matched']) conv = scipy.empty(m.size - arc.size + 1) for i in range(conv.size): tmp = m[i:i + arc.size].copy() ratio = arc.max() / tmp.max() if tmp.max() < 1.: conv[i] = 0. continue tmp *= ratio conv[i] = (tmp * arc).sum() pt = conv[mid - 50:mid + 51].argmax() + mid - 50 initial_pars = [wave[pt], scale] for i in range(order + 1 - len(initial_pars)): initial_pars.append(0.) modellines = get_lines(wave, m, std=10.) modellines = modellines[modellines > wave[pt]] modellines = arcmodel['lines'] modellines = modellines[modellines > wave[pt]] fit = {'coeff': scipy.atleast_2d(initial_pars).T, 'type': 'polynomial'} for o in [1, 2]: w = sf.genfunc(arclines, 0., fit) matches = [] for j in range(w.size): diff = abs(w[j] - modellines) if diff.min() < 5. * scale: matches.append([arclines[j], modellines[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', o) left_matches = [i for i in matches] wmin = sf.genfunc(a.size * 0.45, 0., fit) arc = a[a.size / 2.:].copy() x = scipy.arange(arc.size).astype(scipy.float32) + a.size / 2. arclines = get_lines(x, arc) fit = scipy.zeros(3 * arclines.size + 1) index = 1 for i in range(arclines.size): fit[index] = 1. fit[index + 1] = arclines[i] fit[index + 2] = 10. * scale index += 3 arc_wide = sf.ngauss(x, fit) """ Do an approximate chi-square between the sky model and the data over a range of offsets using a broadened data and sky model. """ max = 0. mid = 0 delta = scale / 10. s = scipy.arange(scale - delta, scale + delta, delta / 10.) for stmp in s: wtmp = scipy.arange(wmin, 10000., stmp) m = interpolate.splev(wtmp, arcmodel['norm']) conv = scipy.empty(m.size - arc_wide.size + 1) for i in range(conv.size): tmp = m[i:i + arc_wide.size].copy() if tmp.max() < 0.1: conv[i] = 0. continue conv[i] = (tmp * arc_wide).sum() curr = conv.max() if curr > max: mid = conv.argmax() scale = stmp max = conv.max() wave = wtmp.copy() """ Refine the starting wavelength position using the 'true' (ie narrow) model of the sky. Searches for a minimum around the minimum found in the previous optimization. """ m = interpolate.splev(wave, arcmodel['matched']) conv = scipy.empty(m.size - arc.size + 1) for i in range(conv.size): tmp = m[i:i + arc.size].copy() ratio = arc.max() / tmp.max() if tmp.max() < 1.: conv[i] = 0. continue tmp *= ratio conv[i] = (tmp * arc).sum() pt = conv[mid - 50:mid + 51].argmax() + mid - 50 wavept = wave[pt] initial_pars = [wavept, scale] for i in range(order + 1 - len(initial_pars)): initial_pars.append(0.) modellines = get_lines(wave, m, std=10.) modellines = modellines[modellines > wavept] modellines = arcmodel['lines'] modellines = modellines[modellines > wavept] fit = {'coeff': scipy.atleast_2d(initial_pars).T, 'type': 'polynomial'} for o in [1, 2]: # The (o-2) bit is to correct the offset after the first loop w = sf.genfunc(arclines + (o - 2) * a.size / 2., 0., fit) matches = [] for j in range(w.size): diff = abs(w[j] - modellines) if diff.min() < 5. * scale: matches.append([arclines[j], modellines[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', o) arc = a.copy() arc_wide = aw.copy() w = sf.genfunc(arclines, 0., fit) for i in range(w.size): diff = abs(w[i] - modellines) if diff.min() < 5. * scale: left_matches.append([arclines[i], modellines[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(left_matches), 'polynomial', order) """ Optimization function for refining the wavelength solution. """ def dofit(p, x, data, model): fit = {'coeff': scipy.atleast_2d(p).T, 'type': 'polynomial'} w = sf.genfunc(x, 0., fit) m = interpolate.splev(w, model) return (m - data) x = scipy.arange(arc.size).astype(scipy.float32) initial_pars = fit['coeff'][:, 0].tolist() coeff, ier = optimize.leastsq(dofit, initial_pars, (x, arc_wide, arcmodel['wide']), maxfev=100000) coeff, ier = optimize.leastsq(dofit, coeff, (x, arc, arcmodel['matched']), maxfev=100000) outcoeff = {'coeff': scipy.atleast_2d(coeff).T, 'type': 'polynomial'} def skycorrect(p, arc, sky, arcmodel, skymodel): fit = {'coeff': scipy.atleast_2d(p[:-1]).T, 'type': 'polynomial'} w = sf.genfunc(x, 0., fit) arcm = interpolate.splev(w + p[-1], arcmodel) chi_arc = (arcm - arc) s = sky[w > 5100.] skym = interpolate.splev(w[w > 5100.], skymodel) skym *= scipy.median(s / skym) chi_sky = 5. * (skym - s) #/abs(m)**0.5 chi = scipy.concatenate((chi_arc, chi_sky)) return chi newcoeff = coeff.tolist() newcoeff.append(0.) coeff, ier = optimize.leastsq( skycorrect, newcoeff, (arc, sky, arcmodel['matched'], skymodel['matched']), maxfev=100000) outcoeff = {'coeff': scipy.atleast_2d(coeff[:-1]).T, 'type': 'polynomial'} """ wave = sf.genfunc(x,0.,outcoeff) sky = sky[wave>5000.] wave = wave[wave>5000.] m = interpolate.splev(wave,wavemodel['matched']) ratio = scipy.median(sky/m) import pylab pylab.plot(wave,sky) pylab.plot(wave,m*ratio) pylab.show() offset,ier = optimize.leastsq(skycorrect,[0.], (wave,sky,wavemodel['matched']),maxfev=100000) print offset outcoeff['coeff'][0] += offset """ return outcoeff
def pipeline(dir, inname, out_prefix, cal_prefix): bias = pyfits.open(cal_prefix + "_bias.fits")[0].data.astype(scipy.float32) bpm = pyfits.open(cal_prefix + "_bpm.fits")[0].data.astype(scipy.float32) flat = pyfits.open(cal_prefix + "_norm.fits")[0].data.astype(scipy.float32) file = open(cal_prefix + "_ycor.dat") orders = load(file) y_soln = load(file) file.close() file = open(cal_prefix + "_wave.dat") wave_soln = load(file) file.close() try: mask = pyfits.open(cal_prefix + "_mask.fits")[0].data.copy() print('mask opened') except: mask = numpy.where(bpm == 1., 1., 0.) mask = straighten(mask, y_soln, orders) mask[mask < 0.7] = numpy.nan mask[mask >= 0.7] = 1. pyfits.PrimaryHDU(mask).writeto(cal_prefix + "_mask.fits") hdu = pyfits.open(dir + inname) data = hdu[0].data.copy() data = biastrim(data, bias, bpm) #*flat blue = [1500, 1400, 1300, 1200, 1100, 900, 600, 200, 0, 0, 0] red = [3000, 3400, 3700, -1, -1, -1, -1, -1, -1, -1] def clip2(arr, nsig=3., edge=0.01): a = arr.flatten() a.sort() a = a[a.size * edge:a.size * (1. - edge)] m, s, l = a.mean(), a.std(), a.size while 1: a = a[abs(a - m) < nsig * s] if a.size == l: return m, s m, s, l = a.mean(), a.std(), a.size try: back = pyfits.open(out_prefix + "_bg.fits")[0].data.astype( scipy.float32) except: strt = straighten(data, y_soln, orders) back = scipy.zeros(strt.shape) for indx in range(len(orders)): i, j = orders[indx] bg = stats.stats.nanmedian(strt[i:j], axis=0) back[i:j] += bg back = curve(back, y_soln, orders) pyfits.PrimaryHDU(back).writeto(out_prefix + "_bg.fits") try: strt = pyfits.open(out_prefix + "_strt.fits")[0].data.astype( scipy.float32) except: bgsub = data - back bgsub[scipy.isnan(bgsub)] = 0. bgsub[scipy.isinf(bgsub)] = 0. model = ndimage.median_filter(bgsub, 3) diff = bgsub - model model = scipy.sqrt(data) model[scipy.isnan(model)] = 0. model[scipy.isinf(model)] = 0. diff[scipy.isnan(diff)] = 0. diff[scipy.isinf(diff)] = 0. crmask = scipy.where(diff > 4. * model, diff, 0.) data -= crmask strt = straighten(data, y_soln, orders) pyfits.PrimaryHDU(strt).writeto(out_prefix + "_strt.fits") hdulist = pyfits.HDUList([pyfits.PrimaryHDU()]) hdulist[0].header = hdu[0].header.copy() varhdu = pyfits.HDUList([pyfits.PrimaryHDU()]) varhdu[0].header = hdu[0].header.copy() for indx in range(len(orders)): i, j = orders[indx] cut = strt[i:j].copy() cut[scipy.isinf(cut)] = scipy.nan bg = stats.stats.nanmedian(cut, axis=0) B = mask[i:j] tmp = (cut - bg) * B T = tmp[3:-3, blue[indx]:red[indx]] T = T[numpy.isfinite(T)] avg, std = clip2(T) slice = stats.stats.nanmedian(tmp[:, blue[indx]:red[indx]], axis=1) slice[:3] = 0. slice[-3:] = 0. avg, std = clip2(slice[3:-3], 3.5, 0.05) good = numpy.where(slice < 2. * std, 1., 0.) good = ndimage.maximum_filter(good, 5) good = ndimage.minimum_filter(good, 15) good = good == 1 xvals = scipy.arange(good.size).astype(scipy.float32) fitdata = scipy.empty((slice[good].size, 2)) fitdata[:, 0] = xvals[good].copy() bgsub = cut.copy() for k in range(cut.shape[1]): fitdata[:, 1] = cut[:, k][good].copy() if fitdata[:, 1][scipy.isfinite(fitdata[:, 1])].size < 4: continue avg, std = clip(fitdata[:, 1]) keep = scipy.where((abs(fitdata[:, 1] - avg) < 4. * std) & scipy.isfinite(fitdata[:, 1]))[0] fit = fitdata[keep] if fit[:, 1][scipy.isfinite(fit[:, 1])].size < 4: continue fit = sf.lsqfit(fit, 'chebyshev', 1) bg = sf.genfunc(xvals, 0, fit) bgsub[:, k] -= bg solution = wave_soln[indx][0] invsoln = wave_soln[indx][1] vals = scipy.arange(cut.shape[1]).astype(scipy.float64) wave = sf.genfunc(vals, 0., solution) delta = (wave[wave.size - 1] - wave[0]) / (wave.size - 1) outc = scipy.arange(wave[0], wave[0] + 2, delta) outc = outc[:cut.shape[1]].copy() pix = sf.genfunc(outc, 0, invsoln) coords = spectools.array_coords(cut.shape).astype(numpy.float64) coords[1] *= 0. coords[1] += pix bgsub[scipy.isnan(bgsub)] = 0. out = ndimage.map_coordinates(bgsub, coords, output=scipy.float64, order=5) hdu = pyfits.ImageHDU(out.copy()) hdu.header.update('CTYPE1', 'WAVE-LOG') hdu.header.update('CRVAL1', wave[0]) hdu.header.update('CRPIX1', 1) hdu.header.update('CDELT1', delta) hdu.header.update('CTYPE2', 'LINEAR') hdu.header.update('CRVAL2', 1) hdu.header.update('CRPIX2', 1) hdu.header.update('CDELT2', 1) hdu.header.update('CD1_1', delta) hdu.header.update('CD2_2', 1) hdu.header.update('CD1_2', 0) hdu.header.update('CD2_1', 0) #hdu.header.update('PC1_1',1) #hdu.header.update('PC1_2',0) #hdu.header.update('PC2_1',0) #hdu.header.update('PC2_2',1) hdu.header.update('DC-FLAG', 1) hdu.header.update('WFITTYPE', 'LOG_LINEAR') hdu.header.update('DISPAXIS', 1) hdulist.append(hdu) cut[scipy.isnan(cut)] = 0. out = ndimage.map_coordinates(cut, coords, output=scipy.float64, order=5) hdu = pyfits.ImageHDU(out.copy() + 5.3) hdu.header.update('CTYPE1', 'WAVE-LOG') hdu.header.update('CRVAL1', wave[0]) hdu.header.update('CRPIX1', 1) hdu.header.update('CD1_1', delta) hdu.header.update('CRVAL2', 1) hdu.header.update('CRPIX2', 1) hdu.header.update('CD2_2', 1) hdu.header.update('DC-FLAG', 1) hdu.header.update('WFITTYPE', 'LOG_LINEAR') hdu.header.update('DISPAXIS', 1) varhdu.append(hdu) hdulist.verify('fix') hdulist.writeto(out_prefix + "_2d.fits") varhdu.verify('fix') varhdu.writeto(out_prefix + "_var.fits")
def arcmatch(curve, sci, arc, yforw, widemodel, finemodel, goodmodel, linemodel, disp, mswave, extra, logfile): """ arcmatch(curve,sci,arc,yforw,widemodel,finemodel,goodmodel,linemodel ,disp,mswave,extra,logfile) Creates the full 2d distortion solution for a slit. Inputs: curve - first pixel (ie bottom) of curved slit sci - biastrimmed, non-ycorrected science data arc - y-corrected arc for this slit yforw - yforw definition for slit widemodel - broad wavelength model finemodel - matched-resolution wavelength model goodmodel - model of the sky linemodel - broad wavelength model (why are there two...?) disp - scale mswave - central wavelength extra - includes name of linefile, starting/ending wavelengths logfile - a file object Outputs: """ """ Do not alter input data! """ sci = sci.copy() nsci = sci.shape[0] """ Parse extra arguments """ linefile = extra[0] cutoff = extra[1] if len(extra) == 3: blue = extra[2] else: blue = 3000. cutoff = 5650. height = arc.shape[0] width = arc.shape[1] """ Straighten science data """ scimodel = scipy.zeros((nsci, height, width)) for k in range(nsci): scimodel[k] = spectools.resampley(sci[k], yforw, curve) """ Skip the bottom and top of the slit, which may have artifacts """ i = 4 j = height - 4 xdim = 2 ydim = 2 avg = scipy.median(scipy.median(arc)) std = clipped_std(arc, 4.) """ The 'fiducial' arc spectrum is taken from the middle of the slit """ arcmid = stats.stats.nanmedian(arc[height / 2 - 2:height / 2 + 3], 0) """ First we straighten the lines out. """ coords = spectools.array_coords((j - i, width)) coords[0] += 4. fitdata = arc[i:j, height:-1 * height].flatten() xdata = coords[1, :, height:-1 * height].flatten() ydata = coords[0, :, height:-1 * height].flatten() """ Only use every third row for the solution (to save time) """ cond = ydata % 3 == 0 fitdata = fitdata[cond] ydata = ydata[cond] xdata = xdata[cond] """ Only use pixels around the peaks (another time saver) """ thresh = scipy.median(arcmid) thresh += 5. * thresh**0.5 mask = ndimage.maximum_filter(arcmid, 15) mask = fitdata > thresh fitdata = fitdata[mask] ydata = ydata[mask] xdata = xdata[mask] p = scipy.zeros((xdim + 1, ydim + 1)) p[1, 0] = 1. p = {'coeff': p, 'type': "polynomial"} xback = arcfitfunc2(p, xdata, ydata, fitdata, arcmid) """ xdata, ydata, yorig, and newxdata hold original/transformed CCD coordinates. We don't need to do a fit to *all* points, so we skip ~every third. """ xdata = coords[1].flatten() ydata = coords[0].flatten() yorig = yforw[i:j].flatten() - curve cond = (ydata + xdata) % 2 == 0 xdata = xdata[cond] ydata = ydata[cond] yorig = yorig[cond] newxdata = special_functions.genfunc(xdata, ydata, xback) tmp = scipy.zeros((xdata.size, 3)) tmp[:, 0] = newxdata.copy() tmp[:, 1] = ydata.copy() tmp[:, 2] = xdata.copy() xforw = special_functions.lsqfit(tmp, "chebyshev", xdim, ydim) bgmodel = scipy.zeros((nsci, sci.shape[2])) for k in range(nsci): tmp = spectools.resample1d(scimodel[k], xforw, "x", -999) scimodel[k] = tmp.copy() tmp.sort(axis=0) tmp[tmp == -999] = scipy.nan bg = stats.stats.nanmedian(tmp, axis=0) bg = scipy.tile(bg, (height, 1)) tmp[scipy.isnan(tmp)] = bg[scipy.isnan(tmp)] tmp[scipy.isnan(tmp)] = 0. bgmodel[k] = tmp[tmp.shape[0] / 4, :] del tmp, bg newarc = spectools.resample1d(arc, xforw, "x", -999) newarc[newarc == -999] = scipy.nan arcmid = stats.stats.nanmedian(newarc, axis=0) arcmid[scipy.isnan(arcmid)] = 0. """ We don't want to do the cross-correlation with all pixels; there are reflections and other badness (like higher order lines). We set the beginning of the 'useable' correlation spectrum to 150 pixels before the part of the arc with substantial flux. We set the final pixel to be 100 pixels after the last substantial peak. Here, 'substantial' means within 20% of the maximum; this in part assumes that the final peak with 20% of the maximum will be the 5460 line, so we are in affect hard-coding that the correlation be carried out to ~ 5500 angstroms. """ mask = scipy.where(arcmid > avg + 4. * std)[0] first = mask[0] - 150 mask = ndimage.maximum_filter(arcmid, 9) mask = scipy.where(mask == arcmid, mask, 0)[:arcmid.argmax() + 460 / disp] last = scipy.where(mask > 0.20 * arcmid.max())[0][-1] + 100 if first < 0.: first = 0. if last > width: last = width """ Set reasonable starting and ending wavelengths for the correlation routines. """ minwave = mswave - disp * width * 1.1 maxwave = mswave + disp * width * 1.1 if minwave < 2000: minwave = 2000. if maxwave > 8000: maxwave = 8000. """ Due a quick fit to define initial guesses...first the starting wavelength. I've used both a correlation and a chi-square-like function to determine the start; it's still not clear which is most robust! The correlation also uses a broadened model of the arcs to make it more robust wrt deviations from a 1st order solution. """ broad = 9. fudge = disp / 1000. nsteps = 35 xmodel = arcmid[first:last].copy() # xmodel = ndimage.gaussian_filter(xmodel,broad/disp) p0 = 0. p1 = disp offset = 0. max = 1e29 c = [] b = [] fitdata = arcmid[first:last].copy() fitx = scipy.arange(fitdata.size).astype(scipy.float64) arclines = findlines(fitx, fitdata, 10) fitmodel = scipy.zeros(3 * len(arclines) + 1) index = 1 for iter in range(len(arclines)): fitmodel[index] = 1. fitmodel[index + 1] = arclines[iter] fitmodel[index + 2] = broad / disp index += 3 xmodel = special_functions.ngauss(fitx, fitmodel) for l in range(nsteps): try_disp = disp + ((l - nsteps / 2) * fudge) skyfit_x = scipy.arange(minwave, maxwave, try_disp) fitmodel = interpolate.splev(skyfit_x, linemodel) # tratio = xmodel.max()/fitmodel.max() # fitmodel *= tratio # fitmodel += scipy.median(xmodel) corr = signal.correlate(xmodel, fitmodel, mode='valid') chi2, off = 1. / corr.max(), corr.argmax() # chi2,off = push(xmodel,fitmodel) if chi2 < max: p1 = try_disp p0 = (off - first) * try_disp + minwave offset = off max = chi2 firstwave = minwave + offset * p1 """ We start with a second order fit: lambda = p0 + p1*x + p2*x*x x = 0 is *not* neccessarily the first pix; it is the first *good* pixel. p2 = 0. should be a reasonable first guess. """ p0 = blue first += (p0 - firstwave) / p1 if first < 0: p0 = p0 - first * p1 first = 0 last = first + (5650. - p0) / p1 if last > width: last = width first = int(first) last = int(last) keeplog(logfile, '\tInitial Solution: %f, %f\n' % (p0, p1)) """ Create a normalized model of the arc data for the first refinement. """ fitdata = arcmid[first:last].copy() fitx = scipy.arange(fitdata.size).astype(scipy.float64) arclines = findlines(fitx, fitdata, 20) fitmodel = scipy.zeros(3 * len(arclines) + 1) index = 1 for iter in range(len(arclines)): fitmodel[index] = 1. fitmodel[index + 1] = arclines[iter] fitmodel[index + 2] = broad / disp index += 3 arcmodel = special_functions.ngauss(fitx, fitmodel) """ We begin with a fit to the normalized spectra. """ xdim = 2 p = scipy.zeros((xdim + 1, 1)) p[0, 0] = p0 p[1, 0] = p1 p = {'coeff': p, 'type': "polynomial"} fitdata = arcmid[first:last].copy() skycoeff = myoptimize(p, fitx, arcmodel, linemodel) # debug(fitx,fitdata,finemodel,skycoeff) skycoeff = matchlines(arclines, p, linefile, order=1, tol=10. * disp) skycoeff = matchlines(arclines, skycoeff, linefile, order=2, tol=10. * disp) skycoeff = matchlines(arclines, skycoeff, linefile, tol=10. * disp) """ Refinement of fit, still with normalized spectra. """ xdim = 3 p = scipy.zeros((xdim + 1, 1)) p[0:skycoeff['coeff'].size, 0] = skycoeff['coeff'][:, 0].copy() skycoeff = {'coeff': p, 'type': "chebyshev"} # skycoeff = myoptimize(skycoeff,fitx,arcmodel,linemodel) skycoeff = myoptimize(skycoeff, fitx, fitdata, finemodel) # debug(fitx,fitdata,finemodel,skycoeff) """ Match lines, progressively constraining the fit """ skycoeff = matchlines(arclines, skycoeff, linefile, order=2, tol=10. * disp) skycoeff = matchlines(arclines, skycoeff, linefile, tol=10. * disp) skycoeff = matchlines(arclines, skycoeff, linefile, tol=10. * disp) skycoeff = matchlines(arclines, skycoeff, linefile, tol=5. * disp) skycoeff = matchlines(arclines, skycoeff, linefile, tol=5. * disp) # debug(fitx,fitdata,finemodel,skycoeff) xvals = scipy.arange(width) - first w = special_functions.genfunc(xvals, 0, skycoeff) """ The wavelength solution might wrap around on itself (ie it might not be a strictly increasing function). This ensures that only the increasing part of the current solution is used. It might be more reasonable to put something in the chi-square to enforce this.... I've added it to arcfitfunc for now.... val = w.argmax() if cutoff>w.max(): cutoff = w.max() w[val:] = cutoff + 1. """ cond = (w >= blue) & (w <= cutoff) # 'good' pixels cond1 = scipy.where(cond)[0] # index of first good pixel fitx = scipy.arange(width).astype(scipy.float64) fitx = fitx[cond] - first fitdata = arcmid[cond] # debug(fitx,fitdata,finemodel,skycoeff) # skycoeff = myoptimize(skycoeff,fitx,fitdata,finemodel) # xdim = 5 # p = scipy.zeros((xdim+1,1)) # p[0:skycoeff['coeff'].size,0] = skycoeff['coeff'][:,0].copy() # skycoeff = {'coeff':p,'type':"polynomial"} # skycoeff = myoptimize(skycoeff,fitx,fitdata,finemodel) # debug(fitx,fitdata,finemodel,skycoeff) coeff = skycoeff['coeff'].copy() arclamppeaks = findlines(fitx, fitdata) skycoeff = matchlines(arclamppeaks, skycoeff, linefile, tol=5. * disp) startx = fitx.copy() startdata = fitdata.copy() wave = special_functions.genfunc(startx, 0., skycoeff) model = interpolate.splrep(wave, startdata, s=0) sky2x = [] sky2y = [] ccd2wave = [] def findoffset(p, x, data, model, coeff): xvals = x + p[0] w = special_functions.genfunc(xvals, 0., coeff) mod = interpolate.splev(w, model) mod = mod * data.max() / mod.max() diff = (mod - data) / scipy.sqrt(abs(mod)) return diff """ If the highest line is greater than 3x the next highest, there was probably some sky activity. This could throw off the pixel-based fitting, so we should choose line-matching instead. A better solution would be to 'dampen' the active line (we can't just alter the residuals because the sky bias and gain offsets were calculated using the line), but that is non-trivial to accomplish without shifting the central wavelength. TURNED OFF--LINE MATCHING ALWAYS USED! """ LINEMATCH = False for k in range(nsci): sky = bgmodel[k] x_vals = scipy.arange(width) skypeaks = findlines(x_vals, sky, 3) amps = ndimage.map_coordinates(sky, [skypeaks], output=scipy.float64, order=5) amps.sort() if amps[-1] > 2. * amps[-2]: LINEMATCH = True # Use LINEMATCH until arcs are better dealt with LINEMATCH = True """ We attempt to determine an offset between the arclamp frame and the sky image due to flexure (or if the arcs were taken in the afternoon...). First we use the arc solution to determine the offset of the bluest sky line. We then shift the arc lines to correct for the offset and find a solution for just the sky. We extrapolate the sky solution to the reddest arc line, determine the arc line's offset and use this as the final offset between sky and arc. We then solve for the full solution using the sky and corrected arc lines. """ for k in range(nsci): sky = bgmodel[k] skycoeff['coeff'] = coeff.copy() x_vals = scipy.arange(width) - first skypeaks = findlines(x_vals, sky, 5) """ Start by trying to match to the 5200 line; if it is too faint (and therefore was not extracted as a skypeak), use the 5577 line. 5197.93,5198.794 -- the line is a blend and the left value is the non-blended wavelength. One issue is that the line exhibits auroral brightening--which appears to change the effective wavelength because the fainter line at 5200. is the one that brightens! BAH! We'll just use 5577. """ # sline = 5198.794 sline = 5577.345 highpeaks = findlines(x_vals, sky, 7) w = special_functions.genfunc(highpeaks, 0., skycoeff) delta = sline - w if abs(delta).min() > 10. * disp: sline = 5577.345 delta = sline - w w = special_functions.genfunc(x_vals, 0., skycoeff) skycond = (w > sline - 40.) & (w < sline + 40.) off = delta[abs(delta).argmin()] / disp tmp = special_functions.genfunc(skypeaks, 0., skycoeff) entry = '\tSky lines found for image %d:' % (k + 1) for t in tmp: entry += ' %7.2f' % t entry += "\n" keeplog(logfile, entry) min = 10. step = None for i in scipy.arange(-3., 3., 0.1): arcpeaks = arclamppeaks + i tmpcoeff, err = doublematch([skypeaks, arcpeaks], skycoeff, ARCLINES, [sline], tol=10. * disp) tmpcoeff, err = doublematch([skypeaks, arcpeaks], tmpcoeff, ARCLINES, SKYLINES, tol=10. * disp) tmpcoeff, err = doublematch([skypeaks, arcpeaks], tmpcoeff, ARCLINES, SKYLINES, tol=5. * disp) if err < min: min = err outcoeff = tmpcoeff.copy() step = i min = 10. for i in scipy.arange(step - 0.5, step + 0.5, 0.01): arcpeaks = arclamppeaks + i tmpcoeff, err = doublematch([skypeaks, arcpeaks], outcoeff, ARCLINES, SKYLINES, tol=5. * disp) if err < min: min = err skycoeff = tmpcoeff.copy() arcoffset = i keeplog(logfile, "\tArc offset for image %d: %6.3f\n" % (k + 1, arcoffset)) skypeaks = findlines(x_vals, sky, 2.) arcpeaks = arclamppeaks + arcoffset skycoeff, e = doublematch([skypeaks, arcpeaks], skycoeff, ARCLINES, SKYLINES, tol=5. * disp, logfile=logfile) w = special_functions.genfunc(x_vals, 0., skycoeff) skycond = (w > 5500.) & (w < 6700.) skyx = x_vals[skycond] skyz = sky[skycond] tmp = skyz.copy() tmp.sort() med = tmp[tmp.size * 0.3] skyx = skyx[skyz > med] skyz = skyz[skyz > med] tmpcoeff, ratio, offset = skyopt(skycoeff, skyx, skyz, goodmodel) skycoeff = myopt2(skycoeff, fitx + arcoffset, fitdata, finemodel, skyx, skyz, goodmodel, ratio, offset) """ Evaluate wavelength of all pixels in straightened frame """ wave = special_functions.genfunc(newxdata - first, 0, skycoeff) """ Now that we have the wavelength solution for the rectified slit we can create the full 2d solutions for: x(lambda,pos) y(lambda,pos) lamba(x,y) where x and y are CCD coordinates and lambda and pos are the 'true' coordinates (wavelength and spatial position along the slit). """ xdim = 5 ydim = 2 revmodel = scipy.zeros((wave.size, 3)) revmodel[:, 0] = wave.copy() revmodel[:, 1] = ydata.copy() revmodel[:, 2] = xdata.copy() sky2x.append( special_functions.lsqfit(revmodel, "chebyshev", xdim, ydim)) revmodel[:, 2] = yorig.copy() sky2y.append( special_functions.lsqfit(revmodel, "chebyshev", xdim, ydim)) revmodel[:, 0] = xdata.copy() revmodel[:, 1] = ydata.copy() revmodel[:, 2] = wave.copy() ccd2wave.append( special_functions.lsqfit(revmodel, "chebyshev", xdim, ydim)) # return sky2x,sky2y,[xforw,xback] # This is for non-2d subtraction return sky2x, sky2y, ccd2wave # Thsi is for 2d subtraction
def extract(image, varimage, outname, width=2., offset=0., pos=None, minwave=None, maxwave=None, response_image=None, response_model=None, regions=[], centroid=None): if outname.lower().find('fits') < 1: outname = "%s.fits" % outname resp = None respwave = None if response_image is not None: if response_model is None: # print "No response model included; not performing response correction." import cPickle resp = cPickle.load(open(response_image)) else: resp = get_response(response_image, response_model, regions) elif response_model is not None: print "No response image included; not performing response correction." if minwave is None: minwave = -10. if maxwave is None: maxwave = 1e99 data = pyfits.open(image)[0].data.copy() if data.ndim == 3: if data.shape[0] > 1: print "Only working on first image plane of 3D image." data = data[0].copy() wave = st.wavelength(image) indx = scipy.where((wave > minwave) & (wave < maxwave))[0] minx, maxx = indx.min(), indx.max() wave = wave[minx:maxx] data = data[:, minx:maxx] tmp = data.copy() tmp[numpy.isnan(data)] = 0. ospec = wave * 0. ovar = wave * 0. x = numpy.arange(data.shape[0]) if centroid is None: n = tmp.shape[1] / 100 peaks = [] bounds = numpy.linspace(0, tmp.shape[1], n + 1) for i in range(n): a, b = bounds[i], bounds[i + 1] p = tmp[:, a:b].sum(1).argmax() peaks.append([(a + b) / 2., p]) peak = sf.lsqfit(numpy.asarray(peaks), 'polynomial', 3) if pos is not None: peak['coeff'][0] = pos peaks = sf.genfunc(numpy.arange(tmp.shape[1]), 0., peak) center = wave * 0. for i in range(center.size): d = data[:, i].copy() peak = peaks[i] if peak < 0: peak = 0 if peak >= d.shape: peak = d.shape - 1. fit = numpy.array([0., d[peak], peak, 1.]) cond = ~numpy.isnan(d) input = numpy.empty((d[cond].size, 2)) input[:, 0] = x[cond].copy() input[:, 1] = d[cond].copy() fit = sf.ngaussfit(input, fit)[0] center[i] = fit[2] fit = sf.lsqfit(ndimage.median_filter(center, 17), 'polynomial', 5) centroid = sf.genfunc(scipy.arange(wave.size), 0., fit) #import pylab #pylab.plot(center-centroid) #pylab.show() xvals = scipy.arange(data.shape[0]) var = pyfits.open(varimage)[0].data.copy() if var.ndim == 3: var = var[0].copy() var = var[:, minx:maxx] cond = (numpy.isnan(var)) | (numpy.isnan(data)) for i in range(ovar.size): c = centroid[i] + offset wid = width mask = xvals * 0. mask[(xvals - c < wid) & (xvals - c > 0.)] = wid - (xvals[ (xvals - c < wid) & (xvals - c > 0.)] - c) mask[(xvals - c < wid - 1) & (xvals - c > 0.)] = 1. mask[(c - xvals < wid + 1) & (c - xvals > 0.)] = (wid + 1) - ( c - xvals[(c - xvals < wid + 1) & (c - xvals > 0.)]) mask[(c - xvals < wid) & (c - xvals > 0.)] = 1. mask[cond[:, i]] = 0. mask /= mask.sum() ospec[i] = (mask[~cond[:, i]] * data[:, i][~cond[:, i]]).sum() ovar[i] = ((mask[~cond[:, i]]**2) * var[:, i][~cond[:, i]]).sum() badpix = numpy.where(numpy.isnan(ospec))[0] ospec[badpix] = 0. ovar[badpix] = ovar[~numpy.isnan(ovar)].max() * 1e6 med = numpy.median(ospec) ospec /= med ovar /= med**2 if resp is not None: # if respwave is None: # resp = sf.genfunc(wave,0,resp) # else: # resp = interpolate.splrep(respwave,resp) resp = interpolate.splev(wave, resp) ospec *= resp ovar *= resp**2 st.make_spec(ospec, ovar, wave, outname, clobber=True) return centroid
def newModMatch(data, model, scale, order, center, sparse=False): import numpy import special_functions as sf from scipy import ndimage, interpolate x0 = numpy.arange(data.size).astype(numpy.float32) datalines = get_lines(x0, data) Xstart = int(datalines.min() - 15.) Xend = int(datalines.max() + 15.) if Xstart < 0: Xstart = 0 if Xend > data.size: Xend = data.size tdata = data[Xstart:Xend].copy() wave = numpy.linspace(center - scale * data.size / 2, center + scale * data.size / 2, data.size)[Xstart:Xend].copy() cond = wave < 10300. mod = interpolate.splev(wave, model['matched']) fftd = numpy.fft.rfft(ndimage.gaussian_filter(tdata[cond], 17)) fftm = numpy.fft.rfft(ndimage.gaussian_filter(mod[cond], 17)[::-1]) T = numpy.fft.fftshift(numpy.fft.irfft(fftd * fftm)).real offset = T.size / 2 - T.argmax() Wstart = wave[offset] pars = numpy.zeros(order + 1) pars[0] = Wstart - Xstart * scale pars[1] = scale cov = numpy.zeros(order + 1) cov[0] = 1. cov[1] = 0.05 for i in range(2, order + 1): cov[i] = (1. / data.size)**i coeff = pars def optfunc(pars, x, d, mod, sig): if numpy.isnan(pars).any(): return -1e200 coeff = numpy.atleast_2d(numpy.array(pars)).T fit = {'coeff': coeff, 'type': 'polynomial'} x0 = sf.genfunc(x, 0., fit) try: model = interpolate.splev(x0, mod) except: return -1e200 model /= model.mean() resid = (model - d) / sig return -0.5 * (resid**2).sum() spec = ndimage.gaussian_filter(tdata, 7) spec /= spec.mean() sig = spec**0.5 logp = optfunc(pars, x0[Xstart:Xend], spec, model['wide'], sig) niter = 2000 for indx in xrange(niter): prop = coeff + numpy.random.randn( cov.size) * cov * 10**(numpy.random.random(cov.size) * 2.3 - 2.) tlogp = optfunc(prop, x0[Xstart:Xend], spec, model['wide'], sig) accept = 0 if tlogp > logp: accept = 1 else: if tlogp - logp > numpy.log(numpy.random.random()): accept = 1 if accept == 1: logp = tlogp coeff = prop.copy() spec = tdata.copy() spec /= spec.mean() sig = spec**0.5 logp = optfunc(pars, x0[Xstart:Xend], spec, model['matched'], sig) c = numpy.empty((niter, coeff.size)) for indx in xrange(niter): prop = coeff + numpy.random.randn( cov.size) * cov * 10**(numpy.random.random(cov.size) * 2.3 - 2.) tlogp = optfunc(prop, x0[Xstart:Xend], spec, model['matched'], sig) accept = 0 if tlogp > logp: accept = 1 else: if tlogp - logp > numpy.log(numpy.random.random()): accept = 1 if accept == 1: logp = tlogp coeff = prop.copy() c[indx] = coeff.copy() for indx in range(cov.size): cov[indx] = c[:, indx].std() mlogp = logp mcoeff = coeff.copy() for indx in xrange(niter): prop = coeff + numpy.random.randn( cov.size) * cov * 10**(numpy.random.random(cov.size) * 2.3 - 2.) tlogp = optfunc(prop, x0[Xstart:Xend], spec, model['matched'], sig) accept = 0 if tlogp > logp: accept = 1 else: if tlogp - logp > numpy.log(numpy.random.random()): accept = 1 if accept == 1: logp = tlogp coeff = prop.copy() if logp > mlogp: mlogp = logp mcoeff = coeff.copy() fit = {'coeff': scipy.atleast_2d(mcoeff).T, 'type': 'polynomial'} w = sf.genfunc(x0[Xstart:Xend], 0., fit) m = interpolate.splev(w, model['matched']) cond = (w < 10400) m = m[cond] m /= m.mean() import pylab pylab.plot(w[cond], m) pylab.plot(w[cond], spec[cond]) pylab.show() return fit #def newModMatch(data,model,scale,order,center,sparse=False): import numpy, pylab from scipy import interpolate, optimize, ndimage x0 = numpy.arange(data.size) datalines = get_lines(x0, data) if datalines.size < 15: sparse = True Xstart = int(datalines.min() - 15.) Xend = int(datalines.max() + 15.) if Xstart < 0: Xstart = 0 if Xend > data.size: Xend = data.size tdata = data[Xstart:Xend].copy() wave = numpy.linspace(center - scale * tdata.size / 2, center + scale * tdata.size / 2, tdata.size) cond = (wave < 10300) mod = interpolate.splev(wave, model['matched']) fftd = numpy.fft.rfft(ndimage.gaussian_filter(tdata[cond], 17)) fftm = numpy.fft.rfft(ndimage.gaussian_filter(mod[cond], 17)[::-1]) fftd[100:] = 0. fftm[100:] = 0. T = numpy.fft.fftshift(numpy.fft.irfft(fftd * fftm)).real offset = T.size / 2 - T.argmax() # offset = T.size/2 - 1747 Wstart = wave[offset] print "Assuming the starting wavelength is %6.1f" % (Wstart) # pylab.plot(T) # pylab.show() if sparse is True: nlines = datalines.size n = order + 2 c = numpy.atleast_2d(numpy.array([Wstart, scale])).T fit = {'coeff': c, 'type': 'polynomial'} first = True while n < nlines: end = datalines[n] + 15. if end > data.size: end = data.size tdata = data[Xstart:end].copy() x = x0[Xstart:end].copy() if first is True: W = sf.genfunc(x - Xstart, 0., fit) else: W = sf.genfunc(x, 0., fit) cond = (W < 10400) W = W[cond] tdata = tdata[cond] x = x[cond] mod = interpolate.splev(W, model['matched']) fftd = numpy.fft.rfft(tdata) fftm = numpy.fft.rfft(mod[::-1]) T = numpy.fft.fftshift(numpy.fft.irfft(fftd * fftm)).real offset = T.size / 2 - T.argmax() Wlines = get_lines(x - offset, mod) Wwave = get_lines(W, mod) matches = [] for i in range(Wlines.size): diff = abs(Wlines[i] - datalines[:n]) if diff.min() < 2: matches.append([datalines[diff.argmin()], Wwave[i]]) if len(matches) < order + 1: n += 1 continue matches = numpy.asarray(matches) fit = sf.lsqfit(matches, 'polynomial', order) n += 1 first = False return fit coeff = numpy.ones(order + 1) rescale = numpy.array([Wstart - Xstart * scale, scale]) for i in range(order - 1): rescale = numpy.append(rescale, 1. / (data.size / 2)**(i + 2)) def optFunc(p, x, spec, mod): if (numpy.isnan(p)).any() or p[0] < 0.5 or p[1] < 0.8 or p[1] > 1.2: return spec coeff = numpy.atleast_2d(numpy.array(p) * rescale).T fit = {'coeff': coeff, 'type': 'polynomial'} w = sf.genfunc(x, 0., fit) m = interpolate.splev(w, mod) cond = (w < 9800.) m = m[cond] m /= m.mean() resid = (spec[cond] - m) / abs(spec[cond] + m)**0.5 return resid / resid.size import pylab from scipy import ndimage n = 2 * order + 2 end = datalines[n] + 15 if end < Xstart + (Xend - Xstart) / 4: end = Xstart + (Xend - Xstart) / 4 widespec = ndimage.gaussian_filter(data, 15) x = x0[Xstart:end].copy() spec = widespec[Xstart:end].copy() spec /= spec.mean() coeff, ier = optimize.leastsq(optFunc, coeff, (x, spec, model['wide']), factor=99., ftol=1e-12, xtol=1e-12, epsfcn=1e-11) while end < Xend: x = x0[Xstart:end].copy() # spec = widespec[Xstart:end].copy() # spec /= spec.mean() # coeff,ier = optimize.leastsq(optFunc,coeff,(x,spec,model['wide']),factor=99.,ftol=1e-12,xtol=1e-12,epsfcn=1e-11) spec = data[Xstart:end].copy() spec /= spec.mean() coeff, ier = optimize.leastsq(optFunc, coeff, (x, spec, model['matched']), factor=99., ftol=1e-12, xtol=1e-12, epsfcn=1e-11) fit = { 'coeff': scipy.atleast_2d(coeff * rescale).T, 'type': 'polynomial' } w = sf.genfunc(x, 0., fit) m = interpolate.splev(w, model['matched']) cond = (w < 10400) m = m[cond] m /= m.mean() coeff, ier = optimize.leastsq(optFunc, coeff, (x, spec, model['matched']), factor=99., ftol=1e-12, xtol=1e-12, epsfcn=1e-11) end += (Xend - Xstart) / 10. x = x0[Xstart:Xend].copy() spec = data[Xstart:Xend].copy() spec /= spec.mean() coeff, ier = optimize.leastsq(optFunc, coeff, (x, spec, model['matched']), ftol=1e-12, xtol=1e-12, epsfcn=1e-7) fit = {'coeff': scipy.atleast_2d(coeff * rescale).T, 'type': 'polynomial'} w = sf.genfunc(x, 0., fit) m = interpolate.splev(w, model['matched']) cond = (w < 10400) m = m[cond] m /= m.mean() pylab.plot(w[cond], m) pylab.plot(w[cond], spec[cond]) pylab.show() return fit w = sf.genfunc(x, 0., fit) m = interpolate.splev(w, model['matched']) rhs = m / err fit = numpy.array(numpy.linalg.lstsq(op, rhs)[0]) d = numpy.dot(linmod.T, fit) pylab.plot(w, m) pylab.plot(w, d) w0 = w.copy() fit = {'coeff': scipy.atleast_2d(pars).T, 'type': 'polynomial'} w = sf.genfunc(x, 0., fit) m = interpolate.splev(w, model['matched']) rhs = m / err fit = numpy.array(numpy.linalg.lstsq(op, rhs)[0]) d2 = numpy.dot(linmod.T, fit) pylab.plot(w, d2) print(w0 - w).std() pylab.figure() pylab.plot(w0 - w) pylab.show() return fit df
def ycorrect(data): # Parameters SUMWIDTH = 41 # Width of summing over columns y_axis = data.shape[0] x_axis = data.shape[1] # Determine the location of the central column central = x_axis / 2 # Determine the centers of the holes in the center column to use as the # reference for all other columns x_min_orig = central - SUMWIDTH / 2 x_max_orig = central + SUMWIDTH / 2 midcol = data[:, x_min_orig:x_max_orig].mean(axis=1) central_edges, threshold, star_cutoff = find_holes(midcol) transform_table = scipy.zeros((1, 3), 'f4') index = 0 for peak in central_edges: if index: transform_table.resize((index + 1, 3)) transform_table[index, 0] = central transform_table[index, 1] = peak transform_table[index, 2] = peak index += 1 offset = scipy.zeros(len(central_edges)) x_min = x_min_orig x_max = x_max_orig current_column = central while current_column > SUMWIDTH + 20: current_column = current_column - SUMWIDTH - 10 x_min = x_min - SUMWIDTH - 10 x_max = x_max - SUMWIDTH - 10 comp_array = data[:, x_min:x_max].mean(axis=1) comp_array.clip(min=-1000., max=star_cutoff) derivative = deriv_1d(comp_array) derivative = ndimage.gaussian_filter1d(derivative, 3) derivative = abs(derivative) for i in range(offset.size): if scipy.isnan(offset[i]): continue ref = central_edges[i] + offset[i] start = int(ref) - 6 end = start + 13 if derivative[start:end].max() < threshold: offset[i] = scipy.nan continue fit = find_peak(derivative[start:end]) if (fit[2] < 0 or fit[2] > 13 or fit[3] < 1 or fit[3] > 6): offset[i] = scipy.nan continue peak = fit[2] + float(start) offset[i] = peak - central_edges[i] transform_table.resize((index + 1, 3)) transform_table[index, 0] = current_column transform_table[index, 1] = central_edges[i] transform_table[index, 2] = peak index += 1 offset = scipy.zeros(offset.size) x_min = x_min_orig x_max = x_max_orig current_column = central while current_column < x_axis - SUMWIDTH - 19: current_column = current_column + SUMWIDTH + 10 x_min = x_min + SUMWIDTH + 10 x_max = x_max + SUMWIDTH + 10 comp_array = data[:, x_min:x_max].mean(axis=1) comp_array.clip(min=-1000., max=star_cutoff) derivative = deriv_1d(comp_array) derivative = ndimage.gaussian_filter1d(derivative, 3) derivative = abs(derivative) for i in range(offset.size): if scipy.isnan(offset[i]): continue ref = central_edges[i] + offset[i] start = int(round(ref)) - 6 end = start + 13 if derivative[start:end].max() < threshold: offset[i] = scipy.nan continue fit = find_peak(derivative[start:end]) if (fit[2] < 0 or fit[2] > 13 or fit[3] < 1 or fit[3] > 6): offset[i] = scipy.nan continue peak = fit[2] + float(start) offset[i] = peak - central_edges[i] transform_table.resize((index + 1, 3)) transform_table[index, 0] = current_column transform_table[index, 1] = central_edges[i] transform_table[index, 2] = peak index += 1 true_coeffs = special_functions.lsqfit(transform_table, "chebyshev", 4, 4) temp = transform_table[:, 1].copy() transform_table[:, 1] = transform_table[:, 2].copy() transform_table[:, 2] = temp.copy() map_coeffs = special_functions.lsqfit(transform_table, "chebyshev", 4, 4) # The xtrue_coeffs give the "true" y value for that pixel. xmap_coeffs # describe where the coordinate should be mapped to in an output grid # (in other words the true pixel value for that y). return true_coeffs, map_coeffs
def wave_arcsky(arc, arcmodel, sky, solution): """ First find the best solution with the skylines, then apply this solution to all arclines within the bounds of the lowest/highest wavelength skylines, solving for the delta_pixel offset between the sky and the arc. Then find the solution for all lines (sky and delta_pixel-offset arcs). """ def clip(arr): a = arr.copy() m, s, l = a.mean(), a.std(), a.size while 1: a = a[abs(a - m) < 3. * s] if a.size == l: return m, s m, s, l = a.mean(), a.std(), a.size STD_LINES = [ 5197.928, 5200.286, 5202.977, 5460.735, 5577.345, 5867.5522, 5915.308, 5932.864, 6257.970, 6300.320, 6363.810, 6533.040, 6553.610, 6863.971, 6912.620, 6923.210, 6939.520, 7303.716, 7329.148, 7340.885, 7358.659, 7392.198, 7586.093, 7808.467, 7821.510, 7841.266, 7993.332, 8310.719, 8344.613, 8399.160, 8415.231, 8430.170, 8791.186, 8885.830, 8943.395, 8988.384, 9038.059, 9337.854, 9375.977, 9419.746, 9439.670, 9458.524 ] x = scipy.arange(sky.size).astype(scipy.float32) lines = get_lines(x, sky) global fit scale = solution['coeff'][1] order = solution['coeff'].size - 1 if scale > 1.5: STD_LINES.insert(6, 5891.) w = sf.genfunc(lines, 0., solution) matches = [] for i in range(w.size): diff = abs(w[i] - STD_LINES) if diff.min() < 5. * scale: matches.append([lines[i], STD_LINES[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order) w = sf.genfunc(lines, 0., fit) matches = [] for i in range(w.size): diff = abs(w[i] - STD_LINES) if diff.min() < 5. * scale: matches.append([lines[i], STD_LINES[diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order) lines = get_lines(x, sky, nstd=7.) w = sf.genfunc(lines, 0., fit) matches = [] for i in range(w.size): diff = abs(w[i] - STD_LINES) if diff.min() < 3. * scale: matches.append([lines[i], STD_LINES[diff.argmin()]]) matches = scipy.asarray(matches) fit = sf.lsqfit(matches, 'polynomial', order) revfit = sf.lsqfit(matches[:, ::-1], 'polynomial', order) ARCS = scipy.sort(arcmodel['lines']) alines = get_lines(x, arc) xmin, xmax = matches[0, 0], matches[-1, 0] arc_x = sf.genfunc(ARCS, 0., revfit) offset = [] for i in range(arc_x.size): if arc_x[i] < xmin - 2 or arc_x[i] > xmax + 2: continue diff = arc_x[i] - alines if abs(diff).min() < 9.: offset.append(diff[abs(diff).argmin()]) offset = scipy.asarray(offset) off, width = clip(offset) aw = sf.genfunc(alines + off, 0., fit) matches = [] for i in range(w.size): diff = abs(w[i] - STD_LINES) if diff.min() < 3. * scale: matches.append([lines[i], STD_LINES[diff.argmin()]]) k = len(matches) for i in range(aw.size): diff = abs(aw[i] - ARCS) if diff.min() < 3. * scale: matches.append([alines[i] + off, ARCS[diff.argmin()]]) matches = scipy.asarray(matches) fit = sf.lsqfit(matches, 'polynomial', order) """ # This code is to optimize the arc offset -- likely unnecessary. def opt(p): global fit off = p[0] w = sf.genfunc(lines,0.,fit) aw = sf.genfunc(alines+off,0.,fit) matches = [] for i in range(w.size): diff = abs(w[i]-STD_LINES) if diff.min()<3.*scale: matches.append([lines[i],STD_LINES[diff.argmin()]]) k = len(matches) for i in range(aw.size): diff = abs(aw[i]-ARCS) if diff.min()<3.*scale: matches.append([alines[i]+off,ARCS[diff.argmin()]]) matches = scipy.asarray(matches) fit = sf.lsqfit(matches,'polynomial',order) return (matches[:,1]-sf.genfunc(matches[:,0],0.,fit)) from scipy import optimize coeff,ier = optimize.leastsq(opt,[off],epsfcn=1e-15) """ return fit
def bgsub(dir, inname, out_prefix, cal_prefix): # Where things begin and end.... blue = [1500, 1400, 1300, 1200, 1100, 900, 600, 200, 0, 0, 0] red = [3000, 3400, 3700, -1, -1, -1, -1, -1, -1, -1] readvar = 7.3 bias = pyfits.open(cal_prefix + "_bias.fits")[0].data.astype(scipy.float32) bpm = pyfits.open(cal_prefix + "_bpm.fits")[0].data.astype(scipy.float32) flat = pyfits.open(cal_prefix + "_norm.fits")[0].data.astype(scipy.float32) orders, y_soln, wideorders = numpy.load(cal_prefix + "_ycor.dat", allow_pickle=True) fullsoln = numpy.load(cal_prefix + "_full.dat", allow_pickle=True) hdu = pyfits.open(dir + inname) data = hdu[0].data.copy() data = biastrim(data, bias, bpm) / flat if data.shape[1] < 3000: blue = [i / 2 for i in blue] red = [1500, 1700, 1850, -1, -1, -1, -1, -1, -1, -1] """ try: back = pyfits.open(out_prefix+"_bg.fits")[0].data.copy() print("Initial background opened") except: strt = straighten(data,y_soln,orders,wideorders) back = scipy.zeros(strt.shape) for indx in range(len(orders)): i,j = orders[indx] bg = numpy.nanmedian(strt[i:j],axis=0) back[i:j] += bg back = curve(back,y_soln,orders,wideorders) pyfits.PrimaryHDU(back).writeto(out_prefix+"_bg.fits") """ try: sub = pyfits.open(out_prefix + "_crsub.fits")[0].data.copy() masks = numpy.load(out_prefix + '_masks.dat', allow_pickle=True) print('Opened CR-subbed image') except: print('Creating CR-subbed image and masks') strt = straighten(data, y_soln, orders, wideorders) back = scipy.zeros(strt.shape) for indx in range(len(orders)): i, j = orders[indx] bg = numpy.nanmedian(strt[i:j], axis=0) back[i:j] += bg back = curve(back, y_soln, orders, wideorders) bgsub = data - back sub = bgsub.copy() omap = sub * 0. for loop in range(3): message('CR iteration %d ...' % (loop + 1)) map = crFind(sub, data + readvar) if map.sum() == 0: message(' no new CRs found.\n') break else: message(' %d pixels flagged.\n' % (map.sum())) inmask = sub.flatten() cond = numpy.where(map.ravel() == 1)[0] inmask[cond] = numpy.where(cond % 2 == 0, 1e5, -1e5).astype(inmask.dtype) med5 = ndimage.median_filter(inmask.reshape(sub.shape), 7) med5 *= map sub = (1. - map) * sub + med5 omap += map crsub = data - (bgsub - sub) bpm = ndimage.minimum_filter(numpy.where(bpm == 1, 1, 0), 3) #mask = numpy.where(numpy.isnan(bpm)|(omap>0),0.,1.) mask = numpy.where((bpm == 0) | (omap > 0), 0, 1) masks = getOrders(mask, orders, wideorders, fullsoln, 1) for i in range(len(masks)): masks[i][0] = numpy.where(masks[i][0] < 0.7, 0, 1) f = open(out_prefix + '_masks.dat', 'wb') dump(masks, f, 2) f.close() pyfits.PrimaryHDU(crsub).writeto(out_prefix + "_crsub.fits") hdulist = pyfits.HDUList([pyfits.PrimaryHDU()]) hdulist[0].header = hdu[0].header.copy() varhdu = pyfits.HDUList([pyfits.PrimaryHDU()]) varhdu[0].header = hdu[0].header.copy() print("Subtracting backgrounds") slits = getOrders(crsub, orders, wideorders, fullsoln) for indx in range(len(orders)): cut, wlo, whigh, disp = slits[indx] slit = cut.copy() mask = masks[indx][0] cut[scipy.isinf(cut)] = numpy.nan cut[mask == 0] = numpy.nan bg = numpy.nanmedian(cut, axis=0) tmp = (cut - bg) * mask T = tmp[3:-3, blue[indx]:red[indx]] T = T[numpy.isfinite(T)] avg, std = clip2(T) slice = numpy.nanmedian(tmp[:, blue[indx]:red[indx]], axis=1) slice[:3] = 0. slice[-3:] = 0. avg, std = clip2(slice[3:-3], 3.5, 0.05) good = numpy.where(slice < 2. * std, 1., 0.) good = ndimage.maximum_filter(good, 5) good = ndimage.minimum_filter(good, 15) good = good == 1 xvals = numpy.arange(good.size).astype(scipy.float32) fitdata = numpy.empty((slice[good].size, 2)) fitdata[:, 0] = xvals[good].copy() bgsub = slit.copy() for k in range(cut.shape[1]): fitdata[:, 1] = cut[:, k][good].copy() if fitdata[:, 1][numpy.isfinite(fitdata[:, 1])].size < 4: continue avg, std = clip(fitdata[:, 1]) keep = numpy.where((abs(fitdata[:, 1] - avg) < 4. * std) & numpy.isfinite(fitdata[:, 1]))[0] fit = fitdata[keep] if fit[:, 1][scipy.isfinite(fit[:, 1])].size < 4: continue fit = sf.lsqfit(fit, 'chebyshev', 1) bg = sf.genfunc(xvals, 0, fit) bgsub[:, k] -= bg hdu = pyfits.ImageHDU(bgsub) hdu.header.set('CTYPE1', 'LINEAR') hdu.header.set('CRVAL1', wlo) hdu.header.set('CRPIX1', 1) hdu.header.set('CDELT1', disp) hdu.header.set('CTYPE2', 'LINEAR') hdu.header.set('CRVAL2', 1) hdu.header.set('CRPIX2', 1) hdu.header.set('CDELT2', 1) hdu.header.set('CD1_1', disp) hdu.header.set('CD2_2', 1) hdu.header.set('CD1_2', 0) hdu.header.set('CD2_1', 0) hdu.header.set('DC-FLAG', 1) hdu.header.set('DISPAXIS', 1) hdulist.append(hdu) hdu = pyfits.ImageHDU(cut + readvar) hdu.header.set('CTYPE1', 'LINEAR') hdu.header.set('CRVAL1', wlo) hdu.header.set('CRPIX1', 1) hdu.header.set('CDELT1', disp) hdu.header.set('CTYPE2', 'LINEAR') hdu.header.set('CRVAL2', 1) hdu.header.set('CRPIX2', 1) hdu.header.set('CDELT2', 1) hdu.header.set('CD1_1', disp) hdu.header.set('CD2_2', 1) hdu.header.set('CD1_2', 0) hdu.header.set('CD2_1', 0) hdu.header.set('DC-FLAG', 1) hdu.header.set('DISPAXIS', 1) varhdu.append(hdu) hdulist.verify('fix') hdulist.writeto(out_prefix + "_bgsub.fits") varhdu.verify('fix') varhdu.writeto(out_prefix + "_var.fits")
def arcwave2(arc, arcmodel, scale, order=3, bcutoff=2e3, rcutoff=1e4): from scipy import ndimage, stats, interpolate, signal, optimize import pylab arc = arc.copy() arc[scipy.isnan(arc)] = 1. arc[arc <= 1.] = arc.mean() wave, model = arcmodel['orig'] model[scipy.isnan(model)] = 0. wave = wave.copy() model = model.copy() cond = (wave > bcutoff) & (wave < rcutoff) corrmodel = model.copy() corrmodel[(wave < bcutoff) | (wave > rcutoff)] = 0. corr = signal.correlate(arc, corrmodel, mode='valid') offset = corr.argmax() lines = arcmodel['lines'].copy() bc = lines.min() - scale * 20. rc = lines.max() + scale * 20. x = scipy.arange(arc[offset:].size) w = wave[offset:offset + x.size].copy() cond = (w > bc) & (w < rc) fit = scipy.empty((x[cond].size, 2)) fit[:, 0] = x[cond].copy() fit[:, 1] = w[cond].copy() pars = sf.lsqfit(fit, 'polynomial', 1) pars['coeff'][0] = wave[offset] pars['coeff'][1] = scale # pars = [wave[offset],scale] # for i in range(2,order+1): # pars.append(1e-5**i) pylab.plot(wave, model) w = sf.genfunc(scipy.arange(x.size), 0., pars) pylab.plot(w, interpolate.splev(w, arcmodel['matched'])) pylab.show() def arcfit(p, x, arc, mod): fit = {'coeff': scipy.atleast_2d(p).T, 'type': 'polynomial'} w = sf.genfunc(x, 0., fit) cond = (w > bcutoff) & (w < rcutoff) m = interpolate.splev(w[cond], mod) chi = (m - arc[cond]) / abs(arc[cond])**0.5 return chi widearc = ndimage.gaussian_filter(arc, 7.) x = scipy.arange(arc[offset:].size) coeff, ier = optimize.leastsq( arcfit, pars, (x, widearc[offset:].copy(), arcmodel['wide']), maxfev=100000) fit = {'coeff': scipy.atleast_2d(coeff).T, 'type': 'polynomial'} x = scipy.arange(arc.size) l = get_lines(x, arc, nstd=15.) lw = sf.genfunc(l - offset, 0., fit) lines = [] for i in range(l.size): diff = abs(lw[i] - arcmodel['lines']) if diff.min() > 5. * scale: continue lines.append([l[i], arcmodel['lines'][diff.argmin()]]) fit = sf.lsqfit(scipy.asarray(lines), 'polynomial', order) pars = fit['coeff'].flatten() coeff, ier = optimize.leastsq( arcfit, pars, (x[offset:], arc[offset:], arcmodel['matched']), maxfev=100000) fit = {'coeff': scipy.atleast_2d(coeff).T, 'type': 'polynomial'} return fit