def dofit(self):
print("Press Enter to input order")
order = raw_input("Order: ")
order = int(order)
lines = scipy.empty(len(self.lines))
wave = scipy.empty(len(self.ids))
for i in range(lines.size):
lines[i] = self.lines[i]
wave[i] = self.ids[i]
fit = scipy.empty((lines.size, 2))
fit[:, 0] = lines.copy()
fit[:, 1] = wave.copy()
self.solution = sf.lsqfit(fit, 'chebyshev', order)
fit[:, 0] = wave.copy()
fit[:, 1] = lines.copy()
self.inverse = sf.lsqfit(fit, 'chebyshev', order)
self.wave = sf.genfunc(self.points, 0., self.solution)
pylab.close()
self.__call__()
for i in self.markers:
wave = sf.genfunc(i, 0., self.solution)
verts = self.markers[i].get_verts()
bottom = verts[0][1]
top = verts[1][1]
wave = wave[0]
mark = mpl.patches.Polygon([(wave, bottom), (wave, top)])
self.markers[wave] = mark
self.ax.add_patch(mark)
del self.markers[i]
pylab.draw()
def combine_xyw(coords, xsoln, ysoln, wsoln, xord, yord):
x = coords[1].flatten()
y = coords[0].flatten()
newy = ysoln.flatten()
from scipy import random
k = random.random(x.size)
args = k.argsort()
x = x[args[:y.size / 10]]
newy = newy[args[:y.size / 10]]
y = y[args[:y.size / 10]]
newx = sf.genfunc(x, y, xsoln['back'])
wave = sf.genfunc(newx, 0., wsoln)
data = scipy.empty((x.size, 3))
data[:, 0] = wave.copy()
data[:, 1] = y.copy()
data[:, 2] = x.copy()
output_to_ccdx = sf.lsqfit(data, 'chebyshev', xord, yord)
data[:, 2] = newy.copy()
output_to_ccdy = sf.lsqfit(data, 'chebyshev', xord, yord)
data[:, 0] = x.copy()
data[:, 1] = y.copy()
data[:, 2] = wave.copy()
ccdx_ycor_to_wave = sf.lsqfit(data, 'chebyshev', xord, yord)
return {
'sky2x': output_to_ccdx,
'sky2y': output_to_ccdy,
'ccd2wave': ccdx_ycor_to_wave
}
def getWsoln(sky, x, wsolution, wmodel):
def refine(p, x, d, sky, model):
pars = {'coeff': numpy.atleast_2d(p).T, 'type': 'polynomial'}
w = sf.genfunc(x, 0., pars)
c = (w > wmodel['blue']) & (w < wmodel['red'])
mod = interpolate.splev(w[c], model)
mod /= mod.mean()
return (d[c] - mod) / abs(sky[c] + mod)**0.5
# For `by hand' fitting
if wsolution is None:
import id_spec
wsolution = id_spec.id_spec(sky, wmodel)
x0 = numpy.arange(sky.size)
w = sf.genfunc(x0, 0., wsolution)
wsolution = sf.lsqfit(numpy.array([x, w]).T, 'polynomial', 3)
else:
import pylab
tmpD = sky.copy()
tmpD /= sky.mean()
sky /= sky.mean()**2
T = FFTFilter(tmpD, 100)
Dlines = spf.get_lines(x, T, nstd=7.)
Dwlines = sf.genfunc(Dlines, 0., wsolution)
w = sf.genfunc(x, 0., wsolution)
c = (w > wmodel['blue']) & (w < wmodel['red'])
mod = interpolate.splev(w[c], wmodel['model'])
Slines = spf.get_lines(w[c], mod, nstd=7.)
matches = []
for j in range(Dwlines.size):
diff = abs(Dwlines[j] - Slines)
if diff.min() < 5. * wmodel['scale']:
matches.append([Dlines[j], Slines[diff.argmin()]])
wsolution = sf.lsqfit(numpy.asarray(matches), 'polynomial', 3)
start = wsolution['coeff'].flatten()
coeff, ier = optimize.leastsq(refine,
start, (x, tmpD, sky, wmodel['model']),
epsfcn=1e-5,
maxfev=10000,
ftol=1e-13,
xtol=1e-13)
wsolution = {'coeff': numpy.atleast_2d(coeff).T, 'type': 'polynomial'}
w = sf.genfunc(x, 0., wsolution)
c = (w > wmodel['blue']) & (w < wmodel['red'])
mod = interpolate.splev(w[c], wmodel['model'])
mod /= mod.mean()
pylab.plot(w[c], mod)
pylab.plot(w[c], tmpD[c])
pylab.show()
w = sf.genfunc(x, 0., wsolution)
rwsoln = sf.lsqfit(numpy.array([w, x]).T, 'polynomial', 3)
return wsolution, rwsoln
def wave_arclines(arc, arcmodel, sky, solution):
from scipy import interpolate
STD_LINES = scipy.sort(arcmodel['lines'])
SKYLINES = [5577.338, 6300.304, 6363.78, 6553.617] #,6912.62]
x = scipy.arange(arc.size).astype(scipy.float32)
lines = get_lines(x, arc)
skycoords = get_lines(x, sky)
arccoords = lines.copy()
scale = arcmodel['scale']
if scale > 1.5:
SKYLINES.insert(1, 5891.)
order = solution['coeff'].size - 1
fit = solution
w = sf.genfunc(lines, 0., fit)
matches = []
for line in STD_LINES:
diff = abs(w - line)
if diff.min() < 5. * scale:
pos = diff.argmin()
matches.append([lines[pos], line])
print matches, order
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order)
order = 1
for i in range(7):
matched = [m for m in matches]
if skycoords.size == 0:
offset = 0.
break
skyline = sf.genfunc(skycoords, 0., fit)
offsets = []
for line in SKYLINES[:i * 2 + 2]:
diff = abs(line - skyline)
if diff.min() < 5. * scale:
offsets.append(line - skyline[diff.argmin()])
if len(offsets) == 0:
offset = 0.
break
offset = scipy.median(scipy.asarray(offsets))
for line in SKYLINES[:i * 2 + 2]:
diff = abs(line - skyline)
if diff.min() < 5. * scale:
pos = diff.argmin()
matched.append([skycoords[pos], line - offset])
fit = sf.lsqfit(scipy.asarray(matched), 'polynomial', order)
fit['coeff'][0] += offset
return fit
def new_xsoln(data, xord=3, yord=3, tol=2.):
data = data.copy()
height, width = data.shape
indx = height / 4
mid = data[indx]
xvals = scipy.arange(mid.size)
middle_lines = get_lines(xvals, mid, False)
ref = []
for row in range(height):
ref.append(get_lines(xvals, data[row], False))
lines = []
for l in middle_lines:
"""
off is used to track the systematic shift of lines as we move
up and down the slit. This is necessary for strongly
curved lines that might be substantially shifted wrt
the central row.
"""
off = 0.
for row in range(indx, height): # The top of the slit
if len(ref[row]) == 0:
continue
p = ref[row] + off
diff = abs(l - p)
if diff.min() < tol:
pos = diff.argmin()
lines.append([ref[row][pos], row, l])
off = l - ref[row][pos]
off = 0.
for row in range(indx - 1, -1, -1): # The bottom of the slit
if len(ref[row]) == 0:
continue
p = ref[row] + off
diff = abs(l - p)
if diff.min() < tol:
pos = diff.argmin()
lines.append([ref[row][pos], row, l])
off = l - ref[row][pos]
lines = scipy.asarray(lines)
soln = sf.lsqfit(lines, 'chebyshev', xord, yord)
tmp = lines[:, 0].copy()
lines[:, 0] = lines[:, 2].copy()
lines[:, 2] = tmp.copy()
soln2 = sf.lsqfit(lines, 'chebyshev', xord, yord)
return {'back': soln, 'forw': soln2}
def matchlines(peaks, solution, linefile, tol=30., order=3, offset=False):
wave = special_functions.genfunc(peaks, 0., solution)
try:
lines = getlines(linefile)
except:
lines = linefile
gooddata = []
goodlines = []
for i in range(wave.size):
delta = 1e9
match = None
for j in lines:
if abs(j - wave[i]) < delta:
delta = abs(j - wave[i])
match = j
else:
break
if delta < tol:
gooddata.append(peaks[i])
goodlines.append(match)
fitdata = scipy.empty((len(gooddata), 2))
fitdata[:, 0] = scipy.asarray(gooddata)
fitdata[:, 1] = scipy.asarray(goodlines)
if offset:
a = special_functions.genfunc(fitdata[:, 0], 0., solution)
return fitdata[:, 1] - a
return special_functions.lsqfit(fitdata, 'chebyshev', order)
def update_xsoln(data, soln, xord=3, yord=3, skip=False):
data = data.copy()
height, width = data.shape
if skip:
slice = data.mean(1)
indx = slice.argsort()[slice.size / 4]
mid = data[indx]
badrow = scipy.where(slice > slice[indx] + 5. * (slice[indx])**0.5)[0]
if badrow.size == 0:
badrow = scipy.array([-1])
else:
indx = height / 2
mid = data[indx]
badrow = scipy.array([-1])
xvals = scipy.arange(mid.size)
middle_lines = get_lines(xvals, mid, False)
straight = sf.genfunc(middle_lines, indx, soln['back'])
ref = []
lines = []
for row in range(height):
if abs(row - badrow).min() == 0:
continue
current = get_lines(xvals, data[row], False)
if current.size == 0:
continue
guess = sf.genfunc(current, row, soln['back'])
for l in straight:
diff = abs(l - guess)
if diff.min() < 2:
pos = diff.argmin()
lines.append([current[pos], row, l])
lines = scipy.asarray(lines)
newsoln = sf.lsqfit(lines, 'chebyshev', xord, yord)
tmp = lines[:, 0].copy()
lines[:, 0] = lines[:, 2].copy()
lines[:, 2] = tmp.copy()
del tmp
newsoln2 = sf.lsqfit(lines, 'chebyshev', xord, yord)
return {'back': newsoln, 'forw': newsoln2}
def doublematch(peaks,
w,
linefile,
skyin,
tol=30.,
order=3,
shift=0.,
logfile=None):
lines = scipy.asarray(skyin)
lines.sort()
p = peaks[0]
wave = special_functions.genfunc(p, 0., w)
goodmatches = []
for i in range(wave.size):
delta = 1e9
match = None
for j in lines:
if abs(j - wave[i]) < delta:
delta = abs(j - wave[i] + shift)
match = j
else:
break
if delta < tol:
goodmatches.append([p[i], match])
p = peaks[1]
wave = special_functions.genfunc(p, 0., w)
lines = scipy.asarray(linefile)
# lines = getlines(linefile)
lines.sort()
for i in range(wave.size):
delta = 1e9
match = None
for arc in lines:
if abs(arc - wave[i]) < delta:
delta = abs(arc - wave[i] + shift)
match = arc
else:
break
if delta < tol:
goodmatches.append([p[i], match])
fitdata = scipy.asarray(goodmatches)
fit = special_functions.lsqfit(fitdata, 'chebyshev', order)
match = special_functions.genfunc(fitdata[:, 0], 0., fit)
diff = match - fitdata[:, 1]
error = stats.stats.std(diff)
if logfile is not None:
logentry = "\tWavelength solution error: %5.3f angstroms from %d lines\n" % (
error, diff.size)
for i in range(diff.size):
logentry += "\t\t%7.2f %7.2f %5.2f\n" % (fitdata[i, 1],
match[i], diff[i])
keeplog(logfile, logentry)
return fit, error
def wave_skylines(sky, solution):
STD_LINES = [
5197.928, 5200.286, 5202.977, 5460.735, 5577.345, 5867.5522, 5915.308,
5932.864, 6257.970, 6300.320, 6363.810, 6533.040, 6553.610, 6863.971,
6912.620, 6923.210, 6939.520, 7303.716, 7329.148, 7340.885, 7358.659,
7392.198, 7586.093, 7808.467, 7821.510, 7841.266, 7993.332, 8310.719,
8344.613, 8399.160, 8415.231, 8430.170, 8791.186, 8885.830, 8943.395,
8988.384, 9038.059, 9337.854, 9375.977, 9419.746, 9439.670, 9458.524
]
x = scipy.arange(sky.size).astype(scipy.float32)
lines = get_lines(x, sky)
scale = solution['coeff'][1]
order = solution['coeff'].size - 1
if scale > 1.5:
STD_LINES.insert(6, 5891.)
w = sf.genfunc(lines, 0., solution)
matches = []
for i in range(w.size):
diff = abs(w[i] - STD_LINES)
if diff.min() < 5. * scale:
matches.append([lines[i], STD_LINES[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order)
w = sf.genfunc(lines, 0., fit)
matches = []
for i in range(w.size):
diff = abs(w[i] - STD_LINES)
if diff.min() < 5. * scale:
matches.append([lines[i], STD_LINES[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order)
lines = get_lines(x, sky, nstd=7.)
w = sf.genfunc(lines, 0., fit)
matches = []
for i in range(w.size):
diff = abs(w[i] - STD_LINES)
if diff.min() < 3. * scale:
matches.append([lines[i], STD_LINES[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order)
return fit
def combine_xy(offset, coords, xsoln, ysoln, xord, yord):
x = coords[1].flatten()
y = coords[0].flatten()
newy = ysoln['back'].flatten()
newx = sf.genfunc(x, y - offset, xsoln['back'])
data = scipy.empty((x.size, 3))
data[:, 0] = x.copy()
data[:, 1] = y.copy()
data[:, 2] = newx.copy()
soln = sf.lsqfit(data, 'chebyshev', xord, yord)
tmp = data[:, 0].copy()
data[:, 0] = data[:, 2].copy()
data[:, 2] = tmp.copy()
soln2 = sf.lsqfit(data, 'chebyshev', xord, yord)
return {'back': soln, 'forw': soln2}
def combine_xw(coords, xsoln, wsoln, xord, yord):
x = coords[1].flatten()
y = coords[0].flatten()
newx = sf.genfunc(x, y, xsoln['back'])
wave = sf.genfunc(newx, 0., wsoln)
data = scipy.empty((x.size, 3))
data[:, 0] = x.copy()
data[:, 1] = y.copy()
data[:, 2] = wave.copy()
soln = sf.lsqfit(data, 'chebyshev', xord, yord)
tmp = data[:, 0].copy()
data[:, 0] = data[:, 2].copy()
data[:, 2] = tmp.copy()
soln2 = sf.lsqfit(data, 'chebyshev', xord, yord)
return {'back': soln, 'forw': soln2}
def medsubtract(image,outname):
data = pyfits.open(image)[0].data.copy()
if data.ndim==3:
data = data[0].copy()
tmp = data.copy()
tmp[numpy.isnan(tmp)] = 0.
tmp -= numpy.sort(tmp,0)[tmp.shape[0]/5]
trace = tmp.sum(1)
peak = trace.argmax()
center = numpy.empty(data.shape[1])
w = center.copy()
for i in range(1+center.size/100):
b = i*100
e = b+100
if e>center.size:
e = center.size
if b==e:
continue
center[b:e] = tmp[:,b:e].sum(1).argmax()
bg = center.copy()
x = numpy.arange(data.shape[0])
for i in range(center.size):
d = tmp[:,i].copy()
peak = center[i]
if numpy.isnan(d[peak]):
center[i] = peak
continue
fit = numpy.array([0.,d[peak],peak,1.])
cond = ~numpy.isnan(d)
input = numpy.empty((d[cond].size,2))
input[:,0] = x[cond].copy()
input[:,1] = d[cond].copy()
fit,chi = sf.ngaussfit(input,fit)
center[i] = fit[2]
w[i] = fit[3]
fit = sf.lsqfit(ndimage.median_filter(center,17),'polynomial',5)
centroid = sf.genfunc(numpy.arange(bg.size),0.,fit)
w = numpy.median(w)
for i in range(bg.size):
d = data[:,i].copy()
d[centroid[i]-w*4:centroid[i]+w*4] = numpy.nan
data[:,i] -= stats.nanmedian(d)
hdu = pyfits.open(image)[0]
hdu.data = data.copy()
hdu.writeto(outname,clobber=True)
def ycor(img):
"""
Follow star traces along the x direction to correct y-distortion
"""
xvals = numpy.arange(img.shape[1])
yvals = numpy.arange(img.shape[0])
mid = spf.get_lines(numpy.arange(img.shape[0]), img[:, yvals.size / 2],
False)
ref = []
for col in range(xvals.size):
ref.append(spf.get_lines(yvals, img[:, col], False))
soln = []
for l in mid:
off = 0.
dy = []
for col in xrange(xvals.size / 2, xvals.max()):
if len(ref[col]) == 0:
continue
p = ref[col] + off
diff = abs(l - p)
if diff.min() < 1.5:
pos = diff.argmin()
dy.append([col, ref[col][pos] - l])
off = l - ref[col][pos]
off = 0.
for col in xrange(xvals.size / 2 - 1, -1, -1):
if len(ref[col]) == 0:
continue
p = ref[col] + off
diff = abs(l - p)
if diff.min() < 1.5:
pos = diff.argmin()
dy.append([col, ref[col][pos] - l])
off = l - ref[col][pos]
dy = numpy.asarray(dy)
fit = sf.lsqfit(dy, 'chebyshev', 3)
soln.append(sf.genfunc(xvals, 0., fit))
soln = numpy.asarray(soln)
soln = numpy.median(soln, 0)
y = numpy.indices(img.shape).astype(numpy.float32)[0] + soln
return y, y - soln * 2
def do_fit(self, fullFit=False):
p = self.peaks[self.selected == 1]
fitdata = scipy.empty((self.xdata.size, 2))
fitdata[:, 0] = scipy.arange(fitdata.shape[0])
fitdata[:, 1] = self.spec.get_xdata()
fitdata = fitdata[(fitdata[:, 0] > p[0] - 20)
& (fitdata[:, 0] < p[-1] + 20)]
ord = int(raw_input('Enter order of fit: '))
fit = sf.lsqfit(fitdata, 'polynomial', ord)
self.soln = fit
if fullFit == True:
from scipy import interpolate, optimize
spec = self.spec.get_ydata()
xvals = numpy.arange(spec.size).astype(numpy.float32)
def opt(p):
p = numpy.array(p)
n = p[-1]
coeff = numpy.atleast_2d(p[:-1]).T
m = {'coeff': coeff, 'type': 'polynomial'}
w = sf.genfunc(xvals, 0., m)
mod = n * interpolate.splev(w, self.skymodel)
return (spec - mod) / abs(spec)**0.5
pars = fit['coeff'].flatten().tolist()
pars.append(1.)
coeff, ier = optimize.leastsq(opt, pars, maxfev=10000, epsfcn=1e-5)
fit['coeff'] = numpy.atleast_2d(coeff[:-1]).T
self.soln = fit
print "pixel fit complete"
else:
print "Fit Complete"
xdata = sf.genfunc(scipy.arange(self.xdata.size), 0., fit)
# self.sky.set_xdata(xdata)
self.data.set_xdata(xdata)
for i in range(self.datapeaks.size):
x = xdata[self.datapeaks[i]]
l = self.datalines[i]
l.set_xdata([x, x])
plt.draw()
def startrace(stars,orders):
from scipy import ndimage
if stars.shape[0]>1500:
WIDTH = 6
fw = 9
else:
WIDTH = 4
fw = 7
nrows = stars.shape[0]
ncols = stars.shape[1]
# This probably isn't necessary....
background = ndimage.percentile_filter(stars,10.,(37,1))
background[numpy.isnan(background)] = 0.
background = ndimage.median_filter(background,7)
smooth = ndimage.gaussian_filter1d(stars-background,1.5,0)
stars -= background
start = stars[:,int(ncols/2)].copy()
# Find the traces down the middle of the image
peaks = ndimage.maximum_filter(start,fw)
peaks2 = ndimage.maximum_filter(start,3)
avg,sig = clip(start,3.)
thresh = sig*5.
all = numpy.where((peaks==start)&(peaks>thresh)&(peaks==peaks2))[0]
all = all.tolist()
yvals = numpy.arange(nrows)*1.
xvals = numpy.arange(ncols)*1.
wideorders = []
solutions = []
# Loop through the orders, following the traces in each order
for i in range(len(orders)):
start,end = orders[i]
while all[0]<start:
del all[0]
ordermatches = numpy.empty((0,3))
while all[0]<end:
peak = all[0]
col = int(ncols/2)
amp = smooth[peak,col]
line = peak
matches = []
# Go towards the blue side of the image
while col>0:
line = int(line)
if line<WIDTH or line+WIDTH>=nrows or stars[line-1:line+2,col].max()<amp/20.:
break
s1 = slice(line-WIDTH,line+WIDTH+1)
line = (stars[s1,col]*yvals[s1]).sum()/stars[s1,col].sum()
matches.append([col,line,peak])
col -= 1
col = int(ncols/2)+1
line = peak
# Go towards the red side of the image
while col<ncols:
line = int(line)
if line<WIDTH or line+WIDTH>=nrows:
break
s1 = slice(line-WIDTH,line+WIDTH+1)
# The red side has some defects....
if numpy.isnan(stars[s1,col]).any():
if i==0:
break
fitData = numpy.array(matches)[:,:2]
fit = lsqfit(fitData,'chebyshev',3)
linePos = genfunc(xvals,0.,fit).astype(numpy.int32)
while col<ncols-WIDTH and numpy.isnan(stars[s1,col]).any():
col += 1
s1 = slice(linePos[col]-WIDTH,linePos[col]+WIDTH+1)
if col>=ncols-WIDTH:
break
line = int((stars[s1,col]*yvals[s1]).sum()/stars[s1,col].sum())
if stars[line-1:line+2,col].max()<amp/20.:
break
matches.append([col,line,peak])
col += 1
# Filter garbage; trace should be within ~2pix of the model
matches = numpy.array(matches)
soln = lsqfit(matches[:,:2],'cheybshev',3)
resid = matches[:,1]-genfunc(matches[:,0],0.,soln)
good = abs(resid)<2.
ordermatches = numpy.concatenate((ordermatches,matches[good]))
del all[0]
matches = ordermatches
ord1,ord2 = 3,3
ymap = lsqfit(matches,"chebyshev",ord1,ord2)
# Clip bad points
resid = matches[:,2]-genfunc(matches[:,0],matches[:,1],ymap)
m,s = clip(resid,3.5)
good = abs(resid-m)<3.5*s
matches = matches[good]
ymap = lsqfit(matches,"chebyshev",ord1,ord2)
tmp = matches[:,1].copy()
matches[:,1] = matches[:,2].copy()
matches[:,2] = tmp.copy()
ytrue = lsqfit(matches,"chebyshev",ord1,ord2)
solutions.append([ytrue,ymap])
lower = int(genfunc(xvals,xvals*0.+start-3,ytrue).min())
upper = int(genfunc(xvals,xvals*0+end+3,ytrue).max())+1
if lower<0:
lower = 0
if upper>stars.shape[1]:
upper = stars.shape[1]
wideorders.append([lower,upper])
return solutions,wideorders
def solve(d,orders):
path = os.path.split(__file__)[0]
lines = {}
lines['cuar'] = numpy.loadtxt(path+"/data/cuar.lines")
lines['hgne'] = numpy.loadtxt(path+"/data/hgne.lines")
lines['xe'] = numpy.loadtxt(path+"/data/xe.lines")
#startsoln = numpy.load(path+"/data/test_wavesol.dat",
# allow_pickle=True)
startsoln = numpy.load(path+"/data/esi_wavesolution.dat",
allow_pickle=True)
#arclist = d.keys() # Under python 3 this does not produce a list
# and so arclist[0] below fails
arclist = list(d)
arclist.sort()
soln = []
if d[arclist[0]].shape[1]>3000:
xvals = numpy.arange(4096.)
cuslice = slice(3860,3940)
fw1 = 75.
fw2 = 9
else:
xvals = numpy.arange(1.,4096.,2.)
cuslice = slice(1930,1970)
fw1 = 37.
fw2 = 5
"""
Do a temporary kludge. In some cases, the finding of the orders
fails, and only 9 orders are found. In this case, we need to skip
the first of the orders
"""
if len(orders) == 9:
ordstart = 1
dord = 1
else:
ordstart = 0
dord = 0
for i in range(ordstart, 10):
solution = startsoln[i]
start,end = orders[(i-dord)]
peaks = {}
trace = {}
fitD = {}
WIDTH = 4
import pylab
for arc in arclist:
data = numpy.nanmedian(d[arc][start:end],axis=0)
data[scipy.isnan(data)] = 0.
if i==0 and arc=='cuar':
data[cuslice] = numpy.median(data)
trace[arc] = data.copy()
bak = ndimage.percentile_filter(data,50.,int(fw1))
bak = getContinuum(bak,40.)
data -= bak
fitD[arc] = data/d[arc][start:end].std(0)
p = ndimage.maximum_filter(data,fw2)
std = clip(scipy.trim_zeros(data),3.)[1]
nsig = ndimage.uniform_filter((data>7.*std)*1.,3)
peak = scipy.where((nsig==1)&(p>10.*std)&(p==data))[0]
peaks[arc] = []
for p in peak:
if p-WIDTH<0 or p+WIDTH+1>xvals.size:
continue
x = xvals[p-WIDTH:p+WIDTH+1].copy()#-xvals[p]
f = data[p-WIDTH:p+WIDTH+1].copy()
fitdata = scipy.array([x,f]).T
fit = scipy.array([0.,f.max(),xvals[p],1.])
fit,chi = sf.ngaussfit(fitdata,fit,weight=1)
peaks[arc].append(fit[2])#+xvals[p])
for converge in range(15):
wave = 10**sf.genfunc(xvals,0.,solution)
refit = []
corrA = {}
err = wave[int(wave.size/2)]-wave[int(wave.size/2-1)]
for arc in arclist:
corr = []
p = 10.**sf.genfunc(peaks[arc],0.,solution)
for k in range(p.size):
cent = p[k]
diff = cent-lines[arc]
corr.append(diff[abs(diff).argmin()])
corr = numpy.array(corr)
if corr.size<4:
continue
m,s = clip(corr)
corr = numpy.median(corr[abs(corr-m)<5.*s])
print(corr)
corrA[arc] = corr
# corr = m
#for arc in arclist:
p = 10.**sf.genfunc(peaks[arc],0.,solution)
for k in range(p.size):
pos = peaks[arc][k]
cent = p[k]
diff = abs(cent-lines[arc]-corr)
if diff.min()<2.*err:
refit.append([pos,lines[arc][diff.argmin()]])
refit = scipy.asarray(refit)
solution = sf.lsqfit(refit,'polynomial',3)
refit = []
err = solution['coeff'][1]
for arc in arclist:
data = trace[arc]
for pos in peaks[arc]:
cent = sf.genfunc(pos,0.,solution)
delta = 1e9
match = None
for j in lines[arc]:
diff = abs(cent-j)
if diff<delta and diff<1.*err:
delta = diff
match = j
if match is not None:
refit.append([pos,match])
refit = scipy.asarray(refit)
refit[:,1] = numpy.log10(refit[:,1])
solution = sf.lsqfit(refit,'chebyshev',3)
#refit[:,0],refit[:,1] = refit[:,1].copy(),refit[:,0].copy()
#refit = numpy.array([refit[:,1],refit[:,0]]).T
#refit = refit[:,::-1]
solution2 = sf.lsqfit(refit[:,::-1],'chebyshev',3)
#soln.append([solution,solution2])
w = 10**sf.genfunc(xvals,0.,solution)
if (w==wave).all() or converge>8:
print("Order %d converged in %d iterations"%(i,converge))
soln.append([solution,solution2])
break
for arc in arclist:
pylab.plot(w,trace[arc])
pylab.plot(w,fitD[arc])
pp = 10**sf.genfunc(peaks[arc],0.,solution)
for p in pp:
pylab.axvline(p,c='k')
for j in 10**refit[:,1]:
if j>w[0] and j<w[-1]:
pylab.axvline(j)
pylab.show()
break
return soln
def jointSolve(d,orders):
import pylab
path = __path__[0]
lines = {}
lines['cuar'] = numpy.loadtxt(path+"/data/cuar.lines")
lines['hgne'] = numpy.loadtxt(path+"/data/hgne.lines")
lines['xe'] = numpy.loadtxt(path+"/data/xe.lines")
startsoln = numpy.load(path+"/data/esi_wavesolution.dat")
startsoln = numpy.load(path+'/data/shao_wave.dat')
startsoln = [i for i,j in startsoln]
alldata = d['arc']
arclist = d.keys()
arclist.remove('arc')
soln = []
if alldata.shape[1]>3000:
xvals = numpy.arange(4096.)
resolve = False
cuslice = slice(3860,3940)
fw1 = 75.
fw2 = 9
WIDTH = 4
else:
xvals = numpy.arange(2048.)
resolve = True
cuslice = slice(1930,1970)
fw1 = 37.
fw2 = 7
WIDTH = 3
for i in range(10):
solution = startsoln[i]
start,end = orders[i]
if resolve==True:
tmp = numpy.arange(0.5,4096.,2.)
w = sf.genfunc(tmp,0.,solution)
solution = sf.lsqfit(numpy.array([xvals,w]).T,'chebyshev',3)
data = numpy.nanmedian(alldata[start:end],axis=0)
data[numpy.isnan(data)] = 0.
if i==0:
data[cuslice] = numpy.median(data)
bak = ndimage.percentile_filter(data,50.,int(fw1))
data -= bak
peaks = []
p = ndimage.maximum_filter(data,fw2)
std = clip(numpy.trim_zeros(data),3.)[1]
peak = numpy.where((p>30.*std)&(p==data))[0]
for p in peak:
if p-WIDTH<0 or p+WIDTH+1>xvals.size:
continue
x = xvals[p-WIDTH:p+WIDTH+1].copy()
f = data[p-WIDTH:p+WIDTH+1].copy()
fitdata = numpy.array([x,f]).T
fit = numpy.array([0.,f.max(),xvals[p],1.])
fit,chi = sf.ngaussfit(fitdata,fit,weight=1)
peaks.append(fit[2])
for converge in range(10):
wave = 10**sf.genfunc(xvals,0.,solution)
refit = []
corr = []
err = wave[wave.size/2]-wave[wave.size/2-1]
p = 10.**sf.genfunc(peaks,0.,solution)
for arc in arclist:
for k in range(p.size):
if i==0 and p[k]>4344.:
continue
cent = p[k]
diff = cent-lines[arc]
corr.append(diff[abs(diff).argmin()])
corr = numpy.array(corr)
corr = corr[abs(corr)<5*err]
m,s = clip(corr)
corr = numpy.median(corr[abs(corr-m)<5.*s])
for arc in arclist:
for k in range(p.size):
if i==0 and p[k]>4344.:
continue
pos = peaks[k]
cent = p[k]
diff = abs(cent-lines[arc]-corr)
if diff.min()<2.*err:
refit.append([pos,lines[arc][diff.argmin()]])
refit = numpy.asarray(refit)
solution = sf.lsqfit(refit,'polynomial',3)
refit = []
err = solution['coeff'][1]
p = sf.genfunc(peaks,0.,solution)
for k in range(p.size):
delta = 1e9
match = None
for arc in arclist:
for j in lines[arc]:
if i==0 and j>4344.:
continue
diff = abs(p[k]-j)
if diff<delta and diff<1.*err:
delta = diff
match = j
if match is not None:
refit.append([peaks[k],match])
refit = numpy.asarray(refit)
refit[:,1] = numpy.log10(refit[:,1])
solution = sf.lsqfit(refit,'chebyshev',3)
solution2 = sf.lsqfit(refit[:,::-1],'chebyshev',3)
g = 10**sf.genfunc(peaks,0.,solution)
g2 = 10**sf.genfunc(peak,0.,solution)
w = 10**sf.genfunc(xvals,0.,solution)
if (w==wave).all():
print("Order %d converged in %d iterations"%(i,converge))
soln.append([solution,solution2])
break
pylab.plot(w,data)
for arc in arclist:
for j in lines[arc]:
if j>w[0] and j<w[-1]:
pylab.axvline(j,c='b')
for j in 10**refit[:,1]:
if j>w[0] and j<w[-1]:
pylab.axvline(j,c='r')
for j in g:
pylab.axvline(j,c='g')
for j in g2:
pylab.axvline(j,c='c')
pylab.show()
break
return soln
def bgsub(dir, inname, out_prefix, cal_prefix):
# Where things begin and end....
blue = [1500, 1400, 1300, 1200, 1100, 900, 600, 200, 0, 0, 0]
red = [3000, 3400, 3700, -1, -1, -1, -1, -1, -1, -1]
readvar = 7.3
bias = pyfits.open(cal_prefix + "_bias.fits")[0].data.astype(scipy.float32)
bpm = pyfits.open(cal_prefix + "_bpm.fits")[0].data.astype(scipy.float32)
flat = pyfits.open(cal_prefix + "_norm.fits")[0].data.astype(scipy.float32)
orders, y_soln, wideorders = numpy.load(cal_prefix + "_ycor.dat")
fullsoln = numpy.load(cal_prefix + "_full.dat")
try:
masks = numpy.loadtxt(cal_prefix + "_bpmOrders.dat")
print('BPM opened')
except:
print("Making BPM")
mask = numpy.where(bpm == 1., 1., 0.)
mask = getOrders(mask, y_soln, orders, wideorders)
for i in range(len(mask)):
mask[i][0] = numpy.where(mask[i][0] < 0.7, numpy.nan, 1.)
f = open(cal_prefix + "_bpmOrders.dat", 'wb')
cPickle.dump(mask, f, 2)
f.close()
hdu = pyfits.open(dir + inname)
data = hdu[0].data.copy()
data = biastrim(data, bias, bpm) / flat
try:
back = pyfits.open(out_prefix + "_bg.fits")[0].data.copy()
except:
strt = straighten(data, y_soln, orders, wideorders)
back = scipy.zeros(strt.shape)
for indx in range(len(orders)):
i, j = orders[indx]
bg = stats.stats.nanmedian(strt[i:j], axis=0)
back[i:j] += bg
back = curve(back, y_soln, orders, wideorders)
pyfits.PrimaryHDU(back).writeto(out_prefix + "_bg.fits")
try:
strt = pyfits.open(out_prefix + "_strt.fits")[0].data.copy()
except:
bgsub = data - back
sub = bgsub.copy()
omap = sub * 0.
for loop in range(3):
message('CR iteration %d ...' % (loop + 1))
map = crFind(sub, data + readvar)
if map.sum() == 0:
message(' no new CRs found.\n')
break
else:
message(' %d pixels flagged.\n' % (map.sum()))
inmask = sub.flatten()
cond = numpy.where(map.ravel() == 1)[0]
inmask[cond] = numpy.where(cond % 2 == 0, 1e5,
-1e5).astype(inmask.dtype)
med5 = ndimage.median_filter(inmask.reshape(sub.shape), 7)
med5 *= map
sub = (1. - map) * sub + med5
oCRS = bgsub - sub
omap += map
pyfits.PrimaryHDU(oCRS).writeto('blah.fits', clobber=True)
omap = omap > 0
sub = numpy.where(numpy.isfinite(bgsub), bgsub, 0.)
ccdBG = numpy.where(numpy.isfinite(back), back, 0.)
orig = numpy.where(numpy.isfinite(data), data, 0.)
readvar = 2.7**2
data -= crmask
strt = straighten(data, y_soln, orders)
pyfits.PrimaryHDU(strt).writeto(out_prefix + "_strt.fits")
hdulist = pyfits.HDUList([pyfits.PrimaryHDU()])
hdulist[0].header = hdu[0].header.copy()
varhdu = pyfits.HDUList([pyfits.PrimaryHDU()])
varhdu[0].header = hdu[0].header.copy()
for indx in range(len(orders)):
i, j = orders[indx]
cut = strt[i:j].copy()
cut[scipy.isinf(cut)] = scipy.nan
bg = stats.stats.nanmedian(cut, axis=0)
B = mask[i:j]
tmp = (cut - bg) * B
T = tmp[3:-3, blue[indx]:red[indx]]
T = T[numpy.isfinite(T)]
avg, std = clip2(T)
slice = stats.stats.nanmedian(tmp[:, blue[indx]:red[indx]], axis=1)
slice[:3] = 0.
slice[-3:] = 0.
avg, std = clip2(slice[3:-3], 3.5, 0.05)
good = numpy.where(slice < 2. * std, 1., 0.)
good = ndimage.maximum_filter(good, 5)
good = ndimage.minimum_filter(good, 15)
good = good == 1
xvals = scipy.arange(good.size).astype(scipy.float32)
fitdata = scipy.empty((slice[good].size, 2))
fitdata[:, 0] = xvals[good].copy()
bgsub = cut.copy()
for k in range(cut.shape[1]):
fitdata[:, 1] = cut[:, k][good].copy()
if fitdata[:, 1][scipy.isfinite(fitdata[:, 1])].size < 4:
continue
avg, std = clip(fitdata[:, 1])
keep = scipy.where((abs(fitdata[:, 1] - avg) < 4. * std)
& scipy.isfinite(fitdata[:, 1]))[0]
fit = fitdata[keep]
if fit[:, 1][scipy.isfinite(fit[:, 1])].size < 4:
continue
fit = sf.lsqfit(fit, 'chebyshev', 1)
bg = sf.genfunc(xvals, 0, fit)
bgsub[:, k] -= bg
solution = wave_soln[indx][0]
invsoln = wave_soln[indx][1]
vals = scipy.arange(cut.shape[1]).astype(scipy.float64)
wave = sf.genfunc(vals, 0., solution)
delta = (wave[wave.size - 1] - wave[0]) / (wave.size - 1)
outc = scipy.arange(wave[0], wave[0] + 2, delta)
outc = outc[:cut.shape[1]].copy()
pix = sf.genfunc(outc, 0, invsoln)
coords = spectools.array_coords(cut.shape).astype(numpy.float64)
coords[1] *= 0.
coords[1] += pix
bgsub[scipy.isnan(bgsub)] = 0.
out = ndimage.map_coordinates(bgsub,
coords,
output=scipy.float64,
order=5)
hdu = pyfits.ImageHDU(out.copy())
hdu.header.update('CTYPE1', 'WAVE-LOG')
hdu.header.update('CRVAL1', wave[0])
hdu.header.update('CRPIX1', 1)
hdu.header.update('CDELT1', delta)
hdu.header.update('CTYPE2', 'LINEAR')
hdu.header.update('CRVAL2', 1)
hdu.header.update('CRPIX2', 1)
hdu.header.update('CDELT2', 1)
hdu.header.update('CD1_1', delta)
hdu.header.update('CD2_2', 1)
hdu.header.update('CD1_2', 0)
hdu.header.update('CD2_1', 0)
#hdu.header.update('PC1_1',1)
#hdu.header.update('PC1_2',0)
#hdu.header.update('PC2_1',0)
#hdu.header.update('PC2_2',1)
hdu.header.update('DC-FLAG', 1)
hdu.header.update('WFITTYPE', 'LOG_LINEAR')
hdu.header.update('DISPAXIS', 1)
hdulist.append(hdu)
cut[scipy.isnan(cut)] = 0.
out = ndimage.map_coordinates(cut,
coords,
output=scipy.float64,
order=5)
hdu = pyfits.ImageHDU(out.copy() + 5.3)
hdu.header.update('CTYPE1', 'WAVE-LOG')
hdu.header.update('CRVAL1', wave[0])
hdu.header.update('CRPIX1', 1)
hdu.header.update('CD1_1', delta)
hdu.header.update('CRVAL2', 1)
hdu.header.update('CRPIX2', 1)
hdu.header.update('CD2_2', 1)
hdu.header.update('DC-FLAG', 1)
hdu.header.update('WFITTYPE', 'LOG_LINEAR')
hdu.header.update('DISPAXIS', 1)
varhdu.append(hdu)
hdulist.verify('fix')
hdulist.writeto(out_prefix + "_2d.fits")
varhdu.verify('fix')
varhdu.writeto(out_prefix + "_var.fits")
def skysub(x, y, z, scale):
"""
skysub(x,y,z,scale)
Routine to determine the 2d background from data. (x,y) are the
coordinates of the data, usually in the *corrected* frame.
Inputs:
x - 1d array describing x-coordinate, usually wavelength
y - 1d array describing y-coordinate, usually corrected spatial
position
z - data each position (x,y)
scale - approximate output scale (for knot placement). It is not, in
general, possible to calculate this from x because the
input coordinates are not on a regular grid.
Outputs:
2d spline model of the background
"""
height = int(y.max() - y.min())
width = int(x.max() - x.min())
npoints = x.size
midpt = y.mean()
"""
Very wide slits need special attention. Here we fit a first order
correction to the slit and subtract it away before doing the high
pixel rejection (the problem is if there is a small gradient across
a wide slit, the top and bottom pixels may differ significantly,
but these pixels may be close in *wavelength* and so locally (on
the CCD) low pixels will be rejected in the smoothing
"""
if height > WIDE:
zbak = z.copy()
args = y.argsort()
revargs = args.argsort()
ymodel = ndimage.percentile_filter(z[args], 30., size=height)[revargs]
fit = special_functions.lsqfit(ymodel, 'polynomial', 1)
if fit['coeff'][1] * float(ymodel.size) / fit['coeff'][0] < 0.05:
pass
else:
ymodel = special_functions.genfunc(scipy.arange(ymodel.size), 0,
fit)
ymodel -= ymodel.mean()
z -= ymodel
# Filter locally (in wavelength space) high points
args = x.argsort()
revargs = args.argsort()
smooth = ndimage.percentile_filter(z[args], 35., size=height)[revargs]
diff = z - smooth
# We assume poisson statistics....
var = scipy.sqrt(scipy.fabs(z))
sigma = diff / var
args = y.argsort()
revargs = args.argsort()
t = ndimage.median_filter(sigma[args], 9)
t = ndimage.gaussian_filter(t, width)[revargs]
# Source detection/rejection
# Reject yvalues > 1. sigma, and weight remaining pixels
w = (1.0 - t) / abs(z)
skycond = ((w > 0.) & (z > 0))
x = x[skycond]
y = y[skycond]
z = z[skycond]
# Reject residual high pixels (and very low pixels too!)
args = x.argsort()
revargs = args.argsort()
smooth = ndimage.median_filter(z[args], height / 4.)[revargs]
diff = z - smooth
var = scipy.sqrt(smooth)
cond = abs(diff) < 4. * var
x = x[cond]
y = y[cond]
z = z[cond]
kx = 3
ky = 1
# If the slit is long, return to original data and increase the order
# of the y-fit.
if height > WIDE:
z = zbak[skycond]
z = z[cond].astype(scipy.float64)
if height > WIDE * 1.5:
ky = 3
cond = z > 0.
x = x[cond]
y = y[cond]
z = z[cond]
w = 1. / z
if x.size < 5. * width:
kx = 1
ky = 1
# Create knots...
innertx = scipy.arange(x.min() + scale / 2., x.max() - scale / 2., scale)
tx = scipy.zeros(innertx.size + kx * 2 + 2)
tx[0:kx + 1] = x.min()
tx[kx + 1:innertx.size + kx + 1] = innertx.copy()
tx[innertx.size + kx + 1:] = x.max()
ty = scipy.zeros(ky * 2 + 2)
ty[0:ky + 1] = y.min()
ty[ky + 1:] = y.max()
# ...and fit.
bgfit = interpolate.bisplrep(x,
y,
z,
w,
tx=tx,
ty=ty,
kx=kx,
ky=ky,
task=-1,
nxest=tx.size,
nyest=ty.size,
s=0)
return bgfit
def skysub(x, y, z, scale):
"""
skysub(x,y,z,scale)
Routine to determine the 2d background from data. (x,y) are the
coordinates of the data, usually in the *corrected* frame.
Inputs:
x - 1d array describing x-coordinate, usually wavelength
y - 1d array describing y-coordinate, usually corrected spatial
position
z - data each position (x,y)
scale - approximate output scale (for knot placement). It is not, in
general, possible to calculate this from x because the
input coordinates are not on a regular grid.
Outputs:
2d spline model of the background
"""
cond = (scipy.isfinite(z)) & (z > 0.)
x = x[cond]
y = y[cond]
z = z[cond]
x0 = x.copy()
y0 = y.copy()
z0 = z.copy()
height = int(y.max() - y.min())
width = int(x.max() - x.min())
npoints = x.size
midpt = y.mean()
"""
Very wide slits need special attention. Here we fit a first order
correction to the slit and subtract it away before doing the high
pixel rejection (the problem is if there is a small gradient across
a wide slit, the top and bottom pixels may differ significantly,
but these pixels may be close in *wavelength* and so locally (on
the CCD) low pixels will be rejected in the smoothing
"""
if height > WIDE:
zbak = z.copy()
args = y.argsort()
revargs = args.argsort()
ymodel = ndimage.percentile_filter(z[args], 30., size=height)[revargs]
fit = special_functions.lsqfit(ymodel, 'polynomial', 1)
if fit['coeff'][1] * float(ymodel.size) / fit['coeff'][0] < 0.05:
pass
else:
ymodel = special_functions.genfunc(scipy.arange(ymodel.size), 0,
fit)
ymodel -= ymodel.mean()
z -= ymodel
# Filter locally (in wavelength space) high points
args = x.argsort()
revargs = args.argsort()
smooth = ndimage.percentile_filter(z[args], 35., size=height)[revargs]
diff = z - smooth
# We assume poisson statistics....
var = scipy.sqrt(scipy.fabs(z))
sigma = diff / var
args = y.argsort()
revargs = args.argsort()
t = ndimage.median_filter(sigma[args], 9)
t = ndimage.gaussian_filter(t, width) #[revargs]
# Source detection/rejection
# Reject yvalues > 1. sigma, and weight remaining pixels
w = (1.0 - t) / abs(z[args])
if AGGRESSIVE:
g = scipy.where(w <= 0, 0, 1)
g = ndimage.maximum_filter(g, width * 3)
g = ndimage.minimum_filter(g, width * 7)
s = sigma[args].copy()
b = ndimage.minimum_filter(g, width * 5)
xi = scipy.arange(t.size)
fitdata = scipy.empty((xi[g == 1].size, 2))
fitdata[:, 0] = xi[g == 1].copy()
fitdata[:, 1] = t[g == 1].copy()
fit = special_functions.lsqfit(fitdata, 'polynomial', 3)
fit = special_functions.genfunc(xi, 0., fit)
diff = (t - fit)[b == 1]
s = diff.std()
while (abs(t - fit)[(g == 1) & (b == 0)] > 2.5 * s).any():
g = b.copy()
b = ndimage.minimum_filter(g, width * 5)
fitdata = scipy.empty((xi[g == 1].size, 2))
fitdata[:, 0] = xi[g == 1].copy()
fitdata[:, 1] = t[g == 1].copy()
fit = special_functions.lsqfit(fitdata, 'polynomial', 3)
fit = special_functions.genfunc(xi, 0., fit)
diff = (t - fit)[b == 1]
s = diff.std()
w *= g
skycond = ((w > 0.) & (z > 0))
x = x[skycond]
y = y[skycond]
z = z[skycond]
# Reject residual high pixels (and very low pixels too!)
args = x.argsort()
revargs = args.argsort()
smooth = ndimage.median_filter(z[args], height / 4.)[revargs]
diff = z - smooth
var = scipy.sqrt(smooth)
cond = abs(diff) < 4. * var
x = x[cond]
y = y[cond]
z = z[cond]
kx = 3
ky = 1
# If the slit is long, return to original data and increase the order
# of the y-fit.
if height > WIDE:
z = zbak[skycond]
z = z[cond].astype(scipy.float64)
# if height>WIDE*1.5:
# ky = 3
cond = z > 0.
x = x[cond]
y = y[cond]
z = z[cond]
w = 1. / z
if x.size < 5. * width:
kx = 1
ky = 1
# Create knots...
innertx = scipy.arange(x.min() + scale / 2.,
x.max() - scale / 2., 3. * scale / 4.)
"""
tx = scipy.zeros(innertx.size+kx*2+2)
tx[0:kx+1] = x.min()
tx[kx+1:innertx.size+kx+1] = innertx.copy()
tx[innertx.size+kx+1:] = x.max()
"""
tx = scipy.linspace(x.min(), x.max(), innertx.size)
xsort = scipy.sort(x)
tmp = [x.min()]
num = []
cnt = 0
j = 1
for i in range(xsort.size):
while xsort[i] > tx[j]:
if cnt > 0:
if len(num) == 0 or cnt > 1 or num[-1] > 1:
tmp.append(tx[j])
num.append(cnt)
cnt = 0
j += 1
cnt += 1
tmp.append(x.max())
tx = scipy.asarray(tmp)
ty = scipy.zeros(ky * 2 + 2)
ty[0:ky + 1] = y.min()
ty[ky + 1:] = y.max()
#del innertx
# ...and fit.
bgfit = interpolate.bisplrep(x,
y,
z,
w,
tx=tx,
ty=ty,
kx=kx,
ky=ky,
task=-1,
nxest=tx.size,
nyest=ty.size)
del x, y, z, w, tx, ty
return bgfit
def arcmatch(curve,sci,arc,yforw,widemodel,finemodel,goodmodel,linemodel,disp,mswave,extra,logfile):
""" Do not alter input data! """
sci = sci.copy()
nsci = sci.shape[0]
""" Parse extra arguments """
linefile = extra[0]
cutoff = extra[1]
if len(extra)==3:
blue = extra[2]
else:
blue = 3000.
height = arc.shape[0]
width = arc.shape[1]
""" Straighten science data """
scimodel = scipy.zeros((nsci,height,width))
for k in range(nsci):
scimodel[k] = spectools.resampley(sci[k],yforw,curve)
""" Skip the bottom and top of the slit, which may have artifacts """
i = 4
j = height-4
xdim = 2
ydim = 2
avg = scipy.median(scipy.median(arc))
m,std = clipped_std(arc,4.)
""" The 'fiducial' arc spectrum is taken from the middle of the slit """
arcmid = stats.stats.nanmedian(arc[height/2-3:height/2+4],0)
""" First we straighten the lines out. """
coords = spectools.array_coords((j-i,width))
coords[0] += 4.
fitdata = arc[i:j,height:-1*height].flatten()
xdata = coords[1,:,height:-1*height].flatten()
ydata = coords[0,:,height:-1*height].flatten()
"""
Only use every third row (but at least 8 rows) for the solution to save
time
"""
nskip = (j-i)/8
if nskip>3:
nskip = 3
cond = ydata%nskip==0
fitdata = fitdata[cond]
ydata = ydata[cond]
xdata = xdata[cond]
""" Only use pixels around the peaks (another time saver) """
thresh = scipy.median(arcmid)
thresh += 5.*thresh**0.5
mask = ndimage.maximum_filter(arcmid,15)
mask = fitdata>thresh
fitdata = fitdata[mask]
ydata = ydata[mask]
xdata = xdata[mask]
p = scipy.zeros((xdim+1,ydim+1))
p[1,0] = 1.
p = {'coeff':p,'type':"chebyshev"}
xback = arcfitfunc2(p,xdata,ydata,fitdata,arcmid)
"""
xdata, ydata, yorig, and newxdata hold original/transformed CCD
coordinates. We don't need to do a fit to *all* points, so we skip
~every third.
"""
xdata = coords[1].flatten()
ydata = coords[0].flatten()
yorig = yforw[i:j].flatten()-curve
cond = (ydata+xdata)%2==0
xdata = xdata[cond]
ydata = ydata[cond]
yorig = yorig[cond]
newxdata = special_functions.genfunc(xdata,ydata,xback)
tmp = scipy.zeros((xdata.size,3))
tmp[:,0] = newxdata.copy()
tmp[:,1] = ydata.copy()
tmp[:,2] = xdata.copy()
xforw = special_functions.lsqfit(tmp,"chebyshev",xdim,ydim)
"""
Sky background model isn't needed if there aren't expected to be any
sky lines.
"""
if cutoff>5200:
bgmodel = scipy.zeros((nsci,sci.shape[2]))
for k in range(nsci):
tmp = spectools.resample1d(scimodel[k],xforw,"x",-999)
scimodel[k] = tmp.copy()
tmp.sort(axis=0)
tmp[tmp==-999] = scipy.nan
bg = stats.stats.nanmedian(tmp,axis=0)
bg = scipy.tile(bg,(height,1))
tmp[scipy.isnan(tmp)] = bg[scipy.isnan(tmp)]
tmp[scipy.isnan(tmp)] = 0.
bgmodel[k] = tmp[tmp.shape[0]/4,:]
del tmp,bg
newarc = spectools.resample1d(arc,xforw,"x",-999)
newarc[newarc==-999] = scipy.nan
arcmid = stats.stats.nanmedian(newarc,axis=0)
arcmid[scipy.isnan(arcmid)] = 0.
"""
We don't want to do the cross-correlation with all pixels; there
are reflections and other badness (like higher order lines). We
set the beginning of the 'useable' correlation spectrum to 150
pixels before the part of the arc with substantial flux. We set
the final pixel to be 100 pixels after the last substantial peak.
Here, 'substantial' means within 20% of the maximum; this in part
assumes that the final peak with 20% of the maximum will be the 5460
line, so we are in affect hard-coding that the correlation be carried
out to ~ 5500 angstroms.
"""
mask = scipy.where(arcmid>avg+4.*std)[0]
first = mask[0]-150
mask = ndimage.maximum_filter(arcmid,9)
mask = scipy.where(mask==arcmid,mask,0)
last = scipy.where(mask>0.20*arcmid.max())[0][-1]+100
if first<0.:
first = 0.
if last>width:
last = width
"""
Set reasonable starting and ending wavelengths for the correlation
routines.
"""
minwave = mswave-disp*width*1.1
maxwave = mswave+disp*width*1.1
if minwave<2000:
minwave = 2000.
if maxwave>8000:
maxwave = 8000.
"""
Due a quick fit to define initial guesses...first the starting
wavelength. I've used both a correlation and a chi-square-like
function to determine the start; It's still not clear which is
most robust! The correlation also uses a broadened model of the
arcs to make it more robust wrt deviations from a linear solution.
"""
broad = 10.
fudge = disp/200.
nsteps = 19
xmodel = arcmid[first:last].copy()
xmodel = ndimage.gaussian_filter(xmodel,broad/disp)
p0 = 0.
p1 = disp
offset = 0.
max = 1e29
for l in range(nsteps):
try_disp = disp + ((l-nsteps/2)*fudge)
skyfit_x = scipy.arange(minwave,maxwave,try_disp)
fitmodel = interpolate.splev(skyfit_x,widemodel)
tratio = xmodel.max()/fitmodel.max()
fitmodel *= tratio
fitmodel += scipy.median(xmodel)
chi2,off = push(xmodel,fitmodel)
if chi2<max:
p1 = try_disp
p0 = (off-first)*try_disp+minwave
offset = off
max = chi2
firstwave = minwave+offset*p1
"""
We start with a second order fit: lambda = p0 + p1*x + p2*x*x
x = 0 is *not* neccessarily the first pix; it is the first *good*
pixel. p2 = 0. should be a reasonable first guess.
"""
p0 = blue
first += (p0-firstwave)/p1
print first, p0, p1
if first<0:
p0 = p0-first*p1
first = 0
last = first+(5650.-p0)/p1
if last>width:
last = width
first = int(first)
last = int(last)
scale = last
"""
Create a normalized model of the arc data for the first refinement.
"""
fitdata = arcmid[first:last].copy()
fitx = scipy.arange(fitdata.size).astype(scipy.float64)
arclines = findlines(fitx,fitdata,20)
fitmodel = scipy.zeros(3*len(arclines)+1)
index = 1
for iter in range(len(arclines)):
fitmodel[index] = 1.
fitmodel[index+1] = arclines[iter]
fitmodel[index+2] = broad/disp
index += 3
arcmodel = special_functions.ngauss(fitx,fitmodel)
""" We begin with a broadened arc-spectrum for the initial fit. """
xdim = 2
p = scipy.zeros((xdim+1,1))
p[0,0] = p0
p[1,0] = p1
p = {'coeff':p,'type':"chebyshev"}
fitdata = arcmid[first:last].copy()
skycoeff = myoptimize(p,fitx,arcmodel,scale,linemodel)
debug(fitx,arcmodel,linemodel,skycoeff)
skycoeff = matchlines(arclines,p,linefile,order=1,tol=10.*disp)
skycoeff = matchlines(arclines,skycoeff,linefile,order=2,tol=10.*disp)
skycoeff = matchlines(arclines,skycoeff,linefile,tol=10.*disp)
""" 3rd order fit """
xdim = 3
p = scipy.zeros((xdim+1,1))
p[0:skycoeff['coeff'].size,0] = skycoeff['coeff'][:,0].copy()
skycoeff = {'coeff':p,'type':"chebyshev"}
skycoeff = myoptimize(skycoeff,fitx,arcmodel,scale,linemodel)
skycoeff = matchlines(arclines,skycoeff,linefile,order=2)
skycoeff = matchlines(arclines,skycoeff,linefile)
skycoeff = matchlines(arclines,skycoeff,linefile)
skycoeff = matchlines(arclines,skycoeff,linefile,tol=10.)
""" debug() is useful here to probe the quality of the line fitting """
# debug(fitx,arcmodel,linemodel,skycoeff)
# debug(fitx,fitdata,finemodel,skycoeff)
xvals = scipy.arange(width)-first
w = special_functions.genfunc(xvals,0,skycoeff)
cond = (w>=blue)&(w<=cutoff) # 'good' pixels
cond1 = scipy.where(cond)[0] # index of first good pixel
fitx = scipy.arange(width).astype(scipy.float64)
fitx = fitx[cond]-first
fitdata = arcmid[cond]
skycoeff = myoptimize(skycoeff,fitx,fitdata,scale,finemodel)
arclines = findlines(fitx,fitdata,10)
skycoeff = matchlines(arclines,skycoeff,linefile,tol=5.*disp)
xdim = 5
ydim = 2
wave = special_functions.genfunc(newxdata-first,0.,skycoeff)
sky2x = []
sky2y = []
ccd2wave = []
"""
We can only use the arcs if the cutoff is too blue.
"""
if cutoff<5200:
revmodel = scipy.zeros((wave.size,3))
revmodel[:,0] = wave.copy()
revmodel[:,1] = ydata.copy()
revmodel[:,2] = xdata.copy()
sky2x.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim))
revmodel[:,2] = yorig.copy()
sky2y.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim))
revmodel[:,0] = xdata.copy()
revmodel[:,1] = ydata.copy()
revmodel[:,2] = wave.copy()
ccd2wave.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim))
for k in range(1,nsci):
sky2x.append(sky2x[0])
sky2y.append(sky2y[0])
ccd2wave.append(ccd2wave[0])
return sky2x,sky2y,ccd2wave
"""
Try to use the 5577 line to refine the wavelength solution.
"""
wlen = wave.copy()
xvals = scipy.arange(width)-first
for k in range(nsci):
peaks = findlines(xvals,bgmodel[k],5.)
w = special_functions.genfunc(peaks,0.,skycoeff)
delta = 5577.345-w
offset = delta[abs(delta).argmin()]
if abs(offset)>3.*disp:
print "Warning: Substantial offset found!"
wave = wlen+offset
revmodel = scipy.zeros((wave.size,3))
revmodel[:,0] = wave.copy()
revmodel[:,1] = ydata.copy()
revmodel[:,2] = xdata.copy()
sky2x.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim))
revmodel[:,2] = yorig.copy()
sky2y.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim))
revmodel[:,0] = xdata.copy()
revmodel[:,1] = ydata.copy()
revmodel[:,2] = wave.copy()
ccd2wave.append(special_functions.lsqfit(revmodel,"chebyshev",xdim,ydim))
return sky2x,sky2y,ccd2wave
def arcwave(sky, arc, arcmodel, skymodel, scale, order):
from scipy import ndimage, stats, interpolate, optimize
sky = sky.copy()
arc = arc.copy()
sky = scipy.median(sky, 0)
x = scipy.arange(sky.size)
x_orig = x.copy()
wave = scipy.arange(3000., 10000., scale)
arc_wide = ndimage.gaussian_filter(arc, 5)
m = interpolate.splev(wave, arcmodel['norm'])
a = arc.copy()
aw = arc_wide.copy()
arc = a[:a.size / 2.]
x = x_orig[:a.size / 2.]
arclines = get_lines(x, arc)
fit = scipy.zeros(3 * arclines.size + 1)
index = 1
for i in range(arclines.size):
fit[index] = 1.
fit[index + 1] = arclines[i]
fit[index + 2] = 15. * scale
index += 3
arc_wide = sf.ngauss(x, fit)
"""
Do an approximate chi-square between the sky model and the data over a
range of offsets using a broadened data and sky model.
"""
max = 0.
mid = 0
delta = scale / 10.
s = scipy.arange(scale - delta, scale + delta, delta / 10.)
for stmp in s:
wtmp = scipy.arange(2000., 10000., stmp)
m = interpolate.splev(wtmp, arcmodel['norm'])
conv = scipy.empty(m.size - arc_wide.size + 1)
for i in range(conv.size):
tmp = m[i:i + arc_wide.size].copy()
if tmp.max() < 0.1:
conv[i] = 0.
continue
conv[i] = (tmp * arc_wide).sum()
conv[i] = 1. / ((tmp - arc_wide)**2).sum()
curr = conv.max()
if curr > max:
mid = conv.argmax()
scale = stmp
max = conv.max()
wave = wtmp.copy()
"""
Refine the starting wavelength position using the 'true' (ie narrow) model
of the sky. Searches for a minimum around the minimum found in the
previous optimization.
"""
m = interpolate.splev(wave, arcmodel['matched'])
conv = scipy.empty(m.size - arc.size + 1)
for i in range(conv.size):
tmp = m[i:i + arc.size].copy()
ratio = arc.max() / tmp.max()
if tmp.max() < 1.:
conv[i] = 0.
continue
tmp *= ratio
conv[i] = (tmp * arc).sum()
pt = conv[mid - 50:mid + 51].argmax() + mid - 50
initial_pars = [wave[pt], scale]
for i in range(order + 1 - len(initial_pars)):
initial_pars.append(0.)
modellines = get_lines(wave, m, std=10.)
modellines = modellines[modellines > wave[pt]]
modellines = arcmodel['lines']
modellines = modellines[modellines > wave[pt]]
fit = {'coeff': scipy.atleast_2d(initial_pars).T, 'type': 'polynomial'}
for o in [1, 2]:
w = sf.genfunc(arclines, 0., fit)
matches = []
for j in range(w.size):
diff = abs(w[j] - modellines)
if diff.min() < 5. * scale:
matches.append([arclines[j], modellines[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', o)
left_matches = [i for i in matches]
wmin = sf.genfunc(a.size * 0.45, 0., fit)
arc = a[a.size / 2.:].copy()
x = scipy.arange(arc.size).astype(scipy.float32) + a.size / 2.
arclines = get_lines(x, arc)
fit = scipy.zeros(3 * arclines.size + 1)
index = 1
for i in range(arclines.size):
fit[index] = 1.
fit[index + 1] = arclines[i]
fit[index + 2] = 10. * scale
index += 3
arc_wide = sf.ngauss(x, fit)
"""
Do an approximate chi-square between the sky model and the data over a
range of offsets using a broadened data and sky model.
"""
max = 0.
mid = 0
delta = scale / 10.
s = scipy.arange(scale - delta, scale + delta, delta / 10.)
for stmp in s:
wtmp = scipy.arange(wmin, 10000., stmp)
m = interpolate.splev(wtmp, arcmodel['norm'])
conv = scipy.empty(m.size - arc_wide.size + 1)
for i in range(conv.size):
tmp = m[i:i + arc_wide.size].copy()
if tmp.max() < 0.1:
conv[i] = 0.
continue
conv[i] = (tmp * arc_wide).sum()
curr = conv.max()
if curr > max:
mid = conv.argmax()
scale = stmp
max = conv.max()
wave = wtmp.copy()
"""
Refine the starting wavelength position using the 'true' (ie narrow) model
of the sky. Searches for a minimum around the minimum found in the
previous optimization.
"""
m = interpolate.splev(wave, arcmodel['matched'])
conv = scipy.empty(m.size - arc.size + 1)
for i in range(conv.size):
tmp = m[i:i + arc.size].copy()
ratio = arc.max() / tmp.max()
if tmp.max() < 1.:
conv[i] = 0.
continue
tmp *= ratio
conv[i] = (tmp * arc).sum()
pt = conv[mid - 50:mid + 51].argmax() + mid - 50
wavept = wave[pt]
initial_pars = [wavept, scale]
for i in range(order + 1 - len(initial_pars)):
initial_pars.append(0.)
modellines = get_lines(wave, m, std=10.)
modellines = modellines[modellines > wavept]
modellines = arcmodel['lines']
modellines = modellines[modellines > wavept]
fit = {'coeff': scipy.atleast_2d(initial_pars).T, 'type': 'polynomial'}
for o in [1, 2]:
# The (o-2) bit is to correct the offset after the first loop
w = sf.genfunc(arclines + (o - 2) * a.size / 2., 0., fit)
matches = []
for j in range(w.size):
diff = abs(w[j] - modellines)
if diff.min() < 5. * scale:
matches.append([arclines[j], modellines[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', o)
arc = a.copy()
arc_wide = aw.copy()
w = sf.genfunc(arclines, 0., fit)
for i in range(w.size):
diff = abs(w[i] - modellines)
if diff.min() < 5. * scale:
left_matches.append([arclines[i], modellines[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(left_matches), 'polynomial', order)
""" Optimization function for refining the wavelength solution. """
def dofit(p, x, data, model):
fit = {'coeff': scipy.atleast_2d(p).T, 'type': 'polynomial'}
w = sf.genfunc(x, 0., fit)
m = interpolate.splev(w, model)
return (m - data)
x = scipy.arange(arc.size).astype(scipy.float32)
initial_pars = fit['coeff'][:, 0].tolist()
coeff, ier = optimize.leastsq(dofit,
initial_pars,
(x, arc_wide, arcmodel['wide']),
maxfev=100000)
coeff, ier = optimize.leastsq(dofit,
coeff, (x, arc, arcmodel['matched']),
maxfev=100000)
outcoeff = {'coeff': scipy.atleast_2d(coeff).T, 'type': 'polynomial'}
def skycorrect(p, arc, sky, arcmodel, skymodel):
fit = {'coeff': scipy.atleast_2d(p[:-1]).T, 'type': 'polynomial'}
w = sf.genfunc(x, 0., fit)
arcm = interpolate.splev(w + p[-1], arcmodel)
chi_arc = (arcm - arc)
s = sky[w > 5100.]
skym = interpolate.splev(w[w > 5100.], skymodel)
skym *= scipy.median(s / skym)
chi_sky = 5. * (skym - s) #/abs(m)**0.5
chi = scipy.concatenate((chi_arc, chi_sky))
return chi
newcoeff = coeff.tolist()
newcoeff.append(0.)
coeff, ier = optimize.leastsq(
skycorrect,
newcoeff, (arc, sky, arcmodel['matched'], skymodel['matched']),
maxfev=100000)
outcoeff = {'coeff': scipy.atleast_2d(coeff[:-1]).T, 'type': 'polynomial'}
"""
wave = sf.genfunc(x,0.,outcoeff)
sky = sky[wave>5000.]
wave = wave[wave>5000.]
m = interpolate.splev(wave,wavemodel['matched'])
ratio = scipy.median(sky/m)
import pylab
pylab.plot(wave,sky)
pylab.plot(wave,m*ratio)
pylab.show()
offset,ier = optimize.leastsq(skycorrect,[0.],
(wave,sky,wavemodel['matched']),maxfev=100000)
print offset
outcoeff['coeff'][0] += offset
"""
return outcoeff
def pipeline(dir, inname, out_prefix, cal_prefix):
bias = pyfits.open(cal_prefix + "_bias.fits")[0].data.astype(scipy.float32)
bpm = pyfits.open(cal_prefix + "_bpm.fits")[0].data.astype(scipy.float32)
flat = pyfits.open(cal_prefix + "_norm.fits")[0].data.astype(scipy.float32)
file = open(cal_prefix + "_ycor.dat")
orders = load(file)
y_soln = load(file)
file.close()
file = open(cal_prefix + "_wave.dat")
wave_soln = load(file)
file.close()
try:
mask = pyfits.open(cal_prefix + "_mask.fits")[0].data.copy()
print('mask opened')
except:
mask = numpy.where(bpm == 1., 1., 0.)
mask = straighten(mask, y_soln, orders)
mask[mask < 0.7] = numpy.nan
mask[mask >= 0.7] = 1.
pyfits.PrimaryHDU(mask).writeto(cal_prefix + "_mask.fits")
hdu = pyfits.open(dir + inname)
data = hdu[0].data.copy()
data = biastrim(data, bias, bpm) #*flat
blue = [1500, 1400, 1300, 1200, 1100, 900, 600, 200, 0, 0, 0]
red = [3000, 3400, 3700, -1, -1, -1, -1, -1, -1, -1]
def clip2(arr, nsig=3., edge=0.01):
a = arr.flatten()
a.sort()
a = a[a.size * edge:a.size * (1. - edge)]
m, s, l = a.mean(), a.std(), a.size
while 1:
a = a[abs(a - m) < nsig * s]
if a.size == l:
return m, s
m, s, l = a.mean(), a.std(), a.size
try:
back = pyfits.open(out_prefix + "_bg.fits")[0].data.astype(
scipy.float32)
except:
strt = straighten(data, y_soln, orders)
back = scipy.zeros(strt.shape)
for indx in range(len(orders)):
i, j = orders[indx]
bg = stats.stats.nanmedian(strt[i:j], axis=0)
back[i:j] += bg
back = curve(back, y_soln, orders)
pyfits.PrimaryHDU(back).writeto(out_prefix + "_bg.fits")
try:
strt = pyfits.open(out_prefix + "_strt.fits")[0].data.astype(
scipy.float32)
except:
bgsub = data - back
bgsub[scipy.isnan(bgsub)] = 0.
bgsub[scipy.isinf(bgsub)] = 0.
model = ndimage.median_filter(bgsub, 3)
diff = bgsub - model
model = scipy.sqrt(data)
model[scipy.isnan(model)] = 0.
model[scipy.isinf(model)] = 0.
diff[scipy.isnan(diff)] = 0.
diff[scipy.isinf(diff)] = 0.
crmask = scipy.where(diff > 4. * model, diff, 0.)
data -= crmask
strt = straighten(data, y_soln, orders)
pyfits.PrimaryHDU(strt).writeto(out_prefix + "_strt.fits")
hdulist = pyfits.HDUList([pyfits.PrimaryHDU()])
hdulist[0].header = hdu[0].header.copy()
varhdu = pyfits.HDUList([pyfits.PrimaryHDU()])
varhdu[0].header = hdu[0].header.copy()
for indx in range(len(orders)):
i, j = orders[indx]
cut = strt[i:j].copy()
cut[scipy.isinf(cut)] = scipy.nan
bg = stats.stats.nanmedian(cut, axis=0)
B = mask[i:j]
tmp = (cut - bg) * B
T = tmp[3:-3, blue[indx]:red[indx]]
T = T[numpy.isfinite(T)]
avg, std = clip2(T)
slice = stats.stats.nanmedian(tmp[:, blue[indx]:red[indx]], axis=1)
slice[:3] = 0.
slice[-3:] = 0.
avg, std = clip2(slice[3:-3], 3.5, 0.05)
good = numpy.where(slice < 2. * std, 1., 0.)
good = ndimage.maximum_filter(good, 5)
good = ndimage.minimum_filter(good, 15)
good = good == 1
xvals = scipy.arange(good.size).astype(scipy.float32)
fitdata = scipy.empty((slice[good].size, 2))
fitdata[:, 0] = xvals[good].copy()
bgsub = cut.copy()
for k in range(cut.shape[1]):
fitdata[:, 1] = cut[:, k][good].copy()
if fitdata[:, 1][scipy.isfinite(fitdata[:, 1])].size < 4:
continue
avg, std = clip(fitdata[:, 1])
keep = scipy.where((abs(fitdata[:, 1] - avg) < 4. * std)
& scipy.isfinite(fitdata[:, 1]))[0]
fit = fitdata[keep]
if fit[:, 1][scipy.isfinite(fit[:, 1])].size < 4:
continue
fit = sf.lsqfit(fit, 'chebyshev', 1)
bg = sf.genfunc(xvals, 0, fit)
bgsub[:, k] -= bg
solution = wave_soln[indx][0]
invsoln = wave_soln[indx][1]
vals = scipy.arange(cut.shape[1]).astype(scipy.float64)
wave = sf.genfunc(vals, 0., solution)
delta = (wave[wave.size - 1] - wave[0]) / (wave.size - 1)
outc = scipy.arange(wave[0], wave[0] + 2, delta)
outc = outc[:cut.shape[1]].copy()
pix = sf.genfunc(outc, 0, invsoln)
coords = spectools.array_coords(cut.shape).astype(numpy.float64)
coords[1] *= 0.
coords[1] += pix
bgsub[scipy.isnan(bgsub)] = 0.
out = ndimage.map_coordinates(bgsub,
coords,
output=scipy.float64,
order=5)
hdu = pyfits.ImageHDU(out.copy())
hdu.header.update('CTYPE1', 'WAVE-LOG')
hdu.header.update('CRVAL1', wave[0])
hdu.header.update('CRPIX1', 1)
hdu.header.update('CDELT1', delta)
hdu.header.update('CTYPE2', 'LINEAR')
hdu.header.update('CRVAL2', 1)
hdu.header.update('CRPIX2', 1)
hdu.header.update('CDELT2', 1)
hdu.header.update('CD1_1', delta)
hdu.header.update('CD2_2', 1)
hdu.header.update('CD1_2', 0)
hdu.header.update('CD2_1', 0)
#hdu.header.update('PC1_1',1)
#hdu.header.update('PC1_2',0)
#hdu.header.update('PC2_1',0)
#hdu.header.update('PC2_2',1)
hdu.header.update('DC-FLAG', 1)
hdu.header.update('WFITTYPE', 'LOG_LINEAR')
hdu.header.update('DISPAXIS', 1)
hdulist.append(hdu)
cut[scipy.isnan(cut)] = 0.
out = ndimage.map_coordinates(cut,
coords,
output=scipy.float64,
order=5)
hdu = pyfits.ImageHDU(out.copy() + 5.3)
hdu.header.update('CTYPE1', 'WAVE-LOG')
hdu.header.update('CRVAL1', wave[0])
hdu.header.update('CRPIX1', 1)
hdu.header.update('CD1_1', delta)
hdu.header.update('CRVAL2', 1)
hdu.header.update('CRPIX2', 1)
hdu.header.update('CD2_2', 1)
hdu.header.update('DC-FLAG', 1)
hdu.header.update('WFITTYPE', 'LOG_LINEAR')
hdu.header.update('DISPAXIS', 1)
varhdu.append(hdu)
hdulist.verify('fix')
hdulist.writeto(out_prefix + "_2d.fits")
varhdu.verify('fix')
varhdu.writeto(out_prefix + "_var.fits")
def arcmatch(curve, sci, arc, yforw, widemodel, finemodel, goodmodel,
linemodel, disp, mswave, extra, logfile):
"""
arcmatch(curve,sci,arc,yforw,widemodel,finemodel,goodmodel,linemodel
,disp,mswave,extra,logfile)
Creates the full 2d distortion solution for a slit.
Inputs:
curve - first pixel (ie bottom) of curved slit
sci - biastrimmed, non-ycorrected science data
arc - y-corrected arc for this slit
yforw - yforw definition for slit
widemodel - broad wavelength model
finemodel - matched-resolution wavelength model
goodmodel - model of the sky
linemodel - broad wavelength model (why are there two...?)
disp - scale
mswave - central wavelength
extra - includes name of linefile, starting/ending wavelengths
logfile - a file object
Outputs:
"""
""" Do not alter input data! """
sci = sci.copy()
nsci = sci.shape[0]
""" Parse extra arguments """
linefile = extra[0]
cutoff = extra[1]
if len(extra) == 3:
blue = extra[2]
else:
blue = 3000.
cutoff = 5650.
height = arc.shape[0]
width = arc.shape[1]
""" Straighten science data """
scimodel = scipy.zeros((nsci, height, width))
for k in range(nsci):
scimodel[k] = spectools.resampley(sci[k], yforw, curve)
""" Skip the bottom and top of the slit, which may have artifacts """
i = 4
j = height - 4
xdim = 2
ydim = 2
avg = scipy.median(scipy.median(arc))
std = clipped_std(arc, 4.)
""" The 'fiducial' arc spectrum is taken from the middle of the slit """
arcmid = stats.stats.nanmedian(arc[height / 2 - 2:height / 2 + 3], 0)
""" First we straighten the lines out. """
coords = spectools.array_coords((j - i, width))
coords[0] += 4.
fitdata = arc[i:j, height:-1 * height].flatten()
xdata = coords[1, :, height:-1 * height].flatten()
ydata = coords[0, :, height:-1 * height].flatten()
""" Only use every third row for the solution (to save time) """
cond = ydata % 3 == 0
fitdata = fitdata[cond]
ydata = ydata[cond]
xdata = xdata[cond]
""" Only use pixels around the peaks (another time saver) """
thresh = scipy.median(arcmid)
thresh += 5. * thresh**0.5
mask = ndimage.maximum_filter(arcmid, 15)
mask = fitdata > thresh
fitdata = fitdata[mask]
ydata = ydata[mask]
xdata = xdata[mask]
p = scipy.zeros((xdim + 1, ydim + 1))
p[1, 0] = 1.
p = {'coeff': p, 'type': "polynomial"}
xback = arcfitfunc2(p, xdata, ydata, fitdata, arcmid)
"""
xdata, ydata, yorig, and newxdata hold original/transformed CCD
coordinates. We don't need to do a fit to *all* points, so we skip
~every third.
"""
xdata = coords[1].flatten()
ydata = coords[0].flatten()
yorig = yforw[i:j].flatten() - curve
cond = (ydata + xdata) % 2 == 0
xdata = xdata[cond]
ydata = ydata[cond]
yorig = yorig[cond]
newxdata = special_functions.genfunc(xdata, ydata, xback)
tmp = scipy.zeros((xdata.size, 3))
tmp[:, 0] = newxdata.copy()
tmp[:, 1] = ydata.copy()
tmp[:, 2] = xdata.copy()
xforw = special_functions.lsqfit(tmp, "chebyshev", xdim, ydim)
bgmodel = scipy.zeros((nsci, sci.shape[2]))
for k in range(nsci):
tmp = spectools.resample1d(scimodel[k], xforw, "x", -999)
scimodel[k] = tmp.copy()
tmp.sort(axis=0)
tmp[tmp == -999] = scipy.nan
bg = stats.stats.nanmedian(tmp, axis=0)
bg = scipy.tile(bg, (height, 1))
tmp[scipy.isnan(tmp)] = bg[scipy.isnan(tmp)]
tmp[scipy.isnan(tmp)] = 0.
bgmodel[k] = tmp[tmp.shape[0] / 4, :]
del tmp, bg
newarc = spectools.resample1d(arc, xforw, "x", -999)
newarc[newarc == -999] = scipy.nan
arcmid = stats.stats.nanmedian(newarc, axis=0)
arcmid[scipy.isnan(arcmid)] = 0.
"""
We don't want to do the cross-correlation with all pixels; there
are reflections and other badness (like higher order lines). We
set the beginning of the 'useable' correlation spectrum to 150
pixels before the part of the arc with substantial flux. We set
the final pixel to be 100 pixels after the last substantial peak.
Here, 'substantial' means within 20% of the maximum; this in part
assumes that the final peak with 20% of the maximum will be the 5460
line, so we are in affect hard-coding that the correlation be carried
out to ~ 5500 angstroms.
"""
mask = scipy.where(arcmid > avg + 4. * std)[0]
first = mask[0] - 150
mask = ndimage.maximum_filter(arcmid, 9)
mask = scipy.where(mask == arcmid, mask, 0)[:arcmid.argmax() + 460 / disp]
last = scipy.where(mask > 0.20 * arcmid.max())[0][-1] + 100
if first < 0.:
first = 0.
if last > width:
last = width
"""
Set reasonable starting and ending wavelengths for the correlation
routines.
"""
minwave = mswave - disp * width * 1.1
maxwave = mswave + disp * width * 1.1
if minwave < 2000:
minwave = 2000.
if maxwave > 8000:
maxwave = 8000.
"""
Due a quick fit to define initial guesses...first the starting
wavelength. I've used both a correlation and a chi-square-like
function to determine the start; it's still not clear which is
most robust! The correlation also uses a broadened model of the
arcs to make it more robust wrt deviations from a 1st order solution.
"""
broad = 9.
fudge = disp / 1000.
nsteps = 35
xmodel = arcmid[first:last].copy()
# xmodel = ndimage.gaussian_filter(xmodel,broad/disp)
p0 = 0.
p1 = disp
offset = 0.
max = 1e29
c = []
b = []
fitdata = arcmid[first:last].copy()
fitx = scipy.arange(fitdata.size).astype(scipy.float64)
arclines = findlines(fitx, fitdata, 10)
fitmodel = scipy.zeros(3 * len(arclines) + 1)
index = 1
for iter in range(len(arclines)):
fitmodel[index] = 1.
fitmodel[index + 1] = arclines[iter]
fitmodel[index + 2] = broad / disp
index += 3
xmodel = special_functions.ngauss(fitx, fitmodel)
for l in range(nsteps):
try_disp = disp + ((l - nsteps / 2) * fudge)
skyfit_x = scipy.arange(minwave, maxwave, try_disp)
fitmodel = interpolate.splev(skyfit_x, linemodel)
# tratio = xmodel.max()/fitmodel.max()
# fitmodel *= tratio
# fitmodel += scipy.median(xmodel)
corr = signal.correlate(xmodel, fitmodel, mode='valid')
chi2, off = 1. / corr.max(), corr.argmax()
# chi2,off = push(xmodel,fitmodel)
if chi2 < max:
p1 = try_disp
p0 = (off - first) * try_disp + minwave
offset = off
max = chi2
firstwave = minwave + offset * p1
"""
We start with a second order fit: lambda = p0 + p1*x + p2*x*x
x = 0 is *not* neccessarily the first pix; it is the first *good*
pixel. p2 = 0. should be a reasonable first guess.
"""
p0 = blue
first += (p0 - firstwave) / p1
if first < 0:
p0 = p0 - first * p1
first = 0
last = first + (5650. - p0) / p1
if last > width:
last = width
first = int(first)
last = int(last)
keeplog(logfile, '\tInitial Solution: %f, %f\n' % (p0, p1))
"""
Create a normalized model of the arc data for the first refinement.
"""
fitdata = arcmid[first:last].copy()
fitx = scipy.arange(fitdata.size).astype(scipy.float64)
arclines = findlines(fitx, fitdata, 20)
fitmodel = scipy.zeros(3 * len(arclines) + 1)
index = 1
for iter in range(len(arclines)):
fitmodel[index] = 1.
fitmodel[index + 1] = arclines[iter]
fitmodel[index + 2] = broad / disp
index += 3
arcmodel = special_functions.ngauss(fitx, fitmodel)
""" We begin with a fit to the normalized spectra. """
xdim = 2
p = scipy.zeros((xdim + 1, 1))
p[0, 0] = p0
p[1, 0] = p1
p = {'coeff': p, 'type': "polynomial"}
fitdata = arcmid[first:last].copy()
skycoeff = myoptimize(p, fitx, arcmodel, linemodel)
# debug(fitx,fitdata,finemodel,skycoeff)
skycoeff = matchlines(arclines, p, linefile, order=1, tol=10. * disp)
skycoeff = matchlines(arclines,
skycoeff,
linefile,
order=2,
tol=10. * disp)
skycoeff = matchlines(arclines, skycoeff, linefile, tol=10. * disp)
""" Refinement of fit, still with normalized spectra. """
xdim = 3
p = scipy.zeros((xdim + 1, 1))
p[0:skycoeff['coeff'].size, 0] = skycoeff['coeff'][:, 0].copy()
skycoeff = {'coeff': p, 'type': "chebyshev"}
# skycoeff = myoptimize(skycoeff,fitx,arcmodel,linemodel)
skycoeff = myoptimize(skycoeff, fitx, fitdata, finemodel)
# debug(fitx,fitdata,finemodel,skycoeff)
""" Match lines, progressively constraining the fit """
skycoeff = matchlines(arclines,
skycoeff,
linefile,
order=2,
tol=10. * disp)
skycoeff = matchlines(arclines, skycoeff, linefile, tol=10. * disp)
skycoeff = matchlines(arclines, skycoeff, linefile, tol=10. * disp)
skycoeff = matchlines(arclines, skycoeff, linefile, tol=5. * disp)
skycoeff = matchlines(arclines, skycoeff, linefile, tol=5. * disp)
# debug(fitx,fitdata,finemodel,skycoeff)
xvals = scipy.arange(width) - first
w = special_functions.genfunc(xvals, 0, skycoeff)
"""
The wavelength solution might wrap around on itself (ie it might not be
a strictly increasing function). This ensures that only the
increasing part of the current solution is used. It might be more
reasonable to put something in the chi-square to enforce this....
I've added it to arcfitfunc for now....
val = w.argmax()
if cutoff>w.max():
cutoff = w.max()
w[val:] = cutoff + 1.
"""
cond = (w >= blue) & (w <= cutoff) # 'good' pixels
cond1 = scipy.where(cond)[0] # index of first good pixel
fitx = scipy.arange(width).astype(scipy.float64)
fitx = fitx[cond] - first
fitdata = arcmid[cond]
# debug(fitx,fitdata,finemodel,skycoeff)
# skycoeff = myoptimize(skycoeff,fitx,fitdata,finemodel)
# xdim = 5
# p = scipy.zeros((xdim+1,1))
# p[0:skycoeff['coeff'].size,0] = skycoeff['coeff'][:,0].copy()
# skycoeff = {'coeff':p,'type':"polynomial"}
# skycoeff = myoptimize(skycoeff,fitx,fitdata,finemodel)
# debug(fitx,fitdata,finemodel,skycoeff)
coeff = skycoeff['coeff'].copy()
arclamppeaks = findlines(fitx, fitdata)
skycoeff = matchlines(arclamppeaks, skycoeff, linefile, tol=5. * disp)
startx = fitx.copy()
startdata = fitdata.copy()
wave = special_functions.genfunc(startx, 0., skycoeff)
model = interpolate.splrep(wave, startdata, s=0)
sky2x = []
sky2y = []
ccd2wave = []
def findoffset(p, x, data, model, coeff):
xvals = x + p[0]
w = special_functions.genfunc(xvals, 0., coeff)
mod = interpolate.splev(w, model)
mod = mod * data.max() / mod.max()
diff = (mod - data) / scipy.sqrt(abs(mod))
return diff
"""
If the highest line is greater than 3x the next highest, there
was probably some sky activity. This could throw off the
pixel-based fitting, so we should choose line-matching
instead. A better solution would be to 'dampen' the active
line (we can't just alter the residuals because the sky bias
and gain offsets were calculated using the line), but that is
non-trivial to accomplish without shifting the central
wavelength. TURNED OFF--LINE MATCHING ALWAYS USED!
"""
LINEMATCH = False
for k in range(nsci):
sky = bgmodel[k]
x_vals = scipy.arange(width)
skypeaks = findlines(x_vals, sky, 3)
amps = ndimage.map_coordinates(sky, [skypeaks],
output=scipy.float64,
order=5)
amps.sort()
if amps[-1] > 2. * amps[-2]:
LINEMATCH = True
# Use LINEMATCH until arcs are better dealt with
LINEMATCH = True
"""
We attempt to determine an offset between the arclamp frame and the sky
image due to flexure (or if the arcs were taken in the afternoon...).
First we use the arc solution to determine the offset of the bluest
sky line. We then shift the arc lines to correct for the offset and
find a solution for just the sky. We extrapolate the sky solution to
the reddest arc line, determine the arc line's offset and use this as
the final offset between sky and arc. We then solve for the full
solution using the sky and corrected arc lines.
"""
for k in range(nsci):
sky = bgmodel[k]
skycoeff['coeff'] = coeff.copy()
x_vals = scipy.arange(width) - first
skypeaks = findlines(x_vals, sky, 5)
"""
Start by trying to match to the 5200 line; if it is too faint
(and therefore was not extracted as a skypeak), use the
5577 line. 5197.93,5198.794 -- the line is a blend and the
left value is the non-blended wavelength. One issue is that
the line exhibits auroral brightening--which appears to
change the effective wavelength because the fainter line at
5200. is the one that brightens! BAH! We'll just use 5577.
"""
# sline = 5198.794
sline = 5577.345
highpeaks = findlines(x_vals, sky, 7)
w = special_functions.genfunc(highpeaks, 0., skycoeff)
delta = sline - w
if abs(delta).min() > 10. * disp:
sline = 5577.345
delta = sline - w
w = special_functions.genfunc(x_vals, 0., skycoeff)
skycond = (w > sline - 40.) & (w < sline + 40.)
off = delta[abs(delta).argmin()] / disp
tmp = special_functions.genfunc(skypeaks, 0., skycoeff)
entry = '\tSky lines found for image %d:' % (k + 1)
for t in tmp:
entry += ' %7.2f' % t
entry += "\n"
keeplog(logfile, entry)
min = 10.
step = None
for i in scipy.arange(-3., 3., 0.1):
arcpeaks = arclamppeaks + i
tmpcoeff, err = doublematch([skypeaks, arcpeaks],
skycoeff,
ARCLINES, [sline],
tol=10. * disp)
tmpcoeff, err = doublematch([skypeaks, arcpeaks],
tmpcoeff,
ARCLINES,
SKYLINES,
tol=10. * disp)
tmpcoeff, err = doublematch([skypeaks, arcpeaks],
tmpcoeff,
ARCLINES,
SKYLINES,
tol=5. * disp)
if err < min:
min = err
outcoeff = tmpcoeff.copy()
step = i
min = 10.
for i in scipy.arange(step - 0.5, step + 0.5, 0.01):
arcpeaks = arclamppeaks + i
tmpcoeff, err = doublematch([skypeaks, arcpeaks],
outcoeff,
ARCLINES,
SKYLINES,
tol=5. * disp)
if err < min:
min = err
skycoeff = tmpcoeff.copy()
arcoffset = i
keeplog(logfile,
"\tArc offset for image %d: %6.3f\n" % (k + 1, arcoffset))
skypeaks = findlines(x_vals, sky, 2.)
arcpeaks = arclamppeaks + arcoffset
skycoeff, e = doublematch([skypeaks, arcpeaks],
skycoeff,
ARCLINES,
SKYLINES,
tol=5. * disp,
logfile=logfile)
w = special_functions.genfunc(x_vals, 0., skycoeff)
skycond = (w > 5500.) & (w < 6700.)
skyx = x_vals[skycond]
skyz = sky[skycond]
tmp = skyz.copy()
tmp.sort()
med = tmp[tmp.size * 0.3]
skyx = skyx[skyz > med]
skyz = skyz[skyz > med]
tmpcoeff, ratio, offset = skyopt(skycoeff, skyx, skyz, goodmodel)
skycoeff = myopt2(skycoeff, fitx + arcoffset, fitdata, finemodel, skyx,
skyz, goodmodel, ratio, offset)
""" Evaluate wavelength of all pixels in straightened frame """
wave = special_functions.genfunc(newxdata - first, 0, skycoeff)
"""
Now that we have the wavelength solution for the rectified slit
we can create the full 2d solutions for:
x(lambda,pos)
y(lambda,pos)
lamba(x,y)
where x and y are CCD coordinates and lambda and pos are the
'true' coordinates (wavelength and spatial position along
the slit).
"""
xdim = 5
ydim = 2
revmodel = scipy.zeros((wave.size, 3))
revmodel[:, 0] = wave.copy()
revmodel[:, 1] = ydata.copy()
revmodel[:, 2] = xdata.copy()
sky2x.append(
special_functions.lsqfit(revmodel, "chebyshev", xdim, ydim))
revmodel[:, 2] = yorig.copy()
sky2y.append(
special_functions.lsqfit(revmodel, "chebyshev", xdim, ydim))
revmodel[:, 0] = xdata.copy()
revmodel[:, 1] = ydata.copy()
revmodel[:, 2] = wave.copy()
ccd2wave.append(
special_functions.lsqfit(revmodel, "chebyshev", xdim, ydim))
# return sky2x,sky2y,[xforw,xback] # This is for non-2d subtraction
return sky2x, sky2y, ccd2wave # Thsi is for 2d subtraction
def extract(image,
varimage,
outname,
width=2.,
offset=0.,
pos=None,
minwave=None,
maxwave=None,
response_image=None,
response_model=None,
regions=[],
centroid=None):
if outname.lower().find('fits') < 1:
outname = "%s.fits" % outname
resp = None
respwave = None
if response_image is not None:
if response_model is None:
# print "No response model included; not performing response correction."
import cPickle
resp = cPickle.load(open(response_image))
else:
resp = get_response(response_image, response_model, regions)
elif response_model is not None:
print "No response image included; not performing response correction."
if minwave is None:
minwave = -10.
if maxwave is None:
maxwave = 1e99
data = pyfits.open(image)[0].data.copy()
if data.ndim == 3:
if data.shape[0] > 1:
print "Only working on first image plane of 3D image."
data = data[0].copy()
wave = st.wavelength(image)
indx = scipy.where((wave > minwave) & (wave < maxwave))[0]
minx, maxx = indx.min(), indx.max()
wave = wave[minx:maxx]
data = data[:, minx:maxx]
tmp = data.copy()
tmp[numpy.isnan(data)] = 0.
ospec = wave * 0.
ovar = wave * 0.
x = numpy.arange(data.shape[0])
if centroid is None:
n = tmp.shape[1] / 100
peaks = []
bounds = numpy.linspace(0, tmp.shape[1], n + 1)
for i in range(n):
a, b = bounds[i], bounds[i + 1]
p = tmp[:, a:b].sum(1).argmax()
peaks.append([(a + b) / 2., p])
peak = sf.lsqfit(numpy.asarray(peaks), 'polynomial', 3)
if pos is not None:
peak['coeff'][0] = pos
peaks = sf.genfunc(numpy.arange(tmp.shape[1]), 0., peak)
center = wave * 0.
for i in range(center.size):
d = data[:, i].copy()
peak = peaks[i]
if peak < 0:
peak = 0
if peak >= d.shape:
peak = d.shape - 1.
fit = numpy.array([0., d[peak], peak, 1.])
cond = ~numpy.isnan(d)
input = numpy.empty((d[cond].size, 2))
input[:, 0] = x[cond].copy()
input[:, 1] = d[cond].copy()
fit = sf.ngaussfit(input, fit)[0]
center[i] = fit[2]
fit = sf.lsqfit(ndimage.median_filter(center, 17), 'polynomial', 5)
centroid = sf.genfunc(scipy.arange(wave.size), 0., fit)
#import pylab
#pylab.plot(center-centroid)
#pylab.show()
xvals = scipy.arange(data.shape[0])
var = pyfits.open(varimage)[0].data.copy()
if var.ndim == 3:
var = var[0].copy()
var = var[:, minx:maxx]
cond = (numpy.isnan(var)) | (numpy.isnan(data))
for i in range(ovar.size):
c = centroid[i] + offset
wid = width
mask = xvals * 0.
mask[(xvals - c < wid) & (xvals - c > 0.)] = wid - (xvals[
(xvals - c < wid) & (xvals - c > 0.)] - c)
mask[(xvals - c < wid - 1) & (xvals - c > 0.)] = 1.
mask[(c - xvals < wid + 1) & (c - xvals > 0.)] = (wid + 1) - (
c - xvals[(c - xvals < wid + 1) & (c - xvals > 0.)])
mask[(c - xvals < wid) & (c - xvals > 0.)] = 1.
mask[cond[:, i]] = 0.
mask /= mask.sum()
ospec[i] = (mask[~cond[:, i]] * data[:, i][~cond[:, i]]).sum()
ovar[i] = ((mask[~cond[:, i]]**2) * var[:, i][~cond[:, i]]).sum()
badpix = numpy.where(numpy.isnan(ospec))[0]
ospec[badpix] = 0.
ovar[badpix] = ovar[~numpy.isnan(ovar)].max() * 1e6
med = numpy.median(ospec)
ospec /= med
ovar /= med**2
if resp is not None:
# if respwave is None:
# resp = sf.genfunc(wave,0,resp)
# else:
# resp = interpolate.splrep(respwave,resp)
resp = interpolate.splev(wave, resp)
ospec *= resp
ovar *= resp**2
st.make_spec(ospec, ovar, wave, outname, clobber=True)
return centroid
def newModMatch(data, model, scale, order, center, sparse=False):
import numpy
import special_functions as sf
from scipy import ndimage, interpolate
x0 = numpy.arange(data.size).astype(numpy.float32)
datalines = get_lines(x0, data)
Xstart = int(datalines.min() - 15.)
Xend = int(datalines.max() + 15.)
if Xstart < 0:
Xstart = 0
if Xend > data.size:
Xend = data.size
tdata = data[Xstart:Xend].copy()
wave = numpy.linspace(center - scale * data.size / 2,
center + scale * data.size / 2,
data.size)[Xstart:Xend].copy()
cond = wave < 10300.
mod = interpolate.splev(wave, model['matched'])
fftd = numpy.fft.rfft(ndimage.gaussian_filter(tdata[cond], 17))
fftm = numpy.fft.rfft(ndimage.gaussian_filter(mod[cond], 17)[::-1])
T = numpy.fft.fftshift(numpy.fft.irfft(fftd * fftm)).real
offset = T.size / 2 - T.argmax()
Wstart = wave[offset]
pars = numpy.zeros(order + 1)
pars[0] = Wstart - Xstart * scale
pars[1] = scale
cov = numpy.zeros(order + 1)
cov[0] = 1.
cov[1] = 0.05
for i in range(2, order + 1):
cov[i] = (1. / data.size)**i
coeff = pars
def optfunc(pars, x, d, mod, sig):
if numpy.isnan(pars).any():
return -1e200
coeff = numpy.atleast_2d(numpy.array(pars)).T
fit = {'coeff': coeff, 'type': 'polynomial'}
x0 = sf.genfunc(x, 0., fit)
try:
model = interpolate.splev(x0, mod)
except:
return -1e200
model /= model.mean()
resid = (model - d) / sig
return -0.5 * (resid**2).sum()
spec = ndimage.gaussian_filter(tdata, 7)
spec /= spec.mean()
sig = spec**0.5
logp = optfunc(pars, x0[Xstart:Xend], spec, model['wide'], sig)
niter = 2000
for indx in xrange(niter):
prop = coeff + numpy.random.randn(
cov.size) * cov * 10**(numpy.random.random(cov.size) * 2.3 - 2.)
tlogp = optfunc(prop, x0[Xstart:Xend], spec, model['wide'], sig)
accept = 0
if tlogp > logp:
accept = 1
else:
if tlogp - logp > numpy.log(numpy.random.random()):
accept = 1
if accept == 1:
logp = tlogp
coeff = prop.copy()
spec = tdata.copy()
spec /= spec.mean()
sig = spec**0.5
logp = optfunc(pars, x0[Xstart:Xend], spec, model['matched'], sig)
c = numpy.empty((niter, coeff.size))
for indx in xrange(niter):
prop = coeff + numpy.random.randn(
cov.size) * cov * 10**(numpy.random.random(cov.size) * 2.3 - 2.)
tlogp = optfunc(prop, x0[Xstart:Xend], spec, model['matched'], sig)
accept = 0
if tlogp > logp:
accept = 1
else:
if tlogp - logp > numpy.log(numpy.random.random()):
accept = 1
if accept == 1:
logp = tlogp
coeff = prop.copy()
c[indx] = coeff.copy()
for indx in range(cov.size):
cov[indx] = c[:, indx].std()
mlogp = logp
mcoeff = coeff.copy()
for indx in xrange(niter):
prop = coeff + numpy.random.randn(
cov.size) * cov * 10**(numpy.random.random(cov.size) * 2.3 - 2.)
tlogp = optfunc(prop, x0[Xstart:Xend], spec, model['matched'], sig)
accept = 0
if tlogp > logp:
accept = 1
else:
if tlogp - logp > numpy.log(numpy.random.random()):
accept = 1
if accept == 1:
logp = tlogp
coeff = prop.copy()
if logp > mlogp:
mlogp = logp
mcoeff = coeff.copy()
fit = {'coeff': scipy.atleast_2d(mcoeff).T, 'type': 'polynomial'}
w = sf.genfunc(x0[Xstart:Xend], 0., fit)
m = interpolate.splev(w, model['matched'])
cond = (w < 10400)
m = m[cond]
m /= m.mean()
import pylab
pylab.plot(w[cond], m)
pylab.plot(w[cond], spec[cond])
pylab.show()
return fit
#def newModMatch(data,model,scale,order,center,sparse=False):
import numpy, pylab
from scipy import interpolate, optimize, ndimage
x0 = numpy.arange(data.size)
datalines = get_lines(x0, data)
if datalines.size < 15:
sparse = True
Xstart = int(datalines.min() - 15.)
Xend = int(datalines.max() + 15.)
if Xstart < 0:
Xstart = 0
if Xend > data.size:
Xend = data.size
tdata = data[Xstart:Xend].copy()
wave = numpy.linspace(center - scale * tdata.size / 2,
center + scale * tdata.size / 2, tdata.size)
cond = (wave < 10300)
mod = interpolate.splev(wave, model['matched'])
fftd = numpy.fft.rfft(ndimage.gaussian_filter(tdata[cond], 17))
fftm = numpy.fft.rfft(ndimage.gaussian_filter(mod[cond], 17)[::-1])
fftd[100:] = 0.
fftm[100:] = 0.
T = numpy.fft.fftshift(numpy.fft.irfft(fftd * fftm)).real
offset = T.size / 2 - T.argmax()
# offset = T.size/2 - 1747
Wstart = wave[offset]
print "Assuming the starting wavelength is %6.1f" % (Wstart)
# pylab.plot(T)
# pylab.show()
if sparse is True:
nlines = datalines.size
n = order + 2
c = numpy.atleast_2d(numpy.array([Wstart, scale])).T
fit = {'coeff': c, 'type': 'polynomial'}
first = True
while n < nlines:
end = datalines[n] + 15.
if end > data.size:
end = data.size
tdata = data[Xstart:end].copy()
x = x0[Xstart:end].copy()
if first is True:
W = sf.genfunc(x - Xstart, 0., fit)
else:
W = sf.genfunc(x, 0., fit)
cond = (W < 10400)
W = W[cond]
tdata = tdata[cond]
x = x[cond]
mod = interpolate.splev(W, model['matched'])
fftd = numpy.fft.rfft(tdata)
fftm = numpy.fft.rfft(mod[::-1])
T = numpy.fft.fftshift(numpy.fft.irfft(fftd * fftm)).real
offset = T.size / 2 - T.argmax()
Wlines = get_lines(x - offset, mod)
Wwave = get_lines(W, mod)
matches = []
for i in range(Wlines.size):
diff = abs(Wlines[i] - datalines[:n])
if diff.min() < 2:
matches.append([datalines[diff.argmin()], Wwave[i]])
if len(matches) < order + 1:
n += 1
continue
matches = numpy.asarray(matches)
fit = sf.lsqfit(matches, 'polynomial', order)
n += 1
first = False
return fit
coeff = numpy.ones(order + 1)
rescale = numpy.array([Wstart - Xstart * scale, scale])
for i in range(order - 1):
rescale = numpy.append(rescale, 1. / (data.size / 2)**(i + 2))
def optFunc(p, x, spec, mod):
if (numpy.isnan(p)).any() or p[0] < 0.5 or p[1] < 0.8 or p[1] > 1.2:
return spec
coeff = numpy.atleast_2d(numpy.array(p) * rescale).T
fit = {'coeff': coeff, 'type': 'polynomial'}
w = sf.genfunc(x, 0., fit)
m = interpolate.splev(w, mod)
cond = (w < 9800.)
m = m[cond]
m /= m.mean()
resid = (spec[cond] - m) / abs(spec[cond] + m)**0.5
return resid / resid.size
import pylab
from scipy import ndimage
n = 2 * order + 2
end = datalines[n] + 15
if end < Xstart + (Xend - Xstart) / 4:
end = Xstart + (Xend - Xstart) / 4
widespec = ndimage.gaussian_filter(data, 15)
x = x0[Xstart:end].copy()
spec = widespec[Xstart:end].copy()
spec /= spec.mean()
coeff, ier = optimize.leastsq(optFunc,
coeff, (x, spec, model['wide']),
factor=99.,
ftol=1e-12,
xtol=1e-12,
epsfcn=1e-11)
while end < Xend:
x = x0[Xstart:end].copy()
# spec = widespec[Xstart:end].copy()
# spec /= spec.mean()
# coeff,ier = optimize.leastsq(optFunc,coeff,(x,spec,model['wide']),factor=99.,ftol=1e-12,xtol=1e-12,epsfcn=1e-11)
spec = data[Xstart:end].copy()
spec /= spec.mean()
coeff, ier = optimize.leastsq(optFunc,
coeff, (x, spec, model['matched']),
factor=99.,
ftol=1e-12,
xtol=1e-12,
epsfcn=1e-11)
fit = {
'coeff': scipy.atleast_2d(coeff * rescale).T,
'type': 'polynomial'
}
w = sf.genfunc(x, 0., fit)
m = interpolate.splev(w, model['matched'])
cond = (w < 10400)
m = m[cond]
m /= m.mean()
coeff, ier = optimize.leastsq(optFunc,
coeff, (x, spec, model['matched']),
factor=99.,
ftol=1e-12,
xtol=1e-12,
epsfcn=1e-11)
end += (Xend - Xstart) / 10.
x = x0[Xstart:Xend].copy()
spec = data[Xstart:Xend].copy()
spec /= spec.mean()
coeff, ier = optimize.leastsq(optFunc,
coeff, (x, spec, model['matched']),
ftol=1e-12,
xtol=1e-12,
epsfcn=1e-7)
fit = {'coeff': scipy.atleast_2d(coeff * rescale).T, 'type': 'polynomial'}
w = sf.genfunc(x, 0., fit)
m = interpolate.splev(w, model['matched'])
cond = (w < 10400)
m = m[cond]
m /= m.mean()
pylab.plot(w[cond], m)
pylab.plot(w[cond], spec[cond])
pylab.show()
return fit
w = sf.genfunc(x, 0., fit)
m = interpolate.splev(w, model['matched'])
rhs = m / err
fit = numpy.array(numpy.linalg.lstsq(op, rhs)[0])
d = numpy.dot(linmod.T, fit)
pylab.plot(w, m)
pylab.plot(w, d)
w0 = w.copy()
fit = {'coeff': scipy.atleast_2d(pars).T, 'type': 'polynomial'}
w = sf.genfunc(x, 0., fit)
m = interpolate.splev(w, model['matched'])
rhs = m / err
fit = numpy.array(numpy.linalg.lstsq(op, rhs)[0])
d2 = numpy.dot(linmod.T, fit)
pylab.plot(w, d2)
print(w0 - w).std()
pylab.figure()
pylab.plot(w0 - w)
pylab.show()
return fit
df
def ycorrect(data):
# Parameters
SUMWIDTH = 41 # Width of summing over columns
y_axis = data.shape[0]
x_axis = data.shape[1]
# Determine the location of the central column
central = x_axis / 2
# Determine the centers of the holes in the center column to use as the
# reference for all other columns
x_min_orig = central - SUMWIDTH / 2
x_max_orig = central + SUMWIDTH / 2
midcol = data[:, x_min_orig:x_max_orig].mean(axis=1)
central_edges, threshold, star_cutoff = find_holes(midcol)
transform_table = scipy.zeros((1, 3), 'f4')
index = 0
for peak in central_edges:
if index:
transform_table.resize((index + 1, 3))
transform_table[index, 0] = central
transform_table[index, 1] = peak
transform_table[index, 2] = peak
index += 1
offset = scipy.zeros(len(central_edges))
x_min = x_min_orig
x_max = x_max_orig
current_column = central
while current_column > SUMWIDTH + 20:
current_column = current_column - SUMWIDTH - 10
x_min = x_min - SUMWIDTH - 10
x_max = x_max - SUMWIDTH - 10
comp_array = data[:, x_min:x_max].mean(axis=1)
comp_array.clip(min=-1000., max=star_cutoff)
derivative = deriv_1d(comp_array)
derivative = ndimage.gaussian_filter1d(derivative, 3)
derivative = abs(derivative)
for i in range(offset.size):
if scipy.isnan(offset[i]):
continue
ref = central_edges[i] + offset[i]
start = int(ref) - 6
end = start + 13
if derivative[start:end].max() < threshold:
offset[i] = scipy.nan
continue
fit = find_peak(derivative[start:end])
if (fit[2] < 0 or fit[2] > 13 or fit[3] < 1 or fit[3] > 6):
offset[i] = scipy.nan
continue
peak = fit[2] + float(start)
offset[i] = peak - central_edges[i]
transform_table.resize((index + 1, 3))
transform_table[index, 0] = current_column
transform_table[index, 1] = central_edges[i]
transform_table[index, 2] = peak
index += 1
offset = scipy.zeros(offset.size)
x_min = x_min_orig
x_max = x_max_orig
current_column = central
while current_column < x_axis - SUMWIDTH - 19:
current_column = current_column + SUMWIDTH + 10
x_min = x_min + SUMWIDTH + 10
x_max = x_max + SUMWIDTH + 10
comp_array = data[:, x_min:x_max].mean(axis=1)
comp_array.clip(min=-1000., max=star_cutoff)
derivative = deriv_1d(comp_array)
derivative = ndimage.gaussian_filter1d(derivative, 3)
derivative = abs(derivative)
for i in range(offset.size):
if scipy.isnan(offset[i]):
continue
ref = central_edges[i] + offset[i]
start = int(round(ref)) - 6
end = start + 13
if derivative[start:end].max() < threshold:
offset[i] = scipy.nan
continue
fit = find_peak(derivative[start:end])
if (fit[2] < 0 or fit[2] > 13 or fit[3] < 1 or fit[3] > 6):
offset[i] = scipy.nan
continue
peak = fit[2] + float(start)
offset[i] = peak - central_edges[i]
transform_table.resize((index + 1, 3))
transform_table[index, 0] = current_column
transform_table[index, 1] = central_edges[i]
transform_table[index, 2] = peak
index += 1
true_coeffs = special_functions.lsqfit(transform_table, "chebyshev", 4, 4)
temp = transform_table[:, 1].copy()
transform_table[:, 1] = transform_table[:, 2].copy()
transform_table[:, 2] = temp.copy()
map_coeffs = special_functions.lsqfit(transform_table, "chebyshev", 4, 4)
# The xtrue_coeffs give the "true" y value for that pixel. xmap_coeffs
# describe where the coordinate should be mapped to in an output grid
# (in other words the true pixel value for that y).
return true_coeffs, map_coeffs
def wave_arcsky(arc, arcmodel, sky, solution):
"""
First find the best solution with the skylines, then apply this solution
to all arclines within the bounds of the lowest/highest wavelength
skylines, solving for the delta_pixel offset between the sky and the
arc. Then find the solution for all lines (sky and delta_pixel-offset
arcs).
"""
def clip(arr):
a = arr.copy()
m, s, l = a.mean(), a.std(), a.size
while 1:
a = a[abs(a - m) < 3. * s]
if a.size == l:
return m, s
m, s, l = a.mean(), a.std(), a.size
STD_LINES = [
5197.928, 5200.286, 5202.977, 5460.735, 5577.345, 5867.5522, 5915.308,
5932.864, 6257.970, 6300.320, 6363.810, 6533.040, 6553.610, 6863.971,
6912.620, 6923.210, 6939.520, 7303.716, 7329.148, 7340.885, 7358.659,
7392.198, 7586.093, 7808.467, 7821.510, 7841.266, 7993.332, 8310.719,
8344.613, 8399.160, 8415.231, 8430.170, 8791.186, 8885.830, 8943.395,
8988.384, 9038.059, 9337.854, 9375.977, 9419.746, 9439.670, 9458.524
]
x = scipy.arange(sky.size).astype(scipy.float32)
lines = get_lines(x, sky)
global fit
scale = solution['coeff'][1]
order = solution['coeff'].size - 1
if scale > 1.5:
STD_LINES.insert(6, 5891.)
w = sf.genfunc(lines, 0., solution)
matches = []
for i in range(w.size):
diff = abs(w[i] - STD_LINES)
if diff.min() < 5. * scale:
matches.append([lines[i], STD_LINES[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order)
w = sf.genfunc(lines, 0., fit)
matches = []
for i in range(w.size):
diff = abs(w[i] - STD_LINES)
if diff.min() < 5. * scale:
matches.append([lines[i], STD_LINES[diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(matches), 'polynomial', order)
lines = get_lines(x, sky, nstd=7.)
w = sf.genfunc(lines, 0., fit)
matches = []
for i in range(w.size):
diff = abs(w[i] - STD_LINES)
if diff.min() < 3. * scale:
matches.append([lines[i], STD_LINES[diff.argmin()]])
matches = scipy.asarray(matches)
fit = sf.lsqfit(matches, 'polynomial', order)
revfit = sf.lsqfit(matches[:, ::-1], 'polynomial', order)
ARCS = scipy.sort(arcmodel['lines'])
alines = get_lines(x, arc)
xmin, xmax = matches[0, 0], matches[-1, 0]
arc_x = sf.genfunc(ARCS, 0., revfit)
offset = []
for i in range(arc_x.size):
if arc_x[i] < xmin - 2 or arc_x[i] > xmax + 2:
continue
diff = arc_x[i] - alines
if abs(diff).min() < 9.:
offset.append(diff[abs(diff).argmin()])
offset = scipy.asarray(offset)
off, width = clip(offset)
aw = sf.genfunc(alines + off, 0., fit)
matches = []
for i in range(w.size):
diff = abs(w[i] - STD_LINES)
if diff.min() < 3. * scale:
matches.append([lines[i], STD_LINES[diff.argmin()]])
k = len(matches)
for i in range(aw.size):
diff = abs(aw[i] - ARCS)
if diff.min() < 3. * scale:
matches.append([alines[i] + off, ARCS[diff.argmin()]])
matches = scipy.asarray(matches)
fit = sf.lsqfit(matches, 'polynomial', order)
"""
# This code is to optimize the arc offset -- likely unnecessary.
def opt(p):
global fit
off = p[0]
w = sf.genfunc(lines,0.,fit)
aw = sf.genfunc(alines+off,0.,fit)
matches = []
for i in range(w.size):
diff = abs(w[i]-STD_LINES)
if diff.min()<3.*scale:
matches.append([lines[i],STD_LINES[diff.argmin()]])
k = len(matches)
for i in range(aw.size):
diff = abs(aw[i]-ARCS)
if diff.min()<3.*scale:
matches.append([alines[i]+off,ARCS[diff.argmin()]])
matches = scipy.asarray(matches)
fit = sf.lsqfit(matches,'polynomial',order)
return (matches[:,1]-sf.genfunc(matches[:,0],0.,fit))
from scipy import optimize
coeff,ier = optimize.leastsq(opt,[off],epsfcn=1e-15)
"""
return fit
def bgsub(dir, inname, out_prefix, cal_prefix):
# Where things begin and end....
blue = [1500, 1400, 1300, 1200, 1100, 900, 600, 200, 0, 0, 0]
red = [3000, 3400, 3700, -1, -1, -1, -1, -1, -1, -1]
readvar = 7.3
bias = pyfits.open(cal_prefix + "_bias.fits")[0].data.astype(scipy.float32)
bpm = pyfits.open(cal_prefix + "_bpm.fits")[0].data.astype(scipy.float32)
flat = pyfits.open(cal_prefix + "_norm.fits")[0].data.astype(scipy.float32)
orders, y_soln, wideorders = numpy.load(cal_prefix + "_ycor.dat",
allow_pickle=True)
fullsoln = numpy.load(cal_prefix + "_full.dat", allow_pickle=True)
hdu = pyfits.open(dir + inname)
data = hdu[0].data.copy()
data = biastrim(data, bias, bpm) / flat
if data.shape[1] < 3000:
blue = [i / 2 for i in blue]
red = [1500, 1700, 1850, -1, -1, -1, -1, -1, -1, -1]
"""
try:
back = pyfits.open(out_prefix+"_bg.fits")[0].data.copy()
print("Initial background opened")
except:
strt = straighten(data,y_soln,orders,wideorders)
back = scipy.zeros(strt.shape)
for indx in range(len(orders)):
i,j = orders[indx]
bg = numpy.nanmedian(strt[i:j],axis=0)
back[i:j] += bg
back = curve(back,y_soln,orders,wideorders)
pyfits.PrimaryHDU(back).writeto(out_prefix+"_bg.fits")
"""
try:
sub = pyfits.open(out_prefix + "_crsub.fits")[0].data.copy()
masks = numpy.load(out_prefix + '_masks.dat', allow_pickle=True)
print('Opened CR-subbed image')
except:
print('Creating CR-subbed image and masks')
strt = straighten(data, y_soln, orders, wideorders)
back = scipy.zeros(strt.shape)
for indx in range(len(orders)):
i, j = orders[indx]
bg = numpy.nanmedian(strt[i:j], axis=0)
back[i:j] += bg
back = curve(back, y_soln, orders, wideorders)
bgsub = data - back
sub = bgsub.copy()
omap = sub * 0.
for loop in range(3):
message('CR iteration %d ...' % (loop + 1))
map = crFind(sub, data + readvar)
if map.sum() == 0:
message(' no new CRs found.\n')
break
else:
message(' %d pixels flagged.\n' % (map.sum()))
inmask = sub.flatten()
cond = numpy.where(map.ravel() == 1)[0]
inmask[cond] = numpy.where(cond % 2 == 0, 1e5,
-1e5).astype(inmask.dtype)
med5 = ndimage.median_filter(inmask.reshape(sub.shape), 7)
med5 *= map
sub = (1. - map) * sub + med5
omap += map
crsub = data - (bgsub - sub)
bpm = ndimage.minimum_filter(numpy.where(bpm == 1, 1, 0), 3)
#mask = numpy.where(numpy.isnan(bpm)|(omap>0),0.,1.)
mask = numpy.where((bpm == 0) | (omap > 0), 0, 1)
masks = getOrders(mask, orders, wideorders, fullsoln, 1)
for i in range(len(masks)):
masks[i][0] = numpy.where(masks[i][0] < 0.7, 0, 1)
f = open(out_prefix + '_masks.dat', 'wb')
dump(masks, f, 2)
f.close()
pyfits.PrimaryHDU(crsub).writeto(out_prefix + "_crsub.fits")
hdulist = pyfits.HDUList([pyfits.PrimaryHDU()])
hdulist[0].header = hdu[0].header.copy()
varhdu = pyfits.HDUList([pyfits.PrimaryHDU()])
varhdu[0].header = hdu[0].header.copy()
print("Subtracting backgrounds")
slits = getOrders(crsub, orders, wideorders, fullsoln)
for indx in range(len(orders)):
cut, wlo, whigh, disp = slits[indx]
slit = cut.copy()
mask = masks[indx][0]
cut[scipy.isinf(cut)] = numpy.nan
cut[mask == 0] = numpy.nan
bg = numpy.nanmedian(cut, axis=0)
tmp = (cut - bg) * mask
T = tmp[3:-3, blue[indx]:red[indx]]
T = T[numpy.isfinite(T)]
avg, std = clip2(T)
slice = numpy.nanmedian(tmp[:, blue[indx]:red[indx]], axis=1)
slice[:3] = 0.
slice[-3:] = 0.
avg, std = clip2(slice[3:-3], 3.5, 0.05)
good = numpy.where(slice < 2. * std, 1., 0.)
good = ndimage.maximum_filter(good, 5)
good = ndimage.minimum_filter(good, 15)
good = good == 1
xvals = numpy.arange(good.size).astype(scipy.float32)
fitdata = numpy.empty((slice[good].size, 2))
fitdata[:, 0] = xvals[good].copy()
bgsub = slit.copy()
for k in range(cut.shape[1]):
fitdata[:, 1] = cut[:, k][good].copy()
if fitdata[:, 1][numpy.isfinite(fitdata[:, 1])].size < 4:
continue
avg, std = clip(fitdata[:, 1])
keep = numpy.where((abs(fitdata[:, 1] - avg) < 4. * std)
& numpy.isfinite(fitdata[:, 1]))[0]
fit = fitdata[keep]
if fit[:, 1][scipy.isfinite(fit[:, 1])].size < 4:
continue
fit = sf.lsqfit(fit, 'chebyshev', 1)
bg = sf.genfunc(xvals, 0, fit)
bgsub[:, k] -= bg
hdu = pyfits.ImageHDU(bgsub)
hdu.header.set('CTYPE1', 'LINEAR')
hdu.header.set('CRVAL1', wlo)
hdu.header.set('CRPIX1', 1)
hdu.header.set('CDELT1', disp)
hdu.header.set('CTYPE2', 'LINEAR')
hdu.header.set('CRVAL2', 1)
hdu.header.set('CRPIX2', 1)
hdu.header.set('CDELT2', 1)
hdu.header.set('CD1_1', disp)
hdu.header.set('CD2_2', 1)
hdu.header.set('CD1_2', 0)
hdu.header.set('CD2_1', 0)
hdu.header.set('DC-FLAG', 1)
hdu.header.set('DISPAXIS', 1)
hdulist.append(hdu)
hdu = pyfits.ImageHDU(cut + readvar)
hdu.header.set('CTYPE1', 'LINEAR')
hdu.header.set('CRVAL1', wlo)
hdu.header.set('CRPIX1', 1)
hdu.header.set('CDELT1', disp)
hdu.header.set('CTYPE2', 'LINEAR')
hdu.header.set('CRVAL2', 1)
hdu.header.set('CRPIX2', 1)
hdu.header.set('CDELT2', 1)
hdu.header.set('CD1_1', disp)
hdu.header.set('CD2_2', 1)
hdu.header.set('CD1_2', 0)
hdu.header.set('CD2_1', 0)
hdu.header.set('DC-FLAG', 1)
hdu.header.set('DISPAXIS', 1)
varhdu.append(hdu)
hdulist.verify('fix')
hdulist.writeto(out_prefix + "_bgsub.fits")
varhdu.verify('fix')
varhdu.writeto(out_prefix + "_var.fits")
def arcwave2(arc, arcmodel, scale, order=3, bcutoff=2e3, rcutoff=1e4):
from scipy import ndimage, stats, interpolate, signal, optimize
import pylab
arc = arc.copy()
arc[scipy.isnan(arc)] = 1.
arc[arc <= 1.] = arc.mean()
wave, model = arcmodel['orig']
model[scipy.isnan(model)] = 0.
wave = wave.copy()
model = model.copy()
cond = (wave > bcutoff) & (wave < rcutoff)
corrmodel = model.copy()
corrmodel[(wave < bcutoff) | (wave > rcutoff)] = 0.
corr = signal.correlate(arc, corrmodel, mode='valid')
offset = corr.argmax()
lines = arcmodel['lines'].copy()
bc = lines.min() - scale * 20.
rc = lines.max() + scale * 20.
x = scipy.arange(arc[offset:].size)
w = wave[offset:offset + x.size].copy()
cond = (w > bc) & (w < rc)
fit = scipy.empty((x[cond].size, 2))
fit[:, 0] = x[cond].copy()
fit[:, 1] = w[cond].copy()
pars = sf.lsqfit(fit, 'polynomial', 1)
pars['coeff'][0] = wave[offset]
pars['coeff'][1] = scale
# pars = [wave[offset],scale]
# for i in range(2,order+1):
# pars.append(1e-5**i)
pylab.plot(wave, model)
w = sf.genfunc(scipy.arange(x.size), 0., pars)
pylab.plot(w, interpolate.splev(w, arcmodel['matched']))
pylab.show()
def arcfit(p, x, arc, mod):
fit = {'coeff': scipy.atleast_2d(p).T, 'type': 'polynomial'}
w = sf.genfunc(x, 0., fit)
cond = (w > bcutoff) & (w < rcutoff)
m = interpolate.splev(w[cond], mod)
chi = (m - arc[cond]) / abs(arc[cond])**0.5
return chi
widearc = ndimage.gaussian_filter(arc, 7.)
x = scipy.arange(arc[offset:].size)
coeff, ier = optimize.leastsq(
arcfit,
pars, (x, widearc[offset:].copy(), arcmodel['wide']),
maxfev=100000)
fit = {'coeff': scipy.atleast_2d(coeff).T, 'type': 'polynomial'}
x = scipy.arange(arc.size)
l = get_lines(x, arc, nstd=15.)
lw = sf.genfunc(l - offset, 0., fit)
lines = []
for i in range(l.size):
diff = abs(lw[i] - arcmodel['lines'])
if diff.min() > 5. * scale:
continue
lines.append([l[i], arcmodel['lines'][diff.argmin()]])
fit = sf.lsqfit(scipy.asarray(lines), 'polynomial', order)
pars = fit['coeff'].flatten()
coeff, ier = optimize.leastsq(
arcfit,
pars, (x[offset:], arc[offset:], arcmodel['matched']),
maxfev=100000)
fit = {'coeff': scipy.atleast_2d(coeff).T, 'type': 'polynomial'}
return fit
