def specpolextract(infilelist, logfile='salt.log', debug=False):
"""Produce a 1-D extract spectra for the O and E beams
This also cleans the 2-D spectra of a number of artifacts, removes the background, accounts for small
spatial shifts in the observation, and resamples the data into a wavelength grid
Parameters
----------
infile_list: list
List of filenames that include an extracted spectra
logfile: str
Name of file for logging
"""
#set up the files
obsdate=os.path.basename(infilelist[0])[8:16]
with logging(logfile, debug) as log:
#create the observation log
obs_dict=obslog(infilelist)
# get rid of arcs
for i in range(len(infilelist))[::-1]:
if (obs_dict['OBJECT'][i].upper().strip()=='ARC'): del infilelist[i]
infiles = len(infilelist)
# contiguous images of the same object and config are grouped together as an observation
obs_dict=obslog(infilelist)
confno_i,confdatlist = configmap(infilelist)
configs = len(confdatlist)
objectlist = list(set(obs_dict['OBJECT']))
objno_i = np.array([objectlist.index(obs_dict['OBJECT'][i]) for i in range(infiles)],dtype=int)
obs_i = np.zeros((infiles),dtype=int)
obs_i[1:] = ((objno_i[1:] != objno_i[:-1]) | (confno_i[1:] != confno_i[:-1]) ).cumsum()
dum,iarg_b = np.unique(obs_i,return_index=True) # gives i for beginning of each obs
obss = iarg_b.shape[0]
obscount_b = np.zeros((obss),dtype=int)
oclist_b = np.array([[objno_i[iarg_b[b]], confno_i[iarg_b[b]]] for b in range(obss)])
if obss>1:
for b in range(1,obss):
obscount_b[b] = (oclist_b[b]==oclist_b[0:b]).all(axis=1).sum()
for b in range(obss):
ilist = np.where(obs_i==b)[0]
outfiles = len(ilist)
outfilelist = [infilelist[i] for i in ilist]
obs_dict=obslog(outfilelist)
imagenolist = [int(os.path.basename(infilelist[i]).split('.')[0][-4:]) for i in ilist]
log.message('\nExtract: '+objectlist[objno_i[ilist[0]]]+' Grating %s Grang %6.2f Artic %6.2f' % \
confdatlist[confno_i[ilist[0]]], with_header=False)
log.message(' Images: '+outfiles*'%i ' % tuple(imagenolist), with_header=False)
hdu0 = pyfits.open(outfilelist[0])
rows,cols = hdu0['SCI'].data.shape[1:3]
cbin,rbin = np.array(obs_dict["CCDSUM"][0].split(" ")).astype(int)
# special version for lamp data
object = obs_dict["OBJECT"][0].strip().upper()
lampid = obs_dict["LAMPID"][0].strip().upper()
if ((object != "ARC") & (lampid != "NONE")) :
specpollampextract(outfilelist, logfile=logfile)
continue
# sum spectra to find target, background artifacts, and estimate sky flat and psf functions
count = 0
for i in range(outfiles):
badbin_orc = pyfits.open(outfilelist[i])['BPM'].data > 0
if count == 0:
count_orc = (~badbin_orc).astype(int)
image_orc = pyfits.open(outfilelist[i])['SCI'].data*count_orc
var_orc = pyfits.open(outfilelist[i])['VAR'].data*count_orc
else:
count_orc += (~badbin_orc).astype(int)
image_orc += pyfits.open(outfilelist[i])['SCI'].data*(~badbin_orc).astype(int)
var_orc += pyfits.open(outfilelist[i])['VAR'].data*(~badbin_orc).astype(int)
count += 1
if count ==0:
print 'No valid images'
continue
image_orc[count_orc>0] /= count_orc[count_orc>0]
badbinall_orc = (count_orc==0) | (image_orc==0) # bin is bad in all images
badbinone_orc = (count_orc < count) | (image_orc==0) # bin is bad in at least one image
var_orc[count_orc>0] /= (count_orc[count_orc>0])**2
wav_orc = pyfits.open(outfilelist[0])['WAV'].data
slitid = obs_dict["MASKID"][0]
okwav_oc = ~((wav_orc == 0).all(axis=1))
if slitid[0] =="P": slitwidth = float(slitid[2:5])/10.
else: slitwidth = float(slitid)
obsname = objectlist[oclist_b[b][0]]+"_c"+str(oclist_b[b][1])+"_"+str(obscount_b[b])
hdusum = pyfits.PrimaryHDU(header=hdu0[0].header)
hdusum = pyfits.HDUList(hdusum)
hdusum[0].header.update('OBJECT',obsname)
header=hdu0['SCI'].header.copy()
hdusum.append(pyfits.ImageHDU(data=image_orc, header=header, name='SCI'))
hdusum.append(pyfits.ImageHDU(data=var_orc, header=header, name='VAR'))
hdusum.append(pyfits.ImageHDU(data=badbinall_orc.astype('uint8'), header=header, name='BPM'))
hdusum.append(pyfits.ImageHDU(data=wav_orc, header=header, name='WAV'))
if debug: hdusum.writeto(obsname+".fits",clobber=True)
psf_orc,skyflat_orc,badbinnew_orc,isbkgcont_orc,maprow_od,drow_oc = \
specpolsignalmap(hdusum,logfile=logfile,debug=debug)
maprow_ocd = maprow_od[:,None,:] + np.zeros((2,cols,4))
maprow_ocd[okwav_oc] += drow_oc[okwav_oc,None]
isedge_orc = (np.arange(rows)[:,None] < maprow_ocd[:,None,:,0]) | \
(np.arange(rows)[:,None] > maprow_ocd[:,None,:,3])
istarget_orc = okwav_oc[:,None,:] & (np.arange(rows)[:,None] > maprow_ocd[:,None,:,1]) & \
(np.arange(rows)[:,None] < maprow_ocd[:,None,:,2])
isbkgcont_orc &= (~badbinall_orc & ~isedge_orc & ~istarget_orc)
badbinall_orc |= badbinnew_orc
badbinone_orc |= badbinnew_orc
hdusum['BPM'].data = badbinnew_orc.astype('uint8')
if debug:
# hdusum.writeto(obsname+".fits",clobber=True)
pyfits.PrimaryHDU(psf_orc.astype('float32')).writeto(obsname+'_psf_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinnew_orc.astype('uint8')).writeto('badbinnew_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinall_orc.astype('uint8')).writeto('badbinall_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinone_orc.astype('uint8')).writeto('badbinone_orc.fits',clobber=True)
# set up wavelength binning
wbin = wav_orc[0,rows/2,cols/2]-wav_orc[0,rows/2,cols/2-1]
wbin = 2.**(np.rint(np.log2(wbin))) # bin to nearest power of 2 angstroms
wmin = (wav_orc.max(axis=1)[okwav_oc].reshape((2,-1))).min(axis=1).max()
wmax = wav_orc.max()
for o in (0,1):
colmax = np.where((wav_orc[o] > 0.).any(axis=0))[0][-1]
row_r = np.where(wav_orc[o,:,colmax] > 0.)[0]
wmax = min(wmax,wav_orc[o,row_r,colmax].min())
wedgemin = wbin*int(wmin/wbin+0.5) + wbin/2.
wedgemax = wbin*int(wmax/wbin-0.5) + wbin/2.
wedge_w = np.arange(wedgemin,wedgemax+wbin,wbin)
wavs = wedge_w.shape[0] - 1
binedge_orw = np.zeros((2,rows,wavs+1))
specrow_or = (maprow_od[:,1:3].mean(axis=1)[:,None] + np.arange(-rows/4,rows/4)).astype(int)
# scrunch and normalize psf from summed images (using badbinone) for optimized extraction
# psf is normalized so its integral over row is 1.
psfnormmin = 0.70 # wavelengths with less than this flux in good bins are marked bad
psf_orw = np.zeros((2,rows,wavs))
for o in (0,1):
for r in specrow_or[o]:
binedge_orw[o,r] = \
interp1d(wav_orc[o,r,okwav_oc[o]],np.arange(cols)[okwav_oc[o]], \
kind='linear',bounds_error=False)(wedge_w)
psf_orw[o,r] = scrunch1d(psf_orc[o,r],binedge_orw[o,r])
if debug:
pyfits.PrimaryHDU(binedge_orw.astype('float32')).writeto(obsname+'_binedge_orw.fits',clobber=True)
pyfits.PrimaryHDU(psf_orw.astype('float32')).writeto(obsname+'_psf_orw.fits',clobber=True)
psf_orw /= psf_orw.sum(axis=1)[:,None,:]
# set up optional image-dependent column shift for slitless data
colshiftfilename = "colshift.txt"
docolshift = os.path.isfile(colshiftfilename)
if docolshift:
img_I,dcol_I = np.loadtxt(colshiftfilename,dtype=float,unpack=True,usecols=(0,1))
shifts = img_I.shape[0]
log.message('Column shift: \n Images '+shifts*'%5i ' % tuple(img_I), with_header=False)
log.message(' Bins '+shifts*'%5.2f ' % tuple(dcol_I), with_header=False)
# background-subtract and extract spectra
for i in range(outfiles):
hdulist = pyfits.open(outfilelist[i])
tnum = os.path.basename(outfilelist[i]).split('.')[0][-3:]
badbin_orc = (hdulist['BPM'].data > 0)
badbinbkg_orc = (badbin_orc | badbinnew_orc | isedge_orc | istarget_orc)
if debug:
pyfits.PrimaryHDU(isedge_orc.astype('uint8')).writeto('isedge_orc_'+tnum+'.fits',clobber=True)
pyfits.PrimaryHDU(istarget_orc.astype('uint8')).writeto('istarget_orc_'+tnum+'.fits',clobber=True)
pyfits.PrimaryHDU(badbinbkg_orc.astype('uint8')).writeto('badbinbkg_orc_'+tnum+'.fits',clobber=True)
target_orc = bkgsub(hdulist,badbinbkg_orc,isbkgcont_orc,skyflat_orc,maprow_ocd,tnum,debug=debug)
target_orc *= (~badbin_orc).astype(int)
if debug:
pyfits.PrimaryHDU(target_orc.astype('float32')).writeto('target_'+tnum+'_orc.fits',clobber=True)
var_orc = hdulist['var'].data
badbin_orc = (hdulist['bpm'].data > 0) | badbinnew_orc
# extract spectrum optimally (Horne, PASP 1986)
target_orw = np.zeros((2,rows,wavs))
var_orw = np.zeros_like(target_orw)
badbin_orw = np.ones((2,rows,wavs),dtype='bool')
wt_orw = np.zeros_like(target_orw)
dcol = 0.
if docolshift:
if int(tnum) in img_I:
dcol = dcol_I[np.where(img_I==int(tnum))] # table has observed shift
for o in (0,1):
for r in specrow_or[o]:
target_orw[o,r] = scrunch1d(target_orc[o,r],binedge_orw[o,r]+dcol)
var_orw[o,r] = scrunch1d(var_orc[o,r],binedge_orw[o,r]+dcol)
badbin_orw[o,r] = scrunch1d(badbin_orc[o,r].astype(float),binedge_orw[o,r]+dcol) > 0.001
badbin_orw |= (var_orw == 0)
badbin_orw |= ((psf_orw*(~badbin_orw)).sum(axis=1)[:,None,:] < psfnormmin)
# pyfits.PrimaryHDU(var_orw.astype('float32')).writeto('var_'+tnum+'_orw.fits',clobber=True)
# pyfits.PrimaryHDU(badbin_orw.astype('uint8')).writeto('badbin_'+tnum+'_orw.fits',clobber=True)
# use master psf shifted in row to allow for guide errors
pwidth = 2*int(1./psf_orw.max())
ok_w = ((psf_orw*badbin_orw).sum(axis=1) < 0.03/float(pwidth/2)).all(axis=0)
crosscor_s = np.zeros(pwidth)
for s in range(pwidth):
crosscor_s[s] = (psf_orw[:,s:s-pwidth]*target_orw[:,pwidth/2:-pwidth/2]*ok_w).sum()
smax = np.argmax(crosscor_s)
s_S = np.arange(smax-pwidth/4,smax-pwidth/4+pwidth/2+1)
polycof = la.lstsq(np.vstack((s_S**2,s_S,np.ones_like(s_S))).T,crosscor_s[s_S])[0]
pshift = -(-0.5*polycof[1]/polycof[0] - pwidth/2)
s = int(pshift+pwidth)-pwidth
sfrac = pshift-s
psfsh_orw = np.zeros_like(psf_orw)
outrow = np.arange(max(0,s+1),rows-(1+int(abs(pshift)))+max(0,s+1))
psfsh_orw[:,outrow] = (1.-sfrac)*psf_orw[:,outrow-s] + sfrac*psf_orw[:,outrow-s-1]
# pyfits.PrimaryHDU(psfsh_orw.astype('float32')).writeto('psfsh_'+tnum+'_orw.fits',clobber=True)
wt_orw[~badbin_orw] = psfsh_orw[~badbin_orw]/var_orw[~badbin_orw]
var_ow = (psfsh_orw*wt_orw*(~badbin_orw)).sum(axis=1)
badbin_ow = (var_ow == 0)
var_ow[~badbin_ow] = 1./var_ow[~badbin_ow]
# pyfits.PrimaryHDU(var_ow.astype('float32')).writeto('var_'+tnum+'_ow.fits',clobber=True)
# pyfits.PrimaryHDU(target_orw.astype('float32')).writeto('target_'+tnum+'_orw.fits',clobber=True)
# pyfits.PrimaryHDU(wt_orw.astype('float32')).writeto('wt_'+tnum+'_orw.fits',clobber=True)
sci_ow = (target_orw*wt_orw).sum(axis=1)*var_ow
badlim = 0.20
psfbadfrac_ow = (psfsh_orw*badbin_orw.astype(int)).sum(axis=1)/psfsh_orw.sum(axis=1)
badbin_ow |= (psfbadfrac_ow > badlim)
cdebug = 83
if debug: np.savetxt("xtrct"+str(cdebug)+"_"+tnum+".txt",np.vstack((psf_orw[:,:,cdebug],var_orw[:,:,cdebug], \
wt_orw[:,:,cdebug],target_orw[:,:,cdebug])).reshape((4,2,-1)).transpose(1,0,2).reshape((8,-1)).T,fmt="%12.5e")
# write O,E spectrum, prefix "s". VAR, BPM for each spectrum. y dim is virtual (length 1)
# for consistency with other modes
hduout = pyfits.PrimaryHDU(header=hdulist[0].header)
hduout = pyfits.HDUList(hduout)
header=hdulist['SCI'].header.copy()
header.update('VAREXT',2)
header.update('BPMEXT',3)
header.update('CRVAL1',wedge_w[0]+wbin/2.)
header.update('CRVAL2',0)
header.update('CDELT1',wbin)
header.update('CTYPE1','Angstroms')
hduout.append(pyfits.ImageHDU(data=sci_ow.reshape((2,1,wavs)), header=header, name='SCI'))
header.update('SCIEXT',1,'Extension for Science Frame',before='VAREXT')
hduout.append(pyfits.ImageHDU(data=var_ow.reshape((2,1,wavs)), header=header, name='VAR'))
hduout.append(pyfits.ImageHDU(data=badbin_ow.astype("uint8").reshape((2,1,wavs)), header=header, name='BPM'))
hduout.writeto('e'+outfilelist[i],clobber=True,output_verify='warn')
log.message('Output file '+'e'+outfilelist[i] , with_header=False)
return
def specpolcombine(infilelist,debug_output=False):
"""combine stokes files
Parameters
----------
infile_list: list
one or more _stokes.fits files
"""
"""
_b observations
_w wavelengths in individual observations
_W wavelengths in combined grid
"""
obss = len(infilelist)
obsdict=obslog(infilelist)
# construct common wavelength grid _W
grating_b = obsdict['GRATING']
grang_b = obsdict['GR-ANGLE']
artic_b = obsdict['CAMANG']
dwav_b = np.empty(obss)
wav0_b = np.empty(obss)
wavs_b = np.empty(obss)
stokeslist_sw = []
varlist_sw = []
oklist_sw = []
for b in range(obss):
hdul = pyfits.open(infilelist[b])
dwav_b[b] = float(hdul['SCI'].header['CDELT1'])
wav0_b[b] = float(hdul['SCI'].header['CRVAL1'])
wavs_b[b] = int(hdul['SCI'].header['NAXIS1'])
stokeslist_sw.append(hdul['SCI'].data[:,0,:])
varlist_sw.append(hdul['VAR'].data[:,0,:])
oklist_sw.append(hdul['BPM'].data[:,0,:] == 0)
dWav = dwav_b.max()
Wav0 = dWav*(wav0_b.min()//dWav)
Wavs = (dWav*((wav0_b + dwav_b*wavs_b).max()//dWav) - Wav0)/dWav
wav_W = np.arange(Wav0,Wav0+dWav*Wavs,dWav)
stokess = stokeslist_sw[0].shape[0]
vars = varlist_sw[0].shape[0]
stokes_bsW = np.zeros((obss,stokess,Wavs))
var_bsW = np.zeros((obss,vars,Wavs))
ok_bsW = np.zeros((obss,stokess,Wavs)).astype(bool)
# get data and put on common grid, combining bins if necessary
for b in range(obss):
if dwav_b[b] == dWav:
W0 = (wav0_b[b] - Wav0)/dWav
stokes_bsW[b,:,W0:W0+wavs_b[b]] = stokeslist_sw[b]
var_bsW[b,:,W0:W0+wavs_b[b]] = varlist_sw[b]
ok_bsW[b,:,W0:W0+wavs_b[b]] = oklist_sw[b]
else:
wbinedge_W = (wav_W - dWav/2. - (wav0_b[b] - dwav_b[b]/2.))/dwav_b
for s in range(stokess):
stokes_bsW[b,s] = scrunch1d(stokeslist_sw[b][s],wbinedge_W)
var_bsW[b,s] = scrunch1d(varlist_sw[b][s],wbinedge_W)
ok_bsW[b,s] = (scrunch1d((oklist_sw[b][s]).astype(int),wbinedge_W) > 0)
if (vars>stokess): var_bsW[b,vars] = scrunch1d(var_sw[vars],wbinedge_W)
if debug_output:
np.savetxt("stokes_bsW.txt",np.vstack((wav_W,stokes_bsW.reshape((6,Wavs)))).T,fmt="%10.3f")
# correct (unfluxed) intensity for grating efficiency to match observations together
for b in range(obss):
greff_W = greff(grating_b[b],grang_b[b],artic_b[b],wav_W)
ok_W = (ok_bsW[b].all(axis=0) & (greff_W > 0.))
stokes_bsW[b][:,ok_W] /= greff_W[ok_W]
var_bsW[b][:,ok_W] /= greff_W[ok_W]**2
# normalize at matching wavelengths _w
# compute ratios at each wavelength, then error-weighted mean of ratio
ismatch_W = ok_bsW.all(axis=0).all(axis=0)
normint_bw = stokes_bsW[:,0,ismatch_W]/stokes_bsW[:,0,ismatch_W].mean(axis=0)
varnorm_bw = var_bsW[:,0,ismatch_W]/stokes_bsW[:,0,ismatch_W].mean(axis=0)**2
normint_b = (normint_bw/varnorm_bw).sum(axis=1)/(1./varnorm_bw).sum(axis=1)
print normint_b
stokes_bsW /= normint_b[:,None,None]
var_bsW /= normint_b[:,None,None]**2
# Do error weighted combine of observations
stokes_sW = np.zeros((stokess,Wavs))
var_sW = np.zeros((vars,Wavs))
for b in range(obss):
ok_W = ok_bsW[b].any(axis=0)
stokes_sW[:,ok_W] += stokes_bsW[b][:,ok_W]/var_bsW[b][:stokess,ok_W]
var_sW[:,ok_W] += 1./var_bsW[b][:,ok_W]
ok_W = (var_sW != 0).all(axis=0)
ok_sW = np.tile(ok_W,(3,1))
var_sW[ok_sW] = 1./var_sW[ok_sW]
stokes_sW[:,ok_W] *= var_sW[:stokess,ok_W]
# Save result, name formed from unique elements of '_'-separated parts of names
namepartlist = []
parts = 100
for file in infilelist:
partlist = os.path.basename(file).split('.')[0].split('_')
parts = min(parts,len(partlist))
namepartlist.append(partlist)
outfile = ''
for part in range(parts):
outfile+='-'.join(sorted(set(zip(*namepartlist)[part])))+'_'
outfile = outfile[:-1]+'.fits'
print "\n",outfile,"\n"
hduout = hdul
for ext in ('SCI','VAR','BPM'):
hduout[ext].header.update('CDELT1',dWav)
hduout[ext].header.update('CRVAL1',Wav0)
hduout['SCI'].data = stokes_sW.astype('float32').reshape((stokess,1,-1))
hduout['VAR'].data = var_sW.astype('float32').reshape((vars,1,-1))
hduout['BPM'].data = (~ok_sW).astype('uint8').reshape((stokess,1,-1))
hduout[0].header.add_history('POLCOMBINE: '+' '.join(infilelist))
hduout.writeto(outfile,clobber=True,output_verify='warn')
return
def specpollampextract(infilelist, logfile='salt.log'):
obsdate=os.path.basename(infilelist[0])[8:16]
with logging(logfile, debug) as log:
log.message('Extraction of Lamp Images' , with_header=False)
obsdict=obslog(infilelist)
hdu0 = pyfits.open(infilelist[0])
rows,cols = hdu0['SCI'].data.shape[1:3]
cbin,rbin = np.array(obsdict["CCDSUM"][0].split(" ")).astype(int)
slitid = obsdict["MASKID"][0]
lampid = obsdict["LAMPID"][0].strip().upper()
lam_c = hdu0['WAV'].data[0,rows/2]
files = len(infilelist)
outfilelist = infilelist
# sum spectra to find target
count = 0
for i in range(files):
badbin_orc = pyfits.open(outfilelist[i])['BPM'].data.astype(bool)
if count == 0:
count_orc = (~badbin_orc).astype(int)
image_orc = pyfits.open(outfilelist[i])['SCI'].data*count_orc
var_orc = pyfits.open(outfilelist[i])['VAR'].data
else:
count_orc += (~badbin_orc).astype(int)
image_orc += pyfits.open(outfilelist[i])['SCI'].data*(~badbin_orc)
var_orc += pyfits.open(outfilelist[i])['VAR'].data
count += 1
if count ==0:
print 'No valid images'
exit()
image_orc[count_orc>0] /= count_orc[count_orc>0]
badbin_orc = (count_orc==0) | (image_orc==0)
okbinpol_orc = (count_orc == count) & (image_orc != 0) # conservative bpm for pol extraction
var_orc[count_orc>0] /= count_orc[count_orc>0]**2
wav_orc = pyfits.open(outfilelist[0])['WAV'].data
# pyfits.PrimaryHDU(image_orc.astype('float32')).writeto('lampsum_orc.fits',clobber=True)
lam_m = np.loadtxt(datadir+"wollaston.txt",dtype=float,usecols=(0,))
rpix_om = np.loadtxt(datadir+"wollaston.txt",dtype=float,unpack=True,usecols=(1,2))
# trace spectrum, compute spatial profile
profile_orc = np.zeros_like(image_orc)
drow_oc = np.zeros((2,cols))
expectrow_oc = np.zeros((2,cols),dtype='float32')
maxrow_oc = np.zeros((2,cols),dtype=int)
maxval_oc = np.zeros((2,cols),dtype='float32')
cross_orC = np.zeros((2,rows,2))
col_cr,row_cr = np.indices(image_orc[0].T.shape)
# sample cross-dispersion at center and on right (_C) to get offset and tilt
Collist = [cols/2,0.8*cols]
for C in (0,1): cross_orC[:,:,C] = np.sum(image_orc[:,:,Collist[C]-cols/16:Collist[C]+cols/16],axis=2)
drow_oC = np.zeros((2,2))
trow_o = np.zeros((2),dtype='int')
okprof_oc = np.zeros((2,cols),dtype='bool')
okprof_orc = np.zeros((2,rows,cols),dtype='bool')
norm_orc = np.zeros((2,rows,cols))
sig_c = np.zeros((cols))
sigmin = 20.; drowmax = 8.
# find spectrum offset and tilt roughly from max of two cross-dispersion samples
for o in (0,1):
expectrow_oc[o] = (1-o)*rows + interp1d(lam_m,rpix_om[o],kind='cubic')(lam_c)/rbin
for C in (0,1):
crossmaxval = np.max(cross_orC[o, \
expectrow_oc[o,Collist[C]]-100/rbin:expectrow_oc[o,Collist[C]]+100/rbin,C])
drow_oC[o,C] = np.where(cross_orC[o,:,C]==crossmaxval)[0][0] - expectrow_oc[o,Collist[C]]
drow_o = drow_oC[:,0]
rowtilt = (drow_oC[:,1]-drow_oC[:,0]).mean()/(Collist[1]-Collist[0])
expectrow_oc += drow_o[:,None] + rowtilt*np.arange(-cols/2,cols/2)
# get trace by finding max in narrow curved aperture and smoothing it
for o in (0,1):
row_c = expectrow_oc[o].astype(int)
aperture_cr = ((row_cr-row_c[:,None])>=-20/rbin) & ((row_cr-row_c[:,None])<=20/rbin)
maxrow_oc[o] = np.argmax(image_orc[o].T[aperture_cr].reshape((cols,-1)),axis=1) + row_c - 20/rbin
maxval_oc[o] = image_orc[o,maxrow_oc[o]].diagonal()
trow_o[o] = maxrow_oc[o,cols/2]
# mark as bad where signal drops too low or position is off
median_c = np.median(image_orc[o].T[aperture_cr].reshape((cols,-1)),axis=1)
var_c = np.mean(var_orc[o].T[aperture_cr].reshape((cols,-1)),axis=1)
sig_c[var_c>0] = (maxval_oc[o] - median_c)[var_c>0]/np.sqrt(var_c[var_c>0])
drow1_c = maxrow_oc[o] -expectrow_oc[o]
okprof_oc[o] = (sig_c > sigmin) & (abs(drow1_c - np.median(drow1_c)) < drowmax)
# divide out spectrum (allowing for spectral curvature and tilt) to make spatial profile
drow2_c = np.polyval(np.polyfit(np.where(okprof_oc[o])[0],drow1_c[okprof_oc[o]],3),(range(cols)))
okprof_orc[o] = (np.abs(drow2_c - drow1_c) < 3) & okprof_oc[o][None,:]
drow_oc[o] = -(expectrow_oc[o] - expectrow_oc[o,cols/2] + drow2_c -drow2_c[cols/2])
for r in range(rows):
norm_orc[o,r] = interp1d(wav_orc[o,trow_o[o],okprof_oc[o]],maxval_oc[o,okprof_oc[o]], \
bounds_error = False, fill_value=0.)(wav_orc[o,r])
log.message('Image tilt: %8.1f arcmin' % (60.*np.degrees(rowtilt*rbin/cbin)), with_header=False)
log.message('Target offset: O %4i E %4i' % tuple(drow_o), with_header=False)
log.message('Target center row: O %4i E %4i' % tuple(trow_o), with_header=False)
okprof_orc &= (norm_orc != 0.)
profile_orc[okprof_orc] = image_orc[okprof_orc]/norm_orc[okprof_orc]
var_orc[okprof_orc] = var_orc[okprof_orc]/norm_orc[okprof_orc]**2
# pyfits.PrimaryHDU(norm_rc.astype('float32')).writeto('norm_rc.fits',clobber=True)
# pyfits.PrimaryHDU(okprof_oc.astype('uint8')).writeto('okprof_oc.fits',clobber=True)
okprof_c = okprof_oc.all(axis=0)
# Sample the normalized row profile at 5 places (_C)
Cols = 5
dcols = 64/cbin
Collist = [np.argmax(okprof_c)+dcols, 0, cols/2, 0, cols-np.argmax(okprof_c[::-1])-dcols]
for C in (1,3): Collist[C] = 0.5*(Collist[C-1] + Collist[C+1])
Collist = map(int,Collist)
profile_Cor = np.zeros((Cols,2,rows))
# Using profile at center, find, mask off fov edge, including possible beam overlap
edgerow_do = np.zeros((2,2),dtype=int)
badrow_or = np.zeros((2,rows),dtype=bool)
axisrow_o = np.zeros(2)
maxoverlaprows = 34/rbin
profile_Cor[Cols/2] = np.median(profile_orc[:,:,cols/2-dcols:cols/2+dcols],axis=2)
for d,o in np.ndindex(2,2): # _d = (0,1) = (bottom,top)
row_y = np.where((d==1) ^ (np.arange(rows) < trow_o[o]))[0][::2*d-1]
edgeval = np.median(profile_Cor[Cols/2,o,row_y],axis=-1)
hist,bin = np.histogram(profile_Cor[Cols/2,o,row_y],bins=32,range=(0,edgeval))
histarg = 32 - np.argmax(hist[::-1]<3) # edge: <3 in hist in decreasing dirn
edgeval = bin[histarg]
edgerow_do[d,o] = trow_o[o] + (2*d-1)*(np.argmax(profile_Cor[Cols/2,o,row_y] <= edgeval))
axisrow_o[o] += edgerow_do[d,o]
edgerow_do[d,o] = np.clip(edgerow_do[d,o],maxoverlaprows,rows-maxoverlaprows)
badrow_or[o] |= ((d==1) ^ (np.arange(rows) < (edgerow_do[d,o]+d)))
axisrow_o /= 2.
# Row profile sample, now background subtracted
profile_orc[okprof_orc] = ((image_orc-np.median(image_orc,axis=1)[:,None,:])[okprof_orc]) \
/(norm_orc-np.median(image_orc,axis=1)[:,None,:])[okprof_orc]
# pyfits.PrimaryHDU(profile_orc.astype('float32')).writeto('profile_orc.fits',clobber=True)
for C in range(Cols):
okcol_c = (profile_orc.sum(axis=0).sum(axis=0)>0) & \
(np.abs(np.arange(cols)-Collist[C])<dcols)
Collist[C] = np.where(okcol_c)[0].mean()
profile_Cor[C] = np.median(profile_orc[:,:,okcol_c],axis=2)
# print 5*"%7.1f " % tuple(Collist)
# pyfits.PrimaryHDU(okprof_orc.astype('uint8')).writeto('okprof_orc.fits',clobber=True)
np.savetxt("profile_oCr.txt",profile_Cor.transpose((1,0,2)).reshape((2*Cols,-1)).T,fmt="%10.6f")
# find edge of target slit, and neighboring slits, if multiple slits
# background picked small enough to miss neighbors in all samples, but matched E and O
isneighbor_d = np.zeros((2),dtype='bool')
edgeoff_doC = np.zeros((2,2,Cols))
for o in (0,1):
plim = 0.35 # slit finder
bkgsafe = 0.90 # avoiding next slit
for C in range(Cols):
leftrow_s = np.flatnonzero((profile_Cor[C,o,:-1] < plim) & (profile_Cor[C,o,1:] > plim))
rightrow_s = np.flatnonzero((profile_Cor[C,o,leftrow_s[0]:-1] > plim) \
& (profile_Cor[C,o,leftrow_s[0]+1:] < plim)) + leftrow_s[0]
slits = rightrow_s.shape[0] # eliminate spikes:
slitrow_s = 0.5*(rightrow_s + leftrow_s[:slits])[(rightrow_s-leftrow_s[:slits]) > 2]
slits = slitrow_s.shape[0]
targetslit = np.where(abs(maxrow_oc[o,Collist[C]] - slitrow_s) < 6)[0][0]
if targetslit > 0:
edgeoff_doC[0,o,C] = maxrow_oc[o,Collist[C]] - slitrow_s[targetslit-1:targetslit+1].mean()
isneighbor_d[0] |= True
if targetslit < slits-1:
edgeoff_doC[1,o,C] = slitrow_s[targetslit:targetslit+2].mean() - maxrow_oc[o,Collist[C]]
isneighbor_d[1] |= True
for d in (0,1):
if isneighbor_d[d]: edgerow_do[d] = trow_o + bkgsafe*(2*d-1)*edgeoff_doC[d].min()
edgerow_doc = (edgerow_do[:,:,None] - drow_oc[None,:,:]).astype(int)
bkgrows_do = ((trow_o - edgerow_do)/2.).astype(int)
bkgrow_doc = edgerow_doc + bkgrows_do[:,:,None]/2
isbkg_dorc = (((np.arange(rows)[:,None] - edgerow_doc[:,:,None,:]) * \
(np.arange(rows)[:,None] - edgerow_doc[:,:,None,:] - bkgrows_do[:,:,None,None])) < 0)
istarg_orc = ((np.arange(rows)[:,None] - edgerow_doc[:,:,None,:]).prod(axis=0) < 0)
istarg_orc &= ~isbkg_dorc.any(axis=0)
okbinpol_orc &= okprof_oc[:,None,:]
# pyfits.PrimaryHDU(image_orc*(isbkg_dorc.sum(axis=0)).astype('float32')).writeto('lampbkg_orc.fits',clobber=True)
# pyfits.PrimaryHDU(istarg_orc.astype('uint8')).writeto('istarg_orc.fits',clobber=True)
log.message('Bottom, top row: O %4i %4i E %4i %4i \n' \
% tuple(edgerow_do.T.flatten()), with_header=False)
# background-subtract and extract spectra
# set up scrunch table and badpixels in wavelength space
wbin = wav_orc[0,rows/2,cols/2]-wav_orc[0,rows/2,cols/2-1]
wbin = float(int(wbin/0.75))
wmin,wmax = wav_orc.min(axis=2).max(),wav_orc.max(axis=2).min()
wedgemin = wbin*int(wmin/wbin+0.5) + wbin/2.
wedgemax = wbin*int(wmax/wbin-0.5) + wbin/2.
wedge_w = np.arange(wedgemin,wedgemax+wbin,wbin)
wavs = wedge_w.shape[0] - 1
badbin_orc = ~okbinpol_orc
binedge_orw = np.zeros((2,rows,wavs+1))
badbin_orw = np.ones((2,rows,wavs),dtype=bool); nottarg_orw = np.ones_like(badbin_orw)
for o in (0,1):
for r in range(edgerow_doc[0,o].min(),edgerow_doc[1,o].max()):
binedge_orw[o,r] = interp1d(wav_orc[o,r],np.arange(cols))(wedge_w)
badbin_orw[o,r] = (scrunch1d(badbin_orc[o,r].astype(int),binedge_orw[o,r]) > 0.)
nottarg_orw[o,r] = (scrunch1d((~istarg_orc[o,r]).astype(int),binedge_orw[o,r]) > 0.)
okbin_orw = ~badbin_orw
istarg_orw = ~nottarg_orw
# wavelengths with bad pixels in targ area are flagged as bad
badcol_ow = (istarg_orw & ~okbin_orw).any(axis=1)
for o in (0,1): okbin_orw[o] &= ~badcol_ow[o]
for i in range(files):
imageno = int(os.path.basename(outfilelist[i]).split('.')[0][-4:])
hdulist = pyfits.open(outfilelist[i])
sci_orc = hdulist['sci'].data
var_orc = hdulist['var'].data
# make background continuum image, linearly interpolated in row
bkg_doc = np.zeros((2,2,cols))
for d,o in np.ndindex(2,2):
bkg_doc[d,o] = np.median(sci_orc[o].T[isbkg_dorc[d,o].T].reshape((cols,-1)),axis=1)
bkgslp_oc = (bkg_doc[1] - bkg_doc[0])/(bkgrow_doc[1] - bkgrow_doc[0])
bkgbase_oc = (bkg_doc[1] + bkg_doc[0])/2. - bkgslp_oc*(bkgrow_doc[1] + bkgrow_doc[0])/2.
bkg_orc = bkgbase_oc[:,None,:] + bkgslp_oc[:,None,:]*np.arange(rows)[:,None]
target_orc = sci_orc-bkg_orc
# np.savetxt('bkg.txt',np.vstack((bkg_doc.reshape((4,-1)),bkgslp_oc,bkgbase_oc)).T,fmt="%11.4f")
# pyfits.PrimaryHDU(bkg_orc.astype('float32')).writeto('bkg_orc_'+str(imageno)+'.fits',clobber=True)
# pyfits.PrimaryHDU(target_orc.astype('float32')).writeto('target_orc_'+str(imageno)+'.fits',clobber=True)
# extract spectrum
target_orw = np.zeros((2,rows,wavs)); var_orw = np.zeros_like(target_orw)
for o in (0,1):
for r in range(edgerow_doc[0,o].min(),edgerow_doc[1,o].max()):
target_orw[o,r] = scrunch1d(target_orc[o,r],binedge_orw[o,r])
var_orw[o,r] = scrunch1d(var_orc[o,r],binedge_orw[o,r])
# columns with negative extracted intensity are marked as bad
sci_ow = (target_orw*okbin_orw).sum(axis=1)
# pyfits.PrimaryHDU((target_orw*okbin_orw).astype('float32')).writeto('sci_orw.fits',clobber=True)
var_ow = (var_orw*okbin_orw).sum(axis=1)
okbin_ow = (okbin_orw.any(axis=1) & (sci_ow > 0.))
bpm_ow = (~okbin_ow).astype('uint8')
# write O,E spectrum, prefix "s". VAR, BPM for each spectrum. y dim is virtual (length 1)
# for consistency with other modes
hduout = pyfits.PrimaryHDU(header=hdulist[0].header)
hduout = pyfits.HDUList(hduout)
hduout[0].header.update('OBJECT',lampid)
header=hdulist['SCI'].header.copy()
header.update('VAREXT',2)
header.update('BPMEXT',3)
header.update('CRVAL1',wedge_w[0]+wbin/2.)
header.update('CRVAL2',0)
header.update('CDELT1',wbin)
header.update('CTYPE1','Angstroms')
hduout.append(pyfits.ImageHDU(data=sci_ow.reshape((2,1,wavs)).astype('float32'), header=header, name='SCI'))
header.update('SCIEXT',1,'Extension for Science Frame',before='VAREXT')
hduout.append(pyfits.ImageHDU(data=var_ow.reshape((2,1,wavs)).astype('float32'), header=header, name='VAR'))
hduout.append(pyfits.ImageHDU(data=bpm_ow.reshape((2,1,wavs)), header=header, name='BPM'))
hduout.writeto('e'+outfilelist[i],clobber=True,output_verify='warn')
log.message('Output file '+'e'+outfilelist[i] , with_header=False)
return
def specpolflux(infilelist, logfile='salt.log', debug=False):
"""Finds/ produces fluxcal tables, and applies to listed _stokes.fits
Parameters
----------
infilelist: list
filename or list of finalstokes filenames
logfile: str
Name of file for logging
"""
# Info on CAL_SPST files:
calspstname_s,calspstfile_s=np.loadtxt(datadir+"spst_filenames.txt", \
dtype=str,usecols=(0,1),unpack=True)
namelen = max(map(len, calspstname_s))
confitemlist = ['GRATING', 'GR-ANGLE', 'CAMANG']
# Find fluxdb files already in this directory
fluxdbtab = Table(names=['no','OBJECT']+confitemlist, \
dtype=[int,'S'+str(namelen),'S6',float,float])
fluxdblist = sorted(glob.glob('fluxdb*.txt'))
olddbentries = len(fluxdblist)
for e, dbfile in enumerate(fluxdblist):
confdat_d = np.genfromtxt(dbfile,usecols=2,comments='?', \
max_rows=4,dtype=str)
fluxdbtab.add_row(np.insert(confdat_d, 0, e + 1))
if olddbentries:
printstdlog('\n Existing Fluxdb:\n' + str(fluxdbtab), logfile)
# Create new fluxdb files if new data present
eclist = sorted(glob.glob('ec*.fits'))
obs_i, config_i, obstab, configtab = configmap(eclist, confitemlist)
obss = len(obstab)
for obs in range(obss):
object, config = obstab[obs]
# Find spst standards
i_j = np.where(obs_i == obs)[0]
if object not in calspstname_s: continue
newfluxdbtab = Table( \
names=['no','OBJECT']+confitemlist,dtype=[int,'S'+str(namelen),'S6',float,float])
newfluxdbtab.add_row([len(fluxdbtab) + 1, object] +
list(configtab[obs]))
if Table(newfluxdbtab.columns[1:]) in Table(fluxdbtab.columns[1:]):
continue
# It's a new flux standard, process it
s = np.where(object == calspstname_s)[0][0]
spstfile = iraf.osfn("pysalt$data/standards/spectroscopic/" +
calspstfile_s[s])
wav_f, ABmag_f = np.loadtxt(spstfile, usecols=(0, 1), unpack=True)
flam_f = 10.**(-0.4 * (ABmag_f + 2.402)) / (wav_f)**2
wbinedge_f = (wav_f[1:] + wav_f[:-1]) / 2.
wbinedge_f = np.insert(wbinedge_f, 0, 2. * wav_f[0] - wbinedge_f[0])
wbinedge_f = np.append(wbinedge_f, 2. * wav_f[-1] - wbinedge_f[-1])
# average multiple samples,E,O
hdul = pyfits.open(eclist[i_j[0]])
grating, grang, artic = configtab[config]
wav0 = hdul['SCI'].header['CRVAL1']
dwav = hdul['SCI'].header['CDELT1']
wavs = hdul['SCI'].data.shape[-1]
phot_w = np.zeros(wavs)
count_w = np.zeros(wavs)
exptime = 0.
samples = i_j.shape[0]
for j in range(samples):
hdul = pyfits.open(eclist[i_j[j]])
phot_w += hdul['SCI'].data.reshape((2, -1)).sum(axis=0)
count_w += (hdul['BPM'].data.reshape((2, -1)) == 0).sum(axis=0)
exptime += hdul['SCI'].header['EXPTIME']
int_w = phot_w / (2 * samples * exptime)
ok_w = (count_w == 2 * samples)
# scrunch onto flux star grid
wav_w = np.arange(wav0, wav0 + wavs * dwav, dwav)
binedge_f = (wbinedge_f - (wav_w[0] - dwav / 2.)) / dwav
int_f = scrunch1d(int_w, binedge_f)
ok_f = (scrunch1d((~ok_w).astype(int),
binedge_f) == 0) # good flux bins have no bad wavs
ok_f &= ((wav_f > wav_w[0]) & (wav_f < wav_w[-1]))
# save flux/intensity mean, extrapolate to edge
fluxcal_F = flam_f[ok_f] / int_f[ok_f]
wav_F = wav_f[ok_f]
fluxcalslope_F = (fluxcal_F[1:] - fluxcal_F[:-1]) / (wav_F[1:] -
wav_F[:-1])
wav_F = np.insert(wav_F, 0, wav_w[0])
wav_F = np.append(wav_F, wav_w[-1])
fluxcal_F = np.insert(
fluxcal_F, 0,
fluxcal_F[0] - fluxcalslope_F[0] * (wav_F[1] - wav_F[0]))
fluxcal_F = np.append(
fluxcal_F,
fluxcal_F[-1] + fluxcalslope_F[-1] * (wav_F[-1] - wav_F[-2]))
fluxdbfile = 'fluxdb_' + calspstname_s[s] + '_c' + str(config) + '.txt'
hdr = ("OBJECT: "+object+"\nGRATING: %s \nARTIC: %s \nGRANG: %s" \
% (grating,grang,artic))
np.savetxt(fluxdbfile,
np.vstack((wav_F, fluxcal_F)).T,
fmt="%8.2f %12.3e ",
header=hdr)
fluxdbtab.add_row(list(newfluxdbtab[0]))
fluxdblist.append(fluxdbfile)
dbentries = len(fluxdbtab)
if (dbentries > olddbentries):
printstdlog(
'\n New Fluxdb entries:\n' + str(fluxdbtab[olddbentries:]),
logfile)
# do fluxcal on listed stokes.fits files
if len(fluxdbtab) == 0:
printstdlog('\n No fluxdb data available', logfile)
return
if (type(infilelist) is str): infilelist = [
infilelist,
]
obs_i, config_i, obstab, configtab = configmap(infilelist, confitemlist)
obss = len(obstab)
fluxdbconftab = fluxdbtab[confitemlist]
cunitfluxed = 'erg/s/cm^2/Ang' # header keyword CUNIT3 if data is already fluxed
for obs in range(obss):
iobs = np.where(obs_i == obs)[0][0]
hdul = pyfits.open(infilelist[iobs])
if 'CUNIT3' in hdul['SCI'].header:
if hdul['SCI'].header['CUNIT3'].replace(' ', '') == cunitfluxed:
printstdlog(
('\n %s already flux calibrated' % infilelist[iobs]),
logfile)
continue
fluxdbentry_e = np.where(configtab[obs] == fluxdbconftab)[0]
if fluxdbentry_e.shape[0] == 0:
printstdlog(('\n No flux calibration available for %s' %
infilelist[iobs]), logfile)
continue
wav0 = hdul['SCI'].header['CRVAL1']
dwav = hdul['SCI'].header['CDELT1']
wavs = hdul['SCI'].data.shape[-1]
wav_w = np.arange(wav0, wav0 + wavs * dwav, dwav)
fluxcal_w = np.zeros(wavs)
# average all applicable fluxdb entries after interpolation onto wavelength grid
fluxcalhistory = "FluxCal: "
fluxcallog = fluxcalhistory
for e in fluxdbentry_e:
wav_F,fluxcal_F = np.loadtxt(fluxdblist[e],skiprows=4,comments='?', \
dtype=float,unpack=True)
fluxcal_w += interp1d(wav_F, fluxcal_F, bounds_error=False)(wav_w)
fluxcalhistory += " " + fluxdblist[e]
fluxcallog += " " + str(e + 1) + " " + fluxdblist[e]
fluxcal_w /= fluxdbentry_e.shape[0]
fluxcal_w = (np.nan_to_num(fluxcal_w))
hdul['SCI'].data *= fluxcal_w
hdul['SCI'].header['CUNIT3'] = cunitfluxed
hdul['VAR'].data *= fluxcal_w**2
hdul['VAR'].header['CUNIT3'] = cunitfluxed
hdul[0].header.add_history(fluxcalhistory)
hdul.writeto(infilelist[iobs], clobber=True)
printstdlog((('\n %s ' + fluxcallog) % infilelist[iobs]), logfile)
return
def specpolflux(infilelist, logfile='salt.log', debug=False):
"""Finds/ produces fluxcal tables, and applies to listed _stokes.fits
Parameters
----------
infilelist: list
filename or list of finalstokes filenames
logfile: str
Name of file for logging
"""
if len(glob.glob('specpol*.log')): logfile = glob.glob('specpol*.log')[0]
fluxcal_w = np.zeros(0)
# Info on CAL_SPST files:
calspstname_s,calspstfile_s=np.loadtxt(datadir+"spst_filenames.txt", \
dtype=str,usecols=(0,1),unpack=True)
namelen = max(map(len, calspstname_s))
confitemList = ['DATE-OBS', 'GRATING', 'GR-ANGLE', 'CAMANG']
# Find fluxdb files already in this directory
fluxdbtab = Table(names=['no','OBJECT']+confitemList, \
dtype=[int,'S'+str(namelen),'S10','S6',float,float])
fluxdblist = sorted(glob.glob('fluxdb*.txt'))
goodfluxdblist = copy.copy(fluxdblist)
for e, dbfile in enumerate(fluxdblist):
if (open(dbfile).read().count("#") < 5):
printstdlog('\n Invalid flux file ' + dbfile + ', not used',
logfile)
goodfluxdblist.remove(dbfile)
continue
confdat_d = np.genfromtxt(dbfile,usecols=2,comments='?', \
max_rows=5,dtype=str)
fluxdbtab.add_row(np.insert(confdat_d, 0, e + 1))
fluxdblist = goodfluxdblist
olddbentries = len(fluxdblist)
if olddbentries:
printstdlog('\n Existing Fluxdb:\n' + str(fluxdbtab), logfile)
# Create new fluxdb files if new data present
eclist = sorted(glob.glob('ec*.fits'))
obss = 0
if len(eclist):
obs_i, config_i, obstab, configtab = configmap(eclist, confitemList)
obss = len(obstab)
for obs in range(obss):
object, config = obstab[obs]
# Find spst standards
i_j = np.where(obs_i == obs)[0]
if object not in calspstname_s: continue
newfluxdbtab = Table( \
names=['no','OBJECT']+confitemList,dtype=[int,'S'+str(namelen),'S10','S6',float,float])
newfluxdbtab.add_row([len(fluxdbtab) + 1, object] +
list(configtab[obs]))
if Table(newfluxdbtab.columns[1:]) in Table(fluxdbtab.columns[1:]):
continue
# It's a new flux standard, process it
printstdlog('\n New Fluxdb entry:\n' + str(newfluxdbtab), logfile)
s = np.where(object == calspstname_s)[0][0]
spstfile = iraf.osfn("pysalt$data/standards/spectroscopic/" +
calspstfile_s[s])
wav_f, ABmag_f = np.loadtxt(spstfile, usecols=(0, 1), unpack=True)
wbinedge_f = (wav_f[1:] + wav_f[:-1]) / 2.
wbinedge_f = np.insert(wbinedge_f, 0, 2. * wav_f[0] - wbinedge_f[0])
wbinedge_f = np.append(wbinedge_f, 2. * wav_f[-1] - wbinedge_f[-1])
flam_f = 10.**(-0.4 * (ABmag_f + 2.402)) / (wav_f)**2
# for HST standards, scrunch down to 50A bins
if (wav_f[0] < 3000.):
wbinedge_F = np.arange(3000., 11000., 50.)
binedge_F = interp1d(wbinedge_f,
np.arange(wbinedge_f.shape[0]))(wbinedge_F)
flam_F = scrunch1d(flam_f, binedge_F) / np.diff(
binedge_F) # mean over new flux bin
wav_f = (wbinedge_F[:-1] + wbinedge_F[1:]) / 2.
flam_f = flam_F
wbinedge_f = wbinedge_F
# average multiple samples,E,O
hdul = pyfits.open(eclist[i_j[0]])
dateobs, grating, grang, artic = configtab[config]
wav0 = hdul['SCI'].header['CRVAL1']
dwav = hdul['SCI'].header['CDELT1']
wavs = hdul['SCI'].data.shape[-1]
phot_w = np.zeros(wavs)
count_w = np.zeros(wavs)
exptime = 0.
samples = i_j.shape[0]
for j in range(samples):
hdul = pyfits.open(eclist[i_j[j]])
phot_w += hdul['SCI'].data.reshape((2, -1)).sum(axis=0)
count_w += (hdul['BPM'].data.reshape((2, -1)) == 0).sum(axis=0)
exptime += hdul['SCI'].header['EXPTIME']
int_w = phot_w / exptime # phot/sec/bin, E+O sum
ok_w = (count_w == 2 * samples)
# check for gain corrections. BPM==2 marks internal ccd amp intersections
aw_pA = np.array(
np.where(hdul['BPM'].data.reshape((2, -1)) == 2))[1].reshape(
(2, -1))
awmin_A, aw_A, awmax_A = (aw_pA.min(axis=0), aw_pA.mean(axis=0),
aw_pA.max(axis=0))
wloList = [
0, awmax_A[0] + 1, aw_A[[0, 1]].mean(), awmax_A[1] + 1,
aw_A[[1, 2]].mean(), awmax_A[2] + 1
]
whiList = [
awmin_A[0] - 1, aw_A[[0, 1]].mean(), awmin_A[1] - 1,
aw_A[[1, 2]].mean(), awmin_A[2] - 1, wavs
]
wList = [
0, aw_A[0], aw_A[[0, 1]].mean(), aw_A[1], aw_A[[1, 2]].mean(),
aw_A[2], wavs
]
photedge_da = np.zeros((2, 6))
for d, a in np.ndindex(2, 6):
w1 = wloList[a] + d * (whiList[a] - wloList[a]) * 2 / 3
w2 = whiList[a] - (1 - d) * (whiList[a] - wloList[a]) * 2 / 3
use_w = (ok_w & (np.arange(wavs) >= w1) & (np.arange(wavs) <= w2))
cof_c = np.polyfit(np.arange(wavs)[use_w], phot_w[use_w], 1)
if debug: print d, a, ('%8.2f %8.0f' % tuple(cof_c))
photedge_da[d, a] = np.polyval(cof_c, wList[a + d])
photrat_A = photedge_da[0, 1:] / photedge_da[1, :-1]
photrat_a = np.insert(np.cumprod(photrat_A), 0, 1.)
historyDict = dict(
[line.split(' ', 1) for line in hdul['SCI'].header['HISTORY']])
if historyDict.has_key('GainCorrection:'):
printstdlog(
('\n Gain cors : ' + historyDict['GainCorrection:']),
logfile)
else:
printstdlog(('\n no gain correction'), logfile)
printstdlog(
(' Gain Ratio: ' + 6 * '%6.4f ' % tuple(photrat_a)),
logfile)
# scrunch onto flux star grid
wav_w = np.arange(wav0, wav0 + wavs * dwav, dwav)
binedge_f = (wbinedge_f - (wav_w[0] - dwav / 2.)) / dwav
int_f = scrunch1d(int_w, binedge_f) / np.diff(
binedge_f) # mean of int_w over flux bin
ok_f = (scrunch1d((~ok_w).astype(int),
binedge_f) == 0) # good flux bins have no bad wavs
ok_f &= ((wav_f > wav_w[0]) & (wav_f < wav_w[-1]))
# save flux/intensity mean over flux standard bin, extrapolate to edge
fluxcal_F = flam_f[ok_f] / int_f[ok_f]
wav_F = wav_f[ok_f]
fluxcalslope_F = (fluxcal_F[1:] - fluxcal_F[:-1]) / (wav_F[1:] -
wav_F[:-1])
wav_F = np.insert(wav_F, 0, wav_w[0])
wav_F = np.append(wav_F, wav_w[-1])
fluxcal_F = np.insert(
fluxcal_F, 0,
fluxcal_F[0] - fluxcalslope_F[0] * (wav_F[1] - wav_F[0]))
fluxcal_F = np.append(
fluxcal_F,
fluxcal_F[-1] + fluxcalslope_F[-1] * (wav_F[-1] - wav_F[-2]))
fluxdbfile = 'fluxdb_' + calspstname_s[s] + '_c' + str(config) + '.txt'
hdr = ("OBJECT: "+object+"\nDATEOBS: %s \nGRATING: %s \nARTIC: %s \nGRANG: %s" \
% (dateobs,grating,grang,artic))
np.savetxt(fluxdbfile,
np.vstack((wav_F, fluxcal_F)).T,
fmt="%8.2f %12.3e ",
header=hdr)
fluxdbtab.add_row(list(newfluxdbtab[0]))
fluxdblist.append(fluxdbfile)
dbentries = len(fluxdbtab)
# do fluxcal on listed stokes.fits files
if len(fluxdbtab) == 0:
printstdlog('\n No fluxdb data available', logfile)
return fluxcal_w
if (type(infilelist) is str): infilelist = [
infilelist,
]
if len(infilelist) == 0:
print "No files to calibrate"
exit()
obs_i, config_i, obstab, configtab = configmap(infilelist, confitemList)
obss = len(obstab)
fluxdbconftab = fluxdbtab[confitemList]
cunitfluxed = 'erg/s/cm^2/Ang' # header keyword CUNIT3 if data is already fluxed
for obs in range(obss):
iobs = np.where(obs_i == obs)[0][0]
hdul = pyfits.open(infilelist[iobs])
if 'CUNIT3' in hdul['SCI'].header:
if hdul['SCI'].header['CUNIT3'].replace(' ', '') == cunitfluxed:
printstdlog(
('\n %s already flux calibrated' % infilelist[iobs]),
logfile)
continue
fluxdbentry_e = []
for e in range(len(fluxdbconftab)):
if ((fluxdbconftab[e]['GRATING']==configtab[obs]['GRATING']) & \
(fluxdbconftab[e]['CAMANG']==configtab[obs]['CAMANG']) & \
(abs(fluxdbconftab[e]['GR-ANGLE']-configtab[obs]['GR-ANGLE']) < 0.1)):
fluxdbentry_e.append(e)
if len(fluxdbentry_e) == 0:
printstdlog(('\n No flux calibration available for %s' %
infilelist[iobs]), logfile)
continue
wav0 = hdul['SCI'].header['CRVAL1']
dwav = hdul['SCI'].header['CDELT1']
wavs = hdul['SCI'].data.shape[-1]
exptime = hdul['SCI'].header['EXPTIME']
wav_w = np.arange(wav0, wav0 + wavs * dwav, dwav)
fluxcal_w = np.zeros(wavs)
# average all applicable fluxdb entries after interpolation onto wavelength grid
# if necessary, block average onto ~50Ang grid, then average onto 50 Ang grid
# interpolate onto configuration for fluxcal
fluxcallog = ''
for e in fluxdbentry_e:
wav_F,fluxcal_F = np.loadtxt(fluxdblist[e],skiprows=5,comments='?', \
dtype=float,unpack=True)
fluxcal_w += interp1d(wav_F, fluxcal_F, bounds_error=False)(wav_w)
hdul[0].header.add_history("FluxCal: " +
fluxdbconftab[e]["DATE-OBS"] + ' ' +
fluxdblist[e])
fluxcallog += ('\n ' + str(e + 1) + ' ' +
fluxdbconftab[e]["DATE-OBS"] + ' ' + fluxdblist[e])
fluxcal_w /= len(fluxdbentry_e)
fluxcal_w = (np.nan_to_num(fluxcal_w)) / exptime
hdul['SCI'].data *= fluxcal_w
hdul['SCI'].header['CUNIT3'] = cunitfluxed
hdul['VAR'].data *= fluxcal_w**2
hdul['VAR'].header['CUNIT3'] = cunitfluxed
hdul['COV'].data *= fluxcal_w**2
hdul['COV'].header['CUNIT3'] = cunitfluxed
hdul['BPM'].data = ((hdul['BPM'].data > 0) |
(fluxcal_w == 0.)).astype('uint8')
hdul.writeto(infilelist[iobs], overwrite=True)
printstdlog((('\n %s Fluxcal:' + fluxcallog) % infilelist[iobs]),
logfile)
return fluxcal_w
def specpolextract(infilelist, logfile='salt.log', debug=False):
"""Produce a 1-D extract spectra for the O and E beams
This also cleans the 2-D spectra of a number of artifacts, removes the background, accounts for small
spatial shifts in the observation, and resamples the data into a wavelength grid
Parameters
----------
infile_list: list
List of filenames that include an extracted spectra
logfile: str
Name of file for logging
"""
with logging(logfile, debug) as log:
config_dict = list_configurations(infilelist, log)
config_count = 0
for config in config_dict:
outfilelist = config_dict[config]['object']
outfiles = len(outfilelist)
obs_dict = obslog(outfilelist)
hdu0 = pyfits.open(outfilelist[0])
rows, cols = hdu0['SCI'].data.shape[1:3]
cbin, rbin = np.array(obs_dict["CCDSUM"][0].split(" ")).astype(int)
object_name = hdu0[0].header['OBJECT']
log.message(
'\nExtract: {3} Grating {0} Grang {1:6.2f} Artic {2:6.2f}'.
format(config[0], config[1], config[2], object_name))
log.message(
' Images: ' +
' '.join([str(image_number(img)) for img in outfilelist]),
with_header=False)
# special version for lamp data
# this is now removed and will not be part of this code
#object = obs_dict["OBJECT"][0].strip().upper()
#ampid = obs_dict["LAMPID"][0].strip().upper()
#f ((object != "ARC") & (lampid != "NONE")) :
# specpollampextract(outfilelist, logfile=logfile)
# continue
# sum spectra to find target, background artifacts, and estimate sky flat and psf functions
count = 0
for i in range(outfiles):
badbin_orc = pyfits.open(outfilelist[i])['BPM'].data > 0
if count == 0:
count_orc = (~badbin_orc).astype(int)
image_orc = pyfits.open(
outfilelist[i])['SCI'].data * count_orc
var_orc = pyfits.open(
outfilelist[i])['VAR'].data * count_orc
else:
count_orc += (~badbin_orc).astype(int)
image_orc += pyfits.open(
outfilelist[i])['SCI'].data * (~badbin_orc).astype(int)
var_orc += pyfits.open(
outfilelist[i])['VAR'].data * (~badbin_orc).astype(int)
count += 1
if count == 0:
print 'No valid images'
continue
image_orc[count_orc > 0] /= count_orc[count_orc > 0]
badbinall_orc = (count_orc == 0) | (image_orc == 0
) # bin is bad in all images
badbinone_orc = (count_orc < count) | (
image_orc == 0) # bin is bad in at least one image
var_orc[count_orc > 0] /= (count_orc[count_orc > 0])**2
wav_orc = pyfits.open(outfilelist[0])['WAV'].data
slitid = obs_dict["MASKID"][0]
okwav_oc = ~((wav_orc == 0).all(axis=1))
obsname = object_name + "_c" + str(config_count) + "_" + str(
outfiles)
hdusum = pyfits.PrimaryHDU(header=hdu0[0].header)
hdusum = pyfits.HDUList(hdusum)
hdusum[0].header['OBJECT'] = obsname
header = hdu0['SCI'].header.copy()
hdusum.append(
pyfits.ImageHDU(data=image_orc, header=header, name='SCI'))
hdusum.append(
pyfits.ImageHDU(data=var_orc, header=header, name='VAR'))
hdusum.append(
pyfits.ImageHDU(data=badbinall_orc.astype('uint8'),
header=header,
name='BPM'))
hdusum.append(
pyfits.ImageHDU(data=wav_orc, header=header, name='WAV'))
if debug: hdusum.writeto(obsname + ".fits", clobber=True)
# run specpolsignalmap on image
psf_orc,skyflat_orc,badbinnew_orc,isbkgcont_orc,maprow_od,drow_oc = \
specpolsignalmap(hdusum,logfile=logfile,debug=debug)
maprow_ocd = maprow_od[:, None, :] + np.zeros((2, cols, 4))
maprow_ocd[okwav_oc] += drow_oc[okwav_oc, None]
isedge_orc = (np.arange(rows)[:,None] < maprow_ocd[:,None,:,0]) | \
(np.arange(rows)[:,None] > maprow_ocd[:,None,:,3])
istarget_orc = okwav_oc[:,None,:] & (np.arange(rows)[:,None] > maprow_ocd[:,None,:,1]) & \
(np.arange(rows)[:,None] < maprow_ocd[:,None,:,2])
isbkgcont_orc &= (~badbinall_orc & ~isedge_orc & ~istarget_orc)
badbinall_orc |= badbinnew_orc
badbinone_orc |= badbinnew_orc
hdusum['BPM'].data = badbinnew_orc.astype('uint8')
psf_orc *= istarget_orc.astype(int)
if debug:
# hdusum.writeto(obsname+".fits",clobber=True)
pyfits.PrimaryHDU(psf_orc.astype('float32')).writeto(
obsname + '_psf_orc.fits', clobber=True)
# pyfits.PrimaryHDU(badbinnew_orc.astype('uint8')).writeto('badbinnew_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinall_orc.astype('uint8')).writeto('badbinall_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinone_orc.astype('uint8')).writeto('badbinone_orc.fits',clobber=True)
# set up wavelength binning
wbin = wav_orc[0, rows / 2, cols / 2] - wav_orc[0, rows / 2,
cols / 2 - 1]
wbin = 2.**(np.rint(np.log2(wbin))
) # bin to nearest power of 2 angstroms
wmin = (wav_orc.max(axis=1)[okwav_oc].reshape(
(2, -1))).min(axis=1).max()
wmax = wav_orc.max()
for o in (0, 1):
colmax = np.where((wav_orc[o] > 0.).any(axis=0))[0][-1]
row_r = np.where(wav_orc[o, :, colmax] > 0.)[0]
wmax = min(wmax, wav_orc[o, row_r, colmax].min())
wedgemin = wbin * int(wmin / wbin + 0.5) + wbin / 2.
wedgemax = wbin * int(wmax / wbin - 0.5) + wbin / 2.
wedge_w = np.arange(wedgemin, wedgemax + wbin, wbin)
wavs = wedge_w.shape[0] - 1
binedge_orw = np.zeros((2, rows, wavs + 1))
specrow_or = (maprow_od[:, 1:3].mean(axis=1)[:, None] +
np.arange(-rows / 4, rows / 4)).astype(int)
# scrunch and normalize psf from summed images (using badbinone) for optimized extraction
# psf is normalized so its integral over row is 1.
psfnormmin = 0.70 # wavelengths with less than this flux in good bins are marked bad
psf_orw = np.zeros((2, rows, wavs))
for o in (0, 1):
for r in specrow_or[o]:
binedge_orw[o,r] = \
interp1d(wav_orc[o,r,okwav_oc[o]],np.arange(cols)[okwav_oc[o]], \
kind='linear',bounds_error=False)(wedge_w)
psf_orw[o, r] = scrunch1d(psf_orc[o, r], binedge_orw[o, r])
if debug:
pyfits.PrimaryHDU(binedge_orw.astype('float32')).writeto(
obsname + '_binedge_orw.fits', clobber=True)
pyfits.PrimaryHDU(psf_orw.astype('float32')).writeto(
obsname + '_psf_orw.fits', clobber=True)
psfnorm_orw = np.repeat(psf_orw.sum(axis=1), rows,
axis=1).reshape(2, rows, -1)
psf_orw[psfnorm_orw > 0.] /= psfnorm_orw[psfnorm_orw > 0.]
pmax = np.minimum(
1.,
np.median(psf_orw[psfnorm_orw > 0.].reshape(
(2, rows, -1)).max(axis=1)))
log.message('Stellar profile width: %8.2f arcsec' %
((1. / pmax) * rbin / 8.),
with_header=False)
pwidth = int(1. / pmax)
if debug:
pyfits.PrimaryHDU(psf_orw.astype('float32')).writeto(
obsname + '_psfnormed_orw.fits', clobber=True)
# set up optional image-dependent column shift for slitless data
colshiftfilename = "colshift.txt"
docolshift = os.path.isfile(colshiftfilename)
if docolshift:
img_I, dcol_I = np.loadtxt(colshiftfilename,
dtype=float,
unpack=True,
usecols=(0, 1))
shifts = img_I.shape[0]
log.message('Column shift: \n Images ' +
shifts * '%5i ' % tuple(img_I),
with_header=False)
log.message(' Bins ' + shifts * '%5.2f ' % tuple(dcol_I),
with_header=False)
log.message('\nArcsec offset Output File', with_header=False)
# background-subtract and extract spectra
for i in range(outfiles):
hdulist = pyfits.open(outfilelist[i])
tnum = image_number(outfilelist[i])
badbin_orc = (hdulist['BPM'].data > 0)
badbinbkg_orc = (badbin_orc | badbinnew_orc | isedge_orc
| istarget_orc)
if debug:
pyfits.PrimaryHDU(isedge_orc.astype('uint8')).writeto(
'isedge_orc_' + tnum + '.fits', clobber=True)
pyfits.PrimaryHDU(istarget_orc.astype('uint8')).writeto(
'istarget_orc_' + tnum + '.fits', clobber=True)
pyfits.PrimaryHDU(badbinbkg_orc.astype('uint8')).writeto(
'badbinbkg_orc_' + tnum + '.fits', clobber=True)
target_orc = bkgsub(hdulist,
badbinbkg_orc,
isbkgcont_orc,
skyflat_orc,
maprow_ocd,
tnum,
debug=debug)
target_orc *= (~badbin_orc).astype(int)
if debug:
pyfits.PrimaryHDU(target_orc.astype('float32')).writeto(
'target_' + tnum + '_orc.fits', clobber=True)
var_orc = hdulist['var'].data
badbin_orc = (hdulist['bpm'].data > 0) | badbinnew_orc
# extract spectrum optimally (Horne, PASP 1986)
target_orw = np.zeros((2, rows, wavs))
var_orw = np.zeros_like(target_orw)
badbin_orw = np.ones((2, rows, wavs), dtype='bool')
wt_orw = np.zeros_like(target_orw)
dcol = 0.
if docolshift:
if int(tnum) in img_I:
dcol = dcol_I[np.where(
img_I == int(tnum))] # table has observed shift
for o in (0, 1):
for r in specrow_or[o]:
target_orw[o, r] = scrunch1d(target_orc[o, r],
binedge_orw[o, r] + dcol)
var_orw[o, r] = scrunch1d(var_orc[o, r],
binedge_orw[o, r] + dcol)
badbin_orw[o, r] = scrunch1d(
badbin_orc[o, r].astype(float),
binedge_orw[o, r] + dcol) > 0.001
badbin_orw |= (var_orw == 0)
badbin_orw |= ((psf_orw *
(~badbin_orw)).sum(axis=1)[:, None, :] <
psfnormmin)
if debug:
# pyfits.PrimaryHDU(var_orw.astype('float32')).writeto('var_'+tnum+'_orw.fits',clobber=True)
pyfits.PrimaryHDU(badbin_orw.astype('uint8')).writeto(
'badbin_' + tnum + '_orw.fits', clobber=True)
# use master psf shifted in row to allow for guide errors
ok_w = ((psf_orw * badbin_orw).sum(axis=1) <
0.03 / float(pwidth / 2)).all(axis=0)
crosscor_s = np.zeros(pwidth)
for s in range(pwidth):
crosscor_s[s] = (psf_orw[:, s:s - pwidth] *
target_orw[:, pwidth / 2:-pwidth / 2] *
ok_w).sum()
smax = np.argmax(crosscor_s)
s_S = np.arange(smax - pwidth / 4,
smax - pwidth / 4 + pwidth / 2 + 1)
polycof = la.lstsq(
np.vstack((s_S**2, s_S, np.ones_like(s_S))).T,
crosscor_s[s_S])[0]
pshift = -(-0.5 * polycof[1] / polycof[0] - pwidth / 2)
s = int(pshift + pwidth) - pwidth
sfrac = pshift - s
psfsh_orw = np.zeros_like(psf_orw)
outrow = np.arange(
max(0, s + 1),
rows - (1 + int(abs(pshift))) + max(0, s + 1))
psfsh_orw[:, outrow] = (
1. - sfrac
) * psf_orw[:, outrow - s] + sfrac * psf_orw[:, outrow - s - 1]
# pyfits.PrimaryHDU(psfsh_orw.astype('float32')).writeto('psfsh_'+tnum+'_orw.fits',clobber=True)
wt_orw[~badbin_orw] = psfsh_orw[~badbin_orw] / var_orw[
~badbin_orw]
var_ow = (psfsh_orw * wt_orw * (~badbin_orw)).sum(axis=1)
badbin_ow = (var_ow == 0)
var_ow[~badbin_ow] = 1. / var_ow[~badbin_ow]
# pyfits.PrimaryHDU(var_ow.astype('float32')).writeto('var_'+tnum+'_ow.fits',clobber=True)
# pyfits.PrimaryHDU(target_orw.astype('float32')).writeto('target_'+tnum+'_orw.fits',clobber=True)
# pyfits.PrimaryHDU(wt_orw.astype('float32')).writeto('wt_'+tnum+'_orw.fits',clobber=True)
sci_ow = (target_orw * wt_orw).sum(axis=1) * var_ow
badlim = 0.20
psfbadfrac_ow = (psfsh_orw * badbin_orw.astype(int)).sum(
axis=1) / psfsh_orw.sum(axis=1)
badbin_ow |= (psfbadfrac_ow > badlim)
cdebug = 83
if debug: np.savetxt("xtrct"+str(cdebug)+"_"+tnum+".txt",np.vstack((psf_orw[:,:,cdebug],var_orw[:,:,cdebug], \
wt_orw[:,:,cdebug],target_orw[:,:,cdebug])).reshape((4,2,-1)).transpose(1,0,2).reshape((8,-1)).T,fmt="%12.5e")
# write O,E spectrum, prefix "s". VAR, BPM for each spectrum. y dim is virtual (length 1)
# for consistency with other modes
hduout = pyfits.PrimaryHDU(header=hdulist[0].header)
hduout = pyfits.HDUList(hduout)
header = hdulist['SCI'].header.copy()
header.update('VAREXT', 2)
header.update('BPMEXT', 3)
header.update('CRVAL1', wedge_w[0] + wbin / 2.)
header.update('CRVAL2', 0)
header.update('CDELT1', wbin)
header.update('CTYPE1', 'Angstroms')
hduout.append(
pyfits.ImageHDU(data=sci_ow.reshape((2, 1, wavs)),
header=header,
name='SCI'))
header.update('SCIEXT',
1,
'Extension for Science Frame',
before='VAREXT')
hduout.append(
pyfits.ImageHDU(data=var_ow.reshape((2, 1, wavs)),
header=header,
name='VAR'))
hduout.append(
pyfits.ImageHDU(data=badbin_ow.astype("uint8").reshape(
(2, 1, wavs)),
header=header,
name='BPM'))
hduout.writeto('e' + outfilelist[i],
clobber=True,
output_verify='warn')
log.message(' %8.2f e%s' %
(pshift * rbin / 8., outfilelist[i]),
with_header=False)
#increate the config count
config_count += 1
return
def specpolextract(infilelist, logfile='salt.log'):
#set up the files
obsdate = os.path.basename(infilelist[0])[8:16]
with logging(logfile, debug) as log:
#create the observation log
obs_dict = obslog(infilelist)
# get rid of arcs
for i in range(len(infilelist))[::-1]:
if (obs_dict['OBJECT'][i].upper().strip() == 'ARC'):
del infilelist[i]
infiles = len(infilelist)
# contiguous images of the same object and config are grouped together
obs_dict = obslog(infilelist)
confno_i, confdatlist = configmap(infilelist)
configs = len(confdatlist)
objectlist = list(set(obs_dict['OBJECT']))
objno_i = np.array(
[objectlist.index(obs_dict['OBJECT'][i]) for i in range(infiles)],
dtype=int)
grp_i = np.zeros((infiles), dtype=int)
grp_i[1:] = ((confno_i[1:] != confno_i[:-1]) |
(objno_i[1:] != objno_i[:-1])).cumsum()
for g in np.unique(grp_i):
ilist = np.where(grp_i == g)[0]
outfiles = len(ilist)
outfilelist = [infilelist[i] for i in ilist]
imagenolist = [
int(os.path.basename(infilelist[i]).split('.')[0][-4:])
for i in ilist
]
log.message('\nExtract: '+objectlist[objno_i[ilist[0]]]+' Grating %s Grang %6.2f Artic %6.2f' % \
confdatlist[confno_i[ilist[0]]], with_header=False)
log.message(' Images: ' + outfiles * '%i ' % tuple(imagenolist),
with_header=False)
hdu0 = pyfits.open(outfilelist[0])
rows, cols = hdu0['SCI'].data.shape[1:3]
cbin, rbin = np.array(obs_dict["CCDSUM"][0].split(" ")).astype(int)
# special version for lamp data
lampid = obs_dict["LAMPID"][0].strip().upper()
if lampid != "NONE":
specpollampextract(outfilelist, logfile=logfile)
continue
# sum spectra to find target, background artifacts, and estimate sky flat and psf functions
count = 0
for i in range(outfiles):
badbin_orc = pyfits.open(outfilelist[i])['BPM'].data > 0
if count == 0:
count_orc = (~badbin_orc).astype(int)
image_orc = pyfits.open(
outfilelist[i])['SCI'].data * count_orc
var_orc = pyfits.open(
outfilelist[i])['VAR'].data * count_orc
else:
count_orc += (~badbin_orc).astype(int)
image_orc += pyfits.open(
infilelist[i])['SCI'].data * (~badbin_orc).astype(int)
var_orc += pyfits.open(
outfilelist[i])['VAR'].data * (~badbin_orc).astype(int)
count += 1
if count == 0:
print 'No valid images'
continue
image_orc[count_orc > 0] /= count_orc[count_orc > 0]
badbinall_orc = (count_orc == 0) | (image_orc == 0
) # bin is bad in all images
badbinone_orc = (count_orc < count) | (
image_orc == 0) # bin is bad in at least one image
var_orc[count_orc > 0] /= (count_orc[count_orc > 0])**2
wav_orc = pyfits.open(outfilelist[0])['WAV'].data
slitid = obs_dict["MASKID"][0]
if slitid[0] == "P": slitwidth = float(slitid[2:5]) / 10.
else: slitwidth = float(slitid)
hdusum = pyfits.PrimaryHDU(header=hdu0[0].header)
hdusum = pyfits.HDUList(hdusum)
header = hdu0['SCI'].header.copy()
hdusum.append(
pyfits.ImageHDU(data=image_orc, header=header, name='SCI'))
hdusum.append(
pyfits.ImageHDU(data=var_orc, header=header, name='VAR'))
hdusum.append(
pyfits.ImageHDU(data=badbinall_orc.astype('uint8'),
header=header,
name='BPM'))
hdusum.append(
pyfits.ImageHDU(data=wav_orc, header=header, name='WAV'))
# hdusum.writeto("groupsum_"+str(g)+".fits",clobber=True)
psf_orc,skyflat_orc,badbinnew_orc,isbkgcont_orc,maprow_od,drow_oc = \
specpolsignalmap(hdusum,logfile=logfile)
maprow_ocd = maprow_od[:, None, :] + np.zeros((2, cols, 4))
maprow_ocd[:, :,
[1, 2
]] -= drow_oc[:, :,
None] # edge is straight, target curved
isedge_orc = (np.arange(rows)[:,None] < maprow_ocd[:,None,:,0]) | \
(np.arange(rows)[:,None] > maprow_ocd[:,None,:,3])
istarget_orc = (np.arange(rows)[:,None] > maprow_ocd[:,None,:,1]) & \
(np.arange(rows)[:,None] < maprow_ocd[:,None,:,2])
isskycont_orc = (((np.arange(rows)[:,None] < maprow_ocd[:,None,:,0]+rows/16) | \
(np.arange(rows)[:,None] > maprow_ocd[:,None,:,3]-rows/16)) & ~isedge_orc)
isbkgcont_orc &= (~badbinall_orc & ~isedge_orc & ~istarget_orc)
badbinall_orc |= badbinnew_orc
badbinone_orc |= badbinnew_orc
# pyfits.PrimaryHDU(var_orc.astype('float32')).writeto('var_orc1.fits',clobber=True)
# pyfits.PrimaryHDU(badbinnew_orc.astype('uint8')).writeto('badbinnew_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinall_orc.astype('uint8')).writeto('badbinall_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinone_orc.astype('uint8')).writeto('badbinone_orc.fits',clobber=True)
# scrunch and normalize psf from summed images (using badbinone) for optimized extraction
psfnormmin = 0.70 # wavelengths with less than this flux in good bins are marked bad
wbin = wav_orc[0, rows / 2, cols / 2] - wav_orc[0, rows / 2,
cols / 2 - 1]
wbin = float(int(wbin / 0.75))
wmin, wmax = wav_orc.min(axis=2).max(), wav_orc.max(axis=2).min()
wedgemin = wbin * int(wmin / wbin + 0.5) + wbin / 2.
wedgemax = wbin * int(wmax / wbin - 0.5) + wbin / 2.
wedge_w = np.arange(wedgemin, wedgemax + wbin, wbin)
wavs = wedge_w.shape[0] - 1
binedge_orw = np.zeros((2, rows, wavs + 1))
psf_orw = np.zeros((2, rows, wavs))
specrow_or = maprow_od[:, 1:3].mean(axis=1)[:, None] + np.arange(
-rows / 4, rows / 4)
# pyfits.PrimaryHDU(var_orc.astype('float32')).writeto('var_orc2.fits',clobber=True)
for o in (0, 1):
for r in specrow_or[o]:
binedge_orw[o, r] = interp1d(wav_orc[o, r],
np.arange(cols))(wedge_w)
psf_orw[o, r] = scrunch1d(psf_orc[o, r], binedge_orw[o, r])
psf_orw /= psf_orw.sum(axis=1)[:, None, :]
# np.savetxt("psfnorm_ow.txt",(psf_orw*okbin_orw).sum(axis=1).T,fmt="%10.4f")
# pyfits.PrimaryHDU(psf_orw.astype('float32')).writeto('psf_orw.fits',clobber=True)
# pyfits.PrimaryHDU(var_orw.astype('float32')).writeto('var_orw.fits',clobber=True)
# set up optional image-dependent column shift for slitless data
colshiftfilename = "colshift.txt"
docolshift = os.path.isfile(colshiftfilename)
if docolshift:
img_I, dcol_I = np.loadtxt(colshiftfilename,
dtype=float,
unpack=True,
usecols=(0, 1))
shifts = img_I.shape[0]
log.message('Column shift: \n Images ' +
shifts * '%5i ' % tuple(img_I),
with_header=False)
log.message(' Bins ' + shifts * '%5.2f ' % tuple(dcol_I),
with_header=False)
# background-subtract and extract spectra
psfbadfrac_iow = np.zeros((outfiles, 2, wavs))
for i in range(outfiles):
hdulist = pyfits.open(outfilelist[i])
sci_orc = hdulist['sci'].data
var_orc = hdulist['var'].data
badbin_orc = (hdulist['bpm'].data > 0) | badbinnew_orc
tnum = os.path.basename(outfilelist[i]).split('.')[0][-3:]
# make background continuum image, smoothed over resolution element
rblk, cblk = int(1.5 * 8. / rbin), int(slitwidth * 8. / cbin)
target_orc = np.zeros_like(sci_orc)
for o in (0, 1):
bkgcont_rc = blksmooth2d(sci_orc[o],
isbkgcont_orc[o],
rblk,
cblk,
0.25,
mode="mean")
# remove sky continuum: ends of bkg continuum * skyflat
skycont_c = (bkgcont_rc.T[isskycont_orc[o].T]/skyflat_orc[o].T[isskycont_orc[o].T]) \
.reshape((cols,-1)).mean(axis=1)
skycont_rc = skycont_c * skyflat_orc[o]
# remove sky lines: image - bkg cont run through 2d sky averaging
obj_data = ((sci_orc[o] - bkgcont_rc) / skyflat_orc)[o]
obj_data[(badbin_orc | isedge_orc
| istarget_orc)[o]] = np.nan
# pyfits.PrimaryHDU(obj_data.astype('float32')).writeto('obj_data.fits',clobber=True)
skylines_rc = make_2d_skyspectrum(obj_data, wav_orc[o],
np.array([
[0, rows],
])) * skyflat_orc[o]
target_orc[o] = sci_orc[o] - skycont_rc - skylines_rc
# pyfits.PrimaryHDU(skylines_rc.astype('float32')).writeto('skylines_rc_'+tnum+'_'+str(o)+'.fits',clobber=True)
# pyfits.PrimaryHDU(skycont_rc.astype('float32')).writeto('skycont_rc_'+tnum+'_'+str(o)+'.fits',clobber=True)
target_orc *= (~badbin_orc).astype(int)
# pyfits.PrimaryHDU(target_orc.astype('float32')).writeto('target_'+tnum+'_orc.fits',clobber=True)
# extract spectrum optimally (Horne, PASP 1986)
target_orw = np.zeros((2, rows, wavs))
var_orw = np.zeros_like(target_orw)
badbin_orw = np.ones((2, rows, wavs), dtype='bool')
wt_orw = np.zeros_like(target_orw)
dcol = 0.
if docolshift:
if int(tnum) in img_I:
dcol = dcol_I[np.where(
img_I == int(tnum))] # table has observed shift
for o in (0, 1):
for r in specrow_or[o]:
target_orw[o, r] = scrunch1d(target_orc[o, r],
binedge_orw[o, r] + dcol)
var_orw[o, r] = scrunch1d(var_orc[o, r],
binedge_orw[o, r] + dcol)
badbin_orw[o, r] = scrunch1d(
badbin_orc[o, r].astype(float),
binedge_orw[o, r] + dcol) > 0.001
badbin_orw |= (var_orw == 0)
badbin_orw |= ((psf_orw *
(~badbin_orw)).sum(axis=1)[:, None, :] <
psfnormmin)
# pyfits.PrimaryHDU(var_orw.astype('float32')).writeto('var_'+tnum+'_orw.fits',clobber=True)
# pyfits.PrimaryHDU(badbin_orw.astype('uint8')).writeto('badbin_'+tnum+'_orw.fits',clobber=True)
# use master psf shifted in row to allow for guide errors
pwidth = 2 * int(1. / psf_orw.max())
ok_w = ((psf_orw * badbin_orw).sum(axis=1) <
0.03 / float(pwidth / 2)).all(axis=0)
crosscor_s = np.zeros(pwidth)
for s in range(pwidth):
crosscor_s[s] = (psf_orw[:, s:s - pwidth] *
target_orw[:, pwidth / 2:-pwidth / 2] *
ok_w).sum()
smax = np.argmax(crosscor_s)
s_S = np.arange(smax - pwidth / 4,
smax - pwidth / 4 + pwidth / 2 + 1)
polycof = la.lstsq(
np.vstack((s_S**2, s_S, np.ones_like(s_S))).T,
crosscor_s[s_S])[0]
pshift = -(-0.5 * polycof[1] / polycof[0] - pwidth / 2)
s = int(pshift + pwidth) - pwidth
sfrac = pshift - s
psfsh_orw = np.zeros_like(psf_orw)
outrow = np.arange(
max(0, s + 1),
rows - (1 + int(abs(pshift))) + max(0, s + 1))
psfsh_orw[:, outrow] = (
1. - sfrac
) * psf_orw[:, outrow - s] + sfrac * psf_orw[:, outrow - s - 1]
# pyfits.PrimaryHDU(psfsh_orw.astype('float32')).writeto('psfsh_'+tnum+'_orw.fits',clobber=True)
wt_orw[~badbin_orw] = psfsh_orw[~badbin_orw] / var_orw[
~badbin_orw]
var_ow = (psfsh_orw * wt_orw * (~badbin_orw)).sum(axis=1)
badbin_ow = (var_ow == 0)
var_ow[~badbin_ow] = 1. / var_ow[~badbin_ow]
# pyfits.PrimaryHDU(var_ow.astype('float32')).writeto('var_'+tnum+'_ow.fits',clobber=True)
# pyfits.PrimaryHDU(target_orw.astype('float32')).writeto('target_'+tnum+'_orw.fits',clobber=True)
# pyfits.PrimaryHDU(wt_orw.astype('float32')).writeto('wt_'+tnum+'_orw.fits',clobber=True)
sci_ow = (target_orw * wt_orw).sum(axis=1) * var_ow
badlim = 0.20
psfbadfrac_iow[i] = (psfsh_orw * badbin_orw.astype(int)).sum(
axis=1) / psfsh_orw.sum(axis=1)
badbin_ow |= (psfbadfrac_iow[i] > badlim)
# cdebug = 39
# np.savetxt("xtrct"+str(cdebug)+"_"+tnum+".txt",np.vstack((psf_orw[:,:,cdebug],var_orw[:,:,cdebug], \
# wt_orw[:,:,cdebug],target_orw[:,:,cdebug])).reshape((4,2,-1)).transpose(1,0,2).reshape((8,-1)).T,fmt="%12.5e")
# write O,E spectrum, prefix "s". VAR, BPM for each spectrum. y dim is virtual (length 1)
# for consistency with other modes
hduout = pyfits.PrimaryHDU(header=hdulist[0].header)
hduout = pyfits.HDUList(hduout)
header = hdulist['SCI'].header.copy()
header.update('VAREXT', 2)
header.update('BPMEXT', 3)
header.update('CRVAL1', wedge_w[0] + wbin / 2.)
header.update('CRVAL2', 0)
header.update('CDELT1', wbin)
header.update('CTYPE1', 'Angstroms')
hduout.append(
pyfits.ImageHDU(data=sci_ow.reshape((2, 1, wavs)),
header=header,
name='SCI'))
header.update('SCIEXT',
1,
'Extension for Science Frame',
before='VAREXT')
hduout.append(
pyfits.ImageHDU(data=var_ow.reshape((2, 1, wavs)),
header=header,
name='VAR'))
hduout.append(
pyfits.ImageHDU(data=badbin_ow.astype("uint8").reshape(
(2, 1, wavs)),
header=header,
name='BPM'))
hduout.writeto('e' + outfilelist[i],
clobber=True,
output_verify='warn')
log.message('Output file ' + 'e' + outfilelist[i],
with_header=False)
# np.savetxt("psfbadfrac_iow.txt",psfbadfrac_iow.reshape((-1,wavs)).T,fmt="%8.5f")
return
def specpolextract(infilelist, logfile='salt.log'):
#set up the files
obsdate=os.path.basename(infilelist[0])[8:16]
with logging(logfile, debug) as log:
#create the observation log
obs_dict=obslog(infilelist)
# get rid of arcs
for i in range(len(infilelist))[::-1]:
if (obs_dict['OBJECT'][i].upper().strip()=='ARC'): del infilelist[i]
infiles = len(infilelist)
# contiguous images of the same object and config are grouped together
obs_dict=obslog(infilelist)
confno_i,confdatlist = configmap(infilelist)
configs = len(confdatlist)
objectlist = list(set(obs_dict['OBJECT']))
objno_i = np.array([objectlist.index(obs_dict['OBJECT'][i]) for i in range(infiles)],dtype=int)
grp_i = np.zeros((infiles),dtype=int)
grp_i[1:] = ((confno_i[1:] != confno_i[:-1]) | (objno_i[1:] != objno_i[:-1])).cumsum()
for g in np.unique(grp_i):
ilist = np.where(grp_i==g)[0]
outfiles = len(ilist)
outfilelist = [infilelist[i] for i in ilist]
imagenolist = [int(os.path.basename(infilelist[i]).split('.')[0][-4:]) for i in ilist]
log.message('\nExtract: '+objectlist[objno_i[ilist[0]]]+' Grating %s Grang %6.2f Artic %6.2f' % \
confdatlist[confno_i[ilist[0]]], with_header=False)
log.message(' Images: '+outfiles*'%i ' % tuple(imagenolist), with_header=False)
hdu0 = pyfits.open(outfilelist[0])
rows,cols = hdu0['SCI'].data.shape[1:3]
cbin,rbin = np.array(obs_dict["CCDSUM"][0].split(" ")).astype(int)
# special version for lamp data
lampid = obs_dict["LAMPID"][0].strip().upper()
if lampid!="NONE":
specpollampextract(outfilelist, logfile=logfile)
continue
# sum spectra to find target, background artifacts, and estimate sky flat and psf functions
count = 0
for i in range(outfiles):
badbin_orc = pyfits.open(outfilelist[i])['BPM'].data > 0
if count == 0:
count_orc = (~badbin_orc).astype(int)
image_orc = pyfits.open(outfilelist[i])['SCI'].data*count_orc
var_orc = pyfits.open(outfilelist[i])['VAR'].data*count_orc
else:
count_orc += (~badbin_orc).astype(int)
image_orc += pyfits.open(infilelist[i])['SCI'].data*(~badbin_orc).astype(int)
var_orc += pyfits.open(outfilelist[i])['VAR'].data*(~badbin_orc).astype(int)
count += 1
if count ==0:
print 'No valid images'
continue
image_orc[count_orc>0] /= count_orc[count_orc>0]
badbinall_orc = (count_orc==0) | (image_orc==0) # bin is bad in all images
badbinone_orc = (count_orc < count) | (image_orc==0) # bin is bad in at least one image
var_orc[count_orc>0] /= (count_orc[count_orc>0])**2
wav_orc = pyfits.open(outfilelist[0])['WAV'].data
slitid = obs_dict["MASKID"][0]
if slitid[0] =="P": slitwidth = float(slitid[2:5])/10.
else: slitwidth = float(slitid)
hdusum = pyfits.PrimaryHDU(header=hdu0[0].header)
hdusum = pyfits.HDUList(hdusum)
header=hdu0['SCI'].header.copy()
hdusum.append(pyfits.ImageHDU(data=image_orc, header=header, name='SCI'))
hdusum.append(pyfits.ImageHDU(data=var_orc, header=header, name='VAR'))
hdusum.append(pyfits.ImageHDU(data=badbinall_orc.astype('uint8'), header=header, name='BPM'))
hdusum.append(pyfits.ImageHDU(data=wav_orc, header=header, name='WAV'))
# hdusum.writeto("groupsum_"+str(g)+".fits",clobber=True)
psf_orc,skyflat_orc,badbinnew_orc,isbkgcont_orc,maprow_od,drow_oc = \
specpolsignalmap(hdusum,logfile=logfile)
maprow_ocd = maprow_od[:,None,:] + np.zeros((2,cols,4))
maprow_ocd[:,:,[1,2]] -= drow_oc[:,:,None] # edge is straight, target curved
isedge_orc = (np.arange(rows)[:,None] < maprow_ocd[:,None,:,0]) | \
(np.arange(rows)[:,None] > maprow_ocd[:,None,:,3])
istarget_orc = (np.arange(rows)[:,None] > maprow_ocd[:,None,:,1]) & \
(np.arange(rows)[:,None] < maprow_ocd[:,None,:,2])
isskycont_orc = (((np.arange(rows)[:,None] < maprow_ocd[:,None,:,0]+rows/16) | \
(np.arange(rows)[:,None] > maprow_ocd[:,None,:,3]-rows/16)) & ~isedge_orc)
isbkgcont_orc &= (~badbinall_orc & ~isedge_orc & ~istarget_orc)
badbinall_orc |= badbinnew_orc
badbinone_orc |= badbinnew_orc
# pyfits.PrimaryHDU(var_orc.astype('float32')).writeto('var_orc1.fits',clobber=True)
# pyfits.PrimaryHDU(badbinnew_orc.astype('uint8')).writeto('badbinnew_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinall_orc.astype('uint8')).writeto('badbinall_orc.fits',clobber=True)
# pyfits.PrimaryHDU(badbinone_orc.astype('uint8')).writeto('badbinone_orc.fits',clobber=True)
# scrunch and normalize psf from summed images (using badbinone) for optimized extraction
psfnormmin = 0.70 # wavelengths with less than this flux in good bins are marked bad
wbin = wav_orc[0,rows/2,cols/2]-wav_orc[0,rows/2,cols/2-1]
wbin = float(int(wbin/0.75))
wmin,wmax = wav_orc.min(axis=2).max(),wav_orc.max(axis=2).min()
wedgemin = wbin*int(wmin/wbin+0.5) + wbin/2.
wedgemax = wbin*int(wmax/wbin-0.5) + wbin/2.
wedge_w = np.arange(wedgemin,wedgemax+wbin,wbin)
wavs = wedge_w.shape[0] - 1
binedge_orw = np.zeros((2,rows,wavs+1))
psf_orw = np.zeros((2,rows,wavs))
specrow_or = maprow_od[:,1:3].mean(axis=1)[:,None] + np.arange(-rows/4,rows/4)
# pyfits.PrimaryHDU(var_orc.astype('float32')).writeto('var_orc2.fits',clobber=True)
for o in (0,1):
for r in specrow_or[o]:
binedge_orw[o,r] = interp1d(wav_orc[o,r],np.arange(cols))(wedge_w)
psf_orw[o,r] = scrunch1d(psf_orc[o,r],binedge_orw[o,r])
psf_orw /= psf_orw.sum(axis=1)[:,None,:]
# np.savetxt("psfnorm_ow.txt",(psf_orw*okbin_orw).sum(axis=1).T,fmt="%10.4f")
# pyfits.PrimaryHDU(psf_orw.astype('float32')).writeto('psf_orw.fits',clobber=True)
# pyfits.PrimaryHDU(var_orw.astype('float32')).writeto('var_orw.fits',clobber=True)
# set up optional image-dependent column shift for slitless data
colshiftfilename = "colshift.txt"
docolshift = os.path.isfile(colshiftfilename)
if docolshift:
img_I,dcol_I = np.loadtxt(colshiftfilename,dtype=float,unpack=True,usecols=(0,1))
shifts = img_I.shape[0]
log.message('Column shift: \n Images '+shifts*'%5i ' % tuple(img_I), with_header=False)
log.message(' Bins '+shifts*'%5.2f ' % tuple(dcol_I), with_header=False)
# background-subtract and extract spectra
psfbadfrac_iow = np.zeros((outfiles,2,wavs))
for i in range(outfiles):
hdulist = pyfits.open(outfilelist[i])
sci_orc = hdulist['sci'].data
var_orc = hdulist['var'].data
badbin_orc = (hdulist['bpm'].data > 0) | badbinnew_orc
tnum = os.path.basename(outfilelist[i]).split('.')[0][-3:]
# make background continuum image, smoothed over resolution element
rblk,cblk = int(1.5*8./rbin), int(slitwidth*8./cbin)
target_orc = np.zeros_like(sci_orc)
for o in (0,1):
bkgcont_rc = blksmooth2d(sci_orc[o],isbkgcont_orc[o],rblk,cblk,0.25,mode="mean")
# remove sky continuum: ends of bkg continuum * skyflat
skycont_c = (bkgcont_rc.T[isskycont_orc[o].T]/skyflat_orc[o].T[isskycont_orc[o].T]) \
.reshape((cols,-1)).mean(axis=1)
skycont_rc = skycont_c*skyflat_orc[o]
# remove sky lines: image - bkg cont run through 2d sky averaging
obj_data = ((sci_orc[o] - bkgcont_rc)/skyflat_orc)[o]
obj_data[(badbin_orc | isedge_orc | istarget_orc)[o]] = np.nan
# pyfits.PrimaryHDU(obj_data.astype('float32')).writeto('obj_data.fits',clobber=True)
skylines_rc = make_2d_skyspectrum(obj_data,wav_orc[o],np.array([[0,rows],]))*skyflat_orc[o]
target_orc[o] = sci_orc[o] - skycont_rc - skylines_rc
# pyfits.PrimaryHDU(skylines_rc.astype('float32')).writeto('skylines_rc_'+tnum+'_'+str(o)+'.fits',clobber=True)
# pyfits.PrimaryHDU(skycont_rc.astype('float32')).writeto('skycont_rc_'+tnum+'_'+str(o)+'.fits',clobber=True)
target_orc *= (~badbin_orc).astype(int)
# pyfits.PrimaryHDU(target_orc.astype('float32')).writeto('target_'+tnum+'_orc.fits',clobber=True)
# extract spectrum optimally (Horne, PASP 1986)
target_orw = np.zeros((2,rows,wavs)); var_orw = np.zeros_like(target_orw)
badbin_orw = np.ones((2,rows,wavs),dtype='bool'); wt_orw = np.zeros_like(target_orw)
dcol = 0.
if docolshift:
if int(tnum) in img_I:
dcol = dcol_I[np.where(img_I==int(tnum))] # table has observed shift
for o in (0,1):
for r in specrow_or[o]:
target_orw[o,r] = scrunch1d(target_orc[o,r],binedge_orw[o,r]+dcol)
var_orw[o,r] = scrunch1d(var_orc[o,r],binedge_orw[o,r]+dcol)
badbin_orw[o,r] = scrunch1d(badbin_orc[o,r].astype(float),binedge_orw[o,r]+dcol) > 0.001
badbin_orw |= (var_orw == 0)
badbin_orw |= ((psf_orw*(~badbin_orw)).sum(axis=1)[:,None,:] < psfnormmin)
# pyfits.PrimaryHDU(var_orw.astype('float32')).writeto('var_'+tnum+'_orw.fits',clobber=True)
# pyfits.PrimaryHDU(badbin_orw.astype('uint8')).writeto('badbin_'+tnum+'_orw.fits',clobber=True)
# use master psf shifted in row to allow for guide errors
pwidth = 2*int(1./psf_orw.max())
ok_w = ((psf_orw*badbin_orw).sum(axis=1) < 0.03/float(pwidth/2)).all(axis=0)
crosscor_s = np.zeros(pwidth)
for s in range(pwidth):
crosscor_s[s] = (psf_orw[:,s:s-pwidth]*target_orw[:,pwidth/2:-pwidth/2]*ok_w).sum()
smax = np.argmax(crosscor_s)
s_S = np.arange(smax-pwidth/4,smax-pwidth/4+pwidth/2+1)
polycof = la.lstsq(np.vstack((s_S**2,s_S,np.ones_like(s_S))).T,crosscor_s[s_S])[0]
pshift = -(-0.5*polycof[1]/polycof[0] - pwidth/2)
s = int(pshift+pwidth)-pwidth
sfrac = pshift-s
psfsh_orw = np.zeros_like(psf_orw)
outrow = np.arange(max(0,s+1),rows-(1+int(abs(pshift)))+max(0,s+1))
psfsh_orw[:,outrow] = (1.-sfrac)*psf_orw[:,outrow-s] + sfrac*psf_orw[:,outrow-s-1]
# pyfits.PrimaryHDU(psfsh_orw.astype('float32')).writeto('psfsh_'+tnum+'_orw.fits',clobber=True)
wt_orw[~badbin_orw] = psfsh_orw[~badbin_orw]/var_orw[~badbin_orw]
var_ow = (psfsh_orw*wt_orw*(~badbin_orw)).sum(axis=1)
badbin_ow = (var_ow == 0)
var_ow[~badbin_ow] = 1./var_ow[~badbin_ow]
# pyfits.PrimaryHDU(var_ow.astype('float32')).writeto('var_'+tnum+'_ow.fits',clobber=True)
# pyfits.PrimaryHDU(target_orw.astype('float32')).writeto('target_'+tnum+'_orw.fits',clobber=True)
# pyfits.PrimaryHDU(wt_orw.astype('float32')).writeto('wt_'+tnum+'_orw.fits',clobber=True)
sci_ow = (target_orw*wt_orw).sum(axis=1)*var_ow
badlim = 0.20
psfbadfrac_iow[i] = (psfsh_orw*badbin_orw.astype(int)).sum(axis=1)/psfsh_orw.sum(axis=1)
badbin_ow |= (psfbadfrac_iow[i] > badlim)
# cdebug = 39
# np.savetxt("xtrct"+str(cdebug)+"_"+tnum+".txt",np.vstack((psf_orw[:,:,cdebug],var_orw[:,:,cdebug], \
# wt_orw[:,:,cdebug],target_orw[:,:,cdebug])).reshape((4,2,-1)).transpose(1,0,2).reshape((8,-1)).T,fmt="%12.5e")
# write O,E spectrum, prefix "s". VAR, BPM for each spectrum. y dim is virtual (length 1)
# for consistency with other modes
hduout = pyfits.PrimaryHDU(header=hdulist[0].header)
hduout = pyfits.HDUList(hduout)
header=hdulist['SCI'].header.copy()
header.update('VAREXT',2)
header.update('BPMEXT',3)
header.update('CRVAL1',wedge_w[0]+wbin/2.)
header.update('CRVAL2',0)
header.update('CDELT1',wbin)
header.update('CTYPE1','Angstroms')
hduout.append(pyfits.ImageHDU(data=sci_ow.reshape((2,1,wavs)), header=header, name='SCI'))
header.update('SCIEXT',1,'Extension for Science Frame',before='VAREXT')
hduout.append(pyfits.ImageHDU(data=var_ow.reshape((2,1,wavs)), header=header, name='VAR'))
hduout.append(pyfits.ImageHDU(data=badbin_ow.astype("uint8").reshape((2,1,wavs)), header=header, name='BPM'))
hduout.writeto('e'+outfilelist[i],clobber=True,output_verify='warn')
log.message('Output file '+'e'+outfilelist[i] , with_header=False)
# np.savetxt("psfbadfrac_iow.txt",psfbadfrac_iow.reshape((-1,wavs)).T,fmt="%8.5f")
return