def mosred(infile_list, slitmask,propcode=None, dy=0, inter=True, guesstype='rss', guessfile='', rstep=100, automethod='Matchlines', preprocess=False): #set up the files infiles=','.join(['%s' % x for x in infile_list]) obsdate=os.path.basename(infile_list[0])[7:15] #set up some files that will be needed logfile='spec'+obsdate+'.log' dbfile='spec%s.db' % obsdate #create the observation log obs_dict=obslog(infile_list) #check the value of dy #apply the mask to the data sets for i in range(len(infile_list)): specslit(image=infile_list[i], outimage='', outpref='s', exttype='rsmt', slitfile=slitmask, outputslitfile='', regprefix='ds_', sections=3, width=25.0, sigma=2.2, thres=6.0, order=1, padding=5, yoffset=dy, inter=False, clobber=True, logfile=logfile, verbose=True) for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip()=='ARC' and (obs_dict['PROPID'][i].upper().strip()==propcode or propcode is None): lamp=obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage='s'+os.path.basename(infile_list[i]) if lamp == 'NONE': lamp='CuAr' lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) specselfid(arcimage, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True) specidentify('a'+arcimage, lampfile, dbfile, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=3, rstep=rstep, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=True, clobber=True, preprocess=True, logfile=logfile, verbose=True) #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', # function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, # blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages='' spec_list=[] for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and \ (obs_dict['PROPID'][i].upper().strip()==propcode or propcode is None) and \ obs_dict['OBSMODE'][i].count('SPECTROSCOPY'): img = infile_list[i] ##rectify it specselfid('s'+img, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True) specrectify('as'+img, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True)
def mosred(infile_list, slitmask,propcode=None, dy=0, inter=True, guesstype='rss', guessfile='', rstep=100, automethod='Matchlines'): #set up the files infiles=','.join(['%s' % x for x in infile_list]) obsdate=os.path.basename(infile_list[0])[7:15] #set up some files that will be needed logfile='spec'+obsdate+'.log' dbfile='spec%s.db' % obsdate #create the observation log obs_dict=obslog(infile_list) #apply the mask to the data sets for i in range(len(infile_list)): specslit(image=infile_list[i], outimage='', outpref='s', exttype='rsmt', slitfile='../../P001423N01.xml', outputslitfile='', regprefix='ds_', sections=3, width=25.0, sigma=2.2, thres=6.0, order=1, padding=5, yoffset=dy, inter=False, clobber=True, logfile=logfile, verbose=True) for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode: lamp=obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage='s'+os.path.basename(infile_list[i]) if lamp == 'NONE': lamp='CuAr' lampfile=iraf.osfn("../../%s.salt" % lamp) specselfid(arcimage, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True) specidentify('a'+arcimage, lampfile, dbfile, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=3, rstep=rstep, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, verbose=True) #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', # function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, # blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages='' spec_list=[] for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and \ obs_dict['PROPID'][i].upper().strip()==propcode and \ obs_dict['OBSMODE'][i].count('SPECTROSCOPY'): img = infile_list[i] ##rectify it specselfid('s'+img, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True) specrectify('as'+img, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True)
def specred(infile_list, target, propcode, calfile=None, inter=True, automethod='Matchlines'): #set up the files infiles=','.join(['%s' % x for x in infile_list]) obsdate=os.path.basename(infile_list[0])[7:15] #set up some files that will be needed logfile='spec'+obsdate+'.log' dbfile='spec%s.db' % obsdate #create the observation log obs_dict=obslog(infile_list) for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode: lamp=obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage=os.path.basename(infile_list[i]) if lamp == 'NONE': lamp='CuAr' lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod=automethod, function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, verbose=True) specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages='' spec_list=[] for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and obs_dict['PROPID'][i].upper().strip()==propcode: img = infile_list[i] ##rectify it specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) #extract the spectra spec_list.append(extract_spectra('x'+img, yc=1030, calfile=calfile, findobject=True, smooth=False, maskzeros=True, clobber=True)) #combine the results w,f,e = speccombine(spec_list, obsdate) outfile = "%s_%s.spec" % (target, obsdate) write_spectra(outfile, w,f,e)
def specred(infile_list, propcode=None, inter=True, automethod='Matchlines'): #set up the files infiles=','.join(['%s' % x for x in infile_list]) obsdate=os.path.basename(infile_list[0])[7:15] #set up some files that will be needed logfile='spec'+obsdate+'.log' dbfile='spec%s.db' % obsdate #create the observation log obs_dict=obslog(infile_list) for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode: lamp=obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage=os.path.basename(infile_list[i]) if lamp == 'NONE': lamp='CuAr' lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) #lampfile='/Users/crawford/research/kepler/Xe.salt' specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod=automethod, function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=inter, clobber=True, logfile=logfile, verbose=True) #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', # function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, # blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages='' spec_list=[] for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and obs_dict['PROPID'][i].upper().strip()==propcode: img = infile_list[i] ##rectify it specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
def pol_wave_map(hduarc, image_no, drow_oc, rows, cols, lampfile, function='legendre', order=3, automethod="Matchlines", log=None, logfile=None): """ Create a wave_map for an arc image For O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map this corrects for the aberration introduced by the beam splitter this will be removed back out when creating the wave map Parameters ---------- hduarc: fits.HDUList Polarimetric arc data. This data should be split into O+E beams image_no: int File number of observations rows: int Nubmer of rows in original data cols: int Nubmer of columns in original data lampfile: str File name containing line list function: str Function used for wavelength fitting order: int Order of fitting function automethod: str Method for automated line identification log: log Log for output Returns ------- wavmap: numpy.ndarray Wave map of wavelengths correspond to pixels """ arc_orc = hduarc[1].data cbin, rbin = [int(x) for x in hduarc[0].header['CCDSUM'].split(" ")] axisrow_o = np.array([rows/4.0, rows/4.0]).astype(int) #set up some output arrays wavmap_orc = np.zeros((2,rows/2.0,cols)) edgerow_od = np.zeros((2,2)) cofrows_o = np.zeros(2) legy_od = np.zeros((2,2)) guessfile=None for o in (0,1): #correct the shape of the arc for the distortions arc_yc = correct_wollaston(arc_orc[o], -drow_oc[o]) # this is used to remove rows outside the slit maxoverlaprows = 34/rbin # beam overlap for 4' longslit in NIR arc_y = arc_yc.sum(axis=1) arc_y[[0,-1]] = 0. edgerow_od[o,0] = axisrow_o[o] - np.argmax(arc_y[axisrow_o[o]::-1] < 0.5*arc_y[axisrow_o[o]]) edgerow_od[o,1] = axisrow_o[o] + np.argmax(arc_y[axisrow_o[o]:] < 0.5*arc_y[axisrow_o[o]]) axisrow_o[o] = edgerow_od[o].mean() if np.abs(edgerow_od[o] - np.array([0,rows/2-1])).min() < maxoverlaprows: edgerow_od[o] += maxoverlaprows*np.array([+1,-1]) #wrtite out temporary image to run specidentify hduarc['SCI'].data = arc_yc arcimage = "arc_"+str(image_no)+"_"+str(o)+".fits" dbfilename = "arcdb_"+str(image_no)+"_"+str(o)+".txt" ystart = axisrow_o[o] if (not os.path.exists(dbfilename)): if guessfile is not None: guesstype = 'file' else: guessfile=dbfilename guesstype = 'rss' hduarc.writeto(arcimage,clobber=True) specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function=function, order=order, rstep=20, rstart=ystart, mdiff=20, thresh=3, niter=5, smooth=3, inter=True, clobber=True, logfile=logfile, verbose=True) if (not debug): os.remove(arcimage) wavmap_yc, cofrows_o[o], legy_od[o], edgerow_od[o] = \ wave_map(dbfilename, edgerow_od[o], rows, cols, ystart, order, log=log) #TODO: Once rest is working, try to switch to pysalt wavemap #soldict = sr.entersolution(dbfilename) #wavmap_yc = wavemap(hduarc, soldict, caltype='line', function=function, # order=order,blank=0, nearest=True, array_only=True, # clobber=True, log=log, verbose=True) # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm) if debug: np.savetxt("drow_wmap_oc.txt",drow_oc.T,fmt="%8.3f %8.3f") wavmap_orc[o] = correct_wollaston(wavmap_yc,drow_oc[o]) y, x = np.indices(wavmap_orc[o].shape) # fixing problem in 0312 sc wavmap notwav_c = np.isnan(drow_oc[o]) drow_oc[o,notwav_c] = 0. mask = (y < edgerow_od[o,0] + drow_oc[o]) | (y > edgerow_od[o,1] + drow_oc[o]) wavmap_orc[o,mask] = 0. wavmap_orc[o][:,notwav_c] = 0. # if log is not None: log.message('\n Wavl coeff rows: O %4i E %4i' % tuple(cofrows_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i' \ % tuple(legy_od.flatten()), with_header=False) log.message('\n Slit axis row: O %4i E %4i' % tuple(axisrow_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i \n' \ % tuple(edgerow_od.flatten()), with_header=False) return wavmap_orc
def specred(infile_list, propcode=None, inter=True, automethod='Matchlines'): #set up the files infiles = ','.join(['%s' % x for x in infile_list]) obsdate = os.path.basename(infile_list[0])[7:15] #set up some files that will be needed logfile = 'spec' + obsdate + '.log' dbfile = 'spec%s.db' % obsdate #create the observation log obs_dict = obslog(infile_list) for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip( ) == 'ARC' and obs_dict['PROPID'][i].upper().strip() == propcode: lamp = obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage = os.path.basename(infile_list[i]) if lamp == 'NONE': lamp = 'CuAr' lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) #lampfile='/Users/crawford/research/kepler/Xe.salt' specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod=automethod, function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=inter, clobber=True, logfile=logfile, verbose=True) #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', # function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, # blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages = '' spec_list = [] for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][ i].count('RSS') and obs_dict['PROPID'][i].upper().strip( ) == propcode: img = infile_list[i] ##rectify it specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
#/usr/bin/env python import sys from pyraf import iraf from pyraf.iraf import pysalt from specidentify import specidentify specidentify(images=sys.argv[1], linelist=sys.argv[2], outfile='db.sol', guesstype='rss', guessfile='', automethod='Matchlines', function='polynomial', order=3,rstep=100 , rstart='middlerow', mdiff=20, thresh=3,niter=5,inter=True, clobber=False, logfile='salt.log', verbose=True)
def pol_wave_map(hduarc, image_no, drow_oc, rows, cols, lampfile, function='legendre', order=3, automethod="Matchlines", log=None, logfile=None): """ Create a wave_map for an arc image For O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map this corrects for the aberration introduced by the beam splitter this will be removed back out when creating the wave map Parameters ---------- hduarc: fits.HDUList Polarimetric arc data. This data should be split into O+E beams image_no: int File number of observations rows: int Nubmer of rows in original data cols: int Nubmer of columns in original data lampfile: str File name containing line list function: str Function used for wavelength fitting order: int Order of fitting function automethod: str Method for automated line identification log: log Log for output Returns ------- wavmap: numpy.ndarray Wave map of wavelengths correspond to pixels """ arc_orc = hduarc[1].data cbin, rbin = [int(x) for x in hduarc[0].header['CCDSUM'].split(" ")] axisrow_o = np.array([rows/4.0, rows/4.0]).astype(int) grating = hduarc[0].header['GRATING'].strip() grang = hduarc[0].header['GR-ANGLE'] artic = hduarc[0].header['CAMANG'] trkrho = hduarc[0].header['TRKRHO'] date = hduarc[0].header['DATE-OBS'].replace('-','') #set up some output arrays wavmap_orc = np.zeros((2,rows/2,cols)) edgerow_od = np.zeros((2,2)) cofrows_o = np.zeros(2) legy_od = np.zeros((2,2)) lam_X = rssmodelwave(grating,grang,artic,trkrho,cbin,cols,date) # np.savetxt("lam_X_"+str(image_no)+".txt",lam_X,fmt="%8.3f") C_f = np.polynomial.legendre.legfit(np.arange(cols),lam_X,3)[::-1] for o in (0,1): #correct the shape of the arc for the distortions arc_yc = correct_wollaston(arc_orc[o], -drow_oc[o]) # this is used to remove rows outside the slit maxoverlaprows = 34/rbin # beam overlap for 4' longslit in NIR arc_y = arc_yc.sum(axis=1) arc_y[[0,-1]] = 0. edgerow_od[o,0] = axisrow_o[o] - np.argmax(arc_y[axisrow_o[o]::-1] < 0.5*arc_y[axisrow_o[o]]) edgerow_od[o,1] = axisrow_o[o] + np.argmax(arc_y[axisrow_o[o]:] < 0.5*arc_y[axisrow_o[o]]) axisrow_o[o] = edgerow_od[o].mean() if np.abs(edgerow_od[o] - np.array([0,rows/2-1])).min() < maxoverlaprows: edgerow_od[o] += maxoverlaprows*np.array([+1,-1]) #write out temporary image to run specidentify hduarc['SCI'].data = arc_yc arcimage = "arc_"+str(image_no)+"_"+str(o)+".fits" dbfilename = "arcdb_"+str(image_no)+"_"+str(o)+".txt" # use for guessfile dbfile for other beam, or, if none, "arcdb_(img)_guess.txt" otherdbfilename = "arcdb_"+str(image_no)+"_"+str(int(not(o==1)))+".txt" if (not os.path.exists(otherdbfilename)): otherdbfilename = "arcdb_"+str(image_no)+"_guess.txt" guessfilename = "" ystart = axisrow_o[o] if (not os.path.exists(dbfilename)): if (os.path.exists(otherdbfilename)): row_Y = np.loadtxt(otherdbfilename,dtype=float,usecols=(0,),ndmin=1) closestrow = row_Y[np.argmin(np.abs(row_Y - ystart))] guessfilename="wavguess_"+str(image_no)+"_"+str(o)+".txt" guessfile=open(guessfilename, 'w') for line in open(otherdbfilename): if (len(line.split())==0): continue # ignore naughty dbfile extra lines if (line[0] == "#"): guessfile.write(line) elif (float(line.split()[0]) == closestrow): guessfile.write(line) guessfile.close() guesstype = 'file' # guesstype = 'rss' else: guesstype = 'rss' hduarc.writeto(arcimage,overwrite=True) specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfilename, automethod=automethod, function=function, order=order, rstep=20, rstart=ystart, mdiff=20, thresh=3, niter=5, smooth=3, inter=True, clobber=True, logfile=logfile, verbose=True) if (not debug): os.remove(arcimage) wavmap_yc, cofrows_o[o], legy_od[o], edgerow_od[o] = \ wave_map(dbfilename, edgerow_od[o], rows, cols, ystart, order, log=log) #TODO: Once rest is working, try to switch to pysalt wavemap #soldict = sr.entersolution(dbfilename) #wavmap_yc = wavemap(hduarc, soldict, caltype='line', function=function, # order=order,blank=0, nearest=True, array_only=True, # overwrite=True, log=log, verbose=True) # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm) if debug: np.savetxt("drow_wmap_oc.txt",drow_oc.T,fmt="%8.3f %8.3f") wavmap_orc[o] = correct_wollaston(wavmap_yc,drow_oc[o]) y, x = np.indices(wavmap_orc[o].shape) # fixing problem in 0312 sc wavmap notwav_c = np.isnan(drow_oc[o]) drow_oc[o,notwav_c] = 0. mask = (y < edgerow_od[o,0] + drow_oc[o]) | (y > edgerow_od[o,1] + drow_oc[o]) wavmap_orc[o,mask] = 0. wavmap_orc[o][:,notwav_c] = 0. # if log is not None: log.message('\n Wavl coeff rows: O %4i E %4i' % tuple(cofrows_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i' \ % tuple(legy_od.flatten()), with_header=False) log.message('\n Slit axis row: O %4i E %4i' % tuple(axisrow_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i \n' \ % tuple(edgerow_od.flatten()), with_header=False) return wavmap_orc
def specred(rawdir, prodir, imreduce=True, specreduce=True, calfile=None, lamp='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True): print rawdir print prodir #get the name of the files infile_list=glob.glob(rawdir+'*.fits') infiles=','.join(['%s' % x for x in infile_list]) #get the current date for the files obsdate=os.path.basename(infile_list[0])[1:9] print obsdate #set up some files that will be needed logfile='spec'+obsdate+'.log' flatimage='FLAT%s.fits' % (obsdate) dbfile='spec%s.db' % obsdate #create the observation log obs_dict=obslog(infile_list) if imreduce: #prepare the data saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True) #bias subtract the data saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='', median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0, niter=10, plotover=False, turbo=False, clobber=True, logfile=logfile, verbose=True) #gain correct the data saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True) #cross talk correct the data saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True) #cosmic ray clean the data #only clean the object data for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'): img='xgbp'+os.path.basename(infile_list[i]) saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0, flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2, gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True) #flat field correct the data flat_imgs='' for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('FLAT'): if flat_imgs: flat_imgs += ',' flat_imgs += 'xgbp'+os.path.basename(infile_list[i]) if len(flat_imgs)!=0: saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \ weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \ hthresh=3, clobber=True, logfile=logfile, verbose=True) saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True) saltflat('xgbpP*fits', '', 'f', flatimage, minflat=500, clobber=True, logfile=logfile, verbose=True) else: flats=None imfiles=glob.glob('cxgbpP*fits') for f in imfiles: shutil.copy(f, 'f'+f) #mosaic the data geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat") saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True) #clean up the images if cleanup: for f in glob.glob('p*fits'): os.remove(f) for f in glob.glob('bp*fits'): os.remove(f) for f in glob.glob('gbp*fits'): os.remove(f) for f in glob.glob('xgbp*fits'): os.remove(f) for f in glob.glob('fxgbp*fits'): os.remove(f) #set up the name of the images if specreduce: for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip()=='ARC': lamp=obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage='mfxgbp'+os.path.basename(infile_list[i]) lampfile=iraf.osfn("pysalt$data/linelists/%s.txt" % lamp) specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod=automethod, function='legendre', order=5, rstep=100, rstart='middlerow', mdiff=10, thresh=3, niter=5, inter=True, clobber=True, logfile=logfile, verbose=True) specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages='' for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'): if objimages: objimages += ',' objimages+='mfxgbp'+os.path.basename(infile_list[i]) if specreduce: #run specidentify on the arc files specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) #create the spectra text files for all of our objects spec_list=[] for img in objimages.split(','): spec_list.extend(createspectra('x'+img, obsdate, smooth=False, skysection=skysection, clobber=True)) print spec_list #determine the spectrophotometric standard extfile=iraf.osfn('pysalt$data/site/suth_extinct.dat') for spec, am, et, pc in spec_list: if pc=='CAL_SPST': stdstar=spec.split('.')[0] print stdstar, am, et stdfile=iraf.osfn('pysalt$data/standards/spectroscopic/m%s.dat' % stdstar.lower().replace('-', '_')) print stdfile ofile=spec.replace('txt', 'sens') calfile=ofile #assumes only one observations of a SP standard specsens(spec, ofile, stdfile, extfile, airmass=am, exptime=et, stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5, clobber=True, logfile='salt.log',verbose=True) for spec, am, et, pc in spec_list: if pc!='CAL_SPST': ofile=spec.replace('txt', 'spec') speccal(spec, ofile, calfile, extfile, airmass=am, exptime=et, clobber=True, logfile='salt.log',verbose=True) #clean up the spectra for bad pixels cleanspectra(ofile)
def auto_arc_lens(arc_image, dbfile='wav.db', ndstep=20, logfile='salt.log'): """Automatically process an arc image for the SALT Lens project """ hdu = fits.open(arc_image) hdr = hdu[0].header if hdr['LAMPID'] == 'Xe' and hdr['GRATING'] == 'PG0900' and hdr['GRTILT'] == 15.875: print('Automatically processing arc image') data = hdu[1].data ystart = int(0.5 * len(data)) xarr = np.arange(len(data[ystart]), dtype='int64') farr = data[ystart] lampfile = os.path.dirname(__file__)+'/data/Xe.lens' guessfile = os.path.dirname(__file__)+'/data/lens.db' slines, sfluxes = st.readlinelist(lampfile) spectrum = Spectrum.Spectrum( slines, sfluxes, dw=0.1, stype='line', sigma=6) swarr = spectrum.wavelength sfarr = spectrum.flux * farr.max() / spectrum.flux.max() soldict = readsolascii(guessfile, {}) soldict = (soldict[soldict.keys()[0]]) ws = WavelengthSolution.WavelengthSolution( xarr, xarr, function=soldict[7], order=soldict[8]) ws.func.func.domain = soldict[11] ws.set_coef(soldict[10][2]) # start the pre processing dcoef = ws.coef * 0.0 dcoef[0] = 0.05 * ndstep dcoef[1] = 0.01 * ws.coef[1] ws = st.findxcor(xarr, farr, swarr, sfarr, ws, dcoef=dcoef, ndstep=ndstep, best=False, inttype='interp') xp, wp = st.crosslinematch(xarr, farr, slines, sfluxes, ws, res=6, mdiff=20, wdiff=10, sections=3, sigma=5, niter=5) ws = st.findfit(np.array(xp), np.array(wp), ws=ws, thresh=ws.thresh) print(ws) with logging(logfile, True) as log: iws = ai.AutoIdentify(xarr, data, slines, sfluxes, ws, farr=farr, method='Matchlines', rstep=100, istart=ystart, nrows=1, res=6, dres=0.25, mdiff=20, sigma=5, smooth=3, niter=5, dc=5, ndstep=ndstep, oneline=False, log=log, verbose=True) # get the basic information about the spectrograph dateobs = saltkey.get('DATE-OBS', hdu[0]) try: utctime = saltkey.get('UTC-OBS', hdu[0]) except SaltError: utctime = saltkey.get('TIME-OBS', hdu[0]) instrume = saltkey.get('INSTRUME', hdu[0]).strip() grating = saltkey.get('GRATING', hdu[0]).strip() grang = saltkey.get('GR-ANGLE', hdu[0]) grasteps = saltkey.get('GRTILT', hdu[0]) arang = saltkey.get('AR-ANGLE', hdu[0]) arsteps = saltkey.get('CAMANG', hdu[0]) rssfilter = saltkey.get('FILTER', hdu[0]) specmode = saltkey.get('OBSMODE', hdu[0]) masktype = saltkey.get('MASKTYP', hdu[0]).strip().upper() slitname = saltkey.get('MASKID', hdu[0]) slit = st.getslitsize(slitname) xbin, ybin = saltkey.ccdbin(hdu[0], arc_image) writeIS(iws, dbfile, dateobs=dateobs, utctime=utctime, instrume=instrume, grating=grating, grang=grang, grasteps=grasteps, arsteps=arsteps, arang=arang, rfilter=rssfilter, slit=slit, xbin=xbin, ybin=ybin, objid=None, filename=arc_image, log=log, verbose=True) print(iws) #self.findfit() else: lamp = hdr['LAMPID'] lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) lampfile = 'Xe.lens' specidentify(arc_image, lampfile, dbfile, guesstype='rss', guessfile=None, automethod='Matchlines', function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=5, niter=5, smooth=3, inter=True, clobber=True, preprocess=True, logfile=logfile, verbose=True) print("Running specidenity in interactive mode")
def specpolwavmap(infilelist, linelistlib="", inter=True, automethod='Matchlines', logfile='salt.log'): obsdate = os.path.basename(infilelist[0])[7:15] with logging(logfile, debug) as log: # create the observation log obs_dict = obslog(infilelist) log.message('Pysalt Version: ' + pysalt.verno, with_header=False) if len(linelistlib) == 0: linelistlib = datadir + "linelistlib.txt" with open(linelistlib) as fd: linelistdict = dict(line.strip().split(None, 1) for line in fd) # eliminate inapplicable images for i in range(len(infilelist)): if int(obs_dict['BS-STATE'][i][1]) != 2: del infilelist[i] obs_dict = obslog(infilelist) # Map out which arc goes with which image. Use arc in closest wavcal block of the config. # wavcal block: neither spectrograph config nor track changes, and no gap in data files infiles = len(infilelist) newtrk = 5. # new track when rotator changes by more (deg) trkrho_i = np.array(map(float, obs_dict['TRKRHO'])) trkno_i = np.zeros((infiles), dtype=int) trkno_i[1:] = ((np.abs(trkrho_i[1:] - trkrho_i[:-1])) > newtrk).cumsum() confno_i, confdatlist = configmap(infilelist) configs = len(confdatlist) imageno_i = np.array([int(os.path.basename(infilelist[i]).split('.')[0][-4:]) \ for i in range(infiles)]) filegrp_i = np.zeros((infiles), dtype=int) filegrp_i[1:] = ((imageno_i[1:] - imageno_i[:-1]) > 1).cumsum() isarc_i = np.array([(obs_dict['OBJECT'][i].upper().strip() == 'ARC') for i in range(infiles)]) wavblk_i = np.zeros((infiles), dtype=int) wavblk_i[1:] = ((filegrp_i[1:] != filegrp_i[:-1]) \ | (trkno_i[1:] != trkno_i[:-1]) \ | (confno_i[1:] != confno_i[:-1])).cumsum() wavblks = wavblk_i.max() + 1 arcs_c = (isarc_i[:, None] & (confno_i[:, None] == range(configs))).sum(axis=0) np.savetxt("wavblktbl.txt",np.vstack((trkrho_i,imageno_i,filegrp_i,trkno_i, \ confno_i,wavblk_i,isarc_i)).T,fmt="%7.2f "+6*"%3i ",header=" rho img grp trk conf wblk arc") for c in range(configs): # worst: no arc for config, remove images if arcs_c[c] == 0: lostimages = imageno_i[confno_i == c] log.message('No Arc for this configuration: ' \ +("Grating %s Grang %6.2f Artic %6.2f" % confdatlist[c]) \ +("\n Images: "+lostimages.shape[0]*"%i " % tuple(lostimages)), with_header=False) wavblk_i[confno_i == c] = -1 if arcs_c.sum() == 0: log.message("Cannot calibrate any images", with_header=False) exit() iarc_i = -np.zeros((infiles), dtype=int) for w in range(wavblks): blkimages = imageno_i[wavblk_i == w] if blkimages.shape[0] == 0: continue iarc_I = np.where((wavblk_i == w) & (isarc_i))[0] if iarc_I.shape[0] > 0: iarc = iarc_I[0] # best: arc is in wavblk, take first else: conf = confno_i[wavblk_i == w][ 0] # fallback: take closest arc of this config iarc_I = np.where((confno_i == conf) & (isarc_i))[0] blkimagepos = blkimages.mean() iarc = iarc_I[np.argmin(imageno_i[iarc_I] - blkimagepos)] iarc_i[wavblk_i == w] = iarc log.message(("For images: "+blkimages.shape[0]*"%i " % tuple(blkimages)) \ +("\n Use Arc %5i" % imageno_i[iarc]), with_header=False) iarc_a = np.unique(iarc_i[iarc_i != -1]) arcs = iarc_a.shape[0] lam_m = np.loadtxt(datadir + "wollaston.txt", dtype=float, usecols=(0, )) rpix_om = np.loadtxt(datadir + "wollaston.txt", dtype=float, unpack=True, usecols=(1, 2)) for a in range(arcs): iarc = iarc_a[a] conf = confno_i[iarc] # use arc to make first-guess wavecal from model, locate beamsplitter split point cbin, rbin = np.array( obs_dict["CCDSUM"][iarc].split(" ")).astype(int) grating, grang, artic = confdatlist[confno_i[iarc]] hduarc = pyfits.open(infilelist[iarc]) arc_rc = hduarc['SCI'].data rows, cols = arc_rc.shape lam_c = rssmodelwave(grating, grang, artic, cbin, cols) arc_r = arc_rc.sum(axis=1) expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \ (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int) foundrow_o = np.zeros((2), dtype=int) for o in (0, 1): foundrow_o[o] = expectrow_o[o]-100/rbin \ + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin]) arcsignal = arc_r[foundrow_o[o]] topedge = foundrow_o[o] + np.argmax( arc_r[foundrow_o[o]:] < 0.75 * arcsignal) botedge = foundrow_o[o] - np.argmax( arc_r[foundrow_o[o]::-1] < 0.75 * arcsignal) foundrow_o[o] = (botedge + topedge) / 2. splitrow = foundrow_o.mean() offset = int(splitrow - rows / 2) # split arc into o/e images padbins = (np.indices((rows, cols))[0] < offset) | (np.indices( (rows, cols))[0] > rows + offset) arc_rc = np.roll(arc_rc, -offset, axis=0) arc_rc[padbins] = 0. arc_orc = arc_rc.reshape((2, rows / 2, cols)) # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map lamp = obs_dict['LAMPID'][iarc].strip().replace(' ', '') if lamp == 'NONE': lamp = 'CuAr' hduarc[0].header.update('MASKTYP', 'LONGSLIT') hduarc[0].header.update('MASKID', 'PL0100N001') del hduarc['VAR'] del hduarc['BPM'] # lampfile=iraf.osfn("pysalt$data/linelists/%s.wav" % lamp) zsalt version: not good lampfile = iraf.osfn("pysalt$data/linelists/" + linelistdict[lamp]) rpix_oc = interp1d(lam_m, rpix_om, kind='cubic')(lam_c) log.message('\nARC: image '+str(imageno_i[iarc])+' GRATING '+grating\ +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False) log.message(' Split Row: ' + ("%4i " % splitrow), with_header=False) wavmap_orc = np.zeros_like(arc_orc) for o in (0, 1): foundrow_o[o] += -offset - o * rows / 2 arc_Rc = np.zeros((rows / 2, cols), dtype='float32') for c in range(cols): shift(arc_orc[o,:,c], \ -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c]) hduarc['SCI'].data = arc_Rc arcimage = "arc_" + str( imageno_i[iarc]) + "_" + str(o) + ".fits" dbfilename = "arcdb_" + str( imageno_i[iarc]) + "_" + str(o) + ".txt" order = 3 if os.path.isfile(dbfilename): # guessfile = "guess_"+str(o)+".txt" # os.rename(dbfilename,guessfile) guesstype = 'file' else: hduarc.writeto(arcimage, clobber=True) guesstype = 'rss' guessfile = '' Rstart = foundrow_o[o] specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=order, rstep=20, rstart=Rstart, mdiff=20, thresh=3, niter=5, smooth=3, inter=inter, clobber=True, logfile=logfile, verbose=True) wavlegR_y = np.loadtxt(dbfilename, dtype=float, usecols=(0, ), ndmin=1) wavlegcof_ly = np.loadtxt(dbfilename, unpack=True, dtype=float, usecols=range(1, order + 2), ndmin=2) if wavlegR_y.min() < 0: wavlegcof_ly = np.delete(wavlegcof_ly, np.where(wavlegR_y < 0)[0], axis=1) wavlegR_y = np.delete(wavlegR_y, np.where(wavlegR_y < 0)[0], axis=0) wavmap_yc = np.polynomial.legendre.legval( np.arange(cols), wavlegcof_ly) mediancof_l = np.median(wavlegcof_ly, axis=1) rms_l = np.sqrt( np.median((wavlegcof_ly - mediancof_l[:, None])**2, axis=1)) sigma_ly = (wavlegcof_ly - mediancof_l[:, None]) / rms_l[:, None] usey = (sigma_ly[0] < 3) & (sigma_ly[1] < 3) wavmap_Rc = np.zeros((rows / 2, cols), dtype='float32') R_y = wavlegR_y[usey] if usey.shape[0] < 5: # for future: if few lines in db, use model shifted to agree, for now just use first wavmap_orc[o] = wavmap_yc[0] else: if usey.shape[0] > 9: aa = np.vstack( (R_y**3, R_y**2, R_y, np.ones_like(R_y))).T else: aa = np.vstack((R_y**2, R_y, np.ones_like(R_y))).T for c in range(cols): polycofs = la.lstsq(aa, wavmap_yc[usey, c])[0] wavmap_orc[o,:,c] \ = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin) # os.remove(arcimage) # for images using this arc,save split data along third fits axis, # add wavmap extension, save as 'w' file hduwav = pyfits.ImageHDU(data=wavmap_orc, header=hduarc['SCI'].header, name='WAV') for i in np.where(iarc_i == iarc_a[a])[0]: hdu = pyfits.open(infilelist[i]) image_rc = np.roll(hdu['SCI'].data, -offset, axis=0) image_rc[padbins] = 0. hdu['SCI'].data = image_rc.reshape((2, rows / 2, cols)) var_rc = np.roll(hdu['VAR'].data, -offset, axis=0) var_rc[padbins] = 0. hdu['VAR'].data = var_rc.reshape((2, rows / 2, cols)) bpm_rc = np.roll(hdu['BPM'].data, -offset, axis=0) bpm_rc[padbins] = 1 hdu['BPM'].data = bpm_rc.reshape((2, rows / 2, cols)) hdu.append(hduwav) for f in ('SCI', 'VAR', 'BPM', 'WAV'): hdu[f].header.update('CTYPE3', 'O,E') hdu.writeto('w' + infilelist[i], clobber='True') log.message('Output file ' + 'w' + infilelist[i], with_header=False) return
def specred(infile_list, propcode=None, inter=True, guessfile = None, automethod='Matchlines', preprocess=False): #set up the files infiles=','.join(['%s' % x for x in infile_list]) obsdate=os.path.basename(infile_list[0])[7:15] #set up some files that will be needed logfile='spec'+obsdate+'.log' dbfile='spec%s.db' % obsdate sdbfile='spec_straight_%s.db' % obsdate straight_function = 'legendre' straight_order = 2 dcoef = [0.50, 1.0, 0.0] #create the observation log obs_dict=obslog(infile_list) for i in range(len(infile_list)): print infile_list[i], obs_dict['OBJECT'][i].upper().strip(), obs_dict['PROPID'][i].upper().strip() if obs_dict['OBJECT'][i].upper().strip()=='ARC' and (propcode is None or obs_dict['PROPID'][i].upper().strip()==propcode): lamp=obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage=os.path.basename(infile_list[i]) if lamp == 'NONE': lamp='CuAr' lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) #lampfile='/Users/crawford/research/kepler/Xe.salt' #straighten the arc #specarcstraighten(arcimage, sdbfile , function=straight_function, order=straight_order, rstep=20, # rstart='middlerow', nrows=1, dcoef=dcoef, ndstep=10, # startext=0, clobber=False, logfile='salt.log', verbose=True) #rectify it #specrectify(arcimage, outimages='', outpref='s', solfile=sdbfile, caltype='line', # function=straight_function, order=straight_order, inttype='interp', w1=None, w2=None, dw=None, nw=None, # nearest=True, blank=0.0, clobber=True, logfile=logfile, verbose=True) #idnetify the line if guessfile is None: guesstype='rss' guessfile=None else: guesstype='file' specidentify(arcimage, lampfile, dbfile, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=5, niter=5, smooth=3, inter=inter, clobber=True, preprocess=preprocess, logfile=logfile, verbose=True) #apply the final rectification specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages='' spec_list=[] for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and (propcode is None or obs_dict['PROPID'][i].upper().strip()==propcode): img = infile_list[i] # rectify it #specrectify(img, outimages='', outpref='s', solfile=sdbfile, caltype='line', # function='legendre', order=2, inttype='interp', w1=None, w2=None, dw=None, nw=None, # blank=0.0, clobber=True, logfile=logfile, verbose=True) specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
def pol_wave_map(hduarc, image_id, drow_oc, rows, cols, lampfile, function='legendre', order=3, automethod="Matchlines", log=None, logfile=None): """ Create a wave_map for an arc image For O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map this corrects for the aberration introduced by the beam splitter this will be removed back out when creating the wave map Parameters ---------- hduarc: fits.HDUList Polarimetric arc data. This data should be split into O+E beams image_id: str Arc identifier rows: int Nubmer of rows in original data cols: int Nubmer of columns in original data lampfile: str File name containing line list function: str Function used for wavelength fitting order: int Order of fitting function automethod: str Method for automated line identification log: log Log for output Returns ------- wavmap: numpy.ndarray Wave map of wavelengths correspond to pixels """ arc_orc = hduarc[1].data cbin, rbin = [int(x) for x in hduarc[0].header['CCDSUM'].split(" ")] axisrow_o = np.array([rows / 4.0, rows / 4.0]).astype(int) grating = hduarc[0].header['GRATING'].strip() grang = hduarc[0].header['GR-ANGLE'] artic = hduarc[0].header['CAMANG'] trkrho = hduarc[0].header['TRKRHO'] date = hduarc[0].header['DATE-OBS'].replace('-', '') #set up some output arrays wavmap_orc = np.zeros((2, rows / 2, cols)) edgerow_od = np.zeros((2, 2)) cofrows_o = np.zeros(2) legy_od = np.zeros((2, 2)) lam_X = rssmodelwave(grating, grang, artic, trkrho, cbin, cols, date) # np.savetxt("lam_X_"+image_id+".txt",lam_X,fmt="%8.3f") C_f = np.polynomial.legendre.legfit(np.arange(cols), lam_X, 3)[::-1] for o in (0, 1): #correct the shape of the arc for the distortions arc_yc = correct_wollaston(arc_orc[o], -drow_oc[o]) # this is used to remove rows outside the slit maxoverlaprows = 34 / rbin # beam overlap for 4' longslit in NIR arc_y = arc_yc.sum(axis=1) arc_y[[0, -1]] = 0. edgerow_od[o, 0] = axisrow_o[o] - np.argmax( arc_y[axisrow_o[o]::-1] < 0.5 * arc_y[axisrow_o[o]]) edgerow_od[o, 1] = axisrow_o[o] + np.argmax( arc_y[axisrow_o[o]:] < 0.5 * arc_y[axisrow_o[o]]) axisrow_o[o] = edgerow_od[o].mean() if np.abs(edgerow_od[o] - np.array([0, rows / 2 - 1])).min() < maxoverlaprows: edgerow_od[o] += maxoverlaprows * np.array([+1, -1]) #write out temporary image to run specidentify hduarc['SCI'].data = arc_yc arcimage = "arc_" + image_id + "_" + str(o) + ".fits" dbfilename = "arcdb_" + image_id + "_" + str(o) + ".txt" # use for guessfile dbfile for other beam, or, if none, "arcdb_(img)_guess.txt" otherdbfilename = "arcdb_" + image_id + "_" + str( int(not (o == 1))) + ".txt" if (not os.path.exists(otherdbfilename)): otherdbfilename = "arcdb_" + image_id + "_guess.txt" guessfilename = "" ystart = axisrow_o[o] if (not os.path.exists(dbfilename)): if (os.path.exists(otherdbfilename)): row_Y = np.loadtxt(otherdbfilename, dtype=float, usecols=(0, ), ndmin=1) closestrow = row_Y[np.argmin(np.abs(row_Y - ystart))] guessfilename = "wavguess_" + image_id + "_" + str(o) + ".txt" guessfile = open(guessfilename, 'w') for line in open(otherdbfilename): if (len(line.split()) == 0): continue # ignore naughty dbfile extra lines if (line[0] == "#"): guessfile.write(line) elif (float(line.split()[0]) == closestrow): guessfile.write(line) guessfile.close() guesstype = 'file' # guesstype = 'rss' else: guesstype = 'rss' hduarc.writeto(arcimage, overwrite=True) specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfilename, automethod=automethod, function=function, order=order, rstep=20, rstart=ystart, mdiff=20, thresh=3, niter=5, smooth=3, inter=True, clobber=True, logfile=logfile, verbose=True) if (not debug): os.remove(arcimage) wavmap_yc, cofrows_o[o], legy_od[o], edgerow_od[o] = \ wave_map(dbfilename, edgerow_od[o], rows, cols, ystart, order, log=log) #TODO: Once rest is working, try to switch to pysalt wavemap #soldict = sr.entersolution(dbfilename) #wavmap_yc = wavemap(hduarc, soldict, caltype='line', function=function, # order=order,blank=0, nearest=True, array_only=True, # overwrite=True, log=log, verbose=True) # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm) if debug: np.savetxt("drow_wmap_oc.txt", drow_oc.T, fmt="%8.3f %8.3f") wavmap_orc[o] = correct_wollaston(wavmap_yc, drow_oc[o]) y, x = np.indices(wavmap_orc[o].shape) # fixing problem in 0312 sc wavmap notwav_c = np.isnan(drow_oc[o]) drow_oc[o, notwav_c] = 0. mask = (y < edgerow_od[o, 0] + drow_oc[o]) | ( y > edgerow_od[o, 1] + drow_oc[o]) wavmap_orc[o, mask] = 0. wavmap_orc[o][:, notwav_c] = 0. # if log is not None: log.message('\n Wavl coeff rows: O %4i E %4i' % tuple(cofrows_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i' \ % tuple(legy_od.flatten()), with_header=False) log.message('\n Slit axis row: O %4i E %4i' % tuple(axisrow_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i \n' \ % tuple(edgerow_od.flatten()), with_header=False) return wavmap_orc
def specpolmap(infile_list, propcode=None, inter=True, automethod='Matchlines',logfile='salt.log'): obsdate=os.path.basename(infile_list[0])[7:15] with logging(logfile, debug) as log: #create the observation log obs_dict=obslog(infile_list) # eliminate inapplicable images for i in range(len(infile_list)): if obs_dict['PROPID'][i].upper().strip()!=propcode or int(obs_dict['BS-STATE'][i][1])!=2: del infile_list[i] obs_dict=obslog(infile_list) # use arc to make first-guess wavecal from model, locate beamsplitter split point for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip()=='ARC' : arcidx = i rbin,cbin = np.array(obs_dict["CCDSUM"][arcidx].split(" ")).astype(int) grating = obs_dict['GRATING'][arcidx].strip() grang = float(obs_dict['GR-ANGLE'][arcidx]) artic = float(obs_dict['CAMANG'][arcidx]) hduarc = pyfits.open(infile_list[arcidx]) arc_rc = hduarc['SCI'].data rows,cols = arc_rc.shape lam_c = rssmodelwave(grating,grang,artic,cbin,cols) arc_r = arc_rc.sum(axis=1) lam_m = np.loadtxt(datadir+"wollaston.txt",dtype=float,usecols=(0,)) rpix_om = np.loadtxt(datadir+"wollaston.txt",dtype=float,unpack=True,usecols=(1,2)) expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \ (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int) foundrow_o = np.zeros((2),dtype=int) for o in (0,1): foundrow_o[o] = expectrow_o[o]-100/rbin \ + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin]) arcsignal = arc_r[foundrow_o[o]] botedge = foundrow_o[o] + np.argmax(arc_r[foundrow_o[o]:] < arcsignal/2.) topedge = foundrow_o[o] - np.argmax(arc_r[foundrow_o[o]::-1] < arcsignal/2.) foundrow_o[o] = (botedge+topedge)/2. splitrow = foundrow_o.mean() offset = int(splitrow - rows/2) log.message('Split Row: '+("%4i " % splitrow), with_header=False) # split arc into o/e images padbins = (np.indices((rows,cols))[0]<offset) | (np.indices((rows,cols))[0]>rows+offset) arc_rc = np.roll(arc_rc,-offset,axis=0) arc_rc[padbins] = 0. arc_orc = arc_rc.reshape((2,rows/2,cols)) # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map lamp=obs_dict['LAMPID'][arcidx].strip().replace(' ', '') if lamp == 'NONE': lamp='CuAr' hduarc[0].header.update('MASKTYP','LONGSLIT') hduarc[0].header.update('MASKID','PL0100N001') del hduarc['VAR'] del hduarc['BPM'] lampfile=iraf.osfn("pysalt$data/linelists/%s.wav" % lamp) rpix_oc = interp1d(lam_m, rpix_om,kind ='cubic')(lam_c) log.message('\nARC: image '+infile_list[i].split('.')[0][-4:]+' GRATING '+grating\ +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False) wavmap_orc = np.zeros_like(arc_orc) for o in (0,1): foundrow_o[o] += -offset + o*rows/2 arc_Rc = np.zeros((rows/2,cols),dtype='float32') for c in range(cols): shift(arc_orc[o,:,c], \ -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c]) hduarc['SCI'].data = arc_Rc arcimage = "arc_"+str(o)+".fits" hduarc.writeto(arcimage,clobber=True) Rstart = foundrow_o[o] order = 3 dbfilename = "arcdb_"+str(o)+".txt" if os.path.isfile(dbfilename): # guessfile = "guess_"+str(o)+".txt" # os.rename(dbfilename,guessfile) guesstype = 'file' else: guessfile = '' guesstype = 'rss' specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=order, rstep=10, rstart=Rstart, mdiff=20, thresh=3, niter=5, smooth=3, inter=inter, clobber=True, logfile=logfile, verbose=True) wavlegR_y = np.loadtxt(dbfilename,dtype=float,usecols=(0,)) wavlegcof_ly = np.loadtxt(dbfilename,unpack=True,dtype=float,usecols=range(1,order+2)) wavlegcof_ly = np.delete(wavlegcof_ly,np.where(wavlegR_y<0)[0],axis=1) wavlegR_y = np.delete(wavlegR_y,np.where(wavlegR_y<0)[0],axis=0) wavmap_yc = np.polynomial.legendre.legval(np.arange(cols),wavlegcof_ly) mediancof_l = np.median(wavlegcof_ly,axis=1) rms_l = np.sqrt(np.median((wavlegcof_ly - mediancof_l[:,None])**2,axis=1)) sigma_ly = (wavlegcof_ly - mediancof_l[:,None])/rms_l[:,None] usey = (sigma_ly[0]<3) & (sigma_ly[1]<3) wavmap_Rc = np.zeros((rows/2,cols),dtype='float32') R_y = wavlegR_y[usey] a = np.vstack((R_y**3,R_y**2,R_y,np.ones_like(R_y))).T for c in range(cols): polycofs = la.lstsq(a,wavmap_yc[usey,c])[0] wavmap_orc[o,:,c] \ = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin) # save split data along third fits axis, add wavmap extension, save as 'w' file hduwav = pyfits.ImageHDU(data=wavmap_orc, header=hduarc['SCI'].header, name='WAV') for i in range(len(infile_list)): hdu = pyfits.open(infile_list[i]) image_rc = np.roll(hdu['SCI'].data,-offset,axis=0) image_rc[padbins] = 0. hdu['SCI'].data = image_rc.reshape((2,rows/2,cols)) var_rc = np.roll(hdu['VAR'].data,-offset,axis=0) var_rc[padbins] = 0. hdu['VAR'].data = var_rc.reshape((2,rows/2,cols)) bpm_rc = np.roll(hdu['BPM'].data,-offset,axis=0) bpm_rc[padbins] = 1 hdu['BPM'].data = bpm_rc.reshape((2,rows/2,cols)) hdu.append(hduwav) for f in ('SCI','VAR','BPM','WAV'): hdu[f].header.update('CTYPE3','O,E') hdu.writeto('w'+infile_list[i],clobber='True') log.message('Output file '+'w'+infile_list[i] , with_header=False)
def specpolmap(infile_list, propcode=None, inter=True, automethod='Matchlines', logfile='salt.log'): obsdate = os.path.basename(infile_list[0])[7:15] with logging(logfile, debug) as log: #create the observation log obs_dict = obslog(infile_list) # eliminate inapplicable images for i in range(len(infile_list)): if obs_dict['PROPID'][i].upper().strip() != propcode or int( obs_dict['BS-STATE'][i][1]) != 2: del infile_list[i] obs_dict = obslog(infile_list) # use arc to make first-guess wavecal from model, locate beamsplitter split point for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip() == 'ARC': arcidx = i rbin, cbin = np.array( obs_dict["CCDSUM"][arcidx].split(" ")).astype(int) grating = obs_dict['GRATING'][arcidx].strip() grang = float(obs_dict['GR-ANGLE'][arcidx]) artic = float(obs_dict['CAMANG'][arcidx]) hduarc = pyfits.open(infile_list[arcidx]) arc_rc = hduarc['SCI'].data rows, cols = arc_rc.shape lam_c = rssmodelwave(grating, grang, artic, cbin, cols) arc_r = arc_rc.sum(axis=1) lam_m = np.loadtxt(datadir + "wollaston.txt", dtype=float, usecols=(0, )) rpix_om = np.loadtxt(datadir + "wollaston.txt", dtype=float, unpack=True, usecols=(1, 2)) expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \ (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int) foundrow_o = np.zeros((2), dtype=int) for o in (0, 1): foundrow_o[o] = expectrow_o[o]-100/rbin \ + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin]) arcsignal = arc_r[foundrow_o[o]] botedge = foundrow_o[o] + np.argmax( arc_r[foundrow_o[o]:] < arcsignal / 2.) topedge = foundrow_o[o] - np.argmax( arc_r[foundrow_o[o]::-1] < arcsignal / 2.) foundrow_o[o] = (botedge + topedge) / 2. splitrow = foundrow_o.mean() offset = int(splitrow - rows / 2) log.message('Split Row: ' + ("%4i " % splitrow), with_header=False) # split arc into o/e images padbins = (np.indices((rows, cols))[0] < offset) | (np.indices( (rows, cols))[0] > rows + offset) arc_rc = np.roll(arc_rc, -offset, axis=0) arc_rc[padbins] = 0. arc_orc = arc_rc.reshape((2, rows / 2, cols)) # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map lamp = obs_dict['LAMPID'][arcidx].strip().replace(' ', '') if lamp == 'NONE': lamp = 'CuAr' hduarc[0].header.update('MASKTYP', 'LONGSLIT') hduarc[0].header.update('MASKID', 'PL0100N001') del hduarc['VAR'] del hduarc['BPM'] lampfile = iraf.osfn("pysalt$data/linelists/%s.wav" % lamp) rpix_oc = interp1d(lam_m, rpix_om, kind='cubic')(lam_c) log.message('\nARC: image '+infile_list[i].split('.')[0][-4:]+' GRATING '+grating\ +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False) wavmap_orc = np.zeros_like(arc_orc) for o in (0, 1): foundrow_o[o] += -offset + o * rows / 2 arc_Rc = np.zeros((rows / 2, cols), dtype='float32') for c in range(cols): shift(arc_orc[o,:,c], \ -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c]) hduarc['SCI'].data = arc_Rc arcimage = "arc_" + str(o) + ".fits" hduarc.writeto(arcimage, clobber=True) Rstart = foundrow_o[o] order = 3 dbfilename = "arcdb_" + str(o) + ".txt" if os.path.isfile(dbfilename): # guessfile = "guess_"+str(o)+".txt" # os.rename(dbfilename,guessfile) guesstype = 'file' else: guessfile = '' guesstype = 'rss' specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=order, rstep=10, rstart=Rstart, mdiff=20, thresh=3, niter=5, smooth=3, inter=inter, clobber=True, logfile=logfile, verbose=True) wavlegR_y = np.loadtxt(dbfilename, dtype=float, usecols=(0, )) wavlegcof_ly = np.loadtxt(dbfilename, unpack=True, dtype=float, usecols=range(1, order + 2)) wavlegcof_ly = np.delete(wavlegcof_ly, np.where(wavlegR_y < 0)[0], axis=1) wavlegR_y = np.delete(wavlegR_y, np.where(wavlegR_y < 0)[0], axis=0) wavmap_yc = np.polynomial.legendre.legval(np.arange(cols), wavlegcof_ly) mediancof_l = np.median(wavlegcof_ly, axis=1) rms_l = np.sqrt( np.median((wavlegcof_ly - mediancof_l[:, None])**2, axis=1)) sigma_ly = (wavlegcof_ly - mediancof_l[:, None]) / rms_l[:, None] usey = (sigma_ly[0] < 3) & (sigma_ly[1] < 3) wavmap_Rc = np.zeros((rows / 2, cols), dtype='float32') R_y = wavlegR_y[usey] a = np.vstack((R_y**3, R_y**2, R_y, np.ones_like(R_y))).T for c in range(cols): polycofs = la.lstsq(a, wavmap_yc[usey, c])[0] wavmap_orc[o,:,c] \ = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin) # save split data along third fits axis, add wavmap extension, save as 'w' file hduwav = pyfits.ImageHDU(data=wavmap_orc, header=hduarc['SCI'].header, name='WAV') for i in range(len(infile_list)): hdu = pyfits.open(infile_list[i]) image_rc = np.roll(hdu['SCI'].data, -offset, axis=0) image_rc[padbins] = 0. hdu['SCI'].data = image_rc.reshape((2, rows / 2, cols)) var_rc = np.roll(hdu['VAR'].data, -offset, axis=0) var_rc[padbins] = 0. hdu['VAR'].data = var_rc.reshape((2, rows / 2, cols)) bpm_rc = np.roll(hdu['BPM'].data, -offset, axis=0) bpm_rc[padbins] = 1 hdu['BPM'].data = bpm_rc.reshape((2, rows / 2, cols)) hdu.append(hduwav) for f in ('SCI', 'VAR', 'BPM', 'WAV'): hdu[f].header.update('CTYPE3', 'O,E') hdu.writeto('w' + infile_list[i], clobber='True') log.message('Output file ' + 'w' + infile_list[i], with_header=False)
def science_red(rawdir, prodir, imreduce=True, specreduce=True, bpmfile=None, calfile=None, lampfile='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True): print rawdir print prodir #get the name of the files infile_list=glob.glob(rawdir+'P*.fits') infiles=','.join(['%s' % x for x in infile_list]) #get the current date for the files obsdate=os.path.basename(infile_list[0])[1:9] print obsdate #set up some files that will be needed logfile='spec'+obsdate+'.log' flatimage='FLAT%s.fits' % (obsdate) dbfile='spec%s.db' % obsdate #create the observation log obs_dict=obslog(infile_list) if imreduce: #prepare the data saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True) #bias subtract the data saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='', median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0, niter=10, plotover=False, turbo=False, clobber=True, logfile=logfile, verbose=True) add_variance('bpP*fits', bpmfile) #gain correct the data saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True) #cross talk correct the data saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True) #flat field correct the data flat_imgs='' for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('FLAT'): if flat_imgs: flat_imgs += ',' flat_imgs += 'xgbp'+os.path.basename(infile_list[i]) if 0: #len(flat_imgs)!=0: saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \ weight=False, blank=0, scale=None, statsec='[200:300, 600:800]', lthresh=3, \ hthresh=3, clobber=True, logfile=logfile, verbose=True) saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True) saltflat('xgbpP*fits', '', 'f', flatimage, minflat=0.8, allext=False, clobber=True, logfile=logfile, verbose=True) else: flats=None imfiles=glob.glob('xgbpP*fits') for f in imfiles: shutil.copy(f, 'f'+f) #cosmic ray clean the data #only clean the object data for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'): img='fxgbp'+os.path.basename(infile_list[i]) saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0, flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2, gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True) #mosaic the data geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat") saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True) #clean up the images if cleanup: for f in glob.glob('p*fits'): os.remove(f) for f in glob.glob('bp*fits'): os.remove(f) for f in glob.glob('gbp*fits'): os.remove(f) for f in glob.glob('xgbp*fits'): os.remove(f) for f in glob.glob('fxgbp*fits'): os.remove(f) #set up the name of the images if specreduce: for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip()=='ARC': lamp=obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage='mfxgbp'+os.path.basename(infile_list[i]) #lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod=automethod, function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=50, thresh=2, niter=5, smooth=3, inter=True, clobber=True, logfile=logfile, verbose=True) specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages='' for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'): if objimages: objimages += ',' objimages+='mfxgbp'+os.path.basename(infile_list[i]) if specreduce: #run specidentify on the arc files specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) return
def specpolwavmap(infilelist, linelistlib="", automethod='Matchlines',logfile='salt.log',debug=False): obsdate=os.path.basename(infilelist[0])[7:15] with logging(logfile, debug) as log: # create the observation log obs_dict=obslog(infilelist) log.message('Pysalt Version: '+pysalt.verno, with_header=False) # eliminate inapplicable images for i in reversed(range(len(infilelist))): if int(obs_dict['BS-STATE'][i][1])!=2: del infilelist[i] obs_dict=obslog(infilelist) # Map out which arc goes with which image. Use arc in closest wavcal block of the config. # wavcal block: neither spectrograph config nor track changes, and no gap in data files infiles = len(infilelist) newtrk = 5. # new track when rotator changes by more (deg) trkrho_i = np.array(map(float,obs_dict['TRKRHO'])) trkno_i = np.zeros((infiles),dtype=int) trkno_i[1:] = ((np.abs(trkrho_i[1:]-trkrho_i[:-1]))>newtrk).cumsum() confno_i,confdatlist = configmap(infilelist) configs = len(confdatlist) imageno_i = np.array([int(os.path.basename(infilelist[i]).split('.')[0][-4:]) \ for i in range(infiles)]) filegrp_i = np.zeros((infiles),dtype=int) filegrp_i[1:] = ((imageno_i[1:]-imageno_i[:-1])>1).cumsum() isarc_i = np.array([(obs_dict['OBJECT'][i].upper().strip()=='ARC') for i in range(infiles)]) wavblk_i = np.zeros((infiles),dtype=int) wavblk_i[1:] = ((filegrp_i[1:] != filegrp_i[:-1]) \ | (trkno_i[1:] != trkno_i[:-1]) \ | (confno_i[1:] != confno_i[:-1])).cumsum() wavblks = wavblk_i.max() +1 arcs_c = (isarc_i[:,None] & (confno_i[:,None]==range(configs))).sum(axis=0) np.savetxt("wavblktbl.txt",np.vstack((trkrho_i,imageno_i,filegrp_i,trkno_i, \ confno_i,wavblk_i,isarc_i)).T,fmt="%7.2f "+6*"%3i ",header=" rho img grp trk conf wblk arc") for c in range(configs): # worst: no arc for config, remove images if arcs_c[c] == 0: lostimages = imageno_i[confno_i==c] log.message('No Arc for this configuration: ' \ +("Grating %s Grang %6.2f Artic %6.2f" % confdatlist[c]) \ +("\n Images: "+lostimages.shape[0]*"%i " % tuple(lostimages)), with_header=False) wavblk_i[confno_i==c] = -1 if arcs_c.sum() ==0: log.message("Cannot calibrate any images", with_header=False) exit() iarc_i = -np.zeros((infiles),dtype=int) for w in range(wavblks): blkimages = imageno_i[wavblk_i==w] if blkimages.shape[0]==0: continue iarc_I = np.where((wavblk_i==w) & (isarc_i))[0] if iarc_I.shape[0] >0: iarc = iarc_I[0] # best: arc is in wavblk, take first else: conf = confno_i[wavblk_i==w][0] # fallback: take closest arc of this config iarc_I = np.where((confno_i==conf) & (isarc_i))[0] blkimagepos = blkimages.mean() iarc = iarc_I[np.argmin(imageno_i[iarc_I] - blkimagepos)] iarc_i[wavblk_i==w] = iarc log.message(("\nFor images: "+blkimages.shape[0]*"%i " % tuple(blkimages)) \ +("\n Use Arc %5i" % imageno_i[iarc]), with_header=False) iarc_a = np.unique(iarc_i[iarc_i != -1]) arcs = iarc_a.shape[0] lam_m = np.loadtxt(datadir+"wollaston.txt",dtype=float,usecols=(0,)) rpix_om = np.loadtxt(datadir+"wollaston.txt",dtype=float,unpack=True,usecols=(1,2)) for a in range(arcs): iarc = iarc_a[a] conf = confno_i[iarc] grating,grang,artic = confdatlist[confno_i[iarc]] if len(linelistlib) ==0: linelistlib=datadir+"linelistlib.txt" if grating=="PG0300": linelistlib=datadir+"linelistlib_300.txt" with open(linelistlib) as fd: linelistdict = dict(line.strip().split(None, 1) for line in fd) # use arc to make first-guess wavecal from model cbin,rbin = np.array(obs_dict["CCDSUM"][iarc].split(" ")).astype(int) hduarc = pyfits.open(infilelist[iarc]) arc_rc = hduarc['SCI'].data rows,cols = arc_rc.shape lam_c = rssmodelwave(grating,grang,artic,cbin,cols) arc_r = arc_rc.sum(axis=1) # if debug: np.savetxt("arc_r_"+str(imageno_i[iarc]+".txt"),arc_r,fmt="%8.2f") # locate beamsplitter split point axisrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic',bounds_error=False) \ (lam_c[cols/2]))/rbin).astype(int) top = axisrow_o[1] + np.argmax(arc_r[axisrow_o[1]:] < 0.5*arc_r[axisrow_o[1]]) bot = axisrow_o[0] - np.argmax(arc_r[axisrow_o[0]::-1] < 0.5*arc_r[axisrow_o[0]]) splitrow = 0.5*(bot + top) offset = int(splitrow - rows/2) # how far split is from center of detector # split arc into o/e images padbins = (np.indices((rows,cols))[0]<offset) | (np.indices((rows,cols))[0]>rows+offset) arc_rc = np.roll(arc_rc,-offset,axis=0) arc_rc[padbins] = 0. arc_orc = arc_rc.reshape((2,rows/2,cols)) # for O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map lamp=obs_dict['LAMPID'][iarc].strip().replace(' ', '') if lamp == 'NONE': lamp='CuAr' hduarc[0].header.update('MASKTYP','LONGSLIT') del hduarc['VAR'] del hduarc['BPM'] lampfile=iraf.osfn("pysalt$data/linelists/"+linelistdict[lamp]) rpix_oc = interp1d(lam_m, rpix_om,kind ='cubic',bounds_error=False,fill_value=0.)(lam_c) drow_oc = (rpix_oc-rpix_oc[:,cols/2][:,None])/rbin log.message('\nARC: image '+str(imageno_i[iarc])+' GRATING '+grating\ +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False) log.message(' Split Row: '+("%4i " % splitrow), with_header=False) wavmap_orc = np.zeros((2,rows/2,cols)) edgerow_od = np.zeros((2,2)) cofrows_o = np.zeros(2) legy_od = np.zeros((2,2)) guessfile=None for o in (0,1): axisrow_o[o] += -offset - o*rows/2 arc_yc = np.zeros((rows/2,cols),dtype='float32') for c in range(cols): shift(arc_orc[o,:,c],-drow_oc[o,c],arc_yc[:,c]) maxoverlaprows = 34/rbin # beam overlap for 4' longslit in NIR arc_yc[(0,rows/2-1)] = 0. arc_y = arc_yc.sum(axis=1) edgerow_od[o,0] = axisrow_o[o] - np.argmax(arc_y[axisrow_o[o]::-1] < 0.5*arc_y[axisrow_o[o]]) edgerow_od[o,1] = axisrow_o[o] + np.argmax(arc_y[axisrow_o[o]:] < 0.5*arc_y[axisrow_o[o]]) axisrow_o[o] = edgerow_od[o].mean() if np.abs(edgerow_od[o] - np.array([0,rows/2-1])).min() < maxoverlaprows: edgerow_od[o] += maxoverlaprows*np.array([+1,-1]) hduarc['SCI'].data = arc_yc order = 3 arcimage = "arc_"+str(imageno_i[iarc])+"_"+str(o)+".fits" dbfilename = "arcdb_"+str(imageno_i[iarc])+"_"+str(o)+".txt" if (not os.path.exists(dbfilename)): if guessfile is not None: guesstype = 'file' else: guessfile=dbfilename guesstype = 'rss' hduarc.writeto(arcimage,clobber=True) ystart = axisrow_o[o] specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=order, rstep=20, rstart=ystart, mdiff=20, thresh=3, niter=5, smooth=3, inter=True, clobber=True, logfile=logfile, verbose=True) if (not debug): os.remove(arcimage) # process dbfile legendre coefs within FOV into wavmap (_Y = line in dbfile) legy_Y = np.loadtxt(dbfilename,dtype=float,usecols=(0,),ndmin=1) dblegcof_lY = np.loadtxt(dbfilename,unpack=True,dtype=float,usecols=range(1,order+2),ndmin=2) # first convert to centered legendre coefficients to remove crosscoupling xcenter = cols/2. legcof_lY = np.zeros_like(dblegcof_lY) legcof_lY[2] = dblegcof_lY[2] + 5.*dblegcof_lY[3]*xcenter legcof_lY[3] = dblegcof_lY[3] legcof_lY[0] = 0.5*legcof_lY[2] + (dblegcof_lY[0]-dblegcof_lY[2]) + \ (dblegcof_lY[1]-1.5*dblegcof_lY[3])*xcenter + 1.5*dblegcof_lY[2]*xcenter**2 + \ 2.5*dblegcof_lY[3]*xcenter**3 legcof_lY[1] = 1.5*legcof_lY[3] + (dblegcof_lY[1]-1.5*dblegcof_lY[3]) + \ 3.*dblegcof_lY[2]*xcenter + 7.5*dblegcof_lY[3]*xcenter**2 # remove rows outside slit and outlier fits argYbad = np.where((legy_Y<edgerow_od[o,0]) | (legy_Y>edgerow_od[o,1]))[0] legy_Y = np.delete(legy_Y, argYbad,axis=0) legcof_lY = np.delete(legcof_lY, argYbad,axis=1) mediancof_l = np.median(legcof_lY,axis=1) rms_l = np.sqrt(np.median((legcof_lY - mediancof_l[:,None])**2,axis=1)) sigma_lY = np.abs((legcof_lY - mediancof_l[:,None]))/rms_l[:,None] argYbad = np.where((sigma_lY>4).any(axis=0))[0] legy_Y = np.delete(legy_Y, argYbad,axis=0) legcof_lY = np.delete(legcof_lY, argYbad,axis=1) cofrows_o[o] = legy_Y.shape[0] legy_od[o] = legy_Y.min(),legy_Y.max() if cofrows_o[o] < 5: # for future: if few lines in db, use model shifted to agree, for now just use mean legcof_l = legcof_lY.mean(axis=1) wavmap_yc = np.polynomial.legendre.legval(np.arange(cols),legcof_l) else: # smooth wavmap along rows by fitting L_0 to quadratic, others to linear fn of row ycenter = rows/4. Y_y = np.arange(-ycenter,ycenter) aa = np.vstack(((legy_Y-ycenter)**2,(legy_Y-ycenter),np.ones(cofrows_o[o]))).T polycofs = la.lstsq(aa,legcof_lY[0])[0] legcof_ly = np.zeros((order+1,rows/2)) legcof_ly[0] = np.polyval(polycofs,Y_y) for l in range(1,order+1): polycofs = la.lstsq(aa[:,1:],legcof_lY[l])[0] legcof_ly[l] = np.polyval(polycofs,Y_y) wavmap_yc = np.zeros((rows/2,cols)) for y in range(rows/2): wavmap_yc[y] = np.polynomial.legendre.legval(np.arange(-cols/2,cols/2),legcof_ly[:,y]) # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm) if debug: np.savetxt("drow_wmap_oc.txt",drow_oc.T,fmt="%8.3f %8.3f") for c in range(cols): shift(wavmap_yc[:,c],drow_oc[o,c],wavmap_orc[o,:,c],order=1) isoffslit_rc = ((np.arange(rows/2)[:,None] < (edgerow_od[o,0]+(rpix_oc[o]-rpix_oc[o,cols/2])/rbin)[None,:]) \ | (np.arange(rows/2)[:,None] > (edgerow_od[o,1]+(rpix_oc[o]-rpix_oc[o,cols/2])/rbin)[None,:])) notwav_rc = (rpix_oc[o]==0.)[None,:] wavmap_orc[o,(isoffslit_rc | notwav_rc)] = 0. log.message('\n Wavl coeff rows: O %4i E %4i' % tuple(cofrows_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i' \ % tuple(legy_od.flatten()), with_header=False) log.message('\n Slit axis row: O %4i E %4i' % tuple(axisrow_o), with_header=False) log.message(' Bottom, top row: O %4i %4i E %4i %4i \n' \ % tuple(edgerow_od.flatten()), with_header=False) # for images using this arc,save split data along third fits axis, # add wavmap extension, save as 'w' file hduwav = pyfits.ImageHDU(data=wavmap_orc.astype('float32'), header=hduarc['SCI'].header, name='WAV') for i in np.where(iarc_i==iarc_a[a])[0]: hdu = pyfits.open(infilelist[i]) image_rc = np.roll(hdu['SCI'].data,-offset,axis=0) image_rc[padbins] = 0. hdu['SCI'].data = image_rc.reshape((2,rows/2,cols)) var_rc = np.roll(hdu['VAR'].data,-offset,axis=0) var_rc[padbins] = 0. hdu['VAR'].data = var_rc.reshape((2,rows/2,cols)) bpm_rc = np.roll(hdu['BPM'].data,-offset,axis=0) bpm_rc[padbins] = 1 bpm_orc = bpm_rc.reshape((2,rows/2,cols)) bpm_orc[wavmap_orc==0.] = 1 hdu['BPM'].data = bpm_orc hdu.append(hduwav) for f in ('SCI','VAR','BPM','WAV'): hdu[f].header.update('CTYPE3','O,E') hdu.writeto('w'+infilelist[i],clobber='True') log.message('Output file '+'w'+infilelist[i] , with_header=False) return
def specpolwavmap(infilelist, linelistlib="", inter=True, automethod='Matchlines',logfile='salt.log'): obsdate=os.path.basename(infilelist[0])[7:15] with logging(logfile, debug) as log: # create the observation log obs_dict=obslog(infilelist) log.message('Pysalt Version: '+pysalt.verno, with_header=False) if len(linelistlib) ==0: linelistlib=datadir+"linelistlib.txt" with open(linelistlib) as fd: linelistdict = dict(line.strip().split(None, 1) for line in fd) # eliminate inapplicable images for i in range(len(infilelist)): if int(obs_dict['BS-STATE'][i][1])!=2: del infilelist[i] obs_dict=obslog(infilelist) # Map out which arc goes with which image. Use arc in closest wavcal block of the config. # wavcal block: neither spectrograph config nor track changes, and no gap in data files infiles = len(infilelist) newtrk = 5. # new track when rotator changes by more (deg) trkrho_i = np.array(map(float,obs_dict['TRKRHO'])) trkno_i = np.zeros((infiles),dtype=int) trkno_i[1:] = ((np.abs(trkrho_i[1:]-trkrho_i[:-1]))>newtrk).cumsum() confno_i,confdatlist = configmap(infilelist) configs = len(confdatlist) imageno_i = np.array([int(os.path.basename(infilelist[i]).split('.')[0][-4:]) \ for i in range(infiles)]) filegrp_i = np.zeros((infiles),dtype=int) filegrp_i[1:] = ((imageno_i[1:]-imageno_i[:-1])>1).cumsum() isarc_i = np.array([(obs_dict['OBJECT'][i].upper().strip()=='ARC') for i in range(infiles)]) wavblk_i = np.zeros((infiles),dtype=int) wavblk_i[1:] = ((filegrp_i[1:] != filegrp_i[:-1]) \ | (trkno_i[1:] != trkno_i[:-1]) \ | (confno_i[1:] != confno_i[:-1])).cumsum() wavblks = wavblk_i.max() +1 arcs_c = (isarc_i[:,None] & (confno_i[:,None]==range(configs))).sum(axis=0) np.savetxt("wavblktbl.txt",np.vstack((trkrho_i,imageno_i,filegrp_i,trkno_i, \ confno_i,wavblk_i,isarc_i)).T,fmt="%7.2f "+6*"%3i ",header=" rho img grp trk conf wblk arc") for c in range(configs): # worst: no arc for config, remove images if arcs_c[c] == 0: lostimages = imageno_i[confno_i==c] log.message('No Arc for this configuration: ' \ +("Grating %s Grang %6.2f Artic %6.2f" % confdatlist[c]) \ +("\n Images: "+lostimages.shape[0]*"%i " % tuple(lostimages)), with_header=False) wavblk_i[confno_i==c] = -1 if arcs_c.sum() ==0: log.message("Cannot calibrate any images", with_header=False) exit() iarc_i = -np.zeros((infiles),dtype=int) for w in range(wavblks): blkimages = imageno_i[wavblk_i==w] if blkimages.shape[0]==0: continue iarc_I = np.where((wavblk_i==w) & (isarc_i))[0] if iarc_I.shape[0] >0: iarc = iarc_I[0] # best: arc is in wavblk, take first else: conf = confno_i[wavblk_i==w][0] # fallback: take closest arc of this config iarc_I = np.where((confno_i==conf) & (isarc_i))[0] blkimagepos = blkimages.mean() iarc = iarc_I[np.argmin(imageno_i[iarc_I] - blkimagepos)] iarc_i[wavblk_i==w] = iarc log.message(("For images: "+blkimages.shape[0]*"%i " % tuple(blkimages)) \ +("\n Use Arc %5i" % imageno_i[iarc]), with_header=False) iarc_a = np.unique(iarc_i[iarc_i != -1]) arcs = iarc_a.shape[0] lam_m = np.loadtxt(datadir+"wollaston.txt",dtype=float,usecols=(0,)) rpix_om = np.loadtxt(datadir+"wollaston.txt",dtype=float,unpack=True,usecols=(1,2)) for a in range(arcs): iarc = iarc_a[a] conf = confno_i[iarc] # use arc to make first-guess wavecal from model, locate beamsplitter split point cbin,rbin = np.array(obs_dict["CCDSUM"][iarc].split(" ")).astype(int) grating,grang,artic = confdatlist[confno_i[iarc]] hduarc = pyfits.open(infilelist[iarc]) arc_rc = hduarc['SCI'].data rows,cols = arc_rc.shape lam_c = rssmodelwave(grating,grang,artic,cbin,cols) arc_r = arc_rc.sum(axis=1) expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \ (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int) foundrow_o = np.zeros((2),dtype=int) for o in (0,1): foundrow_o[o] = expectrow_o[o]-100/rbin \ + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin]) arcsignal = arc_r[foundrow_o[o]] topedge = foundrow_o[o] + np.argmax(arc_r[foundrow_o[o]:] < 0.75*arcsignal) botedge = foundrow_o[o] - np.argmax(arc_r[foundrow_o[o]::-1] < 0.75*arcsignal) foundrow_o[o] = (botedge+topedge)/2. splitrow = foundrow_o.mean() offset = int(splitrow - rows/2) # split arc into o/e images padbins = (np.indices((rows,cols))[0]<offset) | (np.indices((rows,cols))[0]>rows+offset) arc_rc = np.roll(arc_rc,-offset,axis=0) arc_rc[padbins] = 0. arc_orc = arc_rc.reshape((2,rows/2,cols)) # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map lamp=obs_dict['LAMPID'][iarc].strip().replace(' ', '') if lamp == 'NONE': lamp='CuAr' hduarc[0].header.update('MASKTYP','LONGSLIT') hduarc[0].header.update('MASKID','PL0100N001') del hduarc['VAR'] del hduarc['BPM'] # lampfile=iraf.osfn("pysalt$data/linelists/%s.wav" % lamp) zsalt version: not good lampfile=iraf.osfn("pysalt$data/linelists/"+linelistdict[lamp]) rpix_oc = interp1d(lam_m, rpix_om,kind ='cubic')(lam_c) log.message('\nARC: image '+str(imageno_i[iarc])+' GRATING '+grating\ +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False) log.message(' Split Row: '+("%4i " % splitrow), with_header=False) wavmap_orc = np.zeros_like(arc_orc) for o in (0,1): foundrow_o[o] += -offset - o*rows/2 arc_Rc = np.zeros((rows/2,cols),dtype='float32') for c in range(cols): shift(arc_orc[o,:,c], \ -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c]) hduarc['SCI'].data = arc_Rc arcimage = "arc_"+str(imageno_i[iarc])+"_"+str(o)+".fits" dbfilename = "arcdb_"+str(imageno_i[iarc])+"_"+str(o)+".txt" order = 3 if os.path.isfile(dbfilename): # guessfile = "guess_"+str(o)+".txt" # os.rename(dbfilename,guessfile) guesstype = 'file' else: hduarc.writeto(arcimage,clobber=True) guesstype = 'rss' guessfile = '' Rstart = foundrow_o[o] specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype, guessfile=guessfile, automethod=automethod, function='legendre', order=order, rstep=20, rstart=Rstart, mdiff=20, thresh=3, niter=5, smooth=3, inter=inter, clobber=True, logfile=logfile, verbose=True) wavlegR_y = np.loadtxt(dbfilename,dtype=float,usecols=(0,),ndmin=1) wavlegcof_ly = np.loadtxt(dbfilename,unpack=True,dtype=float,usecols=range(1,order+2),ndmin=2) if wavlegR_y.min()<0: wavlegcof_ly = np.delete(wavlegcof_ly,np.where(wavlegR_y<0)[0],axis=1) wavlegR_y = np.delete(wavlegR_y,np.where(wavlegR_y<0)[0],axis=0) wavmap_yc = np.polynomial.legendre.legval(np.arange(cols),wavlegcof_ly) mediancof_l = np.median(wavlegcof_ly,axis=1) rms_l = np.sqrt(np.median((wavlegcof_ly - mediancof_l[:,None])**2,axis=1)) sigma_ly = (wavlegcof_ly - mediancof_l[:,None])/rms_l[:,None] usey = (sigma_ly[0]<3) & (sigma_ly[1]<3) wavmap_Rc = np.zeros((rows/2,cols),dtype='float32') R_y = wavlegR_y[usey] if usey.shape[0] < 5: # for future: if few lines in db, use model shifted to agree, for now just use first wavmap_orc[o] = wavmap_yc[0] else: if usey.shape[0] > 9: aa = np.vstack((R_y**3,R_y**2,R_y,np.ones_like(R_y))).T else: aa = np.vstack((R_y**2,R_y,np.ones_like(R_y))).T for c in range(cols): polycofs = la.lstsq(aa,wavmap_yc[usey,c])[0] wavmap_orc[o,:,c] \ = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin) # os.remove(arcimage) # for images using this arc,save split data along third fits axis, # add wavmap extension, save as 'w' file hduwav = pyfits.ImageHDU(data=wavmap_orc, header=hduarc['SCI'].header, name='WAV') for i in np.where(iarc_i==iarc_a[a])[0]: hdu = pyfits.open(infilelist[i]) image_rc = np.roll(hdu['SCI'].data,-offset,axis=0) image_rc[padbins] = 0. hdu['SCI'].data = image_rc.reshape((2,rows/2,cols)) var_rc = np.roll(hdu['VAR'].data,-offset,axis=0) var_rc[padbins] = 0. hdu['VAR'].data = var_rc.reshape((2,rows/2,cols)) bpm_rc = np.roll(hdu['BPM'].data,-offset,axis=0) bpm_rc[padbins] = 1 hdu['BPM'].data = bpm_rc.reshape((2,rows/2,cols)) hdu.append(hduwav) for f in ('SCI','VAR','BPM','WAV'): hdu[f].header.update('CTYPE3','O,E') hdu.writeto('w'+infilelist[i],clobber='True') log.message('Output file '+'w'+infilelist[i] , with_header=False) return
import sys from pyraf import iraf from iraf import pysalt from specidentify import specidentify specidentify(images=sys.argv[1], linelist=sys.argv[2], outfile='wav.db', guesstype='rss', guessfile='', automethod='Matchlines', function='poly', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=True, startext=0 ,clobber=True, textcolor='black', logfile='salt.log', verbose=True)
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, geomfile=None,subover=True,trim=True,masbias=None, subbias=False, median=False, function='polynomial', order=5,rej_lo=3, rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='', clobber=False, cleanup=True, logfile='salt.log', verbose=True): """SALTADVANCE provides advanced data reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover=False #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) infiles.sort() # create list of output files outpath=saltio.abspath(outpath) #log into the database sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #does the gain database file exist if gaindb: dblist= saltio.readgaindb(gaindb) else: dblist=[] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') obsmode=obsstruct.data.field('OBSMODE') ccdtype=obsstruct.data.field('CCDTYPE') propcode=obsstruct.data.field('PROPID') masktype=obsstruct.data.field('MASKTYP') #set the bias list of objects biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')] masterbias_dict={} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=fits.open(img) bimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist=filename[ccdtype=='FLAT'] masterflat_dict={} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct=fits.open(img) fimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys=masterflat_dict[i][0] flist=masterflat_dict[i][1:] mflatname=outpath+createmasterflatname(flist, fkeys) ffiles=','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the arc data arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')] for i, img in enumerate(infiles): nimg=os.path.basename(img) if nimg in arclist: #open the image struct=fits.open(img) simg=outpath+'bxgp'+os.path.basename(img) obsdate=os.path.basename(img)[1:9] #print the message if log: message='Processing ARC frame %s' % img log.message(message, with_stdout=verbose) struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the images mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #measure the arcdata arcimage=outpath+'mbxgp'+nimg dbfile=outpath+obsdate+'_specid.db' lamp = obsstruct.data.field('LAMPID')[i] lamp = lamp.replace(' ', '') lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) print arcimage, lampfile, os.getcwd() specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod='Matchlines', function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, verbose=verbose) try: ximg = outpath+'xmbxgp'+os.path.basename(arcimage) specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except: pass #process the science data for i, img in enumerate(infiles): nimg=os.path.basename(img) if not (nimg in flatlist or nimg in biaslist or nimg in arclist): #open the image struct=fits.open(img) if struct[0].header['PROPID'].count('CAL_GAIN'): continue simg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #Check to see if it is RSS 2x2 and add bias subtraction instrume=saltkey.get('INSTRUME', struct[0]).strip() gainset = saltkey.get('GAINSET', struct[0]) rospeed = saltkey.get('ROSPEED', struct[0]) target = saltkey.get('OBJECT', struct[0]).strip() exptime = saltkey.get('EXPTIME', struct[0]) obsmode = saltkey.get('OBSMODE', struct[0]).strip() detmode = saltkey.get('DETMODE', struct[0]).strip() masktype = saltkey.get('MASKTYP', struct[0]).strip() xbin, ybin = saltkey.ccdbin( struct[0], img) obsdate=os.path.basename(img)[1:9] bstruct=None crtype=None thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=5 subbias=False if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW': bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin) if os.path.exists(bfile): bstruct=fits.open(bfile) subbias=True if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5: crtype='edge' thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=3 #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \ bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise, bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #if the file is spectroscopic mode, apply the wavelength correction if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT': dbfile=outpath+obsdate+'_specid.db' try: ximg = outpath+'xmbxgp'+os.path.basename(img) specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except Exception, e: log.message('%s' % e) #clean up the results if cleanup: #clean up the bias frames for i in masterbias_dict.keys(): blist=masterbias_dict[i][1:] for b in blist: saltio.delete(b) #clean up the flat frames for i in masterflat_dict.keys(): flist=masterflat_dict[i][1:] for f in flist: saltio.delete(f)
def specred(infile_list, target, propcode, calfile=None, inter=True, automethod='Matchlines'): #set up the files infiles = ','.join(['%s' % x for x in infile_list]) obsdate = os.path.basename(infile_list[0])[7:15] #set up some files that will be needed logfile = 'spec' + obsdate + '.log' dbfile = 'spec%s.db' % obsdate #create the observation log obs_dict = obslog(infile_list) for i in range(len(infile_list)): if obs_dict['OBJECT'][i].upper().strip( ) == 'ARC' and obs_dict['PROPID'][i].upper().strip() == propcode: lamp = obs_dict['LAMPID'][i].strip().replace(' ', '') arcimage = os.path.basename(infile_list[i]) if lamp == 'NONE': lamp = 'CuAr' lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod=automethod, function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, verbose=True) specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) objimages = '' spec_list = [] for i in range(len(infile_list)): if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][ i].count('RSS') and obs_dict['PROPID'][i].upper().strip( ) == propcode: img = infile_list[i] ##rectify it specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, clobber=True, logfile=logfile, verbose=True) #extract the spectra spec_list.append( extract_spectra('x' + img, yc=1030, calfile=calfile, findobject=True, smooth=False, maskzeros=True, clobber=True)) #combine the results w, f, e = speccombine(spec_list, obsdate) outfile = "%s_%s.spec" % (target, obsdate) write_spectra(outfile, w, f, e)
import sys from pyraf import iraf from iraf import pysalt from specidentify import specidentify from astropy.io import fits header = fits.open(sys.argv[1])[0].header lampid = header['LAMPID'].strip().replace(' ', '') print lampid specidentify(images=sys.argv[1], linelist='/Users/crawford/programs/pysalt/data/linelists/%s.salt' % lampid, outfile='wav.db', guesstype='rss', guessfile='', automethod='Matchlines', function='poly', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=5, inter=True, startext=0 ,clobber=True, textcolor='black', logfile='salt.log', verbose=True)
import sys from pyraf import iraf from iraf import pysalt from specidentify import specidentify specidentify(images=sys.argv[1], linelist=sys.argv[2], outfile='wav.db', guesstype='rss', guessfile='', automethod='Matchlines', function='poly', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=True, startext=0, clobber=True, textcolor='black', logfile='salt.log', verbose=True)