def do_psf_photometry(list_files, satmag, photfilesuffix,psfstarfile='', thresh=6.,psfcleaningradius=10.): for datafile in list_files[:]: print datafile iraf.cd(os.path.dirname(datafile)) remove_previous_psfphot_files(datafile) # extract sources. psf_photometry_pairitel(datafile, satmag, photfilesuffix,psfstarfile,psfcleaningradius, thresh)
def remove_previous_psfphot_files(filename): # remove old files from previous runs (pst files = psf fitting stars, sub = subtracted images for psf fitting, als and arj = detected sources with psf fitting, nrj and nst = peak finding detections) iraf.cd(os.path.dirname(filename)) for fl in glob.glob(filename + '*.pst.*'): os.remove(fl) for fl in glob.glob(filename + '*.sub.*'): os.remove(fl) for fl in glob.glob(filename + '*.als.*'): os.remove(fl) for fl in glob.glob(filename + '*.arj.*'): os.remove(fl) for fl in glob.glob(filename + '*.nst.*'): os.remove(fl) for fl in glob.glob(filename + '*.nrj.*'): os.remove(fl) for fl in glob.glob(filename + '*.psg.*'): os.remove(fl) for fl in glob.glob(filename + '*.psf.*.fits'): os.remove(fl) for fl in glob.glob(filename + '*.secondrun.mag.*'): os.remove(fl) for fl in glob.glob(filename + '*.mag.*'): os.remove(fl)
def wspectext(): if not os.path.exists(Path + '/test3/'): os.mkdir(Path + '/test3/') fs = glob(Path + '/test/' + '/sci*.fits') for i, f in enumerate(fs): iraf.cd(Path + '/test3/') iraf.wspectext(f, ('sci' + str(i) + '.dat'))
def do_psf_photometry_with_coo(list_files, satmag, photfilesuffix,psfstarlist, thresh=6.,psfcleaningradius=10.): print '!!!!!!!!!!!!'+str(psfcleaningradius) for i in np.arange(0, len(list_files)): print list_files[i] iraf.cd(os.path.dirname(list_files[i])) remove_previous_psfphot_files(list_files[i]) # extract sources. psf_photometry_pairitel_with_coo(list_files[i], satmag, photfilesuffix,psfstarlist[i],psfcleaningradius=psfcleaningradius, thresh=thresh)
def mask_norm(mask): # normalizes mask. # test if trimmed mask exists: if os.path.isfile(mask): iraf.cd(os.path.dirname(mask)) hdulist = pyfits.open(mask) imdata = hdulist[0].data hdulist.close() outname = mask.replace('_trimmed.fits', '_normed.fits') print imdata.max() iraf.imarith(os.path.basename(mask),"/",imdata.max(),os.path.basename(outname))
def sorting(fs=None): # Run the pysalt pipeline on the raw data. if fs is None: fs = glob('mbxgpP*.fits') if len(fs) == 0: fs = glob('mbxpP*.fits') if len(fs) ==0: print "WARNING: No raw files to run PySALT pre-processing." return # Copy the raw files into a raw directory if not os.path.exists('product'): os.mkdir('product') if not os.path.exists('work'): os.mkdir('work') if not os.path.exists('work/srt'): os.mkdir('work/srt') for f in fs: shutil.copy2(f, 'product/') scifs, scigas = get_ims(fs, 'sci') arcfs, arcgas = get_ims(fs, 'arc') ims = np.append(scifs, arcfs) gas = np.append(scigas, arcgas) for i, f in enumerate(ims): ga = gas[i] if f in scifs: typestr = 'sci' else: typestr = 'arc' # by our naming convention, imnum should be the last 4 characters # before the '.fits' imnum = f[-9:-5] outname = 'srt/' + typestr outname += '%05.2fmos%04i.fits' % (float(ga), int(imnum)) shutil.move(f, 'work/'+outname) iraf.cd('work') h = pyfits.open(outname, 'update') maskim = h[1].data.copy() maskim[:, :] = 0.0 maskim[abs(h[1].data) < 1e-5] = 1 imhdu = pyfits.ImageHDU(maskim) h.append(imhdu) h[1].header['BPMEXT'] = 2 h[2].header['EXTNAME'] = 'BPM' h[2].header['CD2_2'] = 1 h.flush() h.close() iraf.cd('..')
def prepare_files(list_files, resultpath, readoutnoise): # copies data to output directory, trim data files and mask files to contain only good exposure parts, normalize masks, normalize images by masks. Yields exposure-normalized, trimmed sky images. for datafile in list_files[:]: #prepare directories and copy image to resultpath filename=os.path.basename(datafile) obsid=os.path.basename(os.path.dirname(datafile)) imagepath=os.path.join(resultpath,obsid) image=os.path.join(imagepath,filename) if not os.access(imagepath,os.F_OK): os.makedirs(imagepath) print image shutil.copy(datafile,imagepath) os.chdir(imagepath) iraf.cd(imagepath) # add readoutnoise keyword to header: iraf.hedit(images=filename, fields="RDNOISE", value=str(readoutnoise), addonly="yes", verify="no")
def trim(fs=None): iraf.cd('work') if fs is None: fs=glob('flx/sci*c?.fits') if not os.path.exists('trm'): os.mkdir('trm') for i,f in enumerate(fs): outfile=f.replace('flx','trm') w=WCS(f) hdu= pyfits.open(f) npix=hdu[0].data.shape[2] W1=int(w.wcs_pix2world(2,0,0,0)[0]) W2=int(w.wcs_pix2world(npix-2,0,0,0)[0]) iraf.sarith(input1=f,op='copy',output=outfile,w1=W1,w2=W2 ,clobber='yes') iraf.cd('..')
def do_aperture_photometry(list_files, photfilesuffix): for datafile in list_files[:]: print datafile iraf.cd(os.path.dirname(datafile)) # delete old aperture extractions for fl in glob.glob(datafile + '.coo.*'): os.remove(fl) for fl in glob.glob(datafile + '.mag.*'): os.remove(fl) for fl in glob.glob(datafile + photfilesuffix): os.remove(fl) # extract sources. aperture_photometry(datafile,photfilesuffix, threshold=4.,wcs='logical',mode='small', telescope='Pairitel')
def createMagFile(inputDir, inputFile, useCoordFile, outputFile, fullyAutomatic=False, aperturesString='5,10,15,20', bestSkyAperture='100'): cr=['\n'] iraf.cd(inputDir) iraf.noao() iraf.digiphot() iraf.apphot() iraf.phot.setParam('interactive','no') iraf.centerpars.setParam('cbox','5') iraf.photpars.setParam('apertures',aperturesString) iraf.fitskypars.setParam('annulus',bestSkyAperture) iraf.fitskypars.setParam('dannulus','10') iraf.phot.setParam('image',inputFile) iraf.phot.setParam('coords',useCoordFile) iraf.phot.setParam('output',outputFile) if fullyAutomatic: ## If you use this option, it will not prompt you for any of the aperture/sky settings iraf.phot(mode='h',Stdin=cr) else: # Otherwise, you can change them at will, and get to press enter a bunch iraf.phot(mode='h')
def fixpix(fs=None): iraf.cd('work') if fs is None: fs = glob('nrm/sci*nrm*.fits') if len(fs) == 0: print "WARNING: No rectified images to fix." iraf.cd('..') return if not os.path.exists('fix'): os.mkdir('fix') for f in fs: outname = f.replace('nrm', 'fix') # Copy the file to the fix directory shutil.copy(f, outname) # Set all of the BPM pixels = 0 h = pyfits.open(outname, mode='update') h['SCI'].data[h['BPM'].data == 1] = 0 # Grab the CRM extension from the lax file laxhdu = pyfits.open(f.replace('nrm', 'lax')) h.append(pyfits.ImageHDU(data=laxhdu['CRM'].data.copy(), header=laxhdu['CRM'].header.copy(), name='CRM')) h.flush() h.close() laxhdu.close() # Run iraf's fixpix on the cosmic rays, not ideal, # but better than nothing because apall doesn't take a bad pixel mask iraf.unlearn(iraf.fixpix) iraf.flpr() iraf.fixpix(outname + '[SCI]', outname + '[CRM]', mode='hl') iraf.cd('..')
def flatten(fs=None, masterflatdir=None): iraf.cd('work') if fs is None: fs = glob('pysalt/bxgpP*.fits') if len(fs) == 0: print "WARNING: No images to flat-field." # Change directories to fail more gracefully iraf.cd('..') return if not os.path.exists('flts'): os.mkdir('flts') # Make sure there are science images or arcs and what grating angles were # used scifs, scigas = get_ims(fs, 'sci') arcfs, arcgas = get_ims(fs, 'arc') ims = np.append(scifs, arcfs) gas = np.append(scigas, arcgas) # For each science and arc image for i, f in enumerate(ims): thishdu = pyfits.open(f) ga = gas[i] # For each chip for c in range(1, 7): flatfile = 'flats/flt%05.2fresc%i.fits' % (ga, c) if len(glob(flatfile)) == 0: if masterflatdir is None: print("No flat field image found for %s"% f) continue # Check for the master flat directory flatfile = masterflatdir+'/flt%05.2fresc%i.fits' % (ga, c) if len(glob(flatfile)) == 0: # Still can't find one? Abort!! print("No flat field image found for %s"% f) continue # open the corresponding response file resphdu = pyfits.open(flatfile) # divide out the illumination correction and the flatfield # make sure divzero = 0.0 thishdu[c].data /= resphdu[0].data.copy() # replace the infinities with 0.0 thishdu[c].data[np.isinf(thishdu[c].data)] = 0.0 resphdu.close() # save the updated file if f in scifs: typestr = 'sci' else: typestr = 'arc' # get the image number # by salt naming convention, these should be the last 4 characters # before the '.fits' imnum = f[-9:-5] outname = 'flts/' + typestr + '%05.2fflt%04i.fits' % (float(ga), int(imnum)) thishdu.writeto(outname) thishdu.close() iraf.cd('..')
def split1d(fs=None): iraf.cd('work') if fs is None: fs = glob('x1d/sci*x1d????.fits') if len(fs) == 0: print "WARNING: No extracted spectra to split." iraf.cd('..') return for f in fs: hdu = pyfits.open(f.replace('x1d', 'fix')) chipgaps = get_chipgaps(hdu) # Throw away the first pixel as it almost always bad chipedges = [[1, chipgaps[0][0]], [chipgaps[0][1] + 1, chipgaps[1][0]], [chipgaps[1][1] + 1, chipgaps[2][0]]] w = WCS(f) # Copy each of the chips out seperately. Note that iraf is 1 indexed # unlike python so we add 1 for i in range(3): # get the wavelengths that correspond to each chip lam, _apnum, _bandnum = w.all_pix2world(chipedges[i], 0, 0, 0) iraf.scopy(f, f[:-5] + 'c%i' % (i + 1), w1=lam[0], w2=lam[1], format='multispec', rebin='no',clobber='yes') hdu.close() iraf.cd('..')
def identify2d(fs=None): iraf.cd('work') if fs is None: fs = glob('mos/arc*mos*.fits') if len(fs) == 0: print "WARNING: No mosaiced (2D) specidentify." # Change directories to fail gracefully iraf.cd('..') return arcfs, arcgas = get_ims(fs, 'arc') if not os.path.exists('id2'): os.mkdir('id2') lampfiles = { 'Th Ar': 'ThAr.salt', 'Xe': 'Xe.salt', 'Ne': 'NeAr.salt', 'Cu Ar': 'CuAr.salt', 'Ar': 'Argon_hires.salt', 'Hg Ar': 'HgAr.salt' } for i, f in enumerate(arcfs): ga = arcgas[i] # find lamp and corresponding linelist lamp = pyfits.getval(f, 'LAMPID') lampfn = lampfiles[lamp] if pyfits.getval(f, 'GRATING') == 'PG0300' and lamp == 'Ar': lampfn = 'Argon_lores.swj' ccdsum = int(pyfits.getval(f, 'CCDSUM').split()[1]) # linelistpath is a global variable defined in beginning, path to # where the line lists are. lamplines = pysaltpath + '/data/linelists/' + lampfn print(lamplines) # img num should be right before the .fits imgnum = f[-9:-5] # run pysalt specidentify idfile = 'id2/arc%05.2fid2%04i' % (float(ga), int(imgnum)) + '.db' iraf.unlearn(iraf.specidentify) iraf.flpr() iraf.specidentify( images=f, linelist=lamplines, outfile=idfile, guesstype='rss', inter=True, # automethod='FitXcor', rstep=600 / ccdsum, rstart=200 / ccdsum, startext=1, clobber='yes', #startext=1, clobber='yes', verbose='no', mode='hl', logfile='salt.log', mdiff=2, function='legendre') iraf.cd('..')
def stdsensfunc(fs=None): iraf.cd('work') if fs is None: fs = glob('x1d/sci*x1d*c?.fits') if len(fs) == 0: print "WARNING: No extracted spectra to create sensfuncs from." iraf.cd('..') return if not os.path.exists('std'): os.mkdir('std') for f in fs: # Put the file in the std directory, but last 3 letters of sens outfile = 'std/' + f.split('/')[1] outfile = outfile.replace('x1d', 'sens').replace('sci', 'std') outfile = outfile.replace('.fits', '.dat') # if the object name is in the list of standard stars from pysalt if isstdstar(f): # We use pysalt here because standard requires a # dispersion correction which was already taken care of above # Write out an ascii file that pysalt.specsens can read asciispec = 'std/std.ascii.dat' spectoascii(f, asciispec) # run specsens stdfile = pysaltpath + '/data/standards/spectroscopic/m%s.dat' % pyfits.getval(f, 'OBJECT').lower().replace('-','_') extfile = pysaltpath + '/data/site/suth_extinct.dat' iraf.unlearn(iraf.specsens) iraf.specsens(asciispec, outfile, stdfile, extfile, airmass=pyfits.getval(f, 'AIRMASS'), exptime=pyfits.getval(f, 'EXPTIME'), function='poly', order=11, clobber=True, mode='h', thresh=1e10) # delete the ascii file os.remove(asciispec) iraf.cd('..')
def fluxcal(stdsfolder='./', fs=None): iraf.cd('work') if fs is None: fs = glob('x1d/sci*x1d*c*.fits') if len(fs) == 0: print "WARNING: No science chip spectra to flux calibrate." iraf.cd('..') return if not os.path.exists('flx'): os.mkdir('flx') extfile = pysaltpath + '/data/site/suth_extinct.dat' stdfiles = glob(stdsfolder + '/std/*sens*c?.dat') print(stdfiles) for f in fs: outfile = f.replace('x1d', 'flx') chip = outfile[-6] hdu = pyfits.open(f) ga = f.split('/')[1][3:8] # Get the standard sensfunc with the same grating angle stdfile = None for stdf in stdfiles: if np.isclose(float(ga), float(stdf.split('/')[stdf.count('/')][3:8]), rtol=1e-2): # Get the right chip number if chip == stdf[-5]: stdfile = stdf break if stdfile is None: print('No standard star with grating-angle %s' % ga) continue # for each extracted aperture for i in range(hdu[0].data.shape[1]): # create an ascii file that pysalt can read asciiname = 'flx/sciflx.dat' outtmpname = 'flx/scical.dat' spectoascii(f, asciiname, i) # Run pysalt.speccal iraf.unlearn(iraf.speccal) iraf.flpr() iraf.speccal(asciiname, outtmpname, stdfile, extfile, airmass=pyfits.getval(f, 'AIRMASS'), exptime=pyfits.getval(f, 'EXPTIME'), clobber=True, mode='h') # read in the flux calibrated ascii file and copy its # contents into a fits file flxcal = np.genfromtxt(outtmpname).transpose() hdu[0].data[0, i] = flxcal[1] hdu[0].data[2, i] = flxcal[2] # delete the ascii file os.remove(asciiname) os.remove(outtmpname) hdu.writeto(outfile, clobber=True) iraf.cd('..')
def mktelluric(fs=None): iraf.cd('work') if fs is None: fs = glob('sci_com.fits') if len(fs) == 0: print "WARNING: No flux-calibrated spectra to make the a telluric correction." iraf.cd('..') return if not os.path.exists('tel'): os.mkdir('tel') # for each file f = fs[0] # if it is a standard star combined file if isstdstar(f): # read in the spectrum and calculate the wavelengths of the pixels hdu = pyfits.open(f) spec = hdu[0].data.copy() hdr = hdu[0].header.copy() hdu.close() waves = fitshdr_to_wave(hdr) template_spectrum = signal.savgol_filter(spec, 21, 3) noise = np.abs(spec - template_spectrum) noise = ndimage.filters.gaussian_filter1d(noise, 100.0) not_telluric = np.ones(spec.shape, dtype=np.bool) # For each telluric region for wavereg in telluricWaves: in_telluric_region = np.logical_and(waves >= wavereg[0], waves <= wavereg[1]) not_telluric = np.logical_and(not_telluric, np.logical_not(in_telluric_region)) # Smooth the spectrum so that the spline doesn't go as crazy # Use the Savitzky-Golay filter to presevere the edges of the # absorption features (both atomospheric and intrinsic to the star) sgspec = signal.savgol_filter(spec, 31, 3) #Get the number of data points to set the smoothing criteria for the # spline m = not_telluric.sum() intpr = interpolate.splrep(waves[not_telluric], sgspec[not_telluric], w = 1/noise[not_telluric], k=2, s=10 * m) # Replace the telluric with the smoothed function smoothedspec = interpolate.splev(waves, intpr) smoothedspec[not_telluric] = spec[not_telluric] # Divide the original and the telluric corrected spectra to # get the correction factor correction = spec / smoothedspec # Save the correction dout = np.ones((2, len(waves))) dout[0] = waves dout[1] = correction np.savetxt('tel/telcor.dat', dout.transpose()) iraf.cd('..')
def lax(fs=None): iraf.cd('work') if fs is None: fs = glob('bkg/*bkg*.fits') if len(fs) == 0: print "WARNING: No background-subtracted files for Lacosmicx." iraf.cd('..') return if not os.path.exists('lax'): os.mkdir('lax') for f in fs: outname = f.replace('bkg','lax') hdu = pyfits.open(f) # Add a CRM extension hdu.append(pyfits.ImageHDU(data=hdu['BPM'].data.copy(), header=hdu['BPM'].header.copy(), name='CRM')) # Set all of the pixels in the CRM mask to zero hdu['CRM'].data[:, :] = 0 # less aggressive lacosmic on standard star observations if not isstdstar(f): objl = 1.0 sigc = 4.0 else: objl = 3.0 sigc = 10.0 chipgaps = get_chipgaps(hdu) chipedges = [[0, chipgaps[0][0]], [chipgaps[0][1] + 1, chipgaps[1][0]], [chipgaps[1][1] + 1, chipgaps[2][0]]] # Run each chip separately for chip in range(3): # Use previously subtracted sky level = 0 as we have already added # a constant sky value in the background task # Gain = 1, readnoise should be small so it shouldn't matter much. # Default value seems to work. chipinds = slice(chipedges[chip][0], chipedges[chip][1]) crmask, _cleanarr = lacosmicx.lacosmicx(hdu[1].data[:, chipinds].copy(), inmask=np.asarray(hdu[2].data[:, chipinds].copy(), dtype = np.uint8), sigclip=sigc, objlim=objl, sigfrac=0.1, gain=1.0, pssl=0.0) # Update the image hdu['CRM'].data[:, chipinds][:, :] = crmask[:,:] # Flag the cosmic ray pixels with a large negative number hdu['SCI'].data[:, chipinds][crmask == 1] = -1000000 # Save the file hdu.writeto(outname, clobber=True) hdu.close() iraf.cd('..')
def correct_wcs(image,clobber=False,minmag=13, radius=20.): '''correct wcs of fitsfile to 2MASS coordinates This method loads the 2MASS catalog in the region covered by the fits file and shifts,stretches, and rotates the wcs in the fits file to match the 2MASS coordinates The initial wcs needs to be approximately correct (within a few arcsec), otherwise the matching will fail. image = filename clobber = overwrite existing 2MASS catalog file? minmag = mag of faintest object retrieved from 2MASS''' object = iraf.hselect(image+'[0]','OBJECT', 'yes', Stdout=1)[0] filter = iraf.hselect(image+'[0]','FILTER', 'yes', Stdout=1)[0] catalogue=os.path.join(os.path.dirname(image),"2mass_"+str(object)+'_'+str(filter)+'_'+str(minmag)+".cat") #Gererate catalogue, if it does not exist in this directory if clobber or not os.path.isfile(catalogue): get_catalogue(catalogue,iraf.hselect(image+'[0]','RA','yes', Stdout=1)[0],iraf.hselect(image+'[0]','DEC','yes', Stdout=1)[0], radius=radius, minmag=minmag) #cfrac: maximum fraction of targets which do not center correctly # a large number is no problem here, since often there are a few 100 sources in the catalog to match. iraf.mscred(_doprint=0) #matchout=iraf.msccmatch(image,catalogue,interactive=False, Stdout=1,cfrac=0.95,rms=3) iraf.cd(os.path.dirname(image)) matchout=iraf.msccmatch(os.path.basename(image),catalogue,interactive=False, Stdout=1,cfrac=0.9,rms=10,maxshif=100., nsearch=100, nfit=5, update='yes') try: print matchout[-5] except IndexError: print matchout
def main(): for workdir in curr_work_dirs: work_path = 'observations/' + workdir if not os.path.exists(work_path): os.makedirs(work_path) if not os.path.exists(work_path+'/database'): os.makedirs(work_path+'/database') if not os.path.exists(work_path+'/cosmics'): os.makedirs(work_path+'/cosmics') # don't ask, we just need this os.system("touch %s/blank.txt" % work_path) # enter working folder, all the way to the end iraf.cd(work_path) # maybe useful, maybe not, not sure iraf.prcacheOff() # main steps, should work individually #check_spectra(workdir, '.ec.vh') create_masterbias(workdir) create_masterflat(workdir) #do_ref_apall(workdir) #normalize_masterflat(workdir) clean_folder_write_header(workdir) bias_correct(workdir) #flat_correct(workdir) #remove_cosmics(workdir) do_apall_for_real(workdir) apply_apall(workdir) wavelength_solution(workdir) #check_wav_solution(workdir) apply_wav_solution(workdir) vhelio_correct(workdir) #check_spectra(workdir, '.ec.vh') combine_normalize_images(workdir, combine=False) get_RV(workdir, multi_ref=False, multi_sample=False) # export_spectrum_to_txt(workdir, suffix='.ec_wvl', export_cal=True) # export_spectrum_to_txt(workdir, suffix='_vh_norm') #check_spectra(workdir) #new_flatfield_lens_problems(workdir) # back to mother folder iraf.cd('../') iraf.cd('../')
def fluxcal(stdsfolder='./', fs=None): iraf.cd('work') if fs is None: fs = glob('x1d/sci*x1d*c*.fits') if len(fs) == 0: print "WARNING: No science chip spectra to flux calibrate." iraf.cd('..') return if not os.path.exists('flx'): os.mkdir('flx') extfile = pysaltpath + '/data/site/suth_extinct.dat' stdfiles = glob(stdsfolder + '/std/*sens*c?.dat') print(stdfiles) for f in fs: outfile = f.replace('x1d', 'flx') chip = outfile[-6] hdu = pyfits.open(f) ga = f.split('/')[1][3:8] # Get the standard sensfunc with the same grating angle stdfile = None for stdf in stdfiles: if ga in stdf: # Get the right chip number if chip == stdf[-5]: stdfile = stdf break if stdfile is None: print('No standard star with grating-angle %s' % ga) continue # for each extracted aperture for i in range(hdu[0].data.shape[1]): # create an ascii file that pysalt can read asciiname = 'flx/sciflx.dat' outtmpname = 'flx/scical.dat' spectoascii(f, asciiname, i) # Run pysalt.speccal iraf.unlearn(iraf.speccal) iraf.flpr() iraf.speccal(asciiname, outtmpname, stdfile, extfile, airmass=pyfits.getval(f, 'AIRMASS'), exptime=pyfits.getval(f, 'EXPTIME'), clobber=True, mode='h') # read in the flux calibrated ascii file and copy its # contents into a fits file flxcal = np.genfromtxt(outtmpname).transpose() hdu[0].data[0, i] = flxcal[1] hdu[0].data[2, i] = flxcal[2] # delete the ascii file os.remove(asciiname) os.remove(outtmpname) hdu.writeto(outfile, clobber=True) iraf.cd('..')
def mosaic(fs=None): iraf.cd('work') # If the file list is not given, grab the default files if fs is None: fs = glob('flts/*.fits') # Abort if there are no files if len(fs) == 0: print "WARNING: No flat-fielded images to mosaic." iraf.cd('..') return if not os.path.exists('mos'): os.mkdir('mos') # Get the images to work with ims, gas = get_scis_and_arcs(fs) for i, f in enumerate(ims): ga = gas[i] fname = f.split('/')[1] typestr = fname[:3] # by our naming convention, imnum should be the last 4 characters # before the '.fits' imnum = fname[-9:-5] outname = 'mos/' + typestr outname += '%05.2fmos%04i.fits' % (float(ga), int(imnum)) # prepare to run saltmosaic iraf.unlearn(iraf.saltmosaic) iraf.flpr() iraf.saltmosaic(images=f, outimages=outname, outpref='', geomfile=pysaltpath + '/data/rss/RSSgeom.dat', clobber=True, mode='h') # Make a bad pixel mask marking where there is no data. h = pyfits.open(outname, 'update') maskim = h[1].data.copy() maskim[:, :] = 0.0 maskim[abs(h[1].data) < 1e-5] = 1 imhdu = pyfits.ImageHDU(maskim) h.append(imhdu) h[1].header['BPMEXT'] = 2 h[2].header['EXTNAME'] = 'BPM' h[2].header['CD2_2'] = 1 h.flush() h.close() iraf.cd('..')
def background(fs=None): iraf.cd('work') # Get rectified science images if fs is None: fs = glob('nrm/sci*nrm*.fits') if len(fs) == 0: print "WARNING: No rectified images for background-subtraction." iraf.cd('..') return if not os.path.exists('bkg'): os.mkdir('bkg') for f in fs: print("Subtracting background for %s" % f) # Make sure dispaxis is set correctly pyfits.setval(f, 'DISPAXIS', value=1) # the outfile name is very similar, just change folder prefix and # 3-char stage substring outfile = f.replace('nrm','bkg') # We are going to use fit1d instead of the background task # Go look at the code for the background task: it is literally a wrapper for 1D # but it removes the BPM option. Annoying. iraf.unlearn(iraf.fit1d) iraf.fit1d(input=f + '[SCI]', output='tmpbkg.fits', bpm=f + '[BPM]', type='difference', sample='52:949', axis=2, interactive='no', naverage='1', function='legendre', order=5, low_reject=1.0, high_reject=1.0, niterate=5, grow=0.0, mode='hl') # Copy the background subtracted frame into the rectified image # structure. # Save the sky spectrum as extension 3 hdutmp = pyfits.open('tmpbkg.fits') hdu = pyfits.open(f) skydata = hdu[1].data - hdutmp[0].data hdu[1].data[:, :] = hdutmp[0].data[:, :] hdu.append(pyfits.ImageHDU(skydata)) hdu[3].header['EXTNAME'] = 'SKY' hdu[3].data[hdu['BPM'] == 1] = 0.0 # Add back in the median sky level for things like apall and lacosmicx hdu[1].data[:, :] += np.median(skydata) hdu[1].data[hdu['BPM'] == 1] = 0.0 hdutmp.close() hdu.writeto(outfile, clobber=True) # saving the updated file # (data changed) os.remove('tmpbkg.fits') iraf.cd('..')
def identify2d(fs=None): iraf.cd('work') if fs is None: fs = glob('mos/arc*mos*.fits') if len(fs) == 0: print "WARNING: No mosaiced (2D) specidentify." # Change directories to fail gracefully iraf.cd('..') return arcfs, arcgas = get_ims(fs, 'arc') if not os.path.exists('id2'): os.mkdir('id2') lampfiles = {'Th Ar': 'ThAr.salt', 'Xe': 'Xe.salt', 'Ne': 'NeAr.salt', 'Cu Ar': 'CuAr.salt', 'Ar': 'Argon_hires.salt', 'Hg Ar': 'HgAr.salt'} for i, f in enumerate(arcfs): ga = arcgas[i] # find lamp and corresponding linelist lamp = pyfits.getval(f, 'LAMPID') lampfn = lampfiles[lamp] if pyfits.getval(f,'GRATING') == 'PG0300' and lamp == 'Ar': lampfn = 'Argon_lores.swj' ccdsum = int(pyfits.getval(f, 'CCDSUM').split()[1]) # linelistpath is a global variable defined in beginning, path to # where the line lists are. lamplines = pysaltpath + '/data/linelists/' + lampfn print(lamplines) # img num should be right before the .fits imgnum = f[-9:-5] # run pysalt specidentify idfile = 'id2/arc%05.2fid2%04i' % (float(ga), int(imgnum)) + '.db' iraf.unlearn(iraf.specidentify) iraf.flpr() iraf.specidentify(images=f, linelist=lamplines, outfile=idfile, guesstype='rss', inter=True, # automethod='FitXcor', rstep=600 / ccdsum, rstart=200 / ccdsum, startext=1, clobber='yes', #startext=1, clobber='yes', verbose='no', mode='hl', logfile='salt.log', mdiff=2, function='legendre') iraf.cd('..')
def slitnormalize(fs=None): iraf.cd('work') if fs is None: fs = glob('rec/*rec*.fits') if len(fs) == 0: print "WARNING: No rectified files for slitnormalize." # Change directories to fail gracefully iraf.cd('..') return if not os.path.exists('nrm'): os.mkdir('nrm') for f in fs: outname = f.replace('rec', 'nrm') iraf.unlearn(iraf.specslitnormalize) iraf.specslitnormalize(images=f, outimages=outname, outpref='', order=5, clobber=True, mode='h') iraf.cd('..')
def telluric(stdsfolder='./', fs=None): iraf.cd('work') if fs is None: fs = glob('sci_com.fits') if len(fs) == 0: print "WARNING: No flux-calibrated spectra to telluric-correct." iraf.cd('..') return f = fs[0] outfile = 'final.fits' # Get the standard to use for telluric correction stdfile = glob(stdsfolder + '/tel/telcor.dat')[0] hdu = pyfits.open(f) spec = hdu[0].data.copy() hdr = hdu[0].header.copy() hdu.close() waves = fitshdr_to_wave(hdr) telwave, telspec = np.genfromtxt(stdfile).transpose() # Cross-correlate the standard star and the sci spectra # to find wavelength shift of standard star. p = fitxcor(waves, spec, telwave, telspec) if abs(p[0] - 1.0) > 0.02 or abs(p[1]) > 10.0: print "Cross-correlation scale/shift too large; won't do it:" print p print " reset to [1.0, 0.0]" p = [1.0, 0.0] # shift and stretch standard star spectrum to match science # spectrum. telcorr = interp(waves, p[0] * telwave + p[1], telspec) # In principle, we should scale by the proper airmasses, but SALT # always observes at ~same airmass # Divide science spectrum by transformed standard star sub-spectrum correct_spec = spec / telcorr # Copy telluric-corrected data to new file. tofits(outfile, correct_spec, hdr=hdr, clobber=True) print "Telluric correction applied; output is " + outfile iraf.cd('..')
def rectify(ids=None, fs=None): iraf.cd('work') if ids is None: ids = np.array(glob('id2/arc*id2*.db')) if fs is None: fs = glob('srt/*mos*.fits') if len(ids) == 0: print "WARNING: No wavelength solutions for rectification." iraf.cd('..') return if len(fs) == 0: print "WARNING: No images for rectification." iraf.cd('..') return # Get the grating angles of the solution files idgas = [] for i, thisid in enumerate(ids): f = open(thisid) idlines = np.array(f.readlines(), dtype=str) f.close() idgaline = idlines[np.char.startswith(idlines, '#graang')][0] idgas.append(float(idgaline.split('=')[1])) ims, gas = get_scis_and_arcs(fs) if not os.path.exists('rec'): os.mkdir('rec') for i, f in enumerate(ims): fname = f.split('/')[1] typestr = fname[:3] ga, imgnum = gas[i], fname[-9:-5] outfile = 'rec/' + typestr + '%05.2frec' % (ga) + imgnum + '.fits' iraf.unlearn(iraf.specrectify) iraf.flpr() print('_____idgas_____') print (np.array(idgas)) print('_____ga_____') print (ga) idfile = ids[np.array(idgas) == ga][0] iraf.specrectify(images=f, outimages=outfile, solfile=idfile, outpref='', function='legendre', order=3, inttype='interp', conserve='yes', clobber='yes', verbose='yes') # Update the BPM to mask any blank regions h = pyfits.open(outfile, 'update') # Cover the chip gaps. The background task etc do better if the chip # gaps are straight # To deal with this we just throw away the min and max of each side of # the curved chip gap chipgaps = get_chipgaps(h) print(" -- chipgaps --") print(chipgaps) # Chip 1 h[2].data[:, chipgaps[0][0]:chipgaps[0][1]] = 1 # Chip 2 h[2].data[:, chipgaps[1][0]:chipgaps[1][1]] = 1 # edge of chip 3 h[2].data[:, chipgaps[2][0]:chipgaps[2][1]] = 1 # Cover the other blank regions h[2].data[[h[1].data == 0]] = 1 # Set all of the data to zero in the BPM h[1].data[h[2].data == 1] = 0.0 h.flush() h.close() iraf.cd('..')
def pysalt(fs=None): # Run the pysalt pipeline on the raw data. if fs is None: fs = glob('P*.fits') if len(fs) == 0: print "WARNING: No raw files to run PySALT pre-processing." return # Copy the raw files into a raw directory if not os.path.exists('raw'): os.mkdir('raw') if not os.path.exists('work'): os.mkdir('work') for f in fs: shutil.copy2(f, 'raw/') shutil.move(f, 'work/') iraf.cd('work') # Run each of the pysalt pipeline steps deleting temporary files as we go # saltprepare iraf.unlearn(iraf.saltprepare) # Currently, there is not a bad pixel mask provided by SALT # so we don't create one here. iraf.saltprepare(images='P*.fits', clobber=True, mode='h') for f in glob('P*.fits'): os.remove(f) # saltgain iraf.unlearn(iraf.saltgain) # Multiply by the gain so that everything is in electrons. iraf.saltgain(images='pP*.fits', gaindb=pysaltpath + '/data/rss/RSSamps.dat', mult=True, usedb=True, mode='h') for f in glob('pP*.fits'): os.remove(f) # write a keyword in the header keyword gain = 1 in each amplifier fs = glob('gpP*.fits') for f in fs: for i in range(1, 7): pyfits.setval(f, 'GAIN', ext=i, value=1.0) # saltxtalk iraf.unlearn(iraf.saltxtalk) iraf.saltxtalk(images='gpP*.fits', clobber=True, usedb=True, xtalkfile=pysaltpath + '/data/rss/RSSxtalk.dat', mode='h') for f in glob('gpP*.fits'): os.remove(f) # saltbias iraf.unlearn(iraf.saltbias) iraf.saltbias(images='xgpP*.fits', clobber=True, mode='h') for f in glob('xgpP*.fits'): os.remove(f) # Put all of the newly created files into the pysalt directory if not os.path.exists('pysalt'): os.mkdir('pysalt') for f in glob('bxgpP*.fits'): shutil.move(f, 'pysalt') iraf.cd('..')
#import pyfits import astropy.io.fits as pyfits from pyraf import iraf iraf.imexamine.unlearn() # If zenity is not working on you machine, use the cat command instead by uncommnting it. DISPLAY_MENU_CMD = 'zenity --text-info --title "Images in {imgdir}" --filename={imglist} --width 600 --height 250 &' #DISPLAY_MENU_CMD = 'echo "Images in {imgdir}" ; cat {imglist}' IMGListFile = 'imgs.text' # This file will be overwritten pwd=os.getcwd() wildcard=sys.argv[-1] dirs=sys.argv[1:-1] for imgdir in dirs : iraf.cd(imgdir) while 1 : print imgdir images=glob.glob(wildcard) images.sort() imgs_txt=open(IMGListFile,'w') i=0 for img in images: hdulist=pyfits.open(img) Exptime=hdulist[0].header.get('ITIME') Commentx=hdulist[0].header.get('TCOMMENT') Obj=hdulist[0].header.get('TARGET') Comment=hdulist[0].header.get('DATE') Filter=hdulist[0].header.get('UPPER') hdulist.close() imgs_txt.write(str(i)+' '+img+' '+str(Obj)+' '+str(Exptime)+' '+str(Commentx)+' '+str(Comment)+' '+str(Filter)+'\n')
# These statements import python modules that will be used below... import glob import os import string from pyraf import iraf from shutil import copyfile # Load necessary IRAF (pyraf) modules... iraf.stsdas() iraf.hst_calib() iraf.synph # cd to current working directory iraf.cd(os.getcwd()) # This creates a list of filter file names... print "Here is the list of CCDs:" filterFileNameList = glob.glob('des_band_20150601a_syn.average.list') filterFileNameList += glob.glob('des_band_20150601a_syn.ccd*.list') filterFileNameList += glob.glob('des_band_20150601a_syn.ccd*_amp1.list') filterFileNameList += glob.glob('des_band_20150601a_syn.ccd*_amp2.list') filterFileNameList.sort() print filterFileNameList # This is the file containing the list of calspec files, # which you created previously using the pyraf "files" # command (see top of this script)... calspecListFileName = 'pickles.list' # Open and the calspec list file... fd = open(calspecListFileName, 'r')
def makeflats(fs=None): # Note the list of files need to not include any paths relative to # the work directory. # Maybe not the greatest convention, but we can update this later iraf.cd('work') if fs is None: fs = glob('pysalt/bxgp*.fits') if len(fs) == 0: print "WARNING: No flat-fields to combine and normalize." # Fail gracefully by going up a directory iraf.cd('..') return # make a flats directory if not os.path.exists('flats'): os.mkdir('flats') # Figure out which images are flats and which grating angles were used allflats, grangles = get_ims(fs, 'flat') # For each grating angle for ga in np.unique(grangles): # grab the flats for this gr angle flats = allflats[grangles == ga] # For each chip for c in range(1, 7): # run imcombine with average and sigclip, weighted by exposure time flatlist = '' for f in flats: flatlist += '%s[%i],' % (f, c) # Add the exptime keyword to each extension pyfits.setval(f, 'EXPTIME', ext=c, value=pyfits.getval(f, 'EXPTIME')) # set the output combined file name combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c) if os.path.exists(combineoutname): os.remove(combineoutname) # initialize the iraf command iraf.unlearn(iraf.imcombine) print(flatlist) # don't forget to remove the last comma in the filelist iraf.imcombine(input=flatlist[:-1], output=combineoutname, combine='average', reject='sigclip', lsigma=3.0, hsigma=3.0, weight='exposure', expname='EXPTIME') pyfits.setval(combineoutname, 'DISPAXIS', value=1) # We want to make an illumination correction file # before running response: illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c) iraf.unlearn(iraf.illumination) iraf.illumination(images=combineoutname, illuminations=illumoutname, interactive=False, naverage=-40, order=11, low_reject=3.0, high_reject=3.0, niterate=5, mode='hl') # Flag any pixels in the illumination correction< 0.1 illumhdu = pyfits.open(illumoutname, mode='update') illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0 illumhdu.flush() # Get 40 pixels out of the middle of the image and # median them to run response combinehdu = pyfits.open(combineoutname) ny = combinehdu[0].data.shape[0] # divide out the illumination correction before running response flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :] / illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :], axis=0) # close the illumination file because we don't need it anymore illumhdu.close() # File stage m1d for median 1-D flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c) tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy()) # run response # r1d = response1d resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c) iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31, interactive=False, naverage=-5, low_reject=3.0, high_reject=3.0, niterate=5, mode='hl') resp1dhdu = pyfits.open(resp1dfname) resp1d = resp1dhdu[0].data.copy() resp1dhdu.close() # After response divide out the response function # normalize the 1d resp to its median resp1d /= np.median(resp1d) # Chuck any outliers flatsig = np.std(resp1d - 1.0) resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0 resp = flat1d / resp1d resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c) resp2d = combinehdu[0].data.copy() / resp tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy()) combinehdu.close() # close the combined flat because we don't need it anymore combinehdu.close() pyfits.setval(resp2dfname, 'DISPAXIS', value=1) # Reset any pixels in the flat field correction< 0.1 # We could flag bad pixels here if we want, but not right now flathdu = pyfits.open(resp2dfname, mode='update') flathdu[0].data[flathdu[0].data <= 0.1] = 0.0 flathdu.flush() flathdu.close() # Step back up to the top directory iraf.cd('..')
iraf.centerpars.maxshift = 1. # now test visually in ds9 for images that have not been properly aligned. # put info for those which have not been properly aligned into the manual wcs correction file. if input_info.badwcs == "yes": corrbyhand = ascii.read(input_info.wcs_corrbyhand_file) for i in np.arange(0, len(corrbyhand)): # remove file if beyond saving if corrbyhand[i]["beyond_saving"] == "yes": print("removing files which are beyond saving...") if os.path.isfile(corrbyhand[i]["filename_incl_path"]): os.remove(corrbyhand[i]["filename_incl_path"]) else: # if saveable, update header with manual coordinates. print("updating headers with manual coordinates...") iraf.cd(os.path.dirname(corrbyhand[i]["filename_incl_path"])) f = os.path.basename(corrbyhand[i]["filename_incl_path"]) iraf.hedit(f, "CRVAL1", corrbyhand[i]["ra_good"], verify="no") iraf.hedit(f, "CRPIX1", corrbyhand[i]["x_pixel"], verify="no") iraf.hedit(f, "CRVAL2", corrbyhand[i]["dec_good"], verify="no") iraf.hedit(f, "CRPIX2", corrbyhand[i]["y_pixel"], verify="no") # run the automatic coordinate correction again: print("running refined coordinate matching for updated files...") iraf.centerpars.cbox = 10. iraf.centerpars.maxshift = 10. photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.) photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.) photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.) photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.) iraf.centerpars.cbox = 9. iraf.centerpars.maxshift = 1.
shift_temp.append([name_temp[id2], 0., 0., 0.]) #f = open(path+galname+'/shift_pix_'+targname+'_lam'+str(lam_temp[id2])+'_n'+str(id1)+'.txt','a') #f.write('\n'+str(0.)+' '+str(0.)+' '+str(refpa)+' '+str(name_temp[id2])) #f.close() shift_fin.append(shift_temp) shift_temp = [] ######################################################### IMSHIFT + IMCOMBINE ############################################################################ # # Perform the shifts with iraf/pyraf # We need the imshift task: imshift input output xshift yshift (or file with shifts in cols 1 & 2) # The initial list will be *x2d.fits[1] and the output list will be *shift.fits # list_temp = [] list_total = [] iraf.cd(path + galname + '/shifts') for index in range(len(shift_fin)): arr1 = shift_fin[index] for index2 in range(len(arr1)): i_nam = arr1[index2][0] # Initial name for imshift xs = arr1[index2][1] ys = arr1[index2][2] posa = arr1[index2][3] list_in = i_nam.replace('.fits', '.fits[1,overwrite+]') list_ot = list_in.replace('x2d', 'shift') list_out = list_ot.replace('.fits[1,overwrite+]', '.fits') list_temp.append(list_out) if not os.path.exists(list_out): iraf.imshift(input=list_in, output=list_out, xshift=xs, yshift=ys) print('File ' + str(i_nam) + ' changed to ' + str(list_out) + '!') list_total.append(list_temp)
def extract(fs=None): iraf.cd('work') if fs is None: fs = glob('fix/*fix*.fits') if len(fs) == 0: print "WARNING: No fixpixed images available for extraction." iraf.cd('..') return if not os.path.exists('x1d'): os.mkdir('x1d') print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line." for f in fs: # Get the output filename without the ".fits" outbase = f.replace('fix', 'x1d')[:-5] # Get the readnoise, right now assume default value of 5 but we could # get this from the header readnoise = 5 # If interactive open the rectified, background subtracted image in ds9 ds9display(f.replace('fix', 'bkg')) # set dispaxis = 1 just in case pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1) iraf.unlearn(iraf.apall) iraf.flpr() iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes', review='no', line='INDEF', nsum=-1000, lower=-5, upper=5, b_function='legendre', b_order=5, b_sample='-200:-100,100:200', b_naverage=-10, b_niterate=5, b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15, t_step=15, t_nlost=200, t_function='legendre', t_order=5, t_niterate=5, t_low_reject=3.0, t_high_reject=3.0, background='fit', weights='variance', pfit='fit2d', clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0, usigma=4.0, mode='hl') # Copy the CCDSUM keyword into the 1d extraction pyfits.setval(outbase + '.fits', 'CCDSUM', value=pyfits.getval(f, 'CCDSUM')) # Extract the corresponding arc arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0] # set dispaxis = 1 just in case pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1) iraf.unlearn(iraf.apsum) iraf.flpr() iraf.apsum(input=arcname + '[SCI]', output='auxext_arc', references=f[:-5] + '[SCI]', interactive='no', find='no', edit='no', trace='no', fittrace='no', extras='no', review='no', background='no', mode='hl') # copy the arc into the 5 column of the data cube arcfs = glob('auxext_arc*.fits') for af in arcfs: archdu = pyfits.open(af) scihdu = pyfits.open(outbase + '.fits', mode='update') d = scihdu[0].data.copy() scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2])) scihdu[0].data[:-1, :, :] = d[:, :, :] scihdu[0].data[-1::, :] = archdu[0].data.copy() scihdu.flush() scihdu.close() archdu.close() os.remove(af) # Add the airmass, exptime, and other keywords back into the # extracted spectrum header kws = ['AIRMASS','EXPTIME', 'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG', 'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX', 'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS', 'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA', 'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL', 'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING', 'FILTER'] for kw in kws: pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw)) iraf.cd('..')
def extract(fs=None): iraf.cd('work') if fs is None: fs = glob('fix/*fix*.fits') if len(fs) == 0: print "WARNING: No fixpixed images available for extraction." iraf.cd('..') return if not os.path.exists('x1d'): os.mkdir('x1d') print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line." for f in fs: # Get the output filename without the ".fits" outbase = f.replace('fix', 'x1d')[:-5] # Get the readnoise, right now assume default value of 5 but we could # get this from the header readnoise = 5 # If interactive open the rectified, background subtracted image in ds9 ds9display(f.replace('fix', 'bkg')) # set dispaxis = 1 just in case pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1) iraf.unlearn(iraf.apall) iraf.flpr() iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes', review='no', line='INDEF', nsum=-1000, lower=-5, upper=5, b_function='legendre', b_order=5, b_sample='-400:-200,200:400', b_naverage=-10, b_niterate=5, b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15, t_step=15, t_nlost=100, t_function='legendre', t_order=5, t_niterate=5, t_low_reject=3.0, t_high_reject=3.0, background='fit', weights='variance', pfit='fit1d', clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0, usigma=4.0, mode='hl') # Copy the CCDSUM keyword into the 1d extraction pyfits.setval(outbase + '.fits', 'CCDSUM', value=pyfits.getval(f, 'CCDSUM')) # Extract the corresponding arc arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0] # set dispaxis = 1 just in case pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1) iraf.unlearn(iraf.apsum) iraf.flpr() iraf.apsum(input=arcname + '[SCI]', output='auxext_arc', references=f[:-5] + '[SCI]', interactive='no', find='no', edit='no', trace='no', fittrace='no', extras='no', review='no', background='no', mode='hl') # copy the arc into the 5 column of the data cube arcfs = glob('auxext_arc*.fits') for af in arcfs: archdu = pyfits.open(af) scihdu = pyfits.open(outbase + '.fits', mode='update') d = scihdu[0].data.copy() scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2])) scihdu[0].data[:-1, :, :] = d[:, :, :] scihdu[0].data[-1::, :] = archdu[0].data.copy() scihdu.flush() scihdu.close() archdu.close() os.remove(af) # Add the airmass, exptime, and other keywords back into the # extracted spectrum header kws = ['AIRMASS','EXPTIME', 'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG', 'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX', 'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS', 'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA', 'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL', 'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING', 'FILTER'] for kw in kws: pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw)) iraf.cd('..')
def speccombine(fs=None): iraf.cd('work') if fs is None: fs = glob('flx/sci*c?.fits') if len(fs)==0: print("No flux calibrated images to combine.") iraf.cd('..') return nsteps = 8001 lamgrid = np.linspace(2000.0, 10000.0, nsteps) nfs = len(fs) # for each aperture # get all of the science images specs = np.zeros((nfs, nsteps)) specerrs = np.zeros((nfs, nsteps)) ap = 0 for i, f in enumerate(fs): hdu = pyfits.open(f) w = WCS(f) # get the wavelengths of the pixels npix = hdu[0].data.shape[2] lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0] # interpolate each spectrum onto a comman wavelength scale specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap], left=0.0, right=0.0) # Also calculate the errors. Right now we assume that the variances # interpolate linearly. This is not stricly correct but it should be # close. Also we don't include terms in the variance for the # uncertainty in the wavelength solution. specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5 # minimize the chi^2 given free parameters are multiplicative factors # We could use linear or quadratic, but for now assume constant p0 = np.ones(nfs) results = optimize.minimize(combine_spec_chi2, p0, args=(lamgrid, specs, specerrs), method='Nelder-Mead', options={'maxfev': 1e5, 'maxiter': 1e5}) # write the best fit parameters into the headers of the files # Dump the list of spectra into a string that iraf can handle iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '') # write the best fit results into a file lines = [] for p in results['x']: lines.append('%f\n' % (1.0 / p)) f = open('flx/scales.dat', 'w') f.writelines(lines) f.close() # run scombine after multiplying the spectra by the best fit parameters combfile = 'sci_com.fits' if os.path.exists(combfile): os.remove(combfile) iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat', reject='avsigclip', lthreshold=-1e-17) # Remove the other apertures [TBD] # remove the sky and arc bands from the combined spectra. (or add back?? TBD) # remove some header keywords that don't make sense in the combined file delkws = ['GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4'] for kw in delkws: pyfits.delval(combfile,kw) # combine JD (average), AIRMASS (average), EXPTIME (sum) # we assume there is a c1.fits file for each image c1fs = [f for f in fs if 'c1.fits' in f] avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs]) pyfits.setval(combfile,'JD',value=avgjd) print "average JD = " + str(avgjd) sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs]) pyfits.setval(combfile,'EXPTIME',value=sumet) print "total EXPTIME = " + str(sumet) avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs]) pyfits.setval(combfile,'AIRMASS',value=avgam) print "avg AIRMASS = " + str(avgam) iraf.cd('..') return specs
@author: cmccully ''' import os, shutil from glob import glob import pyfits import numpy as np from astroscrappy import detect_cosmics from pyraf import iraf from scipy import interpolate, ndimage, signal, optimize import pf_model as pfm import statsmodels as sm from astropy.modeling import models, fitting import astropy iraf.cd(os.getcwd()) iraf.gemini() iraf.gmos() iraf.onedspec() bluecut = 3450 iraf.gmos.logfile = "log.txt" iraf.gmos.mode = 'h' iraf.set(clobber='yes') iraf.set(stdimage='imtgmos') dooverscan = False is_GS = False
def extract(arcs, objs,ccd,mode=1): """ mode=1: 1D extraction mode=2: 2D extraction """ #apall na vseh luckah print " + identifing arcs" for i in arcs: iraf.hedit(images='tmp/arcs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null") iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null") if mode==1: pass else: iraf.cd("tmp/arcs") geometry_prep(arcs,ccd) os.system("cp transformations* ../objs/") #sys.exit(0) iraf.cd("../..") for i in arcs: os.system("rm tmp/arcs/%s.ms.fits" % (i[1][:-5])) os.system("rm tmp/arcs/%s_cut*.fits" % (i[1][:-5])) os.system("rm tmp/arcs/%s_t*.fits" % (i[1][:-5])) pass os.system("rm tmp/arcs/calibrations/idarcs_cut*") os.system("rm tmp/arcs/arcs_cut*") #extract 2d arcs and objects for i in arcs: iraf.hedit(images='tmp/arcs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null") iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null") #forget apertures for j in range(8,1000): iraf.hedit(images='tmp/arcs/%s.ms.fits' % (i[1][:-5]), fields="APNUM%s" % (str(j)), value='', delete="yes", verify="no", Stdout="/dev/null") for i in objs: iraf.hedit(images='tmp/objs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null") iraf.apall(input='tmp/objs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null") #forget apertures for j in range(8,1000): iraf.hedit(images='tmp/objs/%s.ms.fits' % (i[1][:-5]), fields="APNUM%s" % (str(j)), value='', delete="yes", verify="no", Stdout="/dev/null") iraf.cd("tmp/arcs") for ii in arcs: geometry_transform(ii) iraf.cd("../..") iraf.cd("tmp/objs") for ii in objs: geometry_transform(ii) iraf.cd("../..") #make normal 1d extraction and copy results into it for i in arcs: os.system("rm -f tmp/arcs/%s" % (i[1][:-5]+".ms.fits")) iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=1, pfit="fit1d", Stdout="/dev/null") os.system("cp tmp/arcs/%s" % (i[1][:-5])+".ms.fits tmp/arcs/%s" % (i[1][:-5])+".ms2.fits") if mode==1: pass else: for j in range(1,393): os.system("rm -f tmp/copy_tmp.fits") try: iraf.blkavg(input="tmp/arcs/"+i[1][:-5]+"_t%s.fits" % (str(j)), output="tmp/copy_tmp", option='sum', b1=1, b2=7, Stdout="/dev/null") iraf.imcopy(input="tmp/copy_tmp", output="tmp/arcs/"+i[1][:-5]+".ms.fits[*,%s]" % (j), Stdout="/dev/null") except: pass for i in objs: os.system("rm -f tmp/objs/%s" % (i[1][:-5]+".ms.fits")) iraf.apall(input='tmp/objs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=1, pfit="fit1d", Stdout="/dev/null") os.system("cp tmp/objs/%s" % (i[1][:-5])+".ms.fits tmp/objs/%s" % (i[1][:-5])+".ms2.fits") if mode==1: pass else: for j in range(1,393): os.system("rm -f tmp/copy_tmp.fits") try: iraf.blkavg(input="tmp/objs/"+i[1][:-5]+"_t%s.fits" % (str(j)), output="tmp/copy_tmp", option='sum', b1=1, b2=7, Stdout="/dev/null") iraf.imcopy(input="tmp/copy_tmp", output="tmp/objs/"+i[1][:-5]+".ms.fits[*,%s]" % (j), Stdout="/dev/null") except: pass
def speccombine(fs=None): iraf.cd('work') if fs is None: fs = glob('flx/sci*c?.fits') if len(fs)==0: print("No flux calibrated images to combine.") iraf.cd('..') return nsteps = 8001 lamgrid = np.linspace(2000.0, 10000.0, nsteps) nfs = len(fs) # for each aperture # get all of the science images specs = np.zeros((nfs, nsteps)) specerrs = np.zeros((nfs, nsteps)) ap = 0 for i, f in enumerate(fs): hdu = pyfits.open(f) w = WCS(f) # get the wavelengths of the pixels npix = hdu[0].data.shape[2] lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0] # interpolate each spectrum onto a comman wavelength scale specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap], left=0.0, right=0.0) # Also calculate the errors. Right now we assume that the variances # interpolate linearly. This is not stricly correct but it should be # close. Also we don't include terms in the variance for the # uncertainty in the wavelength solution. specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5 # minimize the chi^2 given free parameters are multiplicative factors # We could use linear or quadratic, but for now assume constant p0 = np.ones(nfs) results = optimize.minimize(combine_spec_chi2, p0, args=(lamgrid, specs, specerrs), method='Nelder-Mead', options={'maxfev': 1e5, 'maxiter': 1e5}) # write the best fit parameters into the headers of the files # Dump the list of spectra into a string that iraf can handle iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '') # write the best fit results into a file lines = [] for p in results['x']: lines.append('%f\n' % (1.0 / p)) f = open('flx/scales.dat', 'w') f.writelines(lines) f.close() # run scombine after multiplying the spectra by the best fit parameters combfile = 'sci_com.fits' if os.path.exists(combfile): os.remove(combfile) iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat', reject='avsigclip', lthreshold=1e-19) # Remove the other apertures [TBD] # remove the sky and arc bands from the combined spectra. (or add back?? TBD) # remove some header keywords that don't make sense in the combined file delkws = ['GRATING','GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4'] for kw in delkws: pyfits.delval(combfile,kw) # combine JD (average), AIRMASS (average), EXPTIME (sum) # we assume there is a c1.fits file for each image c1fs = [f for f in fs if 'c1.fits' in f] avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs]) pyfits.setval(combfile,'JD',value=avgjd) print "average JD = " + str(avgjd) sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs]) pyfits.setval(combfile,'EXPTIME',value=sumet) print "total EXPTIME = " + str(sumet) avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs]) pyfits.setval(combfile,'AIRMASS',value=avgam) print "avg AIRMASS = " + str(avgam) iraf.cd('..') return specs
def speccombine(fs=None): iraf.cd('work') if fs is None: fs = glob('trm/sci*c?.fits') if len(fs)==0: print("No flux calibrated images to combine.") iraf.cd('..') return #diagnostic() nsteps = 8001 lamgrid = np.linspace(2000.0, 10000.0, nsteps) nfs = len(fs) # for each aperture # get all of the science images specs = np.zeros((nfs, nsteps)) specerrs = np.zeros((nfs, nsteps)) ap = 0 for i, f in enumerate(fs): hdu = pyfits.open(f) # print ('---hdu.data---') # print (hdu[0].data) w=WCS(f) # print ('-----w-----') # print(w) # get the wavelengths of the pixels npix = hdu[0].data.shape[2] # print('-----npix-----') # print(npix) lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0] # print('-----lam-----') # print(lam) # interpolate each spectrum onto a comman wavelength scale specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap], left=0.0, right=0.0) # Also calculate the errors. Right now we assume that the variances # interpolate linearly. This is not stricly correct but it should be # close. Also we don't include terms in the variance for the # uncertainty in the wavelength solution. specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5 #print ('-----specs-----') #print (specs) # minimize the chi^2 given free parameters are multiplicative factors # We could use linear or quadratic, but for now assume constant p0 = np.ones(nfs) results = optimize.minimize(combine_spec_chi2, p0, args=(lamgrid, specs, specerrs), method='Nelder-Mead', options={'maxfev': 1e5, 'maxiter': 1e5}) # write the best fit parameters into the headers of the files # Dump the list of spectra into a string that iraf can handle iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '') # write the best fit results into a file lines = [] for p in results['x']: lines.append('%f\n' % (1.0 / p)) f = open('flx/scales.dat', 'w') f.writelines(lines) f.close() # run scombine after multiplying the spectra by the best fit parameters combfile = 'sci_com.fits' if os.path.exists(combfile): os.remove(combfile) iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat', reject='avsigclip', lthreshold=-2e-16) # Remove the other apertures [TBD] # remove the sky and arc bands from the combined spectra. (or add back?? TBD) # remove some header keywords that don't make sense in the combined file delkws = ['GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4'] for kw in delkws: pyfits.delval(combfile,kw) # combine JD (average), AIRMASS (average), EXPTIME (sum) # we assume there is a c1.fits file for each image c1fs = [f for f in fs if 'c1.fits' in f] avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs]) pyfits.setval(combfile,'JD',value=avgjd, comment='average of multiple exposures') print "average JD = " + str(avgjd) sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs]) pyfits.setval(combfile,'EXPTIME',value=sumet,comment='sum of multiple exposures') print "total EXPTIME = " + str(sumet) avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs]) pyfits.setval(combfile,'AIRMASS',value=avgam,comment='average of multiple exposures') print "avg AIRMASS = " + str(avgam) # update this to used avg jd midpoint of all exposures? print "barycentric velocity correction (km/s) = ", iraf.bcvcorr(spectra=combfile,keytime='UTC-OBS',keywhen='mid', obslong="339:11:16.8",obslat="-32:22:46.2",obsalt='1798',obsname='saao', savebcv='yes',savejd='yes',printmode=2) pyfits.setval(combfile,'UTMID',comment='added by RVSAO task BCVCORR') pyfits.setval(combfile,'GJDN',comment='added by RVSAO task BCVCORR') pyfits.setval(combfile,'HJDN',comment='added by RVSAO task BCVCORR') pyfits.setval(combfile,'BCV',comment='added by RVSAO task BCVCORR (km/s)') pyfits.setval(combfile,'HCV',comment='added by RVSAO task BCVCORR (km/s)') iraf.dopcor(input=combfile,output='',redshift=-iraf.bcvcorr.bcv,isvelocity='yes', add='no',dispersion='yes',flux='no',verbose='yes') pyfits.setval(combfile,'DOPCOR01',comment='barycentric velocity correction applied') iraf.cd('..')
def cd(target): """Changes to a directory, pyraf included""" os.chdir(target) iraf.cd(target)
def reduce_science(rawdir, rundir, flat, arc, twilight, twilight_flat, sciimg, starimg, bias, overscan, vardq, observatory, lacos, apply_lacos, lacos_xorder, lacos_yorder, lacos_sigclip, lacos_objlim, bpm, instrument, slits, fl_gscrrej, wltrim_frac, grow_gap, cube_bit_mask): """ Reduction pipeline for standard star. Parameters ---------- rawdir: string Directory containing raw images. rundir: string Directory where processed files are saved. flat: string Names of the files containing flat field images. arc: string Arc images. twilight: string Twilight flat images. twilight_flat: string Flat field for twilight image. starimg: string Name of the file containing the image to be reduced. bias: string Bias images. grow_gap: number Number of pixels by which to grow the bad pixel mask around the chip gaps. """ iraf.set(stdimage='imtgmos') iraf.gemini() iraf.unlearn('gemini') iraf.gmos() iraf.unlearn('gmos') iraf.gemtools() iraf.unlearn('gemtools') # os.path.isfile('arquivo') iraf.unlearn('gemini') iraf.unlearn('gmos') iraf.task(lacos_spec=lacos) # set directories iraf.set(caldir=rawdir) # iraf.set(rawdir=rawdir) # raw files iraf.set(procdir=rundir) # processed files iraf.gmos.logfile = 'logfile.log' iraf.gfextract.verbose = 'no' iraf.cd('procdir') flat = flat.replace('.fits', '') twilight = twilight.replace('.fits', '') twilight_flat = twilight_flat.replace('.fits', '') arc = arc.replace('.fits', '') starimg = starimg.replace('.fits', '') sciimg = sciimg.replace('.fits', '') mdffile = 'mdf' + flat + '.fits' iraf.gfreduce.bias = 'caldir$' + bias iraf.gfreduce.fl_fulldq = 'yes' iraf.gfreduce.fl_fixgaps = 'yes' iraf.gfreduce.grow = grow_gap iraf.gireduce.bpm = 'rawdir$' + bpm iraf.gfextract.verbose = 'no' cal_reduction(rawdir=rawdir, rundir=rundir, flat=flat, arc=arc, twilight=twilight, bias=bias, bpm=bpm, overscan=overscan, vardq=vardq, instrument=instrument, slits=slits, twilight_flat=twilight_flat, grow_gap=grow_gap) # # Actually reduce science # image_name = 'rg' + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: iraf.gfreduce(sciimg, slits='header', rawpath='rawdir$', fl_inter='no', fl_addmdf='yes', key_mdf='MDF', mdffile=mdffile, weights='no', fl_over=overscan, fl_trim='yes', fl_bias='yes', trace='no', recenter='no', fl_fulldq='yes', fl_flux='no', fl_gscrrej='no', fl_extract='no', fl_gsappwave='no', fl_wavtran='no', fl_novl='yes', fl_skysub='no', fl_vardq=vardq, mdfdir='procdir$') prefix = 'rg' # Gemfix image_name = 'p' + prefix + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: iraf.gemfix(prefix + sciimg, out='p' + prefix + sciimg, method='fit1d', bitmask=65535, axis=1) prefix = 'p' + prefix # LA Cosmic - cosmic ray removal if apply_lacos: image_name = 'l' + prefix + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: iraf.gemcrspec(prefix + sciimg, out='l' + prefix + sciimg, sigfrac=0.5, niter=4, fl_vardq=vardq, xorder=lacos_xorder, yorder=lacos_yorder, sigclip=lacos_sigclip, objlim=lacos_objlim) prefix = 'l' + prefix if fl_gscrrej: image_name = 'ex' + prefix + sciimg + '.fits' else: image_name = 'e' + prefix + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: iraf.gfreduce(prefix + sciimg, slits='header', rawpath='./', fl_inter='no', fl_addmdf='no', key_mdf='MDF', mdffile=mdffile, fl_over='no', fl_trim='no', fl_bias='no', trace='no', recenter='no', fl_flux='no', fl_gscrrej=fl_gscrrej, fl_extract='yes', fl_gsappwave='yes', fl_wavtran='no', fl_novl='no', fl_skysub='no', grow=grow_gap, reference='eprg' + flat, weights='no', wavtraname='eprg' + arc, response='eprg' + flat + '_response.fits', fl_vardq=vardq, fl_fulldq='yes', fl_fixgaps='yes') if fl_gscrrej: prefix = 'ex' + prefix else: prefix = 'e' + prefix # if wl2 > 7550.0: # wl2 = 7550.0 # # Apply wavelength transformation # wl1, wl2 = wl_lims(prefix + sciimg + '.fits', wltrim_frac) image_name = 't' + prefix + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: iraf.gftransform( prefix + sciimg, wavtraname='eprg' + arc, fl_vardq=vardq, w1=wl1, w2=wl2, ) prefix = 't' + prefix # # Sky subtraction # image_name = 's' + prefix + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: iraf.gfskysub( prefix + sciimg, expr='default', combine='median', reject='avsigclip', scale='none', zero='none', weight='none', sepslits='yes', fl_inter='no', lsigma=1, hsigma=1, ) prefix = 's' + prefix # # Apply flux calibration to galaxy # image_name = 'c' + prefix + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: iraf.gscalibrate(prefix + sciimg, sfuncti=starimg, extinct='onedstds$ctioextinct.dat', observatory=observatory, fluxsca=1, fl_vardq=vardq) prefix = 'c' + prefix # # Remove spurious data with PCA # image_name = 'x' + prefix + sciimg + '.fits' print(os.getcwd()) print(image_name) if os.path.isfile(image_name): pipe.skipwarn(image_name) else: t = pca.Tomography(prefix + sciimg + '.fits') t.decompose() t.remove_cosmic_rays(sigma_threshold=10.0) t.write(image_name) prefix = 'x' + prefix # # Create data cubes # image_name = 'd' + prefix + sciimg + '.fits' if os.path.isfile(image_name): pipe.skipwarn(image_name) else: data_cube = CubeBuilder(prefix + sciimg + '.fits') data_cube.build_cube() data_cube.fit_refraction_function() data_cube.fix_atmospheric_refraction() data_cube.write(image_name)
import os,string,sys import pylab import numpy as numarray from numpy import zeros,where,add,sqrt,array,any,nan,arange thisdir = os.getcwd() os.chdir('/home/durant') from pyraf import iraf os.chdir(thisdir) iraf.cd(thisdir) iraf.stsdas() iraf.digiphot() iraf.daophot() import glob import pyfits import astLib class star_photometry(object): """reads photometry information from IRAF .als or .mag file, and returns information from it as arrays of magnitudes etc, including conversions of co-ordinates etc. self.data is a float array of the data, containing: id,x,y,mag,magerr,chi,ra,dec. self.image is the corresponding FITS file with WCS information self.photfile is the original file self.nstar is number of stars (=len(self.data))""" mynames={"id":0,"x":1,"y":2,"mag":3,"magerr":4, "sky":5,"chi":6,"ra":7,"dec":8} #record structure def __init__(self,photfile,imagefile=None,calcradec=True): """load data from photfile, and store in self.stars, associate with imagefile if it exists, or try to find it"""
def identify_edge(infile, overwrite=False): print('\n#############################') print('Identifying the edges.') binfct1 = fits.getval(infile, 'BIN-FCT1') coordlist = fi.filibdir + 'edge' + str(binfct1) + '.dat' section = 'middle line' verbose = 'yes' nsum = 50 match = -10. fwidth = 6. / binfct1 cradius = 20. / binfct1 threshold = 0. function = 'chebyshev' order = 2 niter = 0 autowrite = 'yes' newaps = 'yes' override = 'yes' refit = 'no' trace = 'yes' step = 50 shift = 0 nlost = 0 minsep = 60. / binfct1 addfeatures = 'no' database = 'database' logfile = 'identify_edge.log' # Not to display items in IRAF packages sys.stdout = open('/dev/null', 'w') iraf.noao() iraf.twodspec() iraf.longslit() sys.stdout = sys.__stdout__ # Back to the stadard output # entering the channel image directory. # os.chdir() does not change the directory for pyraf only in this function. print('\t Entering the channel image directory, \"' + fi.chimagedir + '\".') iraf.cd(fi.chimagedir) basename = fits.getval('../' + infile, 'FRAMEID') idfile = database + '/id' + basename + '.ch01edge' if os.path.isfile(idfile) and not overwrite: print('\t Edge identification files already exist, ' + idfile \ + '. Skipping.') else: if os.path.isfile(idfile) and overwrite: print('\t Removing ' + idfile) try: os.remove(idfile) except: pass print('\t Identifying: ' + basename + '.ch01edge.fits') iraf.identify(basename + '.ch01edge', section=section, database=database, coordlist=coordlist, units='', nsum=nsum, match=match, maxfeat=2, ftype='emission', fwidth=fwidth, cradius=cradius, threshold=threshold, function=function, order=order, sample='*', niter=niter, autowrite=autowrite) print('\t Reidentifying: ' + basename + '.ch01edge.fits') iraf.reidentify(basename + '.ch01edge', basename + '.ch01edge', interac='no', section=section, newaps=newaps, override=override, refit=refit, trace=trace, step=step, nsum=nsum, shift=shift, nlost=nlost, cradius=cradius, threshold=threshold, addfeatures=addfeatures, coordlist=coordlist, match=match, maxfeat=2, minsep=minsep, database=database, logfile=logfile, plotfile='', verbose=verbose, cursor='') for i in range(2, 25): print('\t Reidentifying: ' + basename + '.ch%02dedge.fits' % i) idfile = database + '/id' + basename + '.ch%02dedge' % i if os.path.isfile(idfile) and not overwrite: print('\t Edge identification files already exist, ' + idfile + '. Skipping.') else: if os.path.isfile(idfile) and overwrite: print('\t Removing ' + idfile) try: os.remove(idfile) except: pass # treatment for VPH650 if i == 12: disperser = fits.getval(basename + '.ch12edge.fits', 'DISPERSR') if disperser == 'SCFCGRHD65': nlost = 1 if i == 13: disperser = fits.getval(basename + '.ch12edge.fits', 'DISPERSR') if disperser == 'SCFCGRHD65': nlost = 0 iraf.reidentify(basename+'.ch%02dedge'%(i-1), \ basename+'.ch%02dedge'%i, \ interac='no', section=section, newaps=newaps, \ override=override, refit=refit, trace=trace, \ step=0.0, nsum=nsum, shift=shift, nlost=nlost, \ cradius=cradius, threshold=threshold, \ addfeatures=addfeatures, coordlist=coordlist, \ match=match, maxfeat=2, minsep=minsep, \ database=database, logfile=logfile, \ plotfile='', verbose=verbose, cursor='') #Check the result iraf.identify(basename+'.ch%02dedge'%i, section=section, \ database=database, coordlist=coordlist, units='', \ nsum=nsum, match=match, maxfeat=2,ftype='emission', \ fwidth=fwidth, cradius=cradius, threshold=threshold, \ function=function, order=order, sample='*', \ niter=niter, autowrite=autowrite) iraf.reidentify(basename+'.ch%02dedge'%i, \ basename+'.ch%02dedge'%i, \ interac='no', section=section, newaps=newaps, \ override=override, refit=refit, trace=trace, \ step=step, nsum=nsum, shift=shift, nlost=nlost, \ cradius=cradius, threshold=threshold, \ addfeatures=addfeatures, coordlist=coordlist, \ match=match, maxfeat=2, minsep=minsep, \ database=database, logfile=logfile, \ plotfile='', verbose=verbose, cursor='') print('\t Go back to the original directory.') iraf.cd('..') disperser = fits.getval(fi.chimagedir + basename + '.ch12edge.fits', 'DISPERSR') if disperser == 'SCFCGRHD65': correct_ch12_edge(basename, overwrite=overwrite) return
def reduce_stdstar(rawdir, rundir, caldir, starobj, stdstar, flat, arc, twilight, twilight_flat, starimg, bias, overscan, vardq, lacos, observatory, apply_lacos, lacos_xorder, lacos_yorder, lacos_objlim, lacos_sigclip, bpm, instrument, slits, fl_gscrrej, wltrim_frac=0.03, sens_order=6, sens_function='spline3', apsum_radius=1): """ Reduction pipeline for standard star. Parameters ---------- rawdir: string Directory containing raw images. rundi: string Directory where processed files are saved. caldir: string Directory containing standard star calibration files. starobj: string Object keyword for the star image. stdstar: string Star name in calibration file. flat: list Names of the files containing flat field images. arc: list Arc images. twilight: list Twilight flat images. starimg: string Name of the file containing the image to be reduced. bias: list Bias images. """ iraf.set(stdimage='imtgmos') iraf.task(lacos_spec=lacos) iraf.gemini() iraf.unlearn('gemini') iraf.gmos() iraf.unlearn('gmos') iraf.gemtools() iraf.unlearn('gemtools') iraf.gmos.logfile = 'logfile.log' iraf.gemtools.gloginit.logfile = 'logfile.log' iraf.gfextract.verbose = 'no' # set directories iraf.set(caldir=rawdir) # iraf.set(rawdir=rawdir) # raw files iraf.set(procdir=rundir) # processed files # os.path.isfile('arquivo') iraf.cd('procdir') flat = flat.strip('.fits') twilight = twilight.strip('.fits') twilight_flat = twilight_flat.strip('.fits') arc = arc.strip('.fits') starimg = starimg.strip('.fits') mdffile = 'mdf' + flat + '.fits' iraf.gfreduce.bias = 'rawdir$' + bias iraf.gfreduce.fl_fulldq = 'yes' iraf.gfreduce.fl_fixgaps = 'yes' iraf.gireduce.bpm = 'rawdir$' + bpm cal_reduction(rawdir=rawdir, rundir=rundir, flat=flat, arc=arc, twilight=twilight, bias=bias, bpm=bpm, overscan=overscan, vardq=vardq, instrument=instrument, slits=slits, twilight_flat=twilight_flat) # # Actually reduce star # imageName = 'rg' + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: imageName = 'g' + starimg + '.fits' if os.path.isfile(imageName): iraf.printlog( 'GIREDS: WARNING: Removing file {:s}'.format(imageName), 'logfile.log', 'yes') iraf.delete(imageName) iraf.gfreduce(starimg, slits='header', rawpath='rawdir$', fl_inter='no', fl_addmdf='yes', key_mdf='MDF', mdffile=mdffile, weights='no', fl_over=overscan, fl_trim='yes', fl_bias='yes', trace='no', recenter='no', fl_flux='no', fl_gscrrej='no', fl_extract='no', fl_gsappwave='no', fl_wavtran='no', fl_novl='yes', fl_skysub='no', fl_vardq=vardq, mdfdir='procdir$') prefix = 'rg' # # Gemfix # imageName = 'p' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gemfix(prefix + starimg, out='p' + prefix + starimg, method='fit1d', bitmask=1, axis=1) prefix = 'p' + prefix # # LA Cosmic # if apply_lacos: imageName = 'l' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: if apply_lacos: iraf.gemcrspec(prefix + starimg, out='l' + prefix + starimg, sigfrac=0.32, niter=4, fl_vardq=vardq, xorder=lacos_xorder, yorder=lacos_yorder, objlim=lacos_objlim, sigclip=lacos_sigclip) prefix = 'l' + prefix # # Extraction and Gemini's comsmic ray rejection # if fl_gscrrej: imageName = 'x' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) fl_gscrrej = False else: imageName = 'ex' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gfreduce(prefix + starimg, slits='header', rawpath='./', fl_inter='no', fl_addmdf='no', key_mdf='MDF', mdffile=mdffile, fl_over='no', fl_trim='no', fl_bias='no', trace='no', recenter='no', fl_flux='no', fl_gscrrej=fl_gscrrej, fl_extract='yes', fl_gsappwave='yes', fl_wavtran='no', fl_novl='no', fl_skysub='no', reference='eprg' + flat, weights='no', wavtraname='eprg' + arc, response='eprg' + flat + '_response.fits', fl_vardq=vardq) prefix = 'ex' + prefix else: imageName = 'e' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gfreduce(prefix + starimg, slits='header', rawpath='./', fl_inter='no', fl_addmdf='no', key_mdf='MDF', mdffile=mdffile, fl_over='no', fl_trim='no', fl_bias='no', trace='no', recenter='no', fl_flux='no', fl_gscrrej=fl_gscrrej, fl_extract='yes', fl_gsappwave='yes', fl_wavtran='no', fl_novl='no', fl_skysub='no', reference='eprg' + flat, weights='no', wavtraname='eprg' + arc, response='eprg' + flat + '_response.fits', fl_vardq=vardq) prefix = 'e' + prefix # # Wavelength transform # wl1, wl2 = wl_lims(prefix + starimg + '.fits', wltrim_frac) imageName = 't' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gfreduce(prefix + starimg, slits='header', rawpath='./', fl_inter='no', fl_addmdf='no', key_mdf='MDF', mdffile=mdffile, fl_over='no', fl_trim='no', fl_bias='no', trace='no', recenter='no', fl_flux='no', fl_gscrrej='no', fl_extract='no', fl_gsappwave='no', fl_wavtran='yes', fl_novl='no', fl_skysub='no', reference='eprg' + flat, weights='no', wavtraname='eprg' + arc, response='eprg' + flat + '_response.fits', fl_vardq=vardq, w1=wl1, w2=wl2) prefix = 't' + prefix # # Sky subtraction # imageName = 's' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gfreduce(prefix + starimg, slits='header', rawpath='./', fl_inter='no', fl_addmdf='no', key_mdf='MDF', mdffile=mdffile, fl_over='no', fl_trim='no', fl_bias='no', trace='no', recenter='no', fl_flux='no', fl_gscrrej='no', fl_extract='no', fl_gsappwave='no', fl_wavtran='no', fl_novl='no', fl_skysub='yes', reference='eprg' + flat, weights='no', wavtraname='eprg' + arc, response='eprg' + flat + '_response.fits', fl_vardq=vardq, w1=wl1, w2=wl2) prefix = 's' + prefix # # Apsumming the stellar spectra # xinst = pf.getdata(prefix + starimg + '.fits', ext=1)['XINST'] if instrument == 'GMOS-N': x0 = np.average(xinst[xinst < 10]) elif instrument == 'GMOS-S': x0 = np.average(xinst[xinst > 10]) ap_expression = '((XINST-{:.2f})**2 + '\ '(YINST-2.45)**2)**0.5 < {:.2f}'.format(x0, apsum_radius) imageName = 'a' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gfapsum(prefix + starimg, fl_inter='no', lthreshold='INDEF', hthreshold='INDEF', reject='avsigclip', expr=ap_expression) # # Building sensibility function # if os.path.isfile('std' + starimg)\ and os.path.isfile('sens' + starimg + '.fits'): skipwarn('std{0:s} and sens{0:s}.fits'.format(starimg)) else: imageName = 'std' + starimg if os.path.isfile(imageName): iraf.printlog( 'GIREDS: WARNING: Removing file {:s}'.format(imageName), 'logfile.log', 'yes') iraf.delete(imageName) imageName = 'sens' + starimg + '.fits' if os.path.isfile(imageName): iraf.printlog( 'GIREDS: WARNING: Removing file {:s}'.format(imageName), 'logfile.log', 'yes') iraf.delete(imageName) iraf.gsstandard('a' + prefix + starimg, starname=stdstar, observatory=observatory, sfile='std' + starimg, sfunction='sens' + starimg, caldir=caldir, order=sens_order, function=sens_function) # # Apply flux calibration to star # imageName = 'c' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gscalibrate(prefix + starimg, sfuncti='sens' + starimg, extinct='onedstds$ctioextinct.dat', observatory=observatory, fluxsca=1, fl_vardq=vardq) # # Create data cubes # imageName = 'dc' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: data_cube = CubeBuilder('c' + prefix + starimg + '.fits') data_cube.build_cube() data_cube.fit_refraction_function() data_cube.fix_atmospheric_refraction() data_cube.write(imageName) # # Test calibration # iraf.cd(caldir) caldata = np.loadtxt(stdstar + '.dat', unpack=True) iraf.cd('procdir') calflux = mag2flux(caldata[0], caldata[1]) imageName = 'ca' + prefix + starimg + '.fits' if os.path.isfile(imageName): skipwarn(imageName) else: iraf.gscalibrate('a' + prefix + starimg, sfuncti='sens' + starimg, extinct='onedstds$ctioextinct.dat', observatory=observatory, fluxsca=1) sumflux = pf.getdata('ca' + prefix + starimg + '.fits', ext=2) sumhead = pf.getheader('ca' + prefix + starimg + '.fits', ext=2) sumwl = sumhead['crval1'] + np.arange( sumhead['naxis1']) * sumhead['cdelt1'] plt.close('all') plt.plot(sumwl, sumflux, 'b', lw=.5) plt.plot(caldata[0], calflux, 'r', lw=1.5) plt.xlim(sumwl[0] * .99, sumwl[-1] * 1.01) plt.ylim(min(calflux) * .8, max(calflux) * 1.2) plt.savefig('calib' + starimg + '.eps')
#!/usr/bin/python # IMP: Keep ds9 open # This script will guide you through the V R and I images directories. # Tell the script which combination of images to align , combine etc. # After that you can do photometry. # Enjoy!!!---------------------- [email protected] import os import glob from pyraf import iraf import pyfits import sys, traceback dirs = open('directories', 'r') for night in dirs.readlines(): night = night.rstrip() iraf.cd(night) print night filters = glob.glob('image[RVI]') #Only going to image[RVI] filters.sort() for imgdir in filters: iraf.cd(imgdir) while 1: print night + ' ' + imgdir images = glob.glob('[sn][nz]*.fits') images.sort() imgs_txt = open('imgs.txt', 'w') i = 0 for img in images: hdulist = pyfits.open(img) Exptime = hdulist[0].header.get('EXPTIME')
def merge_cubes(rawdir, rundir, name, observatory, imgcube, xoff, yoff, crval3, cdelt3, cdelt1): """ Merge cubes. Parameters ---------- rawdir: string Directory containing raw images. rundir: string Directory where processed files are saved. name: string Name of the object. observatory: string Gemini-South/Gemini-North. imgcube: list of strings Cube file for each object cube. xoff: list of floats x-offset for each object cube. yoff: list of floats y-offset for each object cube. crval3: list of floats crval3 for each object cube. cdelt3: list of floats cdelt3 for each object cube. cdelt1: list of floats cdelt1 for each object cube. """ rundir = rundir + '/' iraf.set(stdimage='imtgmos') iraf.gemini() iraf.unlearn('gemini') iraf.gmos() iraf.unlearn('gmos') iraf.gemtools() iraf.unlearn('gemtools') iraf.gmos.logfile = 'logfile.log' iraf.gemtools.gloginit.logfile = 'logfile.log' # set directories iraf.set(caldir=rawdir) # iraf.set(rawdir=rawdir) # raw files iraf.set(procdir=rundir) # processed files iraf.cd('procdir') # # Creation of file/offset files # nCubes = len(imgcube) in_filesSCI = 'files_' + name + '_SCI' in_filesVAR = 'files_' + name + '_VAR' in_offset = 'offsets_' + name with open(in_filesSCI, 'w') as f: for img in imgcube: f.write(rundir + img + '[1]' + '\n') with open(in_filesVAR, 'w') as f: for img in imgcube: f.write(rundir + img + '[2]' + '\n') # Invert (x,y)offsets if in gemini-north sign = -1 if (observatory.lower() == 'gemini-north') else 1 with open(in_offset, 'w') as f: for k in range(nCubes): f.write("{:.5f} {:.5f} {:.5f}\n".format( sign * (xoff[k] - xoff[0]) / cdelt1[k], sign * (yoff[k] - yoff[0]) / cdelt1[k], (crval3[k] - crval3[0]) / cdelt3[k])) # # Definition of in/output files. And header modification. # in_sci = [img + '[1]' for img in imgcube] in_var = [img + '[2]' for img in imgcube] in_dq = [img + '[3]' for img in imgcube] out_sci = name + '_SCI.fits' out_var = name + '_VAR.fits' out_sigIN = name + '_SIGIN.fits' out_exp = name + '_EXP' # Convert DQ extension to 'pl' and add the its filename to 'bpm' key # --- Change to other key. (Other rotines use this key) - Improve # --- Change also the key for bpm used by 'fixpix' ------ Improve out_dqPL = [img[:-5] + '_DQ.pl' for img in imgcube] for k in range(nCubes): print(in_sci[k], in_var[k], in_dq[k], out_dqPL[k]) iraf.imcopy(in_dq[k], out_dqPL[k]) iraf.hedit(in_sci[k], 'BPM', out_dqPL[k], add='yes', verify='no') iraf.hedit(in_var[k], 'BPM', out_dqPL[k], add='yes', verify='no') # # Merge sci/var cubes # iraf.imcombine("@" + in_filesSCI, out_sci, offsets=in_offset, combine='average', reject='avsigclip', masktype='goodvalue', maskvalue=0, expmasks=out_exp, sigmas=out_sigIN) iraf.imcombine("@" + in_filesVAR, out_var, offsets=in_offset, combine='sum', reject='none', masktype='goodvalue', maskvalue=0) # # Criate correct error cube # iraf.imcopy(out_exp + '.pl', out_exp.replace('.pl', '.fits')) # Read cubes sci_cube = pf.getdata(out_sci) var_cube = pf.getdata(out_var) sigIN_cube = pf.getdata(out_sigIN) exp_cube = pf.getdata(out_exp + '.fits') # --- Identify problem with negative values ---- Improve # RuntimeWarning: invalid value encountered in divide exp_MASK = np.ma.array(exp_cube, mask=(exp_cube == 0)) err_cube = np.sqrt(abs(var_cube / exp_MASK**2).data) # # Criate hypercube # # ---- Maybe don't need header for each extension -- Improve pry = pf.PrimaryHDU(header=pf.getheader(out_sci)) hdu1 = pf.ImageHDU(sci_cube, header=pf.getheader(out_sci), name='SCI') hdu2 = pf.ImageHDU(err_cube, header=pf.getheader(out_var), name='ERR') hdu4 = pf.ImageHDU(sigIN_cube, header=pf.getheader(out_sigIN), name='SIG_IN') hdu3 = pf.ImageHDU(exp_cube, header=pf.getheader(out_exp + '.fits'), name='NCUBE') hdu = pf.HDUList([pry, hdu1, hdu2, hdu3, hdu4]) hdu.writeto(name + '_HYPERCUBE.fits')
def rectify(ids=None, fs=None): iraf.cd('work') if ids is None: ids = np.array(glob('id2/arc*id2*.db')) if fs is None: fs = glob('mos/*mos*.fits') if len(ids) == 0: print "WARNING: No wavelength solutions for rectification." iraf.cd('..') return if len(fs) == 0: print "WARNING: No images for rectification." iraf.cd('..') return # Get the grating angles of the solution files idgas = [] for i, thisid in enumerate(ids): f = open(thisid) idlines = np.array(f.readlines(), dtype=str) f.close() idgaline = idlines[np.char.startswith(idlines, '#graang')][0] idgas.append(float(idgaline.split('=')[1])) ims, gas = get_scis_and_arcs(fs) if not os.path.exists('rec'): os.mkdir('rec') for i, f in enumerate(ims): fname = f.split('/')[1] typestr = fname[:3] ga, imgnum = gas[i], fname[-9:-5] outfile = 'rec/' + typestr + '%05.2frec' % (ga) + imgnum + '.fits' iraf.unlearn(iraf.specrectify) iraf.flpr() idfile = ids[np.array(idgas) == ga][0] iraf.specrectify(images=f, outimages=outfile, solfile=idfile, outpref='', function='legendre', order=3, inttype='interp', conserve='yes', clobber='yes', verbose='yes') # Update the BPM to mask any blank regions h = pyfits.open(outfile, 'update') # Cover the chip gaps. The background task etc do better if the chip # gaps are straight # To deal with this we just throw away the min and max of each side of # the curved chip gap chipgaps = get_chipgaps(h) # Chip 1 h[2].data[:, chipgaps[0][0]:chipgaps[0][1]] = 1 # Chip 2 h[2].data[:, chipgaps[1][0]:chipgaps[1][1]] = 1 # edge of chip 3 h[2].data[:, chipgaps[2][0]:chipgaps[2][1]] = 1 # Cover the other blank regions h[2].data[[h[1].data == 0]] = 1 # Set all of the data to zero in the BPM h[1].data[h[2].data == 1] = 0.0 h.flush() h.close() iraf.cd('..')