def get_cal_spec_line(filename1, filename2):
print 'Run get_cal_spec_line, apall...'
outname = 'a' + filename1
if os.path.isfile(outname):
print outname, 'is already exist'
else:
print 'display %s 1' % filename1
iraf.display(image = filename1, frame = 1)
iraf.apall(input = filename1
, output = outname, apertures = '', format = 'multispec'
, references = filename2, profiles = '', interactive = True
, find = False, recenter = False, resize = False
, edit = False, trace = False, fittrace = False
, extract = True, extras = False, review = True
, line = 'INDEF', nsum = 10, lower = -15.0
, upper = 15.0, apidtable = '', b_function = 'chebyshev'
, b_order = 1, b_sample = '-30:-20,20:30', b_naverage = -3
, b_niterate = 0, b_low_reject = 3.0, b_high_reject = 3.0
, b_grow = 0.0, width = 5.0, radius = 10.0
, threshold = 0.0, nfind = 1, minsep = 5.0
, maxsep = 1000.0, order = 'increasing', aprecenter = ''
, npeaks = 'INDEF', shift = True, llimit ='INDEF'
, ulimit = 'INDEF', ylevel = 0.1, peak = True
, bkg = True, r_grow = 0.0, avglimits = False
, t_nsum = 10, t_step = 10, t_nlost = 3, t_function = 'spline3'
, t_order = 3, t_sample = '*', t_naverage = 1
, t_niterate = 0, t_low_reject = 3.0, t_high_reject = 3.0
, t_grow = 0.0, background = 'none', skybox = 1
, weights = 'none', pfit = 'fit1d', clean = False
, saturation = 'INDEF', readnoise = 9.4, gain = 0.35
, lsigma = 4.0, usigma = 4.0, nsubaps = 1)
print 'splot %s' % outname
iraf.splot(images = outname)
return outname
def lris_standard(standards, xcorr=yes):
'''Extract standard stars and calculate sensitivity functions.'''
for ofile, nstd in standards:
shutil.copy(ofile, "%s.fits" % nstd)
# Extract standard
if get_head("%s.fits" % nstd, "SKYSUB"):
iraf.apall(nstd, output="", inter=yes, find=yes, recenter=yes, resize=yes,
edit=yes, trace=yes, fittrace=yes, extract=yes, extras=yes,
review=no, background="none")
else:
iraf.apall(nstd, output="", inter=yes, find=yes, recenter=yes, resize=yes,
edit=yes, trace=yes, fittrace=yes, extract=yes, extras=yes,
review=no, background="fit")
if xcorr:
# Cross correlate to tweak wavelength
iraf.xcsao("%s.ms" % nstd, templates=XCTEMPLATE,
correlate="wavelength", logfiles="%s.xcsao" % ofile)
# If a solution was found, apply shift
try:
xcsaof = open("%s.xcsao" % ofile)
shift = float(xcsaof.readlines()[6].split()[14])
except:
shift = 0.0
iraf.specshift("%s.ms" % nstd, -shift)
# Create telluric
iraf.splot("%s.ms" % nstd) # Remove telluric and absorption, save as a%s.ms
iraf.sarith("%s.ms" % nstd, "/", "a%s.ms" % nstd, "telluric.%s.fits" % nstd)
iraf.imreplace("telluric.%s.fits" % nstd, 1.0, lower=0.0, upper=0.0)
iraf.splot("telluric.%s.fits" % nstd) # Remove stellar features and resave
# Create smoothed standard
iraf.gauss("a%s.ms[*,1,1]" % nstd, "s%s.ms" % nstd, 5.0)
iraf.sarith("%s.ms" % nstd, "/", "s%s.ms" % nstd, "ds%s.ms" % nstd)
# Apply telluric correction
iraf.telluric("ds%s.ms" % nstd, "tds%s.ms" % nstd, "telluric.%s.fits" % nstd,
xcorr=no, tweakrms=no, interactive=no,
sample='4000:4010,6850:6975,7150:7350,7575:7725,8050:8400,8900:9725')
# Define bandpasses for standard star calculation
obj=get_head("%s.fits" % nstd, "OBJECT")
iraf.standard("tds%s.ms" % nstd, "%s.std" % nstd,
extinction='home$extinct/maunakeaextinct.dat',
caldir='home$standards/', observatory='Keck', interac=yes,
star_name=obj, airmass='', exptime='')
# Determine sensitivity function
iraf.sensfunc('%s.std' % nstd, '%s.sens' % nstd,
extinction='home$extinct/maunakeaextinct.dat',
newextinction='extinct.dat', observatory='Keck',
function='legendre', order=4, interactive=yes)
return
def red_science(image, flats, spath, object=None, arc='Arc-Red.fits',
smooth='BD284211.smooth.fits', telluric='telluric.fits',
sens='BD284211.sens', biassec=REDBIAS, trimsec=REDTRIM,
outflat='Flat-Red.fits', gain=REDGAIN, rdnoise=REDRDNOISE):
'''Full reduction of KAST science spectra on red CCD'''
# Bias subtract everything first
redbias(image, biassec=biassec, trimsec=trimsec)
redbias(flats, biassec=biassec, trimsec=trimsec)
# Create and apply flat-field
make_flat(flats, outflat, gain=gain, rdnoise=rdnoise)
iraf.ccdproc(image[0], ccdtype='', noproc=no, fixpix=no, overscan=no,
trim=no, zerocor=no, darkcor=no, flatcor=yes, illumcor=no,
fringecor=no, readcor=no, scancor=no, flat=outflat)
# Cosmic ray rejection
#iraf.lacos_spec(image[0], 'c%s' % image[0], 'cm%s' % image[0],
# gain=gain, readn=rdnoise, xorder=9, yorder=3,
# sigclip=4.5, sigfrac=0.5, objlim=1.0, niter=3)
# Extract spectrum
if object==None:
object=get_head(image, 'OBJECT')
iraf.apall('%s' % image[0], output=object, references='', interactive=yes,
find=yes, recenter=yes, resize=yes, edit=yes, trace=yes,
fittrace=yes, extract=yes, extras=yes, review=no,
background='fit', weights='variance', pfit='fit1d',
readnoise=rdnoise, gain=gain)
# Apply wavelength solution to standard
shutil.copy('%s/%s' % (spath, arc), '.')
shutil.copy('%s/database/id%s' % (spath, arc.rstrip('.fits')), 'database')
iraf.refspec(object, references=arc, sort="", group="", override=yes,
confirm=no, assign=yes)
iraf.dispcor(object, '%s.w' % object, confirm=no, listonly=no)
# Smooth
shutil.copy('%s/%s' % (spath, smooth), '.')
iraf.sarith('%s.w' % object, '/', smooth, '%s.s' % object)
# Create and apply telluric correction
shutil.copy('%s/%s' % (spath, telluric), '.')
iraf.telluric('%s.s' % object, '%s.t' % object, telluric, xcorr=yes,
tweakrms=yes, interactive=yes, sample='6850:6950,7575:7700')
# Flux calibration
shutil.copy('%s/%s.0001.fits' % (spath, sens), '.')
iraf.calibrate('%s.t' % object, '%s.f' % object, extinct=yes, flux=yes,
extinction='onedstds$kpnoextinct.dat',
observatory='Lick', sensitivity=sens, airmass='',
exptime='')
return
def reference_arc(image, output, reference,
coordlist='home$linelists/licklinelist.dat'):
'''Construct reference wavelength solution from arc lamps'''
iraf.apall(image, output=output, references=reference, interactive=no,
find=no, recenter=no, resize=no, edit=no, trace=no, fittrace=no,
extract=yes, extras=yes, review=no, background='none')
iraf.identify(output, coordlist=coordlist)
return
def apall(images, out_images, data_path='.', params={}):
"""
Run iraf apall to extract 1d spectrum.
Parameters
----------
images : str
Images to transform.
out_images : str
Output image file names.
data_path : str
Directory that contains the data.
params : dict
Iraf apall parameters.
"""
_check_working_dir(data_path)
logger.info('running iraf apall')
iraf.apall.unlearn()
iraf.apall.format = params.pop('format', 'onedspec')
iraf.apall.interactive = params.pop('interactive', 'yes')
iraf.apall.find = params.pop('find', 'yes')
iraf.apall.recenter = params.pop('recenter', 'no')
iraf.apall.resize = params.pop('resize', 'no')
iraf.apall.edit = params.pop('edit', 'yes')
iraf.apall.trace = params.pop('trace', 'yes')
iraf.apall.fittrace = params.pop('fittrace', 'yes')
iraf.apall.extract = params.pop('extract', 'yes')
iraf.apall.extras = params.pop('extras', 'no')
iraf.apall.review = params.pop('review', 'no')
iraf.apall.review = params.pop('background', 'no')
# aperture
iraf.apall.line = params.pop('line', 3500)
iraf.apall.nsum = params.pop('nsum', 500)
iraf.apall.lower = params.pop('lower', -20)
iraf.apall.upper = params.pop('upper', 20)
# trace
iraf.apall.t_nsum = params.pop('t_nsum', 300)
iraf.apall.t_step = params.pop('t_step', 10)
iraf.apall.t_nlost = params.pop('t_nlost', 10)
# aperture centering
iraf.apall.width = params.pop('width', 20.0)
iraf.apall.radius= params.pop('radius', 10.0)
iraf.apall.readnoise = params.pop('readnoise', 2.5)
iraf.apall.gain= params.pop('gain', 2.5)
iraf.apall(input=images, output=out_images)
def blue_science(image, spath, object=None, flat='Flat-Blue.fits',
arc='Arc-Blue.fits', smooth='BD284211.smooth.fits',
sens='BD284211.sens', biassec1=BLUEBIAS1, trimsec1=BLUETRIM1,
biassec2=BLUEBIAS2, trimsec2=BLUETRIM2, gain=BLUEGAIN,
rdnoise=BLUERDNOISE):
'''Full reduction of KAST science spectra on blue CCD'''
# Bias subtract everything first
bluebias(image, biassec1=biassec1, trimsec1=trimsec1, biassec2=biassec2,
trimsec2=trimsec2)
# Apply flat-field
shutil.copy('%s/%s' % (spath, flat), '.')
iraf.ccdproc('j%s' % image[0], ccdtype='', noproc=no, fixpix=no,
overscan=no, trim=no, zerocor=no, darkcor=no, flatcor=yes,
illumcor=no, fringecor=no, readcor=no, scancor=no,
flat=flat)
# Cosmic ray rejection
#iraf.lacos_spec('j%s' % image[0], 'cj%s' % image[0], 'cmj%s' % image[0],
# gain=gain, readn=rdnoise, xorder=9, yorder=3,
# sigclip=4.5, sigfrac=0.5, objlim=1.0, niter=3)
# Extract spectrum
if object==None:
object=get_head(image, 'OBJECT')
iraf.apall('j%s' % image[0], output=object, references='',
interactive=yes, find=yes, recenter=yes, resize=yes, edit=yes,
trace=yes, fittrace=yes, extract=yes, extras=yes, review=no,
background='fit', weights='variance', pfit='fit1d',
readnoise=rdnoise, gain=gain)
# Apply wavelength solution to standard
shutil.copy('%s/%s' % (spath, arc), '.')
shutil.copy('%s/database/id%s' % (spath, arc.rstrip('.fits')), 'database')
iraf.refspec(object, references=arc, sort="", group="", override=yes,
confirm=no, assign=yes)
iraf.dispcor(object, '%s.w' % object, confirm=no, listonly=no)
# Smooth
shutil.copy('%s/%s' % (spath, smooth), '.')
iraf.sarith('%s.w' % object, '/', smooth, '%s.s' % object)
# Flux calibration
shutil.copy('%s/%s.0001.fits' % (spath, sens), '.')
iraf.calibrate('%s.s' % object, '%s.f' % object, extinct=yes, flux=yes,
extinction='onedstds$kpnoextinct.dat',
observatory='Lick', sensitivity=sens, airmass='',
exptime='')
return
def extractComps(assignments, dorefspec=1):
'''This task takes a dictionary 'assignments' which has keywords equal
to the objects spectrum and value equal to the comparison spectrum
which should be used for the object spectrum. You could use
closest_time_pos() to figure this out, for example. The apertures
of each comp are then extracted using apall and the object's spectrum
as a reference. Finally, the object's headers are updated using
the REFSPEC1 keyword (unless dorefspec=0).'''
iraf.noao()
iraf.imred()
iraf.echelle()
# First things first, we save the old paramter list.
iraf.apall.saveParList(filename='temp.apall.par')
# iraf.apall.unlearn()
iraf.apall.format = 'echelle'
# now, get the associated list of references and comps:
stuff = assignments
for ref in stuff.keys():
print 'extracting %s using %s as reference' % (stuff[ref], ref)
comp = stuff[ref]
if iraf.imaccess(re.sub('\.fits', '.ec.fits', comp)):
print 'Extracted comparison exists, skipping...'
else:
iraf.apall(input=comp,
references=re.sub('\.ec', '', ref),
interactive=0,
find=0,
recenter=0,
resize=0,
edit=0,
trace=0,
fittrace=0,
extract=1,
extras=0,
review=0)
# Now, since we've used them to extract the spectra, we can assign
# them using refspec:
# print "now assigning reference spectrum"
if dorefspec:
iraf.hedit(images=ref,
fields='REFSPEC1',
value=re.sub('\.fits', '.ec', comp),
add=1,
addonly=1,
verify=0,
show=1)
iraf.apall.setParList(ParList='temp.apall.par')
def apall(lstfile):
iraf.noao()
iraf.twodspec()
iraf.apextract(dispaxis = 2, database = 'database')
f = open(lstfile)
l = f.readlines()
f.close()
l = [tmp.split('\n')[0] for tmp in l]
path = os.getcwd()
for i in l:
infile = 'wftbo' + i
outfile = 'awftbo' + i
while True:
if os.path.isfile(outfile):
print('remove ' + outfile)
os.remove(outfile)
delfile = path + os.sep + 'database/ap' + infile[0:-5]
if os.path.isfile(delfile):
print('remove ' + delfile)
os.remove(delfile)
iraf.apall(input = infile
, output = outfile, apertures = 2, format = 'multispec'
, references = '', profiles = '', interactive = True
, find = True, recenter = True, resize = False
, edit = True, trace = True, fittrace = True
, extract = True, extras = True, review = True
, line = 'INDEF', nsum = 10, lower = -15.0
, upper = 15.0, apidtable = '', b_function = 'chebyshev'
, b_order = 2, b_sample = '-50:-26,26:50', b_naverage = -25
, b_niterate = 1, b_low_reject = 3.0, b_high_reject = 3.0
, b_grow = 0.0, width = 5.0, radius = 10.0
, threshold = 0.0, nfind = 2, minsep = 5.0
, maxsep = 100000.0, order = 'increasing', aprecenter = ''
, npeaks = 'INDEF', shift = True, llimit ='INDEF'
, ulimit = 'INDEF', ylevel = 0.1, peak = True
, bkg = True, r_grow = 0.0, avglimits = False
, t_nsum = 20, t_step = 10, t_nlost = 3, t_function = 'legendre'
, t_order = 7, t_sample = '*', t_naverage = 1
, t_niterate = 1, t_low_reject = 3.0, t_high_reject = 3.0
, t_grow = 0.0, background = 'fit', skybox = 1
, weights = 'variance', pfit = 'fit1d', clean = True
, saturation = 'INDEF', readnoise = 9.4, gain = 0.35
, lsigma = 4.0, usigma = 4.0, nsubaps = 1)
iraf.flpr()
sspecplot.sspecplot(outfile)
getval = raw_input('Are you need repeat apall,may be clean should be close(r/n)')
if getval != 'r':
break
def do_apall_for_real(WORK_DIR):
print '\n + Doing apall for real\n'
#do apall for real with background and all
#This tracing will be used from now on. Pay attention to the background subtraction. You will have to verify each tracing on each reference image (first images in the log.txt).
try:
shutil.copy(observations[WORK_DIR]['REF_AP'], 'database/' + observations[WORK_DIR]['REF_AP'].split('/')[-1])
except:
pass
for obj in observations[WORK_DIR]['objects']:
if observations[WORK_DIR]['REF_AP']:
ref = observations[WORK_DIR]['REF_AP'].split('/')[-1].replace('ap', '')
else:
ref = obj
iraf.apall(input=obj, referen=ref, format='echelle', interac='yes', find='yes', recente='yes', resize='yes', edit='yes', trace='yes', fittrac='yes', extract='no', extras='no', review='no', line=600, nsum=30, lower=-6, upper=6, width=11, radius=11, thresho=0.15, nfind=35, minsep=10, maxsep=255, t_func='chebyshev', t_order=9, t_niter=5, backgro='median', b_order=1, b_sampl=AP_LIM[observations[WORK_DIR]['ap_position']][2], ylevel='INDEF', llimit=-5, ulimit=5, weights='none', clean='yes', order='increasing')#, apidtable='apert.txt')
def apall(ilst, olst):
iraf.noao()
iraf.twodspec()
iraf.apextract(dispaxis=1, database='database')
for i, infile in enumerate(ilst):
outfile = olst[i]
laper, raper, back_samp = -7, 7, '-30:-15,15:30'
while True:
if os.path.isfile(outfile):
print('remove ' + outfile)
os.remove(outfile)
delfile = os.getcwd()+os.sep+'database/ap'+infile[0:-5]
if os.path.isfile(delfile):
print('remove ' + delfile)
os.remove(delfile)
iraf.apall(input=infile, output=outfile, apertures=2,
format='multispec', references='', profiles='',
interactive=True, find=True, recenter=True,
resize=False, edit=True, trace=True, fittrace=True,
extract=True, extras=True, review=True, line='INDEF',
nsum=10, lower=laper, upper=raper, apidtable='',
b_function='chebyshev', b_order=2, b_sample=back_samp,
b_naverage=-25, b_niterate=1, b_low_reject=3.0,
b_high_reject=3.0, b_grow=0.0, width=5.0, radius=10.0,
threshold=0.0, nfind=2, minsep=5.0, maxsep=100000.0,
order='increasing', aprecenter='', npeaks='INDEF',
shift=True, llimit='INDEF', ulimit='INDEF', ylevel=0.1,
peak=True, bkg=True, r_grow=0.0, avglimits=False,
t_nsum=20, t_step=10, t_nlost=3, t_function='legendre',
t_order=12, t_sample='*', t_naverage=1, t_niterate=1,
t_low_reject=3.0, t_high_reject=3.0, t_grow=0.0,
background='median', skybox=1, weights='none',
pfit='fit1d', clean=True, saturation='INDEF',
readnoise='CCDRON', gain='CCDGAIN', lsigma=4.0, usigma=4.0,
nsubaps=1)
iraf.flpr()
getval = raw_input(('Are you need repeat apall,'
'may be clean should be close(r/n)'))
if getval != 'r':
break
def rssstdred(self, obj, arc):
rt.rmexist([obj[0].sensout,obj[0].sensout+'.fits'])
for std in obj:
self.reduce2d(std, arc, do_error=False)
rt.loadparam(self.rssconfig, ['iraf.apall'])
std_data = self.caldir+'/'+std.object+".dat"
rt.rmexist([std.oned,std.oned0])
while not os.path.isfile(std_data):
print("Cannot find file ", std_data)
std_data = raw_input('Enter a star database: ')
iraf.apall(std.bkg, b_niterate=5, b_sample='-100:-40,40:100', b_high_reject=1.5, b_low_reject=2)
os.system('mv %s %s' % (std.oned0,std.oned))
if not rt.header(std.oned,'AIRMASS'): rt.rssairmass(std.oned)
iraf.standard(input=std.oned, output=std.sensout, caldir=self.caldir, interact='yes',
star_name=std.object, extinct=self.extinction)
# Sensitivity files
iraf.sensfunc(std.sensout, std.sensout+'.fits', interactive= 'yes', extinct=self.extinction,
newextinction='extinct.dat')
rt.getsh('cp %s %s' % (std.sensout+'.fits',self.rssdatadir))
return std.sensout+'.fits'
def do_ref_apall(WORK_DIR):
print '\n + Doing apall for a reference to the masterflat\n'
#do quick apall
#This will serve as the reference where the apertures are in the process of creating a flat. 30 apertures are found automatically, including the red-most aperture that goes off the image. Change the position of apertures if needed. It is assumed that the object is positioned at roughly 1/3 from the left edge of the slit (1/3 from the right edge of the aperture).
# d - briši aperturo
# m - dodaj aperturo
# o - posortiraj aperture po vrsti na novo
# z - odstrani interval fitanja
# s - levo in desno, dodaj interval fitanja
iraf.unlearn('apall')
try: os.remove(observations[WORK_DIR]['objects'][0]+'.fits')
except: pass
iraf.imcopy(input=observations[WORK_DIR]['ORIG_DIR']+observations[WORK_DIR]['objects'][0], output=observations[WORK_DIR]['objects'][0])
iraf.apall(input=observations[WORK_DIR]['objects'][0], referen='', format='echelle', interac='yes', find='yes', recente='yes', resize='no', edit='yes', trace='yes', fittrac='yes',extract='no', extras='no', review='no', line=550, nsum=10, lower=-6, upper=6, width=11, radius=11, thresho=0.1, nfind=32, minsep=20, maxsep=155, t_func='chebyshev', t_order=4, t_sampl='51:2098', t_niter=5, backgro='none', ylevel='INDEF', llimit=AP_LIM[observations[WORK_DIR]['ap_position']][0], ulimit=AP_LIM[observations[WORK_DIR]['ap_position']][1])
iraf.apresize(input=observations[WORK_DIR]['objects'][0], interac='no', find='no', recente='no', resize='yes', edit='yes', ylevel='INDEF', llimit=AP_LIM[observations[WORK_DIR]['ap_position']][0], ulimit=AP_LIM[observations[WORK_DIR]['ap_position']][1])
def apply_apall(WORK_DIR):
print '\n + Applying apall to rest of the objects and arcs\n'
for obj in observations[WORK_DIR]['objects']:
print ' - Processing image %s \n' % obj
try:
os.remove(obj+'.ec.fits')
os.remove(obj+'.ec.nosky.fits')
except: pass
for j in [['.ec', 'median'], ['.ec.nosky', 'none']]:
iraf.apall(input=obj, output=obj+j[0], format='echelle', referen=obj, interac='no', find='yes', recente='no', resize='no', edit='yes', trace='no', fittrac='no', extract='yes', extras='no', review='no', line=600, nsum=30, lower=-5, upper=5, width=8, radius=10, thresho=0.1, nfind=35, minsep=10, maxsep=255, t_func='chebyshev', t_order=9, t_niter=5, backgro=j[1], b_order=1, b_sampl=AP_LIM[observations[WORK_DIR]['ap_position']][2], ylevel='INDEF', llimit=-5, ulimit=5, weights='none', clean='yes', order='increasing')#, apidtable='apert.txt')
for i, cal in enumerate(observations[WORK_DIR]['calibs']):
print ' - Processing arc %s \n' % cal
try: os.remove(cal+'.ec.fits')
except: pass
print ' input:',cal,' referen:',observations[WORK_DIR]['objects'][i]
iraf.apall(input=cal, format='echelle', referen=observations[WORK_DIR]['objects'][i], interac='no', find='no', recente='no', resize='no', edit='no',trace='no', fittrac='no', extract='yes', extras='no', review='no', line=600, nsum=30, lower=-5, upper=5, width=8, radius=10, thresho=0.1, nfind=35, minsep=10, maxsep=255, t_func='chebyshev', t_order=9, t_niter=5, backgro='none', ylevel='INDEF', llimit=-5, ulimit=5, order='increasing')#, apidtable='apert.txt')
def reference_arc(image,
output,
reference,
coordlist='home$linelists/licklinelist.dat'):
'''Construct reference wavelength solution from arc lamps'''
iraf.apall(image,
output=output,
references=reference,
interactive=no,
find=no,
recenter=no,
resize=no,
edit=no,
trace=no,
fittrace=no,
extract=yes,
extras=yes,
review=no,
background='none')
iraf.identify(output, coordlist=coordlist)
return
def salt_apall(obj):
outname = string.split(obj, sep='.')[0] + "1d.fits"
iraf.apall(input=obj, output=' ')
os.system('mv ' + string.split(obj, sep='.')[0] + '.0001.fits ' + outname)
err_obj = string.split(obj, sep='.')[0] + '_var.fits'
err_obj1d = string.split(obj, sep='.')[0] + '1d_var.fits'
if os.path.isfile(err_obj):
iraf.apall.references = obj
iraf.apall.interactive = 'no'
iraf.apall.find = 'no'
iraf.apall.recenter = 'no'
iraf.apall.resize = 'no'
iraf.apall.edit = 'no'
iraf.apall.trace = 'no'
iraf.apall.fittrace = 'no'
iraf.apall.extract = 'yes'
iraf.apall.extras = 'no'
iraf.apall.review = 'no'
iraf.apall(input=err_obj, output=err_obj1d)
os.system('mv ' + string.split(obj, sep='.')[0] +
'1d_var.0001.fits ' + err_obj1d)
def produce_thar(filelist_new, thar_reference):
current_directory = os.getcwd()
thar_directory = os.path.dirname(filelist_new[0])
filelistText = open(os.path.join(thar_directory, "ThAr.extracted.lis"),
"w")
for i in range(len(filelist_new)):
trim = trim_remove_bias(filelist_new[i])
extracted = trim.replace(".trim_will_be_removed.fits",
".extracted.fits")
iraf.apall(trim,
output=extracted,
references="find_orders.fits",
profiles="find_orders.fits",
apertures="1-51")
os.remove(trim)
extracted_without_path = extracted.replace(
os.path.join(thar_directory, ""), "")
filelistText.write(extracted_without_path + "\n")
filelistText.close()
os.chdir(thar_directory)
thar_reference_short = thar_reference.replace(".fits", "")
iraf.ecreidentify("@ThAr.extracted.lis",
reference=thar_reference_short,
shift="INDEF")
os.chdir(current_directory)
print("Success! Produced ThAr reference files.")
def deimos_extract(science, standard, dostandard=yes):
'''Extract and flux calibrate DEIMOS spectra (assuming they have been
processed into 2D images using deimos_pipe above).'''
if dostandard:
deimos_standard(standard)
for source in science:
images = iraffiles("%s_??_B.fits" % source)
joinstr = ""
for image in images:
bimage = image.split(".")[0]; rimage = bimage[:-1] + "R"
update_head(bimage, "FLUX_OBJ", standard)
update_head(rimage, "FLUX_OBJ", standard)
# Extract 1D spectra
if get_head("%s.fits" % bimage, "SKYSUB"):
iraf.apall(bimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="none",
reference="%s_01_B" % standard)
iraf.apall(rimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="none",
reference="%s_01_R" % standard)
else:
iraf.apall(bimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="fit",
reference="%s_01_B" % standard)
iraf.apall(rimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="fit",
reference="%s_01_R" % standard)
# Normalize by the standard continuua
iraf.sarith("%s.ms" % bimage, "/", "s%s_01_B.ms.fits" % standard,
"s%s.ms" % bimage)
iraf.sarith("%s.ms" % rimage, "/", "s%s_01_R.ms.fits" % standard,
"s%s.ms" % rimage)
# Telluric correct
bstd = "%s_01_B" % standard; rstd = "%s_01_R" % standard
iraf.telluric("s%s.ms" % bimage, "ts%s.ms" % bimage,
"telluric.B.%s.fits" % bstd, tweakrms=yes,
interactive=yes, sample='6850:6950,7575:7700')
iraf.telluric("s%s.ms" % rimage, "ts%s.ms" % rimage,
"telluric.R.%s.fits" % rstd, tweakrms=yes,
interactive=yes, sample='6850:6950,7575:7700')
# Flux calibration
iraf.calibrate("ts%s.ms" % bimage, "fts%s.ms" % bimage, extinct=yes,
flux=yes, extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck", sensitivity="%s.B.sens" % standard,
airmass='', exptime='')
iraf.calibrate("ts%s.ms" % rimage, "fts%s.ms" % rimage, extinct=yes,
flux=yes, extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck", sensitivity="%s.R.sens" % standard,
airmass='', exptime='')
joinstr += "fts%s.ms,fts%s.ms," % (bimage, rimage)
# Combine
iraf.scombine(joinstr[:-1], "%s.ms.fits" % source, combine="average",
reject="avsigclip", w1=INDEF, w2=INDEF, dw=INDEF, nw=INDEF,
scale="none", zero="none", weight="none", lsigma=3.0,
hsigma=3.0, gain=CCDGAIN, rdnoise=CCDRNOISE)
# Plot to enable final tweaks
iraf.splot("%s.ms.fits" % source)
return
def red_standard(image, arcs, flats, object=None, biassec=REDBIAS,
trimsec=REDTRIM, outflat='Flat-Red.fits', gain=REDGAIN,
rdnoise=REDRDNOISE, arc='Arc-Red.fits',
caldir='home$standards/'):
'''Reduce and calibrate standard star observation with red CCD'''
# Bias subtract everything first
redbias(image, biassec=biassec, trimsec=trimsec)
redbias(arcs, biassec=biassec, trimsec=trimsec)
redbias(flats, biassec=biassec, trimsec=trimsec)
# Create and apply flat-field
make_flat(flats, outflat, gain=gain, rdnoise=rdnoise)
iraf.ccdproc(image[0], ccdtype='', noproc=no, fixpix=no, overscan=no,
trim=no, zerocor=no, darkcor=no, flatcor=yes, illumcor=no,
fringecor=no, readcor=no, scancor=no, flat=outflat)
arcimages=','.join(arcs)
iraf.ccdproc(arcs, ccdtype='', noproc=no, fixpix=no, overscan=no,
trim=no, zerocor=no, darkcor=no, flatcor=yes, illumcor=no,
fringecor=no, readcor=no, scancor=no, flat=outflat)
# Extract spectrum of standard
if object==None:
object=get_head(image, 'OBJECT')
iraf.apall(image[0], output=object, references='', interactive=yes,
find=yes, recenter=yes, resize=yes, edit=yes, trace=yes,
fittrace=yes, extract=yes, extras=yes, review=no,
background='fit', weights='variance', pfit='fit1d',
readnoise=rdnoise, gain=gain)
# Extract arc and fit wavelength solution
iraf.imarith(arcs[0], '+', arcs[1], 'Arc-Sum.fits')
reference_arc('Arc-Sum.fits', arc, image[0])
# Apply wavelength solution to standard
iraf.refspec(object, references=arc, sort="", group="", override=yes,
confirm=no, assign=yes)
iraf.dispcor(object, '%s.w' % object, confirm=no, listonly=no)
# Remove absorption features and smooth
iraf.splot('%s.w' % object, 1, 1)
iraf.gauss('temp1[*,1,1]', '%s.smooth' % object, 3.0)
iraf.sarith('%s.w' % object, '/', '%s.smooth' % object, '%s.s' % object)
# Create and apply telluric correction
iraf.sarith('%s.w' % object, '/', 'temp1', 'telluric')
iraf.splot('telluric', 1, 1)
iraf.telluric('%s.s' % object, '%s.t' % object, 'telluric', xcorr=no,
tweakrms=no, interactive=no, sample='6850:6950,7575:7700')
# Define bandpasses for standard star calculation
iraf.standard('%s.t' % object, '%s.std' % object,
extinction='onedstds$kpnoextinct.dat', caldir=caldir,
observatory='Lick', interact=yes, star_name=object,
airmass='', exptime='')
# Determine sensitivity function
iraf.sensfunc('%s.std' % object, '%s.sens' % object,
extinction='onedstds$kpnoextinct.dat',
newextinction='extinct.dat', observatory='Lick',
function='legendre', order=3, interactive=yes)
return
def extract(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('fix/*fix*.fits')
if len(fs) == 0:
print "WARNING: No fixpixed images available for extraction."
iraf.cd('..')
return
if not os.path.exists('x1d'):
os.mkdir('x1d')
print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line."
for f in fs:
# Get the output filename without the ".fits"
outbase = f.replace('fix', 'x1d')[:-5]
# Get the readnoise, right now assume default value of 5 but we could
# get this from the header
readnoise = 5
# If interactive open the rectified, background subtracted image in ds9
ds9display(f.replace('fix', 'bkg'))
# set dispaxis = 1 just in case
pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apall)
iraf.flpr()
iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes',
review='no', line='INDEF', nsum=-1000, lower=-5, upper=5,
b_function='legendre', b_order=5,
b_sample='-400:-200,200:400', b_naverage=-10, b_niterate=5,
b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15,
t_step=15, t_nlost=100, t_function='legendre', t_order=5,
t_niterate=5, t_low_reject=3.0, t_high_reject=3.0,
background='fit', weights='variance', pfit='fit1d',
clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0,
usigma=4.0, mode='hl')
# Copy the CCDSUM keyword into the 1d extraction
pyfits.setval(outbase + '.fits', 'CCDSUM',
value=pyfits.getval(f, 'CCDSUM'))
# Extract the corresponding arc
arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0]
# set dispaxis = 1 just in case
pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apsum)
iraf.flpr()
iraf.apsum(input=arcname + '[SCI]', output='auxext_arc',
references=f[:-5] + '[SCI]', interactive='no', find='no',
edit='no', trace='no', fittrace='no', extras='no',
review='no', background='no', mode='hl')
# copy the arc into the 5 column of the data cube
arcfs = glob('auxext_arc*.fits')
for af in arcfs:
archdu = pyfits.open(af)
scihdu = pyfits.open(outbase + '.fits', mode='update')
d = scihdu[0].data.copy()
scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2]))
scihdu[0].data[:-1, :, :] = d[:, :, :]
scihdu[0].data[-1::, :] = archdu[0].data.copy()
scihdu.flush()
scihdu.close()
archdu.close()
os.remove(af)
# Add the airmass, exptime, and other keywords back into the
# extracted spectrum header
kws = ['AIRMASS','EXPTIME',
'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG',
'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX',
'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS',
'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA',
'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL',
'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING',
'FILTER']
for kw in kws:
pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw))
iraf.cd('..')
iraf.apall(input="J105829_250_3500_4_Grism_7_2015_06_17_yf170018bfc.fits",
output="J105829_250_3500_4_Grism_7_2015_06_17_yf170018bfcap.fits",
apertures="",
format="multispec",
references="",
profiles="",
interactive="yes",
find="yes",
recenter="yes",
resize="yes",
edit="yes",
trace="yes",
fittrace="yes",
extract="yes",
extras="yes",
review="yes",
line="INDEF",
nsum=20,
lower=-5.,
upper=5.,
apidtable="",
b_function="chebyshev",
b_order=1,
b_sample="-10:-6,6:10",
b_naverage=-3,
b_niterate=0,
b_low_reject=3.,
b_high_rejec=3.,
b_grow=0.,
width=5.,
radius=10.,
threshold=0.,
minsep=5.,
maxsep=1000.,
order="increasing",
aprecenter="",
npeaks="INDEF",
shift="yes",
llimit="INDEF",
ulimit="INDEF",
ylevel=0.1,
peak="yes",
bkg="yes",
r_grow=0.,
avglimits="no",
t_nsum=10,
t_step=10,
t_nlost=3,
t_function="legendre",
t_order=2,
t_sample="*",
t_naverage=1,
t_niterate=0,
t_low_reject=3.,
t_high_rejec=3.,
t_grow=0.,
background="fit",
skybox=1,
weights="variance",
pfit="fit1d",
clean="yes",
saturation="INDEF",
readnoise=4.8,
gain=1.22,
lsigma=4.,
usigma=4.,
nsubaps=1)
def blue_standard(image, arcs, flats, object=None, biassec1=BLUEBIAS1,
trimsec1=BLUETRIM1, biassec2=BLUEBIAS2,
trimsec2=BLUETRIM2, outflat='Flat-Blue.fits', gain=BLUEGAIN,
rdnoise=BLUERDNOISE, arc='Arc-Blue.fits',
caldir='home$standards/'):
'''Reduce and calibrate standard star observation with blue CCD'''
# Bias subtract everything first
bluebias(image, biassec1=biassec1, trimsec1=trimsec1, biassec2=biassec2,
trimsec2=trimsec2)
bluebias(arcs, biassec1=biassec1, trimsec1=trimsec1, biassec2=biassec2,
trimsec2=trimsec2)
bluebias(flats, biassec1=biassec1, trimsec1=trimsec1, biassec2=biassec2,
trimsec2=trimsec2)
# Create and apply flat-field
for i in range(len(flats)):
flats[i]='j%s' % flats[i]
make_flat(flats, outflat, gain=gain, rdnoise=rdnoise)
iraf.ccdproc('j%s' % image[0], ccdtype='', noproc=no, fixpix=no,
overscan=no, trim=no, zerocor=no, darkcor=no, flatcor=yes,
illumcor=no, fringecor=no, readcor=no, scancor=no,
flat=outflat)
iraf.ccdproc('j%s' % arcs[0], ccdtype='', noproc=no, fixpix=no,
overscan=no, trim=no, zerocor=no, darkcor=no, flatcor=yes,
illumcor=no, fringecor=no, readcor=no, scancor=no,
flat=outflat)
# Extract spectrum of standard
if object==None:
object=get_head(image, 'OBJECT')
iraf.apall('j%s' % image[0], output=object, references='', interactive=yes,
find=yes, recenter=yes, resize=yes, edit=yes, trace=yes,
fittrace=yes, extract=yes, extras=yes, review=no,
background='fit', weights='variance', pfit='fit1d',
readnoise=rdnoise, gain=gain)
# Extract arc and fit wavelength solution
reference_arc('j%s' % arcs[0], arc, 'j%s' % image[0])
# Apply wavelength solution to standard
iraf.refspec(object, references=arc, sort="", group="", override=yes,
confirm=no, assign=yes)
iraf.dispcor(object, '%s.w' % object, confirm=no, listonly=no)
# Remove absorption features and smooth
iraf.splot('%s.w' % object, 1, 1)
iraf.gauss('temp1[*,1,1]', '%s.smooth' % object, 3.0)
iraf.sarith('%s.w' % object, '/', '%s.smooth' % object, '%s.s' % object)
# Define bandpasses for standard star calculation
iraf.standard('%s.s' % object, '%s.std' % object,
extinction='onedstds$kpnoextinct.dat', caldir=caldir,
observatory='Lick', interact=yes, star_name=object,
airmass='', exptime='')
# Determine sensitivity function
iraf.sensfunc('%s.std' % object, '%s.sens' % object,
extinction='onedstds$kpnoextinct.dat',
newextinction='extinct.dat', observatory='Lick',
function='legendre', order=3, interactive=yes)
return
def extract(arcs, objs,ccd,mode=1):
"""
mode=1: 1D extraction
mode=2: 2D extraction
"""
#apall na vseh luckah
print " + identifing arcs"
for i in arcs:
iraf.hedit(images='tmp/arcs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null")
iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null")
if mode==1:
pass
else:
iraf.cd("tmp/arcs")
geometry_prep(arcs,ccd)
os.system("cp transformations* ../objs/")
#sys.exit(0)
iraf.cd("../..")
for i in arcs:
os.system("rm tmp/arcs/%s.ms.fits" % (i[1][:-5]))
os.system("rm tmp/arcs/%s_cut*.fits" % (i[1][:-5]))
os.system("rm tmp/arcs/%s_t*.fits" % (i[1][:-5]))
pass
os.system("rm tmp/arcs/calibrations/idarcs_cut*")
os.system("rm tmp/arcs/arcs_cut*")
#extract 2d arcs and objects
for i in arcs:
iraf.hedit(images='tmp/arcs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null")
iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null")
#forget apertures
for j in range(8,1000):
iraf.hedit(images='tmp/arcs/%s.ms.fits' % (i[1][:-5]), fields="APNUM%s" % (str(j)), value='', delete="yes", verify="no", Stdout="/dev/null")
for i in objs:
iraf.hedit(images='tmp/objs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null")
iraf.apall(input='tmp/objs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null")
#forget apertures
for j in range(8,1000):
iraf.hedit(images='tmp/objs/%s.ms.fits' % (i[1][:-5]), fields="APNUM%s" % (str(j)), value='', delete="yes", verify="no", Stdout="/dev/null")
iraf.cd("tmp/arcs")
for ii in arcs:
geometry_transform(ii)
iraf.cd("../..")
iraf.cd("tmp/objs")
for ii in objs:
geometry_transform(ii)
iraf.cd("../..")
#make normal 1d extraction and copy results into it
for i in arcs:
os.system("rm -f tmp/arcs/%s" % (i[1][:-5]+".ms.fits"))
iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=1, pfit="fit1d", Stdout="/dev/null")
os.system("cp tmp/arcs/%s" % (i[1][:-5])+".ms.fits tmp/arcs/%s" % (i[1][:-5])+".ms2.fits")
if mode==1:
pass
else:
for j in range(1,393):
os.system("rm -f tmp/copy_tmp.fits")
try:
iraf.blkavg(input="tmp/arcs/"+i[1][:-5]+"_t%s.fits" % (str(j)), output="tmp/copy_tmp", option='sum', b1=1, b2=7, Stdout="/dev/null")
iraf.imcopy(input="tmp/copy_tmp", output="tmp/arcs/"+i[1][:-5]+".ms.fits[*,%s]" % (j), Stdout="/dev/null")
except:
pass
for i in objs:
os.system("rm -f tmp/objs/%s" % (i[1][:-5]+".ms.fits"))
iraf.apall(input='tmp/objs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=1, pfit="fit1d", Stdout="/dev/null")
os.system("cp tmp/objs/%s" % (i[1][:-5])+".ms.fits tmp/objs/%s" % (i[1][:-5])+".ms2.fits")
if mode==1:
pass
else:
for j in range(1,393):
os.system("rm -f tmp/copy_tmp.fits")
try:
iraf.blkavg(input="tmp/objs/"+i[1][:-5]+"_t%s.fits" % (str(j)), output="tmp/copy_tmp", option='sum', b1=1, b2=7, Stdout="/dev/null")
iraf.imcopy(input="tmp/copy_tmp", output="tmp/objs/"+i[1][:-5]+".ms.fits[*,%s]" % (j), Stdout="/dev/null")
except:
pass
def combine(do_cti=False, doreduce=True, doshifts=True):
if do_cti:
os.system('stis_cti --crds_update')
if doreduce:
# Defringing didn't seem to converge because of the low S/N
stistools.ocrreject.ocrreject('oc0102070_flc.fits','oc0102070_crc.fits')
iraf.normspflat(inflat='oc0102070_crc.fits',outflat='oc0102070_nsp.fits', do_cal='no')
iraf.imcalc(input='oc0102070_nsp.fits', output='temp_nsp.fits', equals='if(x .lt. 250) then 1 else im1')
iraf.imcopy('temp_nsp.fits[1][1:250,*]', 'oc0102070_nsp.fits[1][1:250,*]')
#iraf.defringe('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102060_dfr.fits')
#for each image
for im in ['oc0102050_flc','oc0102060_flc']:
outbase = 'blue'
if im[:-4] == 'oc0102060':
outbase = 'red'
#reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im +'.fits','APERTAB',value='oref$y2r1559to_apt.fits')
pyfits.setval(im +'.fits', 'SPTRCTAB', value='oref$qa31608go_1dt.fits')
# fixpix any negative values. In principle some of this noise
# could be real, but I have found that is often not the case
hdu = fits.open(im+ '.fits')
mask1 = hdu[1].data < -20
mask2 = hdu[4].data < -20
hdu.close()
fits.writeto(outbase+'mask1.fits', mask1.astype('i'), clobber=True)
fits.writeto(outbase+'mask2.fits', mask2.astype('i'), clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[1]', outbase+'mask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[4]', outbase+'mask2.fits')
# Subtract off the median value
hdu = fits.open(im+ '.fits', mode='update')
hdu[1].data -= np.median(hdu[1].data)
hdu[4].data -= np.median(hdu[4].data)
readnoise1 = 1.4826 * np.median(np.abs(hdu[1].data))
readnoise2 = 1.4826 * np.median(np.abs(hdu[4].data))
# Cosmic ray reject both images using scrappy
# Make sure to treat the noise in a sensible way
crmask1, clean1 = detect_cosmics(hdu[1].data, readnoise=readnoise1,
sigclip=5, objlim=5, sigfrac=0.8,
fsmode='median', psfmodel='gaussy',
psffwhm=2., cleantype='idw')
crmask2, clean2 = detect_cosmics(hdu[4].data, readnoise=readnoise2,
sigclip=5, objlim=5, sigfrac=0.8,
fsmode='median', psfmodel='gaussy',
psffwhm=2., cleantype='idw')
hdu.flush()
hdu.close()
fits.writeto(outbase + '_crmask1.fits', crmask1.astype('i'), clobber=True)
fits.writeto(outbase + '_crmask2.fits', crmask2.astype('i'), clobber=True)
# Run fixpix on the frames
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[1]', outbase+'_crmask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[4]', outbase+'_crmask2.fits')
if outbase=='red':
iraf.mkfringeflat('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102070_frr.fits',
beg_scale=0.6, end_scale=1.5, scale_step=0.01,
beg_shift=-3.0, end_shift=3.0,shift_step=0.05)
iraf.defringe('oc0102060_flc.fits', 'oc0102070_frr.fits', 'oc0102060_dfr.fits')
#Run x2d on the flt frame
stistools.x2d.x2d(input='oc0102060_dfr.fits',output=im[:-4]+'x2d.fits')
else:
stistools.x2d.x2d(input='oc0102050_flc.fits',output=im[:-4]+'x2d.fits')
h = pyfits.open(im[:-4]+'x2d.fits', mode='update')
#Replace all of the bad pixels in the image by -666 based on the DQ array
#save them to a new file
#Throw away bad reference file pixels and saturated pixels. None of the other error codes
#were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(bitwise_and(d,256) == 256,bitwise_and(d,512) == 512)
h[1].data[badpix] = -10000
d = h[6].data
badpix = logical_and(bitwise_and(d,256) == 256,bitwise_and(d,512) == 512)
h[4].data[badpix] = -10000
h.flush()
# Trim the images
for i in range(1,7):
h[i].data = h[i].data[100:-100, 100:-100]
h[i].header['CRPIX1'] -= 100
h[i].header['CRPIX2'] -= 100
h.flush()
h.close()
# Combine the images
iraf.unlearn(iraf.imcombine)
iraf.imcombine(input=im[:-4]+'x2d[1],'+im[:-4]+'x2d[4]', output=outbase+'_com.fits',
reject='crreject')
hdu = pyfits.open(outbase +'_com.fits')
mask = hdu[0].data == 0.0
hdu.close()
fits.writeto(outbase+'_mask.fits', mask.astype('i'), clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(outbase+'_com.fits', outbase+'_mask.fits')
iraf.unlearn(iraf.apall)
iraf.apall(input=outbase+'_com',output='13dh_'+outbase, review='no',
nsum = -500, b_order = 1,
b_function='legendre',b_niterate=30, b_naverage = -21,
nfind=1,t_order=3,background='median',weights='variance',
skybox=100 )
iraf.splot(outbase+'[SCI]')
def apall(imlist_name, reference=''):
"""
Extract 1d spectrum by tracing column of 2d data.
Interactive mode is turned on by default.
On the interactive mode;
(1) Aperture selection
:w -> e1, e2 : Zoom in
:w -> a : Zoom out
:m : Select aperture
:l -> Click left of the aper. : Set lower limit of the aperture.
:u -> Click right of the aper. : Set upper limit of the aperture.
(2) Background selection
:z : Delete previously-selected background region.
:b : Enter the background mode
:s1, s2 (on the left part of bkg) : Set the region of left bkg.
:s3, s4 (on the right part of bkg) : Set the region of right bkg.
And then enter q --> Yes, Yes
(3) Fitting mode
:d : Exclude a point
:function : Change the fitting function. ("legendre" by default)
:order=15 : Change the function order into 15.
:niterate : Cliping iteration (3 times by default)
:f : Go fitting
:q : Confirm
And the enter q --> Yes, Yes
Output image will be saved as "XXX.ms.fits"
reference='' : Not using specified aperture used.
reference='??.ms' : Use the same aperture of ??.ms. Do copy of database of the reference into the same location of input image***
"""
import glob
import os, sys
from pyraf import iraf
iraf.noao()
iraf.twodspec()
iraf.apextract()
imlist = glob.glob(imlist_name)
imlist.sort()
for i in range(len(imlist)):
inim = imlist[i]
print('1d extraction for ' + inim + '...')
if reference == '':
iraf.apall(input=inim,
apertures='',
interactive='yes',
find='no',
t_function="legendre",
t_order=15,
t_niterate=3,
t_low_reject=3,
t_high_reject=3.)
elif reference != '':
iraf.apall(input=inim,
apertures='',
reference=reference,
interactive='no',
find='no',
recenter='no',
trace='no',
t_function="legendre",
t_order=15,
t_niterate=3,
t_low_reject=3,
t_high_reject=3.)
print('Done.')
def reduce_data(filelist_new, filelist_masterflats, filelist_masterthars):
if len(filelist_new) == 0:
print("Please, choose science files to reduce first!")
elif len(filelist_masterflats) == 0:
print("Please, choose masterflat files first!")
elif len(filelist_masterthars) == 0:
print("Please, choose Thorium-Argon reference files first!")
else:
print("Starting reduction process...")
extracted_science_files = []
current_directory = os.getcwd()
years = [
'{:04d}'.format(year) for year in range(2005,
date.today().year + 1)
]
month = ['{:02d}'.format(mon) for mon in range(1, 13)]
for i in range(len(filelist_new)):
trim = trim_remove_bias(filelist_new[i])
chosen_masterflat = None
dummyI = trim.replace(".trim_will_be_removed.fits", ".dummyI.fits")
dummyII = trim.replace(".trim_will_be_removed.fits",
".dummyII.fits")
if os.path.exists(dummyI):
os.remove(dummyI)
if os.path.exists(dummyII):
os.remove(dummyII)
for y in years:
for mon in month:
date_check_1 = y + "_" + mon
date_check_2 = y + "-" + mon
if ((date_check_1 in trim)
or (date_check_2 in trim)) == True:
for masterflat in filelist_masterflats:
if ((date_check_1 in masterflat)
or (date_check_2 in masterflat)) == True:
chosen_masterflat = masterflat
if chosen_masterflat is None:
print("Masterflat File not found!!")
continue
iraf.imarith(trim, "/", chosen_masterflat, dummyI)
iraf.apscatter(dummyI,
output=dummyII,
interactive="No",
apertures="1-51",
references="find_orders.fits")
clean = trim.replace(".trim_will_be_removed.fits", ".clean.fits")
mask = trim.replace(".trim_will_be_removed.fits", ".mask.fits")
mean_image = iraf.imstat(dummyII,
Stdout=1,
fields="mean",
format="no")
print(mean_image)
if float(mean_image[0]) > 5000:
array, header = cosmics.fromfits(dummyII)
sigclip = 50.0
objlim = 50.0
c = cosmics.cosmicsimage(array,
gain=0.368,
readnoise=3.7,
sigclip=sigclip,
sigfrac=10.0,
objlim=objlim)
c.run(maxiter=0)
cosmics.tofits(clean, c.cleanarray, header)
cosmics.tofits(mask, c.mask, header)
elif 5000 > float(mean_image[0]) > 1000:
array, header = cosmics.fromfits(dummyII)
sigclip = 20.0
objlim = 20.0
c = cosmics.cosmicsimage(array,
gain=0.368,
readnoise=3.7,
sigclip=sigclip,
sigfrac=4.0,
objlim=objlim)
c.run(maxiter=2)
cosmics.tofits(clean, c.cleanarray, header)
cosmics.tofits(mask, c.mask, header)
else:
array, header = cosmics.fromfits(dummyII)
sigclip = 5.0
objlim = 5.0
c = cosmics.cosmicsimage(array,
gain=0.368,
readnoise=3.7,
sigclip=sigclip,
sigfrac=1.8,
objlim=objlim)
c.run(maxiter=2)
cosmics.tofits(clean, c.cleanarray, header)
cosmics.tofits(mask, c.mask, header)
extract = trim.replace(".trim_will_be_removed.fits",
".extracted.fits")
extracted_science_files.append(extract)
iraf.apall(clean,
output=extract,
references="find_orders",
profiles="find_orders",
apertures="1-51")
# iraf.apall(clean, output=extract, references= "find_orders", profiles= "find_orders", apertures = "3-50" )
# iraf.apall(clean, output=extract, references= "find_orders", profiles= "find_orders", apertures = "3-50" )
os.remove(dummyI)
os.remove(dummyII)
os.remove(trim)
write_reftable(filelist_new, filelist_masterthars)
thar_directory = os.path.dirname(filelist_masterthars[0])
if thar_directory == os.path.dirname(filelist_new[0]):
pass
else:
remove_file(thar_directory, "science.lis")
remove_file(thar_directory, "reftable.dat")
shutil.move("science.lis", thar_directory)
shutil.move("reftable.dat", thar_directory)
os.chdir(thar_directory)
iraf.refspectra("@science.lis",
references="reftable.dat",
ignoreaps="Yes",
sort="",
group="",
override="yes",
confirm="no",
assign="yes")
iraf.dispcor("@science.lis", output="@science.lis")
if thar_directory == os.path.dirname(filelist_new[0]):
pass
else:
remove_file(current_directory, "science.lis")
remove_file(current_directory, "reftable.dat")
shutil.move("science.lis", current_directory)
shutil.move("reftable.dat", current_directory)
os.chdir(current_directory)
normalize_and_merge(extracted_science_files)
print("Success! Reduction process complete.")
str_v = data_dir + ('[0]' + ',' + data_dir).join(v) + '[0]'
str_v_mod = temp_dir + (',' + temp_dir).join(np.array(v_mod))
iraf.imarith(str_v, '-', cal_dir + 'master_dark.fits', str_v_mod)
star_file = star.replace('*', '') + '.fits'
print "**** combine star files ****"
iraf.imcombine(str_v_mod,
temp_dir + star_file,
combine="sum",
reject="sigclip")
print "**** trace the stars orders ****"
iraf.apall(temp_dir + star_file,
extract='no',
nfind=62,
interactive="no",
find="yes")
#print "**** flatten spectra ****"
#v_mod_norm =SRP.ouputnames(v,'tnrm')
#str_v_mod_flats=temp_dir+(','+temp_dir).join(v_mod_norm)+''
#for j,x in enumerate(v_mod):
# iraf.apflatten(cal_dir+'master_flat.fits',output=temp_dir+v_mod_norm[j],reference=temp_dir+star_file,clobber=True)
iraf.reset(use_new_imt="no")
iraf.flpr("0")
#print "trace the stars orders"
#iraf.apall(temp_dir+star_file,extract='no',nfind=62,interactive="no",find="yes")
#adds DISPAXIS=1 to the header
iraf.hedit(images=flat, fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null")
# <codecell>
iraf.unlearn('apall') #restores initial paramenters to apall
# <codecell>
shift_master(ap_ref[ccd], "database/ap%s" % ap_ref[ccd][:-5])
# <codecell>
check=iraf.apall(input=flat, format='multispec', referen=flat, interac='no', find='no',
recenter='yes', resize='no', edit='yes', trace='yes', fittrac='yes',
extract='yes', extras='no', review='yes', line=2000, lower=-2, upper=2,
nfind=392, maxsep=45, minsep=3, width=5, radius=2, ylevel=0.3, shift='yes',
t_order=7, t_niter=10, t_low_r=3, t_high_r=3, t_sampl='1:4095', t_nlost=1,
npeaks=392, bkg='no', b_order=7, nsum=-10,Stdout=1)
# <codecell>
flat
# <codecell>
#create flats with different aperture widths
iraf.apall(input=flat, output='flat_1', referen=flat, format='multispec', interac='no', find='no', recenter='no', resize='yes', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='no', line=2000, lower=-1, upper=1, bkg='no', nsum=-10, ylevel="INDEF", llimit=-1, ulimit=1,Stdout="/dev/null")
# iraf.apall(input='tmp/masterflat.fits', output='tmp/flat_2', referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='yes', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='no', line=2000, lower=-2, upper=2, bkg='no', nsum=-10, ylevel="INDEF", llimit=-2, ulimit=2,Stdout="/dev/null")
# iraf.apall(input='tmp/masterflat.fits', output='tmp/flat_3', referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='yes', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='no', line=2000, lower=-3, upper=3, bkg='no', nsum=-10, ylevel="INDEF", llimit=-3, ulimit=3,Stdout="/dev/null")
# <codecell>
#multispec
ptfsn1ms = 'ptfsn1.ms'
ptfsn2ms = 'ptfsn2.ms'
ptfstar1ms = 'ptfstar1.ms'
ptfstar2ms = 'ptfstar2.ms'
#Reduce with apall
iraf.apall(input=ptfsn1cr,
out=ptfsn1ms,
nfind=1,
interactive=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
t_nsum=50,
t_nlost=50,
extract=yes,
extras=yes,
review=yes,
background='fit')
iraf.apall(input=ptfsn2cr,
out=ptfsn2ms,
nfind=1,
interactive=yes,
find=yes,
recenter=yes,
resize=yes,
def deimos_standard(standard):
'''Extract standard star and calculate sensitivit function.'''
bstd = "%s_01_B" % standard
rstd = "%s_01_R" % standard
# Extract standard
if get_head("%s.fits" % bstd, "SKYSUB"):
iraf.apall(bstd,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="none")
iraf.apall(rstd,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="none")
else:
iraf.apall(bstd,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="fit")
iraf.apall(rstd,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="fit")
# Fit the continuum
#iraf.continuum("%s.ms" % bstd, output="s%s.ms" % bstd, lines=1, bands=1,
# type="fit", sample="*", naverage=1, function="chebyshev",
# order=15, low_reject=3.0, high_reject=5.0, niterate=10, grow=1.0,
# interac=yes)
#iraf.continuum("%s.ms" % rstd, output="s%s.ms" % rstd, lines=1, bands=1,
# type="fit", sample="*", naverage=1, function="chebyshev",
# order=15, low_reject=3.0, high_reject=5.0, niterate=10, grow=1.0,
# interac=yes)
# Create telluric
iraf.splot("%s.ms" %
bstd) # Remove telluric and absorption, save as a%s.ms
iraf.sarith("%s.ms" % bstd, "/", "a%s.ms" % bstd,
"telluric.B.%s.fits" % bstd)
iraf.imreplace("telluric.B.%s.fits" % bstd, 1.0, lower=0.0, upper=0.0)
iraf.splot("telluric.B.%s.fits" %
bstd) # Remove stellar features and resave
iraf.splot("%s.ms" %
rstd) # Remove telluric and absorption, save as a%s.ms
iraf.sarith("%s.ms" % rstd, "/", "a%s.ms" % rstd,
"telluric.R.%s.fits" % rstd)
iraf.imreplace("telluric.R.%s.fits" % rstd, 1.0, lower=0.0, upper=0.0)
iraf.splot("telluric.R.%s.fits" %
rstd) # Remove stellar features and resave
# Create smoothed standard
iraf.gauss("a%s.ms[*,1,1]" % bstd, "s%s.ms" % bstd, 5.0)
iraf.sarith("%s.ms" % bstd, "/", "s%s.ms" % bstd, "ds%s.ms" % bstd)
iraf.gauss("a%s.ms[*,1,1]" % rstd, "s%s.ms" % rstd, 5.0)
iraf.sarith("%s.ms" % rstd, "/", "s%s.ms" % rstd, "ds%s.ms" % rstd)
# Divide through by smoothed standard
#iraf.sarith("%s.ms" % bstd, "/", "s%s.ms" % bstd, "ds%s.ms" % bstd)
#iraf.sarith("%s.ms" % rstd, "/", "s%s.ms" % rstd, "ds%s.ms" % rstd)
# Create and apply telluric correction
#iraf.splot("%s.ms" % bstd) # Remove telluric, save as a%s.ms
#iraf.splot("%s.ms" % rstd) # Remove telluric, save as a%s.ms
#iraf.sarith("%s.ms" % bstd, "/", "a%s.ms" % bstd, "telluric.B.fits")
#iraf.sarith("%s.ms" % rstd, "/", "a%s.ms" % rstd, "telluric.R.fits")
#iraf.imreplace("telluric.B.fits", 1.0, lower=0.0, upper=0.0)
#iraf.imreplace("telluric.R.fits", 1.0, lower=0.0, upper=0.0)
iraf.telluric("ds%s.ms" % bstd,
"tds%s.ms" % bstd,
"telluric.B.%s.fits" % bstd,
xcorr=no,
tweakrms=no,
interactive=no,
sample='6850:6950,7575:7700')
iraf.telluric("ds%s.ms" % rstd,
"tds%s.ms" % rstd,
"telluric.R.%s.fits" % rstd,
xcorr=no,
tweakrms=no,
interactive=no,
sample='6850:6950,7575:7700')
# Define bandpasses for standard star calculation
iraf.standard("tds%s.ms" % bstd,
"%s.B.std" % standard,
extinction='home$extinct/maunakeaextinct.dat',
caldir='home$standards/',
observatory='Keck',
interac=yes,
star_name=standard,
airmass='',
exptime='')
iraf.standard("tds%s.ms" % rstd,
"%s.R.std" % standard,
extinction='home$extinct/maunakeaextinct.dat',
caldir='home$standards/',
observatory='Keck',
interac=yes,
star_name=standard,
airmass='',
exptime='')
# Determine sensitivity function
iraf.sensfunc('%s.B.std' % standard,
'%s.B.sens' % standard,
extinction='home$extinct/maunakeaextinct.dat',
newextinction='extinct.dat',
observatory='Keck',
function='legendre',
order=4,
interactive=yes)
iraf.sensfunc('%s.R.std' % standard,
'%s.R.sens' % standard,
extinction='home$extinct/maunakeaextinct.dat',
newextinction='extinct.dat',
observatory='Keck',
function='legendre',
order=4,
interactive=yes)
return
def deimos_extract(science, standard, dostandard=yes):
'''Extract and flux calibrate DEIMOS spectra (assuming they have been
processed into 2D images using deimos_pipe above).'''
if dostandard:
deimos_standard(standard)
for source in science:
images = iraffiles("%s_??_B.fits" % source)
joinstr = ""
for image in images:
bimage = image.split(".")[0]
rimage = bimage[:-1] + "R"
update_head(bimage, "FLUX_OBJ", standard)
update_head(rimage, "FLUX_OBJ", standard)
# Extract 1D spectra
if get_head("%s.fits" % bimage, "SKYSUB"):
iraf.apall(bimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="none",
reference="%s_01_B" % standard)
iraf.apall(rimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="none",
reference="%s_01_R" % standard)
else:
iraf.apall(bimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="fit",
reference="%s_01_B" % standard)
iraf.apall(rimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="fit",
reference="%s_01_R" % standard)
# Normalize by the standard continuua
iraf.sarith("%s.ms" % bimage, "/", "s%s_01_B.ms.fits" % standard,
"s%s.ms" % bimage)
iraf.sarith("%s.ms" % rimage, "/", "s%s_01_R.ms.fits" % standard,
"s%s.ms" % rimage)
# Telluric correct
bstd = "%s_01_B" % standard
rstd = "%s_01_R" % standard
iraf.telluric("s%s.ms" % bimage,
"ts%s.ms" % bimage,
"telluric.B.%s.fits" % bstd,
tweakrms=yes,
interactive=yes,
sample='6850:6950,7575:7700')
iraf.telluric("s%s.ms" % rimage,
"ts%s.ms" % rimage,
"telluric.R.%s.fits" % rstd,
tweakrms=yes,
interactive=yes,
sample='6850:6950,7575:7700')
# Flux calibration
iraf.calibrate("ts%s.ms" % bimage,
"fts%s.ms" % bimage,
extinct=yes,
flux=yes,
extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck",
sensitivity="%s.B.sens" % standard,
airmass='',
exptime='')
iraf.calibrate("ts%s.ms" % rimage,
"fts%s.ms" % rimage,
extinct=yes,
flux=yes,
extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck",
sensitivity="%s.R.sens" % standard,
airmass='',
exptime='')
joinstr += "fts%s.ms,fts%s.ms," % (bimage, rimage)
# Combine
iraf.scombine(joinstr[:-1],
"%s.ms.fits" % source,
combine="average",
reject="avsigclip",
w1=INDEF,
w2=INDEF,
dw=INDEF,
nw=INDEF,
scale="none",
zero="none",
weight="none",
lsigma=3.0,
hsigma=3.0,
gain=CCDGAIN,
rdnoise=CCDRNOISE)
# Plot to enable final tweaks
iraf.splot("%s.ms.fits" % source)
return
def deimos_standard(standard):
'''Extract standard star and calculate sensitivit function.'''
bstd = "%s_01_B" % standard; rstd = "%s_01_R" % standard
# Extract standard
if get_head("%s.fits" % bstd, "SKYSUB"):
iraf.apall(bstd, output="", inter=yes, find=yes, recenter=yes, resize=yes,
edit=yes, trace=yes, fittrace=yes, extract=yes, extras=yes,
review=no, background="none")
iraf.apall(rstd, output="", inter=yes, find=yes, recenter=yes, resize=yes,
edit=yes, trace=yes, fittrace=yes, extract=yes, extras=yes,
review=no, background="none")
else:
iraf.apall(bstd, output="", inter=yes, find=yes, recenter=yes, resize=yes,
edit=yes, trace=yes, fittrace=yes, extract=yes, extras=yes,
review=no, background="fit")
iraf.apall(rstd, output="", inter=yes, find=yes, recenter=yes, resize=yes,
edit=yes, trace=yes, fittrace=yes, extract=yes, extras=yes,
review=no, background="fit")
# Fit the continuum
#iraf.continuum("%s.ms" % bstd, output="s%s.ms" % bstd, lines=1, bands=1,
# type="fit", sample="*", naverage=1, function="chebyshev",
# order=15, low_reject=3.0, high_reject=5.0, niterate=10, grow=1.0,
# interac=yes)
#iraf.continuum("%s.ms" % rstd, output="s%s.ms" % rstd, lines=1, bands=1,
# type="fit", sample="*", naverage=1, function="chebyshev",
# order=15, low_reject=3.0, high_reject=5.0, niterate=10, grow=1.0,
# interac=yes)
# Create telluric
iraf.splot("%s.ms" % bstd) # Remove telluric and absorption, save as a%s.ms
iraf.sarith("%s.ms" % bstd, "/", "a%s.ms" % bstd, "telluric.B.%s.fits" % bstd)
iraf.imreplace("telluric.B.%s.fits" % bstd, 1.0, lower=0.0, upper=0.0)
iraf.splot("telluric.B.%s.fits" % bstd) # Remove stellar features and resave
iraf.splot("%s.ms" % rstd) # Remove telluric and absorption, save as a%s.ms
iraf.sarith("%s.ms" % rstd, "/", "a%s.ms" % rstd, "telluric.R.%s.fits" % rstd)
iraf.imreplace("telluric.R.%s.fits" % rstd, 1.0, lower=0.0, upper=0.0)
iraf.splot("telluric.R.%s.fits" % rstd) # Remove stellar features and resave
# Create smoothed standard
iraf.gauss("a%s.ms[*,1,1]" % bstd, "s%s.ms" % bstd, 5.0)
iraf.sarith("%s.ms" % bstd, "/", "s%s.ms" % bstd, "ds%s.ms" % bstd)
iraf.gauss("a%s.ms[*,1,1]" % rstd, "s%s.ms" % rstd, 5.0)
iraf.sarith("%s.ms" % rstd, "/", "s%s.ms" % rstd, "ds%s.ms" % rstd)
# Divide through by smoothed standard
#iraf.sarith("%s.ms" % bstd, "/", "s%s.ms" % bstd, "ds%s.ms" % bstd)
#iraf.sarith("%s.ms" % rstd, "/", "s%s.ms" % rstd, "ds%s.ms" % rstd)
# Create and apply telluric correction
#iraf.splot("%s.ms" % bstd) # Remove telluric, save as a%s.ms
#iraf.splot("%s.ms" % rstd) # Remove telluric, save as a%s.ms
#iraf.sarith("%s.ms" % bstd, "/", "a%s.ms" % bstd, "telluric.B.fits")
#iraf.sarith("%s.ms" % rstd, "/", "a%s.ms" % rstd, "telluric.R.fits")
#iraf.imreplace("telluric.B.fits", 1.0, lower=0.0, upper=0.0)
#iraf.imreplace("telluric.R.fits", 1.0, lower=0.0, upper=0.0)
iraf.telluric("ds%s.ms" % bstd, "tds%s.ms" % bstd, "telluric.B.%s.fits" % bstd,
xcorr=no, tweakrms=no, interactive=no, sample='6850:6950,7575:7700')
iraf.telluric("ds%s.ms" % rstd, "tds%s.ms" % rstd, "telluric.R.%s.fits" % rstd,
xcorr=no, tweakrms=no, interactive=no, sample='6850:6950,7575:7700')
# Define bandpasses for standard star calculation
iraf.standard("tds%s.ms" % bstd, "%s.B.std" % standard,
extinction='home$extinct/maunakeaextinct.dat',
caldir='home$standards/', observatory='Keck', interac=yes,
star_name=standard, airmass='', exptime='')
iraf.standard("tds%s.ms" % rstd, "%s.R.std" % standard,
extinction='home$extinct/maunakeaextinct.dat',
caldir='home$standards/', observatory='Keck', interac=yes,
star_name=standard, airmass='', exptime='')
# Determine sensitivity function
iraf.sensfunc('%s.B.std' % standard, '%s.B.sens' % standard,
extinction='home$extinct/maunakeaextinct.dat',
newextinction='extinct.dat', observatory='Keck',
function='legendre', order=4, interactive=yes)
iraf.sensfunc('%s.R.std' % standard, '%s.R.sens' % standard,
extinction='home$extinct/maunakeaextinct.dat',
newextinction='extinct.dat', observatory='Keck',
function='legendre', order=4, interactive=yes)
return
def oned_extract(object_b_fn, object_b_fn_ec, order_def, colour):
# Import IRAF modules:
iraf.noao(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.apextract(_doprint=0)
# Check input file and reference extraction exist before proceeding:
if os.path.isfile(object_b_fn) == True:
if os.path.isfile(order_def) == True:
# Extract one dimensional spectrum from bias- and background scatter-subtracted
# normalized flat-fielded red object frame using IRAF task apall:
if colour == 'red':
parList = "r_oned_extraction_apall.par"
if os.path.isfile(parList) == True:
iraf.apall.setParList(ParList = "r_oned_extraction_apall.par")
iraf.apall(input=object_b_fn, output=object_b_fn_ec, references=order_def)
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'One dimensional spectral extraction of bias- and '
print 'background-subtracted, normalized flat-fielded '
print 'red object ' + object_b_fn
print 'to create ' + object_b_fn_ec + '.'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'One dimensional extraction IRAF .par file '
print str(parList)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
# Extract one dimensional spectrum from bias- and background scatter-subtracted
# normalized flat-fielded blue object frame using IRAF task apall:
if colour == 'blue':
parList = "b_oned_extraction_apall.par"
if os.path.isfile(parList) == True:
iraf.apall.setParList(ParList = "b_oned_extraction_apall.par")
iraf.apall(input=object_b_fn, output=object_b_fn_ec, references=order_def)
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'One dimensional spectral extraction of bias- and '
print 'background-subtracted, normalized flat-fielded '
print 'blue object ' + object_b_fn
print 'to create ' + object_b_fn_s + '.'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'One dimensional extraction IRAF .par file '
print str(parList)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Order definition frame '
print str(order_def)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Input frame '
print str(object_b_fn)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
if False: # for testing
input = calfns[0]
output = redfns[0]
else:
input = '@tempjunkcal'
output = '@tempjunkred'
if len(calfns)==0: input = None
if doextract and (input is None):
print "Beginning spectral extraction, but no valid files found for extraction."
elif doextract:
print "Beginning spectral extraction for %i files" % (len(calfns))
for fniter in range(nfiles):
iraf.apall(input+('%i'%fniter), output=output+('%i'%fniter), format='onedspec', recenter='YES',resize='YES',extras='yes', nfind=1, nsubaps=1, minsep=100, weights='variance', bkg='yes', b_function='chebyshev', b_order=1, b_naverage=-3, b_niterate=2, t_order=4, t_niterate=3, t_naverage=t_naverage, background='fit', clean='yes', interactive=True, review=False, b_sample='-40:-15,15:40', trace='YES',edit='YES',fittrace='YES',extract='YES', find='YES', t_sample=horizsamp)
print "... and we're done!\n"
print "Verify that all datasets were correctly matched up:\n"
for dataset in datasets:
datfns = [fn.replace('.fits', '.0001.fits') for fn in dataset[0]]
ra, dec, oo = [], [], []
for fn in datfns:
if os.path.isfile(fn):
ra.append(pyfits.getval(fn, 'RA'))
dec.append(pyfits.getval(fn, 'DEC'))
def combine(do_cti=False, doreduce=True, doshifts=True):
if do_cti:
os.system('stis_cti --crds_update')
if doreduce:
# Defringing didn't seem to converge because of the low S/N
stistools.ocrreject.ocrreject('oc0102070_flc.fits',
'oc0102070_crc.fits')
iraf.normspflat(inflat='oc0102070_crc.fits',
outflat='oc0102070_nsp.fits',
do_cal='no')
iraf.imcalc(input='oc0102070_nsp.fits',
output='temp_nsp.fits',
equals='if(x .lt. 250) then 1 else im1')
iraf.imcopy('temp_nsp.fits[1][1:250,*]',
'oc0102070_nsp.fits[1][1:250,*]')
#iraf.defringe('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102060_dfr.fits')
#for each image
for im in ['oc0102050_flc', 'oc0102060_flc']:
outbase = 'blue'
if im[:-4] == 'oc0102060':
outbase = 'red'
#reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im + '.fits',
'APERTAB',
value='oref$y2r1559to_apt.fits')
pyfits.setval(im + '.fits',
'SPTRCTAB',
value='oref$qa31608go_1dt.fits')
# fixpix any negative values. In principle some of this noise
# could be real, but I have found that is often not the case
hdu = fits.open(im + '.fits')
mask1 = hdu[1].data < -20
mask2 = hdu[4].data < -20
hdu.close()
fits.writeto(outbase + 'mask1.fits',
mask1.astype('i'),
clobber=True)
fits.writeto(outbase + 'mask2.fits',
mask2.astype('i'),
clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[1]', outbase + 'mask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[4]', outbase + 'mask2.fits')
# Subtract off the median value
hdu = fits.open(im + '.fits', mode='update')
hdu[1].data -= np.median(hdu[1].data)
hdu[4].data -= np.median(hdu[4].data)
readnoise1 = 1.4826 * np.median(np.abs(hdu[1].data))
readnoise2 = 1.4826 * np.median(np.abs(hdu[4].data))
# Cosmic ray reject both images using scrappy
# Make sure to treat the noise in a sensible way
crmask1, clean1 = detect_cosmics(hdu[1].data,
readnoise=readnoise1,
sigclip=5,
objlim=5,
sigfrac=0.8,
fsmode='median',
psfmodel='gaussy',
psffwhm=2.,
cleantype='idw')
crmask2, clean2 = detect_cosmics(hdu[4].data,
readnoise=readnoise2,
sigclip=5,
objlim=5,
sigfrac=0.8,
fsmode='median',
psfmodel='gaussy',
psffwhm=2.,
cleantype='idw')
hdu.flush()
hdu.close()
fits.writeto(outbase + '_crmask1.fits',
crmask1.astype('i'),
clobber=True)
fits.writeto(outbase + '_crmask2.fits',
crmask2.astype('i'),
clobber=True)
# Run fixpix on the frames
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[1]', outbase + '_crmask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[4]', outbase + '_crmask2.fits')
if outbase == 'red':
iraf.mkfringeflat('oc0102060_flc.fits',
'oc0102070_nsp.fits',
'oc0102070_frr.fits',
beg_scale=0.6,
end_scale=1.5,
scale_step=0.01,
beg_shift=-3.0,
end_shift=3.0,
shift_step=0.05)
iraf.defringe('oc0102060_flc.fits', 'oc0102070_frr.fits',
'oc0102060_dfr.fits')
#Run x2d on the flt frame
stistools.x2d.x2d(input='oc0102060_dfr.fits',
output=im[:-4] + 'x2d.fits')
else:
stistools.x2d.x2d(input='oc0102050_flc.fits',
output=im[:-4] + 'x2d.fits')
h = pyfits.open(im[:-4] + 'x2d.fits', mode='update')
#Replace all of the bad pixels in the image by -666 based on the DQ array
#save them to a new file
#Throw away bad reference file pixels and saturated pixels. None of the other error codes
#were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[1].data[badpix] = -10000
d = h[6].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[4].data[badpix] = -10000
h.flush()
# Trim the images
for i in range(1, 7):
h[i].data = h[i].data[100:-100, 100:-100]
h[i].header['CRPIX1'] -= 100
h[i].header['CRPIX2'] -= 100
h.flush()
h.close()
# Combine the images
iraf.unlearn(iraf.imcombine)
iraf.imcombine(input=im[:-4] + 'x2d[1],' + im[:-4] + 'x2d[4]',
output=outbase + '_com.fits',
reject='crreject')
hdu = pyfits.open(outbase + '_com.fits')
mask = hdu[0].data == 0.0
hdu.close()
fits.writeto(outbase + '_mask.fits',
mask.astype('i'),
clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(outbase + '_com.fits', outbase + '_mask.fits')
iraf.unlearn(iraf.apall)
iraf.apall(input=outbase + '_com',
output='13dh_' + outbase,
review='no',
nsum=-500,
b_order=1,
b_function='legendre',
b_niterate=30,
b_naverage=-21,
nfind=1,
t_order=3,
background='median',
weights='variance',
skybox=100)
iraf.splot(outbase + '[SCI]')
def extractSpectra():
"""
Extract 1D spectra using IRAF interactively
Interpolate across the two arcs either side to
get the most accurate wavelength solution
TODO: Finish docstring
Add method of using super arc for inital
identify
"""
# load IRAF from the location of the login.cl file
here = os.getcwd()
os.chdir(loginCl_location)
from pyraf import iraf
os.chdir(here)
time.sleep(2)
# make a list of the science images to be analysed
templist = g.glob('i_s*')
# import IRAF packages for spectroscopy
iraf.imred(_doprint=0)
iraf.kpnoslit(_doprint=0)
# apall parameters
iraf.apall.setParam('format', 'multispec')
iraf.apall.setParam('interac', 'yes')
iraf.apall.setParam('find', 'yes')
iraf.apall.setParam('recen', 'yes')
iraf.apall.setParam('resize', 'yes')
iraf.apall.setParam('trace', 'yes')
iraf.apall.setParam('fittrac', 'yes')
iraf.apall.setParam('extract', 'yes')
iraf.apall.setParam('extras', 'yes')
iraf.apall.setParam('review', 'yes')
iraf.apall.setParam('line', 'INDEF')
iraf.apall.setParam('nsum', '12')
iraf.apall.setParam('lower', '-6')
iraf.apall.setParam('upper', '6')
iraf.apall.setParam('b_funct', 'chebyshev')
iraf.apall.setParam('b_order', '1')
iraf.apall.setParam('b_sampl', '-25:-15,15:25')
iraf.apall.setParam('b_naver', '-100')
iraf.apall.setParam('b_niter', '0')
iraf.apall.setParam('b_low_r', '3')
iraf.apall.setParam('b_high', '3')
iraf.apall.setParam('b_grow', '0')
iraf.apall.setParam('width', '10')
iraf.apall.setParam('radius', '10')
iraf.apall.setParam('threshold', '0')
iraf.apall.setParam('nfind', '1')
iraf.apall.setParam('t_nsum', '10')
iraf.apall.setParam('t_step', '10')
iraf.apall.setParam('t_nlost', '3')
iraf.apall.setParam('t_niter', '7')
iraf.apall.setParam('t_funct', 'spline3')
iraf.apall.setParam('t_order', '3')
iraf.apall.setParam('backgro', 'fit')
iraf.apall.setParam('skybox', '1')
iraf.apall.setParam('weights', 'variance')
iraf.apall.setParam('pfit', 'fit1d')
iraf.apall.setParam('clean', 'yes')
iraf.apall.setParam('saturat', SATURATION)
iraf.apall.setParam('readnoi', RDNOISE)
iraf.apall.setParam('gain', GAIN)
iraf.apall.setParam('lsigma', '4.0')
iraf.apall.setParam('usigma', '4.0')
iraf.apall.setParam('nsubaps', '1')
iraf.apall.saveParList(filename="apall.pars")
# make reference arc for reidentify
if '.' in args.refarc:
args.refarc = args.refarc.split('.')[0]
refarc = "a_s_{}_t.fits".format(args.refarc)
refarc_out = "a_s_{}_t.ms.fits".format(args.refarc)
# loop over all the the spectra
for i in range(0, len(templist)):
hdulist = fits.open(templist[i])
prihdr = hdulist[0].header
target_id = prihdr['CAT-NAME']
spectrum_id = int(templist[i].split('_')[2].split('r')[1])
if args.ds9:
os.system('xpaset fuckingds9 fits < {}'.format(templist[i]))
# extract the object spectrum
print("[{}/{}] Extracting spectrum of {} from image {}".format(i+1, len(templist), target_id, templist[i]))
print("[{}/{}] Check aperture and background. Change if required".format(i+1, len(templist)))
print("[{}/{}] AP: m = mark aperture, d = delete aperture".format(i+1, len(templist)))
print("[{}/{}] SKY: s = mark sky, t = delete sky, f = refit".format(i+1, len(templist)))
print("[{}/{}] q = continue".format(i+1, len(templist)))
iraf.apall(input=templist[i])
print("Spectrum extracted!")
# find the arcs either side of the object
arclist = []
arc1 = "a_s_r{0:d}_t.fits".format(spectrum_id-1)
arc2 = "a_s_r{0:d}_t.fits".format(spectrum_id+1)
arc1_out = "a_s_r{0:d}_t.ms.fits".format(spectrum_id-1)
arc2_out = "a_s_r{0:d}_t.ms.fits".format(spectrum_id+1)
# predict the arc names
print("\nPredicting arcs names...")
print("Arc1: {}".format(arc1))
print("Arc2: {}".format(arc2))
# setup a reference filename for the arc conditions in database
reffile = templist[i].split('.fits')[0]
# extract the arcs
print("\nExtracting arcs under the same conditions...")
if os.path.exists(arc1):
iraf.apall(input=arc1,
reference=reffile,
recente="no",
trace="no",
backgro="no",
interac="no")
print("Arc1 {} extracted".format(arc1))
arclist.append(arc1_out)
else:
print("\n\nArc1 {} FILE NOT FOUND\n\n".format(arc1))
if os.path.exists(arc2):
iraf.apall(input=arc2,
reference=reffile,
recente="no",
trace="no",
backgro="no",
interac="no")
print("Arc2 {} extracted".format(arc2))
arclist.append(arc2_out)
else:
print("\n\nArc2 {} FILE NOT FOUND\n\n".format(arc2))
# get a list of the extracted arcs and objects
spectrum_out = "i_s_r{0:d}_t.ms.fits".format(spectrum_id)
if i == 0:
# extract the master reference arc
print("\nExtracting master arc {} under the same conditions...".format(refarc))
iraf.apall(input=refarc,
reference=reffile,
recente="no",
trace="no",
backgro="no",
interac="no")
print("Reference arc {} extracted".format(refarc))
# identify the lines in it
print("\nIdentify arc lines:")
print("Enter the following in the splot window")
print("\t:thres 500")
print("\t:order 1, max = 3")
print("\tfwidth 2")
print("Select 3-5 arc lines from line atlas")
print("Press 'm' to mark, then enter wavelength")
print("Then press 'l' to automatically ID the other lines")
print("Press 'f' to fit the dispersion correction")
print("Use 'd' to remove bad points, 'f' to refit")
print("'q' from fit, then 'q' from identify to continue\n")
iraf.identify(images=refarc_out, coordlist=lineList_location)
# use the refarc to ID all the subsequent arcs
for arc in arclist:
print("\nReidentifying arclines from {}".format(arc))
iraf.reidentify(reference=refarc_out, images=arc)
# add the refspec keywords to the image header for dispcor
# refspec_factor tells IRAF how to interpolate the arcs
refspec_factor = round((1./len(arclist)), 1)
for i in range(0, len(arclist)):
refspec = "{} {}".format(arclist[i].split(".fits")[0], refspec_factor)
print("REFSPEC{}: {}".format(i+1, refspec))
iraf.hedit(images=spectrum_out,
fields="REFSPEC{}".format(i+1),
value=refspec,
add="yes",
verify="no",
show="yes")
print("Headers updated!\n")
# apply the dispersion correction
print("Applying the dispersion correction")
iraf.dispcor(input=spectrum_out,
output=spectrum_out,
lineari="yes",
databas="database",
table="")
print("Correction applied!")
# normalize the spectrum using continuum
normspec_out = "{}n.ms.fits".format(spectrum_out.split('.ms')[0])
iraf.continuum(input=spectrum_out,
output=normspec_out,
logfile="logfile",
interac="yes",
functio="spline3",
order="5",
niterat="10",
markrej="yes")
print("\n\n")
# Extract raw spectral data from the echelle images
ir.imdelete('@' + speccal)
ir.apall('@' + proccal,
output='@' + speccal,
format='echelle',
recenter='yes',
resize='yes',
extras='yes',
nfind=n_ap,
nsubaps=1,
minsep=10,
weights='variance',
bkg='yes',
b_function=bfunc,
b_order=bord,
b_sample=bsamp,
b_naverage=-3,
b_niterate=2,
t_order=3,
t_sample=horizsamp,
t_niterate=3,
t_naverage=3,
background='fit',
clean='yes',
interactive=interactive,
nsum=-10,
t_function='chebyshev')
if verbose: print "Done extracting spectra from cal stars!"
def lris_extract(science, standards, dostandard=yes):
'''Extract and flux calibrate LRIS spectra (assuming they have been
processed into 2D images using LRIS_pipe above).'''
if dostandard:
lris_standard(standards)
for src in science:
bimages = iraffiles("%s_??_B.fits" % src)
rimages = iraffiles("%s_??_R.fits" % src)
joinstr = ""
for bimage in bimages:
bimage = bimage.split(".")[0]
# Find appropriate standard for this image
bstd = get_head("%s.fits" % bimage, "STDNAME")
# Extract 1D spectra
if get_head("%s.fits" % bimage, "SKYSUB"):
iraf.apall(bimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="none",
reference=bstd)
else:
iraf.apall(bimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="fit",
reference=bstd)
# Normalize by the standard continuua
iraf.sarith("%s.ms" % bimage, "/", "../standards/s%s.ms.fits" % bstd,
"s%s.ms" % bimage, w1=INDEF, w2=INDEF)
# Telluric correct
iraf.telluric("s%s.ms" % bimage, "ts%s.ms" % bimage,
"../standards/telluric.%s.fits" % bstd, tweakrms=yes,
interactive=yes, sample='6850:6950,7575:7700,8750:9925')
# Flux calibration
iraf.calibrate("ts%s.ms" % bimage, "fts%s.ms" % bimage, extinct=yes,
flux=yes, extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck", sens="../standards/%s.sens" % bstd,
airmass='', exptime='')
joinstr += "fts%s.ms," % bimage
for rimage in rimages:
rimage = rimage.split(".")[0]
# Find appropriate standard for this image
rstd = get_head("%s.fits" % rimage, "STDNAME")
# Extract 1D spectra
if get_head("%s.fits" % rimage, "SKYSUB"):
iraf.apall(rimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="none",
reference=rstd)
else:
iraf.apall(rimage, output="", inter=yes, find=yes, recenter=yes,
resize=yes, edit=yes, trace=yes, fittrace=yes, extract=yes,
extras=yes, review=no, background="fit",
reference=rstd)
# Normalize by the standard continuua
iraf.sarith("%s.ms" % rimage, "/", "../standards/s%s.ms.fits" % rstd,
"s%s.ms" % rimage, w1=5500, w2=INDEF)
# Telluric correct
iraf.telluric("s%s.ms" % rimage, "ts%s.ms" % rimage,
"../standards/telluric.%s.fits" % rstd, tweakrms=yes,
interactive=yes, sample='6850:6950,7575:7700')
# Flux calibration
iraf.calibrate("ts%s.ms" % rimage, "fts%s.ms" % rimage, extinct=yes,
flux=yes, extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck", sens="../standards/%s.sens" % rstd,
airmass='', exptime='')
joinstr += "fts%s.ms," % rimage
# Combine
iraf.scombine(joinstr[:-1], "%s.ms.fits" % src, combine="average",
reject="avsigclip", w1=INDEF, w2=INDEF, dw=INDEF, nw=INDEF,
scale="median", zero="none", weight="none", sample="5450:5600",
lsigma=3.0, hsigma=3.0, gain=RCCDGAIN, rdnoise=RCCDRNOISE)
# Plot to enable final tweaks
iraf.splot("%s.ms.fits" % src)
return
file1 = open('listafinal', 'w')
file1.writelines(["%s\n" % item for item in final])
file1.close()
f = open('listafinal')
i = 1
for item in f:
file = item.split('\n')[0]
shutil.copy(file, 'spec'+str(i)+'.fits')
i += 1
#FROM NOW -> iraf
iraf.noao.twodspec.apextract
iraf.apall('spec*.fits')
#assign dispersion solution - from now IRAF
shutil.copy('./arc/database/idarc1.ms','./database/.')
for file in os.listdir(os.getcwd()):
if file.endswith('ms.fits'):
iraf.hedit(file,fields="REFSPEC1",value="arc1.ms",add='yes',ver='no',show='yes')
#wavelength calinration
for file in os.listdir(os.getcwd()):
if file.endswith('ms.fits'):
iraf.dispcor(file, 'd'+file)
#flux calibration
def extract_spectrum(targetdir,
trace,
arcspec,
refspec,
t_nsum,
t_step,
line,
ylevel,
interactive):
"""
Extract spectrum
Must be in target directory.
"""
iraf.noao(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.apextract(_doprint=0)
basedir = '/data/lc585/WHT_20150331/OBS/'
targetdir = os.path.join(basedir,targetdir,'Reduced')
print 'Target directory is ' + targetdir
print 'Extracting spectrum...'
if os.path.exists( os.path.join(targetdir, 'imcomb.ms.fits') ):
os.remove( os.path.join( targetdir, 'imcomb.ms.fits') )
print 'Removing file ' + os.path.join( targetdir, 'imcomb.ms.fits')
# If can't fit trace use trace from nearby object
if trace == 'no':
dest = os.path.join(targetdir,'database')
if not os.path.exists(dest):
os.makedirs(dest)
db = os.path.join(basedir,refspec,'Reduced','database','ap_data_lc585_WHT_20150331_OBS_'+refspec+'_Reduced_imcomb')
shutil.copy(db,dest)
iraf.apall.setParam('references',os.path.join(basedir,refspec,'Reduced','imcomb.fit')) # List of aperture reference images
print 'Using trace from reference spectra ' + refspec
# Since frame is averaged I think we need to scale down read nosie but gain will stay the same.
names = []
for n in os.listdir(targetdir):
if (n.endswith('.fit')) & (n.startswith('r')):
names.append(n)
nframes = float(len(names))
# Doesn't seem to work if I give it absolute path to input!
iraf.apall.setParam('input','imcomb.fit') # List of input images
iraf.apall.setParam('output','') # List of output spectra
iraf.apall.setParam('apertur','') # Apertures
iraf.apall.setParam('format','multispec') # Extracted spectra format
iraf.apall.setParam('referen','') # List of aperture reference images
iraf.apall.setParam('profile','') # List of aperture profile images
iraf.apall.setParam('interac',interactive) # Run task interactively?
iraf.apall.setParam('find','no') # Find apertures?
iraf.apall.setParam('recente','no') # Recenter apertures?
iraf.apall.setParam('resize','no') # Resize apertures?
iraf.apall.setParam('edit','yes') # Edit apertures?
iraf.apall.setParam('trace',trace) # Trace apertures?
iraf.apall.setParam('fittrac',interactive) # Fit the traced points interactively?
iraf.apall.setParam('extract','yes') # Extract spectra?
iraf.apall.setParam('extras','yes') # Extract sky, sigma, etc.?
iraf.apall.setParam('review',interactive) # Review extractions?
iraf.apall.setParam('line',line) # Dispersion line
iraf.apall.setParam('nsum',20) # Number of dispersion lines to sum or median
# DEFAULT APERTURE PARAMETERS
iraf.apall.setParam('lower',-5.) # Lower aperture limit relative to center
iraf.apall.setParam('upper',5.) # Upper aperture limit relative to center
iraf.apall.setParam('apidtab','') # Aperture ID table (optional)
# DEFAULT BACKGROUND PARAMETERS
# Background is now a constant at each wavelength
iraf.apall.setParam('b_funct','chebyshev') # Background function
iraf.apall.setParam('b_order',1) # Background function order
iraf.apall.setParam('b_sampl','-10:-6,6:10') # Background sample regions
iraf.apall.setParam('b_naver',-3) # Background average or median
iraf.apall.setParam('b_niter',2) # Background rejection iterations
iraf.apall.setParam('b_low_r',3.) # Background lower rejection sigma
iraf.apall.setParam('b_high_',3.) # Background upper rejection sigma
iraf.apall.setParam('b_grow',0.) # Background rejection growing radius
# APERTURE CENTERING PARAMETERS
iraf.apall.setParam('width',5.) # Profile centering width
iraf.apall.setParam('radius',10.) # Profile centering radius
iraf.apall.setParam('thresho',0.) # Detection threshold for profile centering
# AUTOMATIC FINDING AND ORDERING PARAMETERS
iraf.apall.setParam('nfind','') # Number of apertures to be found automatically
iraf.apall.setParam('minsep',5.) # Minimum separation between spectra
iraf.apall.setParam('maxsep',100000.) # Maximum separation between spectra
iraf.apall.setParam('order','increasing') # Order of apertures
# RECENTERING PARAMETERS
iraf.apall.setParam('aprecen','') # Apertures for recentering calculation
iraf.apall.setParam('npeaks','INDEF') # Select brightest peaks
iraf.apall.setParam('shift','yes') # Use average shift instead of recentering?
# RESIZING PARAMETERS
iraf.apall.setParam('llimit','INDEF') # Lower aperture limit relative to center
iraf.apall.setParam('ulimit','INDEF') # Upper aperture limit relative to center
iraf.apall.setParam('ylevel',0.2) # Fraction of peak or intensity for automatic widt
iraf.apall.setParam('peak','yes') # Is ylevel a fraction of the peak?
iraf.apall.setParam('bkg','yes') # Subtract background in automatic width?
iraf.apall.setParam('r_grow',0.) # Grow limits by this factor
iraf.apall.setParam('avglimi','no') # Average limits over all apertures?
# TRACING PARAMETERS
iraf.apall.setParam('t_nsum',20) # Number of dispersion lines to sum
iraf.apall.setParam('t_step', 20) # Tracing step
iraf.apall.setParam('t_nlost',3) # Number of consecutive times profile is lost befo
iraf.apall.setParam('t_funct','spline3') # Trace fitting function
iraf.apall.setParam('t_order',2) # Trace fitting function order
iraf.apall.setParam('t_sampl','*') # Trace sample regions
iraf.apall.setParam('t_naver',1) # Trace average or median
iraf.apall.setParam('t_niter',2) # Trace rejection iterations
iraf.apall.setParam('t_low_r',3.) # Trace lower rejection sigma
iraf.apall.setParam('t_high_',3.) # Trace upper rejection sigma
iraf.apall.setParam('t_grow',0.) # Trace rejection growing radius
# EXTRACTION PARAMETERS
iraf.apall.setParam('backgro','none') # Background to subtract
iraf.apall.setParam('skybox',1) # Box car smoothing length for sky
iraf.apall.setParam('weights','variance') # Extraction weights (none|variance)
iraf.apall.setParam('pfit','fit1d') # Profile fitting type (fit1d|fit2d)
iraf.apall.setParam('clean','yes') # Detect and replace bad pixels?
iraf.apall.setParam('saturat',300000.) # Saturation level
# iraf.apall.setParam('readnoi',17.0)
iraf.apall.setParam('readnoi',17./np.sqrt(nframes)) # Read out noise sigma (photons)
iraf.apall.setParam('gain',4.) # Photon gain (photons/data number)
iraf.apall.setParam('lsigma',4.) # Lower rejection threshold
iraf.apall.setParam('usigma',4.) # Upper rejection threshold
iraf.apall.setParam('nsubaps',1) # Number of subapertures per aperture
iraf.apall.setParam('mode','q') # h = hidden, q = query, l = learn
iraf.apall()
# Now extract arc through same aperture for wavelength calibration
print '\n' '\n' '\n'
print 'Extracting Arc through same aperture...'
if os.path.exists( os.path.join(targetdir,'aimcomb.fits')):
os.remove( os.path.join(targetdir, 'aimcomb.fits') )
print 'Removing file ' + os.path.join(targetdir, 'aimcomb.fits')
arcspec = os.path.join(basedir,arcspec)
iraf.apall.setParam('input', arcspec)
iraf.apall.setParam('output', 'aimcomb')
iraf.apall.setParam('references', 'imcomb.fit' )
iraf.apall.setParam('recenter','no')
iraf.apall.setParam('trace','no')
iraf.apall.setParam('background','no')
iraf.apall.setParam('interactive','no')
iraf.apall()
if os.path.exists( os.path.join(targetdir, 'imcomb+bkgd.ms.fits') ):
os.remove( os.path.join( targetdir, 'imcomb+bkgd.ms.fits') )
print 'Removing file ' + os.path.join( targetdir, 'imcomb+bkgd.ms.fits')
iraf.apall.setParam('input','imcomb+bkgd.fit') # List of input images
iraf.apall.setParam('output','') # List of output spectra
iraf.apall.setParam('referen','imcomb.fit') # List of aperture reference images
iraf.apall.setParam('interac','yes') # Run task interactively?
iraf.apall.setParam('find','yes') # Find apertures?
iraf.apall.setParam('recenter','no') # Recenter apertures?
iraf.apall.setParam('resize','no') # Resize apertures?
iraf.apall.setParam('edit','yes') # Edit apertures?
iraf.apall.setParam('trace','no') # Trace apertures?
iraf.apall.setParam('fittrac',interactive) # Fit the traced points interactively?
iraf.apall.setParam('extract','yes') # Extract spectra?
iraf.apall.setParam('extras','yes') # Extract sky, sigma, etc.?
iraf.apall.setParam('review','yes') # Review extractions?
# DEFAULT BACKGROUND PARAMETERS
# Background is now a constant at each wavelength
iraf.apall.setParam('b_funct','chebyshev') # Background function
iraf.apall.setParam('b_order',1) # Background function order
iraf.apall.setParam('b_sampl','-10:-6,6:10') # Background sample regions
iraf.apall.setParam('b_naver',-3) # Background average or median
iraf.apall.setParam('b_niter',2) # Background rejection iterations
iraf.apall.setParam('b_low_r',3.) # Background lower rejection sigma
iraf.apall.setParam('b_high_',3.) # Background upper rejection sigma
iraf.apall.setParam('b_grow',0.) # Background rejection growing radius
# EXTRACTION PARAMETERS
# before i wasn't dividing by the square root of the frames, but surely this must be true if I'm taking the average
iraf.apall.setParam('backgro','median') # Background to subtract
iraf.apall.setParam('skybox',1) # Box car smoothing length for sky
iraf.apall.setParam('weights','variance') # Extraction weights (none|variance)
iraf.apall.setParam('pfit','fit1d') # Profile fitting type (fit1d|fit2d)
iraf.apall.setParam('clean','yes') # Detect and replace bad pixels?
iraf.apall.setParam('saturat',300000.) # Saturation level
# iraf.apall.setParam('readnoi',17.0)
iraf.apall.setParam('readnoi',17.0/np.sqrt(nframes)) # Read out noise sigma (photons)
iraf.apall.setParam('gain',4.) # Photon gain (photons/data number)
iraf.apall.setParam('lsigma',4.) # Lower rejection threshold
iraf.apall.setParam('usigma',4.) # Upper rejection threshold
iraf.apall.setParam('nsubaps',1) # Number of subapertures per aperture
iraf.apall()
hdulist = fits.open(os.path.join(targetdir, 'imcomb+bkgd.ms.fits'))
sigma = hdulist[0].data[3,0,:]
hdulist.close()
hdulist = fits.open(os.path.join(targetdir, 'imcomb.ms.fits'), mode='update')
hdulist[0].data[2,0,:] = sigma
hdulist.flush()
hdulist.close()
return None
iraf.apall(
input = "t"+im_slice+"_"+file_name, \
output = "",\
format = "multispec",\
references = "",\
apertures = 1,\
interactive = interact, \
find = 1,\
recenter = 1,\
resize = 1,\
edit =interact,\
trace = 1,\
fittrace = 0,\
extract = 1,\
extras = 1,\
review = 0,\
line = "INDEF",\
nsum = -100,\
lower = -10.0,\
upper = 10.0,\
b_sample = "-15:-8,8:15",\
b_order = 1,\
b_naverage = -5,\
b_niterate = 0,\
b_low_reject = 3.0,\
b_high_reject = 3.0,\
b_grow = 0.0,\
width = 5.0,\
radius = 10.0,\
threshold = 0.0,\
nfind = 1,\
ylevel = 0.3,\
peak = 1,\
bkg = 1,\
t_nsum = 40,\
t_step = 5,\
t_nlost = 3,\
t_function = "spline3",\
t_order = 5,\
t_naverage = -10,\
t_niterate = 5,\
t_low_reject = 2.0,\
t_high_reject = 2.0,\
t_grow = 0.0,\
background = "fit",\
weights = "variance",\
clean = 1,\
lsigma = 20.0,\
usigma = 3.0,\
saturation = 64000,\
readnoise = 3.8,\
gain = 0.9)
def combine(doreduce=True, doshifts=True):
if doreduce:
ims = glob('oc01020[1-4]*_raw.fits')
ims += glob('oc016*_raw.fits')
# for each image
for im in ims:
print im
stistools.basic2d.basic2d(im, im.replace('raw', 'flt'))
im = im.replace('raw', 'flt')
print im
# Update the target position at 0.0
for i in range(4):
pyfits.setval(im, 'POSTARG2', value=0.0, ext=i)
# reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im, 'APERTAB', value='oref$y2r1559to_apt.fits')
# Reset the wcs to have CRPIX2 along the trace
# Run x2d on the flt frame
stistools.x2d.x2d(input=im, output=im.replace('flt', 'x2d'))
h = pyfits.open(im.replace('flt', 'x2d'), mode='update')
# Replace all of the bad pixels in the image by -10000 based on the DQ array
# save them to a new file
# Throw away bad reference file pixels and saturated pixels. None of the other error codes
# were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[1].data[badpix] = -10000
h.flush()
# Trim the image
for i in range(1, 4):
h[i].data = h[i].data[100:-100, 120:-100]
h[i].header['CRPIX1'] -= 120
h[i].header['CRPIX2'] -= 100
h.close()
ims = glob('oc01020[1-4]*_x2d.fits')
ims += glob('oc01610[1-2]*_x2d.fits')
if doshifts:
init_guesses = [501, 542, 522, 523, 541, 524]
centroids = []
for i, im in enumerate(ims):
print(im)
h = pyfits.open(im)
d = average(h[1].data[:, 915:925], axis=1)
popt, _pcov = curve_fit(gauss,
arange(len(d)),
d,
p0=[10, init_guesses[i], 1.5, 0])
centroids.append(popt[1])
shift = centroids[0] - popt[1]
from matplotlib import pyplot as pl
pl.ion()
pl.clf()
pl.plot(arange(len(d)), d)
pl.plot(arange(len(d)),
gauss(arange(len(d)), popt[0], popt[1], popt[2], popt[3]))
_w = raw_input('Press return to continue')
# watch the sign convention
# This gives what you need to shift the input image by to get to the reference image
iraf.unlearn(iraf.imshift)
iraf.imshift(im + '[1]',
im[:-8] + 'shift1.fits',
0.0,
shift,
interp_type='drizzle')
# Run imcombine on the rectified (but not flux scaled) images with crreject
iraf.unlearn(iraf.imcombine)
imlist = ''
for im in ims:
imlist += im[:-8] + 'shift1.fits,'
# drop the last comma
imlist = imlist[:-1]
iraf.imcombine(input=imlist,
output='13dh_uv_com.fits',
reject='none',
lthreshold=-20,
hthreshold=300)
# run apall on the combined flux scaled image
iraf.unlearn(iraf.apall)
iraf.apall(input='13dh_uv_com.fits',
output='13dh_uv',
review='no',
line=1024,
nsum=-50,
b_order=2,
b_function='legendre',
b_niterate=30,
b_naverage=-21,
nfind=1,
t_order=2,
background='fit',
weights='variance')
def process(root,dir='./',verbose=True) :
# setup
imred.setup(root,dir=dir,idet=16)
# create bias frames
biases = imred.getfiles('zero',verbose=verbose)
bias = imred.combine(biases,verbose=verbose)
# create flat field
flats = imred.getfiles('flat',filter='Open',verbose=verbose)
flat = imred.combine(flats,bias=bias,trim=True,verbose=verbose)
flats = imred.getfiles('flat',filter='Blue',verbose=verbose)
flat += imred.combine(flats,bias=bias,trim=True,verbose=verbose)
hdu=fits.PrimaryHDU(flat)
hdu.writeto('flat.fits',clobber=True)
# create normalized 1D flat field
for file in ['flat_mag.fits','flat_mag.ec.fits','flat.ec.fits','norm_flat.ec.fits'] :
if os.path.isfile(file) : os.remove(file)
iraf.magnify(input='flat.fits',output='flat_mag.fits',xmag=1,ymag=4)
iraf.hedit('flat_mag.fits',fields='CCDSEC',value='[200:1850,1:8189]',
add=no,addonly=no,delete=no,verify=no,show=yes,update=yes)
iraf.hedit('flat_mag.fits','dispaxis','1',add=yes,verify=no,show=yes,update=yes)
if verbose: print( 'Modeling and extracting the superflat...')
iraf.apall(input='flat_mag.fits',ref='echtrace130522',format='echelle',
interactive=no,find=no,recenter=yes,resize=yes,edit=no,trace=yes,
fittrace=no,extract=yes,extras=no,review=no,line=825,nsum=10,
lower=-14.0,upper=14.0,b_function='chebyshev',b_order=2,
b_niterate=3,b_naverage=-3,b_sample='-22:-15,15:22',width=18.0,
radius=18.0,npeaks=INDEF,shift=no,ylevel=.05,t_nsum=5,t_step=1,
t_function='legendre',t_order=5,t_naverage=3,t_niterate=3,
t_low_reject=2.5,t_high_reject=2.5,t_nlost=3,t_sample='*',
background='fit',weights='none',clean=no,saturation=40000.0)
flat=fits.open('flat_mag.ec.fits')[0].data
specflat = imred.specflat(flat,indiv=True)
hdu=fits.PrimaryHDU(specflat)
hdu.writeto('normflat.ec.fits',clobber=True)
#iraf.sfit (input="flat_mag.ec.fits",output="norm_flat_mag.ec.fits",type="ratio",
# replace=no,wavesca=no,logscal=no,override=yes,interac=no,sample="*",
# naverag=1,funct="spline3",order=5,low_rej=2,high_rej=0,niterate=10,grow=1)
pdb.set_trace()
# arcs
arcs = imred.getfiles('comp',verbose=verbose,listfile='arcs.lis')
for arc in arcs :
if verbose: print('arc: ', arc)
data=imred.reduce(arc,bias=bias,trim=True)
arcfile='arc.{:04d}.fits'.format(arc)
outfile='arc.{:04d}.ec.fits'.format(arc)
data.writeto(arcfile,clobber=True)
if verbose: print('Resampling the arc by a factor of 4 in the y direction...')
iraf.magnify(input=arcfile,output=arcfile,xmag=1,ymag=4)
iraf.hedit(images=arcfile,fields='CCDSEC',value='[200:1850,1:8189]',add=no,
addonly=no,delete=no,verify=no,show=yes,update=yes)
iraf.hedit(arcfile,'dispaxis','1',add=yes,verify=no,show=yes,update=yes)
if verbose: print('Applying model apertures to the arc and extracting spectra...')
if os.path.isfile(outfile) : os.remove(outfile)
iraf.apall(input=arcfile,output=outfile,reference='flat_mag',format='echelle',
interactive=no,find=no,recenter=no,resize=no,edit=no,
trace=no,fittrace=no,extract=yes,extras=no,review=no,
shift=no,background='none',weights='none')
iraf.ecreidentify(outfile,reference='arcnewref.ec',
shift=INDEF,cradius=2,threshold=45,refit=yes)
pdb.set_trace()
# loop over objects
objs = imred.getfiles('object',verbose=verbose)
for obj in objs :
data=imred.reduce(obj,bias=bias,flat=specflat,trim=True)
pdb.set_trace()
def extract(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('fix/*fix*.fits')
if len(fs) == 0:
print "WARNING: No fixpixed images available for extraction."
iraf.cd('..')
return
if not os.path.exists('x1d'):
os.mkdir('x1d')
print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line."
for f in fs:
# Get the output filename without the ".fits"
outbase = f.replace('fix', 'x1d')[:-5]
# Get the readnoise, right now assume default value of 5 but we could
# get this from the header
readnoise = 5
# If interactive open the rectified, background subtracted image in ds9
ds9display(f.replace('fix', 'bkg'))
# set dispaxis = 1 just in case
pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apall)
iraf.flpr()
iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes',
review='no', line='INDEF', nsum=-1000, lower=-5, upper=5,
b_function='legendre', b_order=5,
b_sample='-200:-100,100:200', b_naverage=-10, b_niterate=5,
b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15,
t_step=15, t_nlost=200, t_function='legendre', t_order=5,
t_niterate=5, t_low_reject=3.0, t_high_reject=3.0,
background='fit', weights='variance', pfit='fit2d',
clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0,
usigma=4.0, mode='hl')
# Copy the CCDSUM keyword into the 1d extraction
pyfits.setval(outbase + '.fits', 'CCDSUM',
value=pyfits.getval(f, 'CCDSUM'))
# Extract the corresponding arc
arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0]
# set dispaxis = 1 just in case
pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apsum)
iraf.flpr()
iraf.apsum(input=arcname + '[SCI]', output='auxext_arc',
references=f[:-5] + '[SCI]', interactive='no', find='no',
edit='no', trace='no', fittrace='no', extras='no',
review='no', background='no', mode='hl')
# copy the arc into the 5 column of the data cube
arcfs = glob('auxext_arc*.fits')
for af in arcfs:
archdu = pyfits.open(af)
scihdu = pyfits.open(outbase + '.fits', mode='update')
d = scihdu[0].data.copy()
scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2]))
scihdu[0].data[:-1, :, :] = d[:, :, :]
scihdu[0].data[-1::, :] = archdu[0].data.copy()
scihdu.flush()
scihdu.close()
archdu.close()
os.remove(af)
# Add the airmass, exptime, and other keywords back into the
# extracted spectrum header
kws = ['AIRMASS','EXPTIME',
'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG',
'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX',
'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS',
'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA',
'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL',
'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING',
'FILTER']
for kw in kws:
pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw))
iraf.cd('..')
def processEachTarg(i, input_list, output_list, apall_kws):
ir.apall(input_list[i], output=output_list[i], **apall_kws)
if saveCorrectedImg == False:
ir.imdelete(input_list[i])
#Reduce with apall
for imname in objects_pars.keys():
print(imname, imname + 'ms')
templlist = list(objects_pars[imname])
templlist.append(imname + 'ms')
objects_pars[imname] = templlist
for imname in objects_pars.keys():
iraf.apall(input=imname,
out=imname + 'ms',
nfind=1,
interactive=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
t_nsum=50,
t_nlost=50,
extract=yes,
extras=yes,
review=yes,
background='fit')
l.write('extracted=yes')
for imname in objects_pars.keys():
if 'ztfstar' in imname:
print(objects_pars[imname][4])
iraf.standard(input=objects_pars[imname][4],
output='std_' + imname,
def apall_std(stdlist):
irf_prm.set_apall_std(iraf.apall)
for std in stdlist:
iraf.apall(input=std,output = 's'+std,nfind=0)#,interactive='no')
return
masterbias.shape[0] * .97)
iraf.apall(masterfn,
output=masterfn.replace('.fits', '_spec.fits'),
format='onedspec',
recenter='YES',
resize='YES',
extras='yes',
nfind=1,
nsubaps=1,
minsep=100,
weights='variance',
bkg='yes',
b_function='chebyshev',
b_order=1,
b_naverage=-3,
b_niterate=2,
t_order=7,
t_niterate=4,
t_naverage=t_naverage,
background='fit',
clean='yes',
interactive=True,
review=False,
b_sample='-30:-10,10:30',
trace='YES',
edit='YES',
fittrace='YES',
extract='YES',
find='YES',
t_sample=horizsamp)
iraf.apall(input='flat_fid_mag.fits',
ref='echtrace130522',
format='echelle',
interactive=no,
find=no,
recenter=yes,
resize=yes,
edit=no,
trace=yes,
fittrace=no,
extract=yes,
extras=no,
review=no,
line=825,
nsum=10,
lower=-14.0,
upper=14.0,
b_function='chebyshev',
b_order=2,
b_niterate=3,
b_naverage=-3,
b_sample='-22:-15,15:22',
width=18.0,
radius=18.0,
npeaks=INDEF,
shift=no,
ylevel=.05,
t_nsum=5,
t_step=1,
t_function='legendre',
t_order=5,
t_naverage=3,
t_niterate=3,
t_low_reject=2.5,
t_high_reject=2.5,
t_nlost=3,
t_sample='*',
background='fit',
weights='variance',
clean=no,
saturation=INDEF,
gain='GAIN',
readnoise='RDNOISE',
pfit='fit2d',
apertures='6-96')
if doextract:
# Construct a MASTER FRAME, and trace it::
if os.path.isfile(_proc + 'masterframe_override.fits'):
masterfn = _proc + 'masterframe_override.fits'
else:
sci_fns = []
for d in datasets: sci_fns+= d[0]
for ii in range(len(sci_fns)): sci_fns[ii] = sci_fns[ii].replace('_spec.fits', '.fits')
sumframe = np.zeros(pyfits.getdata(sci_fns[0]).shape, dtype=float)
for fn in sci_fns: sumframe += pyfits.getdata(fn)
masterfn = _proc + 'masterframe.fits'
pyfits.writeto(masterfn, sumframe, clobber=clobber)
horizsamp='%i:%i' % (masterbias.shape[0]*.02, masterbias.shape[0]*.97)
iraf.apall(masterfn, output=masterfn.replace('.fits', '_spec.fits'), format='onedspec', recenter='YES',resize='YES',extras='yes', nfind=1, nsubaps=1, minsep=100, weights='variance', bkg='yes', b_function='chebyshev', b_order=1, b_naverage=-3, b_niterate=2, t_order=7, t_niterate=4, t_naverage=t_naverage, background='fit', clean='yes', interactive=True, review=False, b_sample='-30:-10,10:30', trace='YES',edit='YES',fittrace='YES',extract='YES', find='YES', t_sample=horizsamp)
# Use the master-trace aperture to extract all spectra (even faint ones!)
if clobber:
for fn in redfns:
fn2 = fn.replace('spec.fits', 'spec.0001.fits')
if os.path.isfile(fn2): os.remove(fn2)
iraf.apall('@tempjunkcal', output='@tempjunkred', format='onedspec', recenter='NO',resize='NO',extras='yes', nfind=1, nsubaps=1, minsep=100, weights='variance', bkg='yes', b_function='chebyshev', b_order=1, b_naverage=-3, b_niterate=2, t_order=3, t_niterate=4, t_naverage=t_naverage, background='fit', clean='yes', interactive=True, review=False, b_sample='-30:-10,10:30', trace='NO',edit='NO',fittrace='NO',extract='YES', find='NO', references='last', t_sample=horizsamp)
#Clobber existing output image /Users/ianc/proj/mdwarfs/data/proc/20150821/EFOSC.2015-08-22T03:07:43.730_spec.0001? ('no'): y
os.chdir(dir0)
print "... and we're done!"
print "Generating plots..."
def lris_extract(science, standards, dostandard=yes):
'''Extract and flux calibrate LRIS spectra (assuming they have been
processed into 2D images using LRIS_pipe above).'''
if dostandard:
lris_standard(standards)
for src in science:
bimages = iraffiles("%s_??_B.fits" % src)
rimages = iraffiles("%s_??_R.fits" % src)
joinstr = ""
for bimage in bimages:
bimage = bimage.split(".")[0]
# Find appropriate standard for this image
bstd = get_head("%s.fits" % bimage, "STDNAME")
# Extract 1D spectra
if get_head("%s.fits" % bimage, "SKYSUB"):
iraf.apall(bimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="none",
reference=bstd)
else:
iraf.apall(bimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="fit",
reference=bstd)
# Normalize by the standard continuua
iraf.sarith("%s.ms" % bimage,
"/",
"../standards/s%s.ms.fits" % bstd,
"s%s.ms" % bimage,
w1=INDEF,
w2=INDEF)
# Telluric correct
iraf.telluric("s%s.ms" % bimage,
"ts%s.ms" % bimage,
"../standards/telluric.%s.fits" % bstd,
tweakrms=yes,
interactive=yes,
sample='6850:6950,7575:7700,8750:9925')
# Flux calibration
iraf.calibrate("ts%s.ms" % bimage,
"fts%s.ms" % bimage,
extinct=yes,
flux=yes,
extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck",
sens="../standards/%s.sens" % bstd,
airmass='',
exptime='')
joinstr += "fts%s.ms," % bimage
for rimage in rimages:
rimage = rimage.split(".")[0]
# Find appropriate standard for this image
rstd = get_head("%s.fits" % rimage, "STDNAME")
# Extract 1D spectra
if get_head("%s.fits" % rimage, "SKYSUB"):
iraf.apall(rimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="none",
reference=rstd)
else:
iraf.apall(rimage,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="fit",
reference=rstd)
# Normalize by the standard continuua
iraf.sarith("%s.ms" % rimage,
"/",
"../standards/s%s.ms.fits" % rstd,
"s%s.ms" % rimage,
w1=5500,
w2=INDEF)
# Telluric correct
iraf.telluric("s%s.ms" % rimage,
"ts%s.ms" % rimage,
"../standards/telluric.%s.fits" % rstd,
tweakrms=yes,
interactive=yes,
sample='6850:6950,7575:7700')
# Flux calibration
iraf.calibrate("ts%s.ms" % rimage,
"fts%s.ms" % rimage,
extinct=yes,
flux=yes,
extinction="home$extinct/maunakeaextinct.dat",
observatory="Keck",
sens="../standards/%s.sens" % rstd,
airmass='',
exptime='')
joinstr += "fts%s.ms," % rimage
# Combine
iraf.scombine(joinstr[:-1],
"%s.ms.fits" % src,
combine="average",
reject="avsigclip",
w1=INDEF,
w2=INDEF,
dw=INDEF,
nw=INDEF,
scale="median",
zero="none",
weight="none",
sample="5450:5600",
lsigma=3.0,
hsigma=3.0,
gain=RCCDGAIN,
rdnoise=RCCDRNOISE)
# Plot to enable final tweaks
iraf.splot("%s.ms.fits" % src)
return
with open('filenames.pickle') as f: # Python 3: open(..., 'rb')
dataset, target, filename,extracted_filename, calibrated_filename, crval, dispersion = pickle.load(f)
# Delete some directories/files from previous runs:
os.system("rm -rf login.cl database pyraf uparm")
# Delete previous results.
os.system("rm "+extracted_filename+" "+calibrated_filename)
# Run the spectral extraction program.
iraf.apextract.setParam("dispaxis", "1")
iraf.apall(input=filename, find="No", recenter="No", resize="No",interactive="Yes")
# Make sure that the dispersion axis is in the header.
iraf.hedit(images=[filename], fields=["DISPAXIS"], value=["1"], add="Yes")
iraf.identify(filename[:-5], coordli=home+"/Downloads/HgNe(1).dat",
section="line 105 125",
crval=crval,
cdelt=dispersion,
fwidth=5)
# Tell the extracted spectrum what the wavelength solutions are.
iraf.hedit(images=[extracted_filename],
def lris_standard(standards, xcorr=yes):
'''Extract standard stars and calculate sensitivity functions.'''
for ofile, nstd in standards:
shutil.copy(ofile, "%s.fits" % nstd)
# Extract standard
if get_head("%s.fits" % nstd, "SKYSUB"):
iraf.apall(nstd,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="none")
else:
iraf.apall(nstd,
output="",
inter=yes,
find=yes,
recenter=yes,
resize=yes,
edit=yes,
trace=yes,
fittrace=yes,
extract=yes,
extras=yes,
review=no,
background="fit")
if xcorr:
# Cross correlate to tweak wavelength
iraf.xcsao("%s.ms" % nstd,
templates=XCTEMPLATE,
correlate="wavelength",
logfiles="%s.xcsao" % ofile)
# If a solution was found, apply shift
try:
xcsaof = open("%s.xcsao" % ofile)
shift = float(xcsaof.readlines()[6].split()[14])
except:
shift = 0.0
iraf.specshift("%s.ms" % nstd, -shift)
# Create telluric
iraf.splot("%s.ms" %
nstd) # Remove telluric and absorption, save as a%s.ms
iraf.sarith("%s.ms" % nstd, "/", "a%s.ms" % nstd,
"telluric.%s.fits" % nstd)
iraf.imreplace("telluric.%s.fits" % nstd, 1.0, lower=0.0, upper=0.0)
iraf.splot("telluric.%s.fits" %
nstd) # Remove stellar features and resave
# Create smoothed standard
iraf.gauss("a%s.ms[*,1,1]" % nstd, "s%s.ms" % nstd, 5.0)
iraf.sarith("%s.ms" % nstd, "/", "s%s.ms" % nstd, "ds%s.ms" % nstd)
# Apply telluric correction
iraf.telluric(
"ds%s.ms" % nstd,
"tds%s.ms" % nstd,
"telluric.%s.fits" % nstd,
xcorr=no,
tweakrms=no,
interactive=no,
sample='4000:4010,6850:6975,7150:7350,7575:7725,8050:8400,8900:9725'
)
# Define bandpasses for standard star calculation
obj = get_head("%s.fits" % nstd, "OBJECT")
iraf.standard("tds%s.ms" % nstd,
"%s.std" % nstd,
extinction='home$extinct/maunakeaextinct.dat',
caldir='home$standards/',
observatory='Keck',
interac=yes,
star_name=obj,
airmass='',
exptime='')
# Determine sensitivity function
iraf.sensfunc('%s.std' % nstd,
'%s.sens' % nstd,
extinction='home$extinct/maunakeaextinct.dat',
newextinction='extinct.dat',
observatory='Keck',
function='legendre',
order=4,
interactive=yes)
return
extracted_filename = 'vega_9.3narrow.ms.fits'
calibrated_filename = 'vega_9.3narrow.calib.fits'
# Delete previous results.
os.system("rm "+extracted_filename+" "+calibrated_filename)
# Make sure that the dispersion axis is in the header.
iraf.hedit(images=[filename], fields=["DISPAXIS"], value=["1"], add="Yes")
# Run the spectral extraction program.
iraf.apextract.setParam("dispaxis", "1")
iraf.apall(input=filename, find="No", recenter="No", resize="No")
# Now identify the spectral lines in the arc lamps. Need to replace l1 l2
# with the range of rows that have the spectral lines.
iraf.identify(filename, section="line 265 285")
# Tell the extracted spectrum what the wavelength solutions are.
iraf.hedit(images=[extracted_filename], fields=["REFSPEC1"], \
value=[filename], add="Yes")
iraf.dispcor(input=extracted_filename, output=calibrated_filename)
# Plot the extracted spectrum?
def combine(doreduce=True, doshifts=True):
if doreduce:
ims = glob('oc01020[1-4]*_raw.fits')
ims += glob('oc016*_raw.fits')
# for each image
for im in ims:
print im
stistools.basic2d.basic2d(im, im.replace('raw','flt'))
im = im.replace('raw', 'flt')
print im
# Update the target position at 0.0
for i in range(4):
pyfits.setval(im, 'POSTARG2', value=0.0, ext=i)
# reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im, 'APERTAB', value='oref$y2r1559to_apt.fits')
# Reset the wcs to have CRPIX2 along the trace
# Run x2d on the flt frame
stistools.x2d.x2d(input=im, output=im.replace('flt','x2d') )
h = pyfits.open(im.replace('flt','x2d'), mode='update')
# Replace all of the bad pixels in the image by -10000 based on the DQ array
# save them to a new file
# Throw away bad reference file pixels and saturated pixels. None of the other error codes
# were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[1].data[badpix] = -10000
h.flush()
# Trim the image
for i in range(1,4):
h[i].data = h[i].data[100:-100, 120:-100]
h[i].header['CRPIX1'] -= 120
h[i].header['CRPIX2'] -= 100
h.close()
ims = glob('oc01020[1-4]*_x2d.fits')
ims += glob('oc01610[1-2]*_x2d.fits')
if doshifts:
init_guesses = [501, 542, 522, 523, 541, 524]
centroids = []
for i, im in enumerate(ims):
print(im)
h = pyfits.open(im)
d = average(h[1].data[:, 915:925], axis=1)
popt, _pcov = curve_fit(gauss, arange(len(d)), d, p0=[10, init_guesses[i], 1.5, 0])
centroids.append(popt[1])
shift = centroids[0] - popt[1]
from matplotlib import pyplot as pl
pl.ion()
pl.clf()
pl.plot(arange(len(d)), d)
pl.plot(arange(len(d)), gauss(arange(len(d)), popt[0], popt[1], popt[2], popt[3]))
_w = raw_input('Press return to continue')
# watch the sign convention
# This gives what you need to shift the input image by to get to the reference image
iraf.unlearn(iraf.imshift)
iraf.imshift(im + '[1]', im[:-8] + 'shift1.fits', 0.0, shift,
interp_type='drizzle')
# Run imcombine on the rectified (but not flux scaled) images with crreject
iraf.unlearn(iraf.imcombine)
imlist = ''
for im in ims:
imlist += im[:-8] + 'shift1.fits,'
# drop the last comma
imlist = imlist[:-1]
iraf.imcombine(input=imlist, output='13dh_uv_com.fits', reject='none',
lthreshold=-20, hthreshold=300)
# run apall on the combined flux scaled image
iraf.unlearn(iraf.apall)
iraf.apall(input='13dh_uv_com.fits', output='13dh_uv', review='no',
line=1024, nsum=-50, b_order=2, b_function='legendre',
b_niterate=30, b_naverage=-21, nfind=1, t_order=2,
background='fit', weights='variance')
