def normalize_quartzes(quartz_list):
'''Normalizes the files in quartz_list using noao>twodspec>longslit>response'''
#This needs more intelligent Failure Handling; current version is alpha.
irf_prm.set_response(iraf.response)
for quartz in quartz_list:
iraf.response(calibrat=quartz,normaliz=quartz,response='n'+quartz)
return
def normalise_flats(flatdir):
"""
Normalise flats
"""
print 'In directory ' + flatdir
print 'Normalising combinined flats...'
if os.path.exists( os.path.join(flatdir,'nFlat.fits') ):
os.remove(os.path.join(flatdir,'nFlat.fits') )
print 'Removing file ' + os.path.join(flatdir,'nFlat.fits')
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.response.setParam('calibration', os.path.join(flatdir,'Flat.fits'))
iraf.response.setParam('normalization', os.path.join( flatdir, 'Flat.fits' ))
iraf.response.setParam('response', os.path.join( flatdir, 'nFlat') )
iraf.response.setParam('low_reject', 3.)
iraf.response.setParam('high_reject', 3.)
iraf.response.setParam('order',40)
iraf.response()
return None
def combine_flat(lstfile):
if os.path.isfile('Halogen.fits'):
print 'remove Halogen.fits'
os.remove('Halogen.fits')
if os.path.isfile('Resp.fits'):
print 'remove Resp.fits'
os.remove('Resp.fits')
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.flatcombine(input='tbo//@' + lstfile,
output='Halogen',
combine='average',
reject='crreject',
ccdtype='',
process=False,
subsets=False,
delete=False,
clobber=False,
scale='mode',
statsec='',
nlow=1,
nhigh=1,
nkeep=1,
mclip=True,
lsigma=3.0,
hsigma=3.0,
rdnoise='rdnoise',
gain='gain',
snoise=0.0,
pclip=-0.5,
blank=1.0)
iraf.twodspec()
iraf.longslit(dispaxis=2,
nsum=1,
observatory='Lijiang',
extinction=func.config_path + os.sep + 'LJextinct.dat',
caldir=func.std_path + os.sep,
interp='poly5')
iraf.response(calibration='Halogen',
normalization='Halogen',
response='Resp',
interactive=True,
threshold='INDEF',
sample='*',
naverage=1,
function='spline3',
order=25,
low_reject=10.0,
high_reject=10.0,
niterate=1,
grow=0.0,
graphics='stdgraph',
cursor='')
def combine_flat(filename):
outname = filename.replace('.lst','.fits')
print 'run function flatcombine...'
print 'make file', outname
iraf.flatcombine(input = 'tbo//@' + filename
, output = outname, combine = 'average', reject = 'avsigclip'
, ccdtype = '', process = False, subsets = True
, delete = False, clobber = False, scale = 'mode'
, statsec = '', nlow = 1, nhigh = 1, nkeep = 1
, mclip = True, lsigma = 3.0, hsigma = 3.0
, rdnoise = 9.4, gain = 0.35, snoise = 0.0
, pclip = -0.5, blank = 1.0)
iraf.noao()
iraf.twodspec()
iraf.longslit()
print 'run function response...'
print 'make file', 're' + outname
iraf.response(calibration = outname
, normalization = outname, response = 're' + outname
, interactive = True, threshold = 'INDEF', sample = '*'
, naverage = 1, function = 'spline3', order = 7
, low_reject = 0.0, high_reject = 0.0, niterate = 1
, grow = 0.0, graphics = 'stdgraph', cursor = '')
print 'run function illumination...'
print 'make file', 'il' + outname
iraf.illumination(images = 're' + outname
, illuminations = 'il' + outname, interactive = False
, bins = '', nbins = 5, sample = '*', naverage = 1
, function = 'spline3', order = 1, low_reject = 0.0
, high_reject = 0.0, niterate = 1, grow = 0.0
, interpolator = 'poly3', graphics = 'stdgraph', cursor = '')
print 'run function imarith...'
print 'make file', 'per' + outname
iraf.imarith(operand1 = 're' + outname
, op = '/', operand2 = 'il' + outname, result = 'per' + outname
, title = '', divzero = 0.0, hparams = '', pixtype = ''
, calctype = '', verbose = True, noact = False)
return outname, 're' + outname, 'il' + outname, 'per' + outname
def gen_Resp_2016():
iraf.twodspec()
iraf.longslit(dispaxis=2,
nsum=1,
observatory='observatory',
extinction=func.extinction_file,
caldir=func.std_path + os.sep,
interp='poly5')
iraf.response(calibration='Halogen',
normalization='Halogen',
response='Resp',
interactive=True,
threshold='INDEF',
sample='*',
naverage=1,
function='spline3',
order=45,
low_reject=10.0,
high_reject=10.0,
niterate=1,
grow=0.0,
graphics='stdgraph',
cursor='')
def combine_flat(lstfile):
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.flatcombine(input = 'tbo//@' + lstfile
, output = 'Halogen', combine = 'average', reject = 'crreject'
, ccdtype = '', process = False, subsets = False
, delete = False, clobber = False, scale = 'mode'
, statsec = '', nlow = 1, nhigh = 1, nkeep = 1
, mclip = True, lsigma = 3.0, hsigma = 3.0
, rdnoise = 'rdnoise', gain = 'gain', snoise = 0.0
, pclip = -0.5, blank = 1.0)
script_path = os.path.split(os.path.realpath(__file__))[0]
iraf.twodspec()
iraf.longslit(dispaxis = 2, nsum = 1, observatory = 'observatory'
, extinction = script_path + os.sep + 'LJextinct.dat'
, caldir = script_path + os.sep + 'standarddir' + os.sep, interp = 'poly5')
iraf.response(calibration = 'Halogen'
, normalization = 'Halogen', response = 'Resp'
, interactive = True, threshold = 'INDEF', sample = '*'
, naverage = 1, function = 'spline3', order = 25
, low_reject = 10.0, high_reject = 10.0, niterate = 1
, grow = 0.0, graphics = 'stdgraph', cursor = '')
flat1d = np.median(flat1dtop / flat1dbot, axis=0)
# close the illumination file because we don't need it anymore
illumhdu.close()
# File stage m1d for median 1-D
flat1dfname = 'flats/flt%05.2fm1d.fits' % (ga)
tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())
# run response
# r1d = response1d
resp1dfname = 'flats/flt%05.2fr1d.fits' % (ga)
iraf.response(flat1dfname,
flat1dfname,
resp1dfname,
order=31,
interactive=False,
naverage=-5,
low_reject=3.0,
high_reject=3.0,
niterate=5,
mode='hl')
resp1dhdu = pyfits.open(resp1dfname)
resp1d = resp1dhdu[0].data.copy()
resp1dhdu.close()
# After response divide out the response function
# normalize the 1d resp to its median
resp1d /= np.median(resp1d)
# Chuck any outliers
flatsig = np.std(resp1d - 1.0)
def process():
ovrsc = '[1:4,*]'
trim = '[200:2046,50:200]'
imglist = glob.glob('FORS2*.fits')
# objdict = {}
# objdict.clear()
for img in imglist:
hdu = pyfits.getheader(img)
if 'OBJECT' in hdu:
obj = hdu['OBJECT']
if obj not in objdict:
objdict[obj] = [img]
else:
objdict[obj].append(img)
if 'STD' not in objdict:
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')
stars = {'specphot-LTT7379':'l7379','specphot-LDS749B':'lds749b','specphot-EG274':'eg274','specphot-G138-31':'g13831','specphot-C-32d9927':'cd32','specphot-LTT9491':'l9491','specphot-LTT7987':'l7987'}
i = 0
for key in objdict.keys():
if 'STD' in key:
for img in objdict[key]:
i = i + 1
numstars = len(objdict['STD'])
hds = pyfits.getheader(img)
_starname = stars[hds['HIERARCH ESO OBS NAME']]
if _starname in ['lds749b','g13831']:
print 'Bad standard, copying from 2016-05-12'
if not os.path.isdir('badstd'):
os.mkdir('badstd')
os.system('mv '+img+' ./badstd')
if i >= numstars:
objdict.pop('STD')
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')
if os.path.isfile('biaslist'):
os.remove('biaslist')
if os.path.isfile('masterbias.fits'):
os.remove('masterbias.fits')
f = open('biaslist','w')
for img in objdict['BIAS']:
f.write(img+'\n')
f.close()
imglist = '@biaslist'
name = 'masterbias.fits'
hdb = pyfits.getheader(objdict['BIAS'][0])
_gain = hdb['HIERARCH ESO DET OUT1 GAIN']
_ron = hdb['HIERARCH ESO DET OUT1 RON']
iraf.zerocombine(imglist,output=name,combine='average',reject='minmax',ccdtype='none',process='no',gain=_gain,rdnoise=_ron,Stdout=1)
if os.path.isfile('flatlist'):
os.remove('flatlist')
if os.path.isfile('sciflatlist'):
os.remove('sciflatlist')
if os.path.isfile('stdflatlist'):
os.remove('stdflatlist')
if os.path.isfile('masterflat.fits'):
os.remove('masterflat.fits')
if os.path.isfile('scimasterflat.fits'):
os.remove('scimasterflat.fits')
if os.path.isfile('stdmasterflat.fits'):
os.remove('stdmasterflat.fits')
f = open('sciflatlist','w')
for img in objdict['FLAT,LAMP']:
hdu = pyfits.getheader(img)
if hdu['HIERARCH ESO DPR TECH'] == 'SPECTRUM':
f.write(img+'\n')
f.close()
j = 0
f = open('stdflatlist','w')
for img in objdict['FLAT,LAMP']:
hdu = pyfits.getheader(img)
if hdu['HIERARCH ESO DPR TECH'] == 'MOS':
f.write(img+'\n')
j = j + 1
f.close()
imglist = '@sciflatlist'
name = 'scimasterflat.fits'
hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
_gain = hdf['HIERARCH ESO DET OUT1 GAIN']
_ron = hdf['HIERARCH ESO DET OUT1 RON']
iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)
if j == 0:
imglist = '@sciflatlist'
elif j >= 1:
imglist = '@stdflatlist'
name = 'stdmasterflat.fits'
hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
_gain = hdf['HIERARCH ESO DET OUT1 GAIN']
_ron = hdf['HIERARCH ESO DET OUT1 RON']
iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)
if os.path.isfile('tmasterbias.fits'):
os.remove('tmasterbias.fits')
fits = 'masterbias.fits'
name = 'tmasterbias.fits'
zcor = 'no'
fcor = 'no'
_zero = ''
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
if os.path.isfile('tmasterflat.fits'):
os.remove('tmasterflat.fits')
if os.path.isfile('tscimasterflat.fits'):
os.remove('tscimasterflat.fits')
if os.path.isfile('tstdmasterflat.fits'):
os.remove('tstdmasterflat.fits')
fits = 'scimasterflat.fits'
name = 'tscimasterflat.fits'
zcor = 'yes'
fcor = 'no'
_zero = 'tmasterbias.fits'
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
fits = 'stdmasterflat.fits'
name = 'tstdmasterflat.fits'
zcor = 'yes'
fcor = 'no'
_zero = 'tmasterbias.fits'
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
if os.path.isfile('ntmasterflat.fits'):
os.remove('ntmasterflat.fits')
if os.path.isfile('ntscimasterflat.fits'):
os.remove('ntscimasterflat.fits')
if os.path.isfile('ntstdmasterflat.fits'):
os.remove('ntstdmasterflat.fits')
cal = 'tscimasterflat.fits'
resp = 'ntscimasterflat.fits'
_order = 100
iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')
cal = 'tstdmasterflat.fits'
resp = 'ntstdmasterflat.fits'
_order = 100
iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')
##########
ovrsc = '[1:4,*]'
trim = '[200:2046,50:200]'
fimg = 'ntscimasterflat'
_area = '[1:150,*]'
_value = 1.0
iraf.imreplace(images=fimg+_area,value=_value)
fimg = 'ntstdmasterflat'
_area = '[1:150,*]'
_value = 1.0
iraf.imreplace(images=fimg+_area,value=_value)
alist = glob.glob('*AGN*.fits')
for f in alist:
os.remove(f)
slist = glob.glob('STD*.fits')
for f in slist:
os.remove(f)
llist = glob.glob('LAMP*.fits')
for f in llist:
os.remove(f)
zcor = 'yes'
fcor = 'yes'
_zero = 'tmasterbias.fits'
sciflat = 'ntscimasterflat.fits'
stdflat = 'ntstdmasterflat.fits'
for key in objdict.keys():
if 'AGN' in key:
for img in objdict[key]:
hds = pyfits.getheader(img)
if 'EXPTIME' in hds:
if hds['EXPTIME'] >= 50.0:
num = img.rsplit('.',2)[1]
name = key+'_'+num
'''
iraf.ccdproc(img,output='trim_'+name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor='no',darkcor='no',flatcor='no',illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero='',flat='',Stdout=1)
_gain = hds['HIERARCH ESO DET OUT1 GAIN']
_ron = hds['HIERARCH ESO DET OUT1 RON']
cosmics.lacos('trim_'+name+'.fits', output='c_'+name+'.fits', gain=_gain, readn=_ron, xorder=9, yorder=9, sigclip=4.5, sigfrac=0.5, objlim=1, verbose=True, interactive=False)
iraf.ccdproc('c_'+name,output=name,ccdtype='',noproc='no',fixpix='no',overscan='no',trim='no',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec='',trimsec='',zero=_zero,flat=sciflat,Stdout=1)
'''
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=sciflat,Stdout=1)
else:
pass
elif 'STD' in key:
for img in objdict[key]:
num = img.rsplit('.',2)[1]
name = key+'_'+num
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)
elif 'WAVE' in key:
for img in objdict[key]:
num = img.rsplit('.',2)[1]
name = 'LAMP_'+num
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)
os.system("ls halogen*b.fits > flat.in")
iraf.imcombine('@flat.in',
'mflat.fits',
combine="median",
rdnoise=4.8,
gain=1.22)
#making master flat.
iraf.response("mflat",
"mflat",
"nmflat",
interactive="yes",
threshold="INDEF",
sample="*",
naverage=1,
function="spline3",
order=20,
low_reject=0.,
high_reject=0.,
niterate=1,
grow=0.,
graphics="stdgraph",
cursor="")
iraf.imstat('nmflat.fits')
os.system("ls J105829*b.fits GD248*b.fits > flatf.in")
os.system("sed s/b.fits/bf.fits/g flatf.in > flatf.out")
iraf.imarith('@flatf.in', '/', 'nmflat.fits', '@flatf.out')
print("***********************************************************")
print("Flat correction done!")
def makeflats(fs=None):
# Note the list of files need to not include any paths relative to
# the work directory.
# Maybe not the greatest convention, but we can update this later
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgp*.fits')
if len(fs) == 0:
print "WARNING: No flat-fields to combine and normalize."
# Fail gracefully by going up a directory
iraf.cd('..')
return
# make a flats directory
if not os.path.exists('flats'):
os.mkdir('flats')
# Figure out which images are flats and which grating angles were used
allflats, grangles = get_ims(fs, 'flat')
# For each grating angle
for ga in np.unique(grangles):
# grab the flats for this gr angle
flats = allflats[grangles == ga]
# For each chip
for c in range(1, 7):
# run imcombine with average and sigclip, weighted by exposure time
flatlist = ''
for f in flats:
flatlist += '%s[%i],' % (f, c)
# Add the exptime keyword to each extension
pyfits.setval(f, 'EXPTIME', ext=c,
value=pyfits.getval(f, 'EXPTIME'))
# set the output combined file name
combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
if os.path.exists(combineoutname):
os.remove(combineoutname)
# initialize the iraf command
iraf.unlearn(iraf.imcombine)
print(flatlist)
# don't forget to remove the last comma in the filelist
iraf.imcombine(input=flatlist[:-1], output=combineoutname,
combine='average', reject='sigclip', lsigma=3.0,
hsigma=3.0, weight='exposure', expname='EXPTIME')
pyfits.setval(combineoutname, 'DISPAXIS', value=1)
# We want to make an illumination correction file
# before running response:
illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
iraf.unlearn(iraf.illumination)
iraf.illumination(images=combineoutname,
illuminations=illumoutname, interactive=False,
naverage=-40, order=11, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
# Flag any pixels in the illumination correction< 0.1
illumhdu = pyfits.open(illumoutname, mode='update')
illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
illumhdu.flush()
# Get 40 pixels out of the middle of the image and
# median them to run response
combinehdu = pyfits.open(combineoutname)
ny = combinehdu[0].data.shape[0]
# divide out the illumination correction before running response
flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
/ illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
axis=0)
# close the illumination file because we don't need it anymore
illumhdu.close()
# File stage m1d for median 1-D
flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())
# run response
# r1d = response1d
resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
interactive=False, naverage=-5, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
resp1dhdu = pyfits.open(resp1dfname)
resp1d = resp1dhdu[0].data.copy()
resp1dhdu.close()
# After response divide out the response function
# normalize the 1d resp to its median
resp1d /= np.median(resp1d)
# Chuck any outliers
flatsig = np.std(resp1d - 1.0)
resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
resp = flat1d / resp1d
resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
resp2d = combinehdu[0].data.copy() / resp
tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
combinehdu.close()
# close the combined flat because we don't need it anymore
combinehdu.close()
pyfits.setval(resp2dfname, 'DISPAXIS', value=1)
# Reset any pixels in the flat field correction< 0.1
# We could flag bad pixels here if we want, but not right now
flathdu = pyfits.open(resp2dfname, mode='update')
flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
flathdu.flush()
flathdu.close()
# Step back up to the top directory
iraf.cd('..')
def makeflats(fs=None):
# Note the list of files need to not include any paths relative to
# the work directory.
# Maybe not the greatest convention, but we can update this later
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgp*.fits')
if len(fs) == 0:
print "WARNING: No flat-fields to combine and normalize."
# Fail gracefully by going up a directory
iraf.cd('..')
return
# make a flats directory
if not os.path.exists('flats'):
os.mkdir('flats')
# Figure out which images are flats and which grating angles were used
allflats, grangles = get_ims(fs, 'flat')
# For each grating angle
for ga in np.unique(grangles):
# grab the flats for this gr angle
flats = allflats[grangles == ga]
# For each chip
for c in range(1, 7):
# run imcombine with average and sigclip, weighted by exposure time
flatlist = ''
for f in flats:
flatlist += '%s[%i],' % (f, c)
# Add the exptime keyword to each extension
pyfits.setval(f, 'EXPTIME', ext=c,
value=pyfits.getval(f, 'EXPTIME'))
# set the output combined file name
combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
if os.path.exists(combineoutname):
os.remove(combineoutname)
# initialize the iraf command
iraf.unlearn(iraf.imcombine)
print(flatlist)
# don't forget to remove the last comma in the filelist
iraf.imcombine(input=flatlist[:-1], output=combineoutname,
combine='average', reject='sigclip', lsigma=3.0,
hsigma=3.0, weight='exposure', expname='EXPTIME')
pyfits.setval(combineoutname, 'DISPAXIS', value=1)
# We want to make an illumination correction file
# before running response:
illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
iraf.unlearn(iraf.illumination)
iraf.illumination(images=combineoutname,
illuminations=illumoutname, interactive=False,
naverage=-40, order=11, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
# Flag any pixels in the illumination correction< 0.1
illumhdu = pyfits.open(illumoutname, mode='update')
illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
illumhdu.flush()
# Get 40 pixels out of the middle of the image and
# median them to run response
combinehdu = pyfits.open(combineoutname)
ny = combinehdu[0].data.shape[0]
# divide out the illumination correction before running response
flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
/ illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
axis=0)
# close the illumination file because we don't need it anymore
illumhdu.close()
# File stage m1d for median 1-D
flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())
# run response
# r1d = response1d
resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
interactive=False, naverage=-5, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
resp1dhdu = pyfits.open(resp1dfname)
resp1d = resp1dhdu[0].data.copy()
resp1dhdu.close()
# After response divide out the response function
# normalize the 1d resp to its median
resp1d /= np.median(resp1d)
# Chuck any outliers
flatsig = np.std(resp1d - 1.0)
resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
resp = flat1d / resp1d
resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
resp2d = combinehdu[0].data.copy() / resp
tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
combinehdu.close()
# close the combined flat because we don't need it anymore
combinehdu.close()
pyfits.setval(resp2dfname, 'DISPAXIS', value=1)
# Reset any pixels in the flat field correction< 0.1
# We could flag bad pixels here if we want, but not right now
flathdu = pyfits.open(resp2dfname, mode='update')
flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
flathdu.flush()
flathdu.close()
# Step back up to the top directory
iraf.cd('..')
def test(instr=None,rawdata_dir='UT160327',reduction_dir=None,dograting=None,dofilter=None,silent=False):
print('='*70)
print("---------- Drew's APO 3.5m reduction script ----------")
print('='*70)
print('loading some basic iraf packages...')
iraf.noao(); iraf.imred(); iraf.ccdred()
print('='*70)
##########################################################################################################
print('1. INSTRUMENT ESTABLISHMENT\n')
if instr is None:
print('Which APO 3.5m instrument did you use (default is ARCTIC)? Choices:')
print(' - enter 1 for ARCTIC (default)')
print(' - enter 2 for DIS')
print(' - enter 3 for something else')
a=raw_input('')
if a=='': a='1'
if a[:1]=='1': instr='arctic'
if a[:1]=='2': instr='dis'
if (a[:1]!='1') & (a[:1]!='2'): sys.exit("You're out of luck for now. Sorry!")
print('\nOk. '+instr.upper()+' data will be reduced')
print('='*70)
##########################################################################################################
print('2. RAW DATA DIRECTORY ESTABLISHMENT\n')
if rawdata_dir is None:
a=raw_input('Enter the name of the directory where your images are stored: ')
if a=='': sys.exit('You must specify a raw data directory. Exiting the APO 3.5m reduction script.')
if a[-1]!='/': a=a+'/'
rawdata_dir=a
check=glob.glob(rawdata_dir)
if len(check)==0: sys.exit('The directory "'+rawdata_dir+'" does not exist. Exiting the APO 3.5m reduction script.')
else:
if rawdata_dir[-1:]!='/': rawdata_dir=rawdata_dir+'/'
check=glob.glob(rawdata_dir)
if len(check)==0: sys.exit('The directory "'+rawdata_dir+'" does not exist. Exiting the APO 3.5m reduction script.')
print('Ok. The raw data directory will be "'+rawdata_dir+'"')
print('='*70)
##########################################################################################################
print('3. RAW DATA CHECK\n')
rawfiles = np.array(glob.glob(rawdata_dir+'*fits')); nraw=len(rawfiles)
if nraw==0: sys.exit('No FITS files found in the raw data directory. Exiting the APO 3.5m reduction script.')
instrlist=[]; imtype=[]
for rawfile in rawfiles:
head=fits.getheader(rawfile)
instrlist.append(head['instrume'].lower())
instrlist=np.array(instrlist)
gd=np.where(instrlist==instr); ngd=len(gd[0])
if ngd==0:
sys.exit('No FITS files from '+instr.upper()+' were found in the raw data directory. Exiting the APO 3.5m reduction script.')
else:
rawfiles=rawfiles[gd]
print('Ok. Found '+str(ngd)+' FITS images with instrument='+instr.upper()+' in the raw data directory.')
print('='*70)
##########################################################################################################
print('4. REDUCTION DIRECTORY ESTABLISHMENT\n')
if reduction_dir is not None:
check=glob.glob(reduction_dir)
if len(check)==0:
a=raw_input('Specified reduction directory does not exist. Create it (Y/N)? ')
if (a=='') | (a[:1].lower()=='y'):
cmnd='mkdir '+reduction_dir
os.system(cmnd)
print('"'+reduction_dir+'" has been created.')
else:
reduction_dir=rawdata_dir[:-1]+'_reduction/'
print('The reduction directory will be '+reduction_dir)
print(' - hit ENTER to accept')
print(' - otherwise enter a different reduction directory name: ')
a=raw_input('')
if a=='':
check=glob.glob(reduction_dir)
if len(check)==0:
cmnd='mkdir '+reduction_dir
os.system(cmnd)
print('\nOk. "'+reduction_dir+'" has been created.')
else:
reduction_dir=a
check=glob.glob(reduction_dir)
if len(check)==0:
cmnd='mkdir '+reduction_dir
os.system(cmnd)
print('\nOk. "'+reduction_dir+'" has been created.')
print('\nOk. Reduction products will be stored in '+reduction_dir+'.')
print('='*70)
##########################################################################################################
if instr=='dis':
print('loading some more IRAF packages for DIS reduction...')
iraf.twod(); iraf.apex(); iraf.longs(); iraf.oned()
print('='*70)
print('5. DIS PARAMETER ESTABLISHMENT\n')
grating=[]; imagetypes=[]
for rawfile in rawfiles:
head=fits.getheader(rawfile)
grating.append(head['grating'])
imagetypes.append(head['imagetyp'].lower())
imagetypes=np.array(imagetypes)
grating=np.array(grating)
uniquegrating=np.unique(grating)
if dograting is None:
print("Which DIS grating do you want to reduce? Here's what you have:")
for i in range(len(uniquegrating)):
if i==0:
print(' - enter '+str(i+1)+' for '+uniquegrating[i]+' (default)')
else:
print(' - enter '+str(i+1)+' for '+uniquegrating[i])
a=raw_input('')
if a=='': a='1'
dograting=uniquegrating[int(a[:1])-1]
gd=np.where(grating==dograting); ngd=len(gd[0])
rawfiles=rawfiles[gd]; imagetypes=imagetypes[gd]
print('\nOk. '+dograting+' data will be reduced: '+str(ngd)+' total images')
else:
gd=np.where(grating==dograting); ngd=len(gd[0])
if ngd==0:
sys.exit('No '+dograting+' images found. Exiting the APO 3.5m reduction script.')
else:
rawfiles=rawfiles[gd]; imagetypes=imagetypes[gd]
print('\nOk. '+dograting+' images will be reduced: '+str(ngd)+' total images')
bias=np.where((imagetypes=='bias') | (imagetypes=='zero')); nbias=len(bias[0])
zerocor='yes'
if nbias==0:
print('\nNo bias images were found. Proceed without bias correction? (y/n)')
a=raw_input('')
if (a[:1]=='y') | (a==''):
zerocor='no'; biasfiles=''; nbias=0
else:
sys.exit('Exiting the APO 3.5m reduction script.')
else:
biasfiles=rawfiles[bias]; nbias=len(bias[0])
# get values depending on R vs B detector
detector=dograting[:1].lower()
if detector=='b':
gain=1.68
rdnoise=4.9
badpixfile=''
fixpix='no'
if detector=='r':
gain=1.88
rdnoise=4.6
badpixfile='badpix_disR.txt'
fixpix='yes'
head=fits.getheader(rawfiles[0])
biassec=head['biassec']
datasec=head['datasec']
genericlabel=dograting
print('\nOk. DIS parameters established.')
print('='*70)
##########################################################################################################
if instr=='arctic':
print('5. ARCTIC PARAMETER ESTABLISHMENT\n')
gain=2.0
rdnoise=3.7
badpixfile=''
fixpix='no'
filters=[]; imagetypes=[]
for rawfile in rawfiles:
head=fits.getheader(rawfile)
filters.append(head['filter'])
imagetypes.append(head['imagetyp'].lower())
imagetypes=np.array(imagetypes)
filters=np.array(filters)
uniquefilters=np.unique(filters)
bias=np.where(imagetypes=='bias'); nbias=len(bias[0])
zcor='yes'
if nbias==0:
print('\nNo bias images were found. Proceed without bias correction? (y/n)')
a=raw_input('')
if (a[:1]=='y') | (a==''):
zcor='no'; biasfiles=''
else:
sys.exit('Exiting the APO 3.5m reduction script.')
else:
biasfiles=rawfiles[bias]; nbias=len(bias[0])
# establish ARCTIC filter
if dofilter is None:
print('Which ARCTIC filter do you want to reduce? ')
for i in range(len(uniquefilters)):
if i==0:
print(' - enter '+str(i+1)+' for '+uniquefilters[i]+' (default)')
else:
print(' - enter '+str(i+1)+' for '+uniquefilters[i])
a=raw_input('')
if a=='': a='1'
dofilter=uniquefilters[int(a[:1])-1]
gd=np.where(filters==dofilter); ngd=len(gd[0])
rawfiles=rawfiles[gd]
print('\nOk. '+dofilter+' filter images will be reduced: '+str(ngd)+' total images')
else:
gd=np.where(filters==dofilter); ngd=len(gd[0])
if ngd==0:
sys.exit('No '+dofilter+' images found. Exiting the APO 3.5m reduction script.')
else:
rawfiles=rawfiles[gd]
print('\nOk. '+dofilter+' filter images will be reduced: '+str(ngd)+' total images')
head=fits.getheader(rawfiles[0])
biassec=head['bsec11']
datasec=head['dsec11']
genericlabel=dofilter.replace(' ','_')
genericlabel=genericlabel.replace('/','_')
print('\nOk. ARCTIC parameters established.')
print('='*70)
##########################################################################################################
print('6. GET IMAGE TYPES\n')
nrawfiles = len(rawfiles)
# gather some more info from headers
imtype = []; exptime = []; ra = []; dec = []
for rawfile in rawfiles:
header = fits.getheader(rawfile)
imtype.append(header['IMAGETYP'].lower())
exptime.append(header['EXPTIME'])
ra.append(header['RA'])
dec.append(header['DEC'])
imtype=np.array(imtype)
exptime=np.array(exptime); ra=np.array(ra)
dec=np.array(dec)
# separate the files into biases, flat, objs, and comps
flat=np.where(imtype=='flat'); nflat=len(flat[0]); flatfiles=rawfiles[flat]
obj=np.where(imtype=='object'); nobj=len(obj[0]); objfiles=rawfiles[obj]
if instr=='dis':
comp=np.where(imtype=='comp'); ncomp=len(comp[0]); compfiles=rawfiles[comp]
print('Ok. You have '+str(nbias)+' biases, '+str(nflat)+' flats, '+str(ncomp)+' comps, and '+str(nobj)+' objects.')
if instr=='arctic':
print('Ok. You have '+str(nbias)+' biases, '+str(nflat)+' flats, and '+str(nobj)+' objects.')
print('='*70)
##########################################################################################################
print('7. MAKE MASTER BIAS\n')
if zcor=='yes':
masterzero=reduction_dir+'Zero_'+genericlabel+'.fits'
a=raw_input('Average combine biases into master bias (y/n)? ')
if (a[:1]=='y') | (a==''):
iraf.imcombine(','.join(biasfiles),output=masterzero,combine='average',reject='avsigclip',lsigma='3',
hsigma='3',rdnoise=rdnoise,gain=gain)
print('\nOk. A master bias has been made: '+masterzero)
print('='*70)
else:
masterzero=''
print('\nOk, you have no bias files. Proceeding')
##########################################################################################################
print('8. BIAS AND OVERSCAN CORRECTION\n')
a=raw_input('Overscan and bias-subtraction of the flats, objs, and if DIS, comps. Do it (y/n)? ')
if (a[:1]=='y') | (a==''):
# set up a list of the input files (flats, comps, objs)
if instr=='dis': infiles=np.concatenate((flatfiles,compfiles,objfiles),axis=0)
if instr=='arctic': infiles=np.concatenate((flatfiles,objfiles),axis=0)
infilesfile=reduction_dir+'inlist_'+genericlabel+'_ccdproc1'
np.savetxt(infilesfile,infiles,fmt='%s')
# We're making changes to the images now, so need to set up a list of output files
outfiles=[]
for infile in infiles:
tmp=infile.split('/')
tmp1=tmp[len(tmp)-1].split('.fits')
outfiles.append(reduction_dir+tmp1[0]+'_cproc1.fits')
outfiles=np.array(outfiles)
outfilesfile=reduction_dir+'outlist_'+genericlabel+'_ccdproc1'
np.savetxt(outfilesfile,outfiles,fmt='%s')
iraf.ccdproc('@'+infilesfile,output='@'+outfilesfile,ccdtype='',fixpix=fixpix,oversca='yes',trim='yes',
zerocor=zcor,darkcor='no',flatcor='no',fixfile=badpixfile,biassec=biassec,trimsec=datasec,
zero=masterzero,interac='no',low_rej='3',high_re='3')
print('\nOk. Overscan and bias subtracting done.')
print('='*70)
##########################################################################################################
print('9. MAKE MASTER FLAT\n')
masterflat=reduction_dir+'Flat_'+genericlabel+'.fits'
a=raw_input('Median combine the flats (y/n)? ')
if (a[:1]=='y') | (a==''):
procfiles=np.array(glob.glob(reduction_dir+'*_cproc1.fits'))
imtyp=[]
for i in range(len(procfiles)):
head=fits.getheader(procfiles[i])
imtyp.append(head['imagetyp'].lower())
imtyp=np.array(imtyp)
gd=np.where(imtyp=='flat')
flats=procfiles[gd]
iraf.imcombine(','.join(flats),output=masterflat,combine='median',reject='avsigclip',lsigma='2',
hsigma='2',rdnoise=rdnoise,gain=gain)
print('\nOk. A master flat has been made: '+masterflat)
print('='*70)
##########################################################################################################
print('10. MAKE NORMALIZED MASTER FLAT\n')
masternormflat=reduction_dir+'NormFlat_'+genericlabel+'.fits'
a=raw_input('Normalize the response of the master flat (y/n)? ')
if (a[:1]=='y') | (a==''):
if instr=='arctic':
im=fits.getdata(masterflat)
imdim=len(im)
meanval=np.mean(im[imdim-100:imdim+100,imdim-100:imdim+100])
iraf.imarith(operand1=masterflat,op='/',operand2=meanval,result=masternormflat)
print('\nOk. A normalized master flat has been created: '+masternormflat)
if instr=='dis':
iraf.response(calibrat=masterflat,normaliz=masterflat+'[*,400:500]',response=masternormflat,
interac='no',functio='spline3',order='5',low_rej='3',high_rej='3')
print('\nOk. A normalized master flat has been created: '+masternormflat)
print('='*70)
##########################################################################################################
print('11. FLAT FIELD THE IMAGES\n')
a=raw_input('Flat field the objs & comps by the normalized master flat (y/n)? ')
if (a[:1]=='y') | (a==''):
tmpfiles=np.array(glob.glob(reduction_dir+'*_cproc1.fits'))
imtyp=[]
for i in range(len(tmpfiles)):
head=fits.getheader(tmpfiles[i])
imtyp.append(head['imagetyp'].lower())
imtyp=np.array(imtyp)
gd1=np.where(imtyp=='object')
if instr=='dis':
gd2=np.where(imtyp=='comp')
infiles=np.concatenate((tmpfiles[gd1],tmpfiles[gd2]),axis=0)
if instr=='arctic':
infiles=tmpfiles[gd1]
infilesfile=reduction_dir+'inlist_'+genericlabel+'_ccdproc2'
np.savetxt(infilesfile,infiles,fmt='%s')
outfiles=[]
for infile in infiles:
outfiles.append(infile.replace('cproc1','cproc2'))
outfiles=np.array(outfiles)
outfilesfile=reduction_dir+'outlist_'+genericlabel+'_ccdproc2'
np.savetxt(outfilesfile,outfiles,fmt='%s')
iraf.ccdproc('@'+infilesfile,output='@'+outfilesfile,ccdtype='',fixpix='no',oversca='no',trim='no',
zerocor='no',darkcor='no',flatcor='yes',fixfile='',biassec=biassec,trimsec=datasec,
flat=masternormflat,interac='no',low_rej='3',high_re='3')
print('\nOk. The objs and comps have been flat fielded.')
print('='*70)
##########################################################################################################
if instr=='dis':
print('Next steps (done manually for now):')
print('='*70)
print('12. Extract apertures via APALL.')
print('13. Run IDENTIFY on one or more comps.')
print('14. (optional) Run REIDENTIFY to transfer solution among comps.')
print('15. Assign solution(s) to objs using REFSPECTRA.')
print('16. Run DISPCOR on the objs to apply the wavelength solution.')
print('='*70)
print('Other things that should be done:')
print('='*70)
print('step ?1. flux calibration.')
print('step ?2. sky/background subtraction.')
print('step ?3. heliocentric velocity correction.')
return imtyp
rdnoise='rdnoise',
gain='gain')
iraf.imstat(flatcup)
iraf.imstat('Flat')
iraf.imstat(flati)
iraf.imstat('iFlat')
iraf.imexamine('Flat')
iraf.imexamine('iFlat')
iraf.response.unlearn()
iraf.response(calibration='Flat',
normalization='Flat',
response='nFlat',
function='legendre',
order='1')
iraf.response.unlearn()
iraf.response(calibration='iFlat',
normalization='iFlat',
response='niFlat',
function='legendre',
order='1')
iraf.imstat('nFlat')
iraf.imexamine('nFlat')
iraf.imstat('niFlat')
iraf.imexamine('niFlat')
