def makeFlat(fileList='flats.lst',inputType=INPUT_LISTFNAME, output='Flat',combine='average', reject='avsigclip', ccdtype='flat', process = iraf.yes, subsets=iraf.yes, scale='mode'): '''create master Flats from fileList, if subsets is iraf.yes, then create a different flat for each subset indicated in the subset header''' iraf.flatcombine(_genIRAFString(fileList, inputType), output=output, combine=combine, process=process, ccdtype=ccdtype,reject=reject, scale=scale, subsets=subsets) return
def mkflatINTWFC(data, combine='median', reject='avsigclip'):
"""
Creates a master flat from a given list of flat frames.
Reads readnoise and gain from the FITS header, as each WFC chip has
different values.
:note: ccdproc
"""
for filter in data:
input1 = ''
for file in data[filter]:
input1 += file + '[1],'
input2 = input1.replace('[1]', '[2]')
input3 = input1.replace('[1]', '[3]')
input4 = input1.replace('[1]', '[4]')
inputs = [input1, input2, input3, input4]
for ext, input in enumerate(inputs):
print input
iraf.unlearn('flatcombine')
iraf.flatcombine(input=input[:-1],
output='FLAT_{0:>s}_{1:d}.fits'.format(filter, ext + 1),
combine=combine,
reject=reject,
rdnoise='READNOIS',
gain='GAIN')
def combine_flat(lstfile):
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.flatcombine(input='tbo//@' + lstfile,
output='Halogen',
combine='average',
reject='crreject',
ccdtype='',
process=False,
subsets=False,
delete=False,
clobber=False,
scale='mode',
statsec='',
nlow=1,
nhigh=1,
nkeep=1,
mclip=True,
lsigma=3.0,
hsigma=3.0,
rdnoise='rdnoise',
gain='gain',
snoise=0.0,
pclip=-0.5,
blank=1.0)
def combine_flat(ftblst):
iraf.flatcombine(input='tb//@' + ftblst,
output='flat',
combine='average',
reject='avsigclip',
ccdtype='',
process=False,
subsets=False,
delete=False,
clobber=False,
scale='mode',
statsec='',
nlow=1,
nhigh=1,
nkeep=1,
mclip=True,
lsigma=3.0,
hsigma=3.0,
rdnoise='CCDRON',
gain='CCDGAIN',
snoise=0.0,
pclip=-0.5,
blank=1.0)
iraf.flpr()
print '<<<<<combine flat successfully>>>>>'
def make_flat(images,
outflat,
gain=1.0,
rdnoise=0.0,
xwindow=50,
ywindow=50,
hmin=0,
hmax=65535,
lowclip=0.7,
highclip=1.3):
'''Construct flat field from individual frames'''
flatimages = ','.join(images)
iraf.flatcombine(flatimages,
output='flat1',
combine='median',
reject='avsigclip',
ccdtype='',
process=no,
subsets=no,
delete=no,
clobber=no,
scale='median',
lsigma=3.0,
hsigma=3.0,
gain=gain,
rdnoise=rdnoise)
iraf.fmedian('flat1', 'flat2', xwindow, ywindow, hmin=hmin, hmax=hmax)
iraf.imarith('flat1', '/', 'flat2', outflat)
iraf.imreplace(outflat, 1.0, lower=INDEF, upper=lowclip)
iraf.imreplace(outflat, 1.0, lower=highclip, upper=INDEF)
return
def mkflatINTWFC(data, combine='median', reject='avsigclip'):
"""
Creates a master flat from a given list of flat frames.
Reads readnoise and gain from the FITS header, as each WFC chip has
different values.
:note: ccdproc
"""
for filter in data:
input1 = ''
for file in data[filter]:
input1 += file + '[1],'
input2 = input1.replace('[1]', '[2]')
input3 = input1.replace('[1]', '[3]')
input4 = input1.replace('[1]', '[4]')
inputs = [input1, input2, input3, input4]
for ext, input in enumerate(inputs):
print input
iraf.unlearn('flatcombine')
iraf.flatcombine(input=input[:-1],
output='FLAT_{0:>s}_{1:d}.fits'.format(
filter, ext + 1),
combine=combine,
reject=reject,
rdnoise='READNOIS',
gain='GAIN')
def make_master(obj, file_type=None):
"""
This routine takes the name of a file containing the raw calibration images in the **current** directory that are to
be combined to form a master calibration image. The type of calibration images (i.e. bias, dark or flat) is specified
by the file_type keyword; this is necessary because the type of calibration image determines how the multiple raw
calibration images are to be combined (eg. pixel averages vs medians).
"""
# Get the name of the master calibration file:
if file_type=='bias':
list_file = obj.bias_list
elif file_type=='dark':
list_file = obj.dark_list
elif file_type=='flat':
list_file = obj.flat_list
dirpath = os.path.dirname(list_file)+'/'
filename = 'master_'+file_type+'.fits'
master_filename = dirpath+filename
if os.path.exists(master_filename): os.remove(master_filename)
# Now make the master calibration file:
if file_type=='bias':
iraf.zerocombine(input='@'+list_file, output=master_filename, combine='average', ccdtype='', process='no', delete='no')
obj.master_bias = master_filename
elif file_type=='dark':
iraf.darkcombine(input='@'+list_file, output=master_filename, combine='average', ccdtype='', process='no', delete='no')
obj.master_dark = master_filename
elif file_type=='flat':
iraf.flatcombine(input='@'+list_file, output=master_filename, combine='median', ccdtype='', process='no', delete='no')
obj.master_flat = master_filename
return None
def combine_flat(lstfile):
if os.path.isfile('Halogen.fits'):
print 'remove Halogen.fits'
os.remove('Halogen.fits')
if os.path.isfile('Resp.fits'):
print 'remove Resp.fits'
os.remove('Resp.fits')
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.flatcombine(input='tbo//@' + lstfile,
output='Halogen',
combine='average',
reject='crreject',
ccdtype='',
process=False,
subsets=False,
delete=False,
clobber=False,
scale='mode',
statsec='',
nlow=1,
nhigh=1,
nkeep=1,
mclip=True,
lsigma=3.0,
hsigma=3.0,
rdnoise='rdnoise',
gain='gain',
snoise=0.0,
pclip=-0.5,
blank=1.0)
iraf.twodspec()
iraf.longslit(dispaxis=2,
nsum=1,
observatory='Lijiang',
extinction=func.config_path + os.sep + 'LJextinct.dat',
caldir=func.std_path + os.sep,
interp='poly5')
iraf.response(calibration='Halogen',
normalization='Halogen',
response='Resp',
interactive=True,
threshold='INDEF',
sample='*',
naverage=1,
function='spline3',
order=25,
low_reject=10.0,
high_reject=10.0,
niterate=1,
grow=0.0,
graphics='stdgraph',
cursor='')
def create_flat(self):
"""Creating Combined Flat Frame"""
self.log.debug("Runnign iraf.flatcombine on image list...")
iraf.unlearn(iraf.flatcombine)
iraf.flatcombine(self.flat.iraf.inatfile(),
output=self.flat.iraf.outfile("Flat"),
combine=self.config["Flat.Combine"],
ccdtype="flat",
reject=self.config["Flat.Reject"],
scale=self.config["Flat.Scale"],
process="no", subsets="no", nlow=0, nhigh=1, nkeep=1, mclip="yes", lsigma=3.0, hsigma=3.0, rdnoise="0.", gain ="1.")
self.flat.iraf.done()
def flatcom(input_flat='zFLAT*.fit'):
import glob
import os, sys
import itertools
import numpy as np
from pyraf import iraf
from astropy.io import fits
from astropy.io import ascii
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.ccdred.setinst(instrume='camera', directo='/iraf/iraf/noao/imred/ccdred/ccddb/', query='q', review='no')
# Filter classification
calflat = glob.glob(input_flat)
allfilter = []
i=0
for i in range(len(calflat)) :
hdr = fits.getheader(calflat[i])
allfilter.append(hdr['filter'])
filterset = set(allfilter)
infilter = list(sorted(filterset))
i=0
for i in range(len(infilter)) :
print('Find images with filter of '+str(infilter[i]))
imlist = []
for j in range(len(calflat)) :
hdr = fits.getheader(calflat[j])
if hdr['filter'] == infilter[i] :
imlist.append(calflat[j])
else :
pass
print(imlist)
imlist.sort()
input_name = str(infilter[i])+'flat.list'
k=0
f=open(input_name,'w+')
for k in range(len(imlist)) :
f.write(imlist[k]+'\n')
f.close()
output_name = input_name[:-5]+'.fits'
#iraf.flatcombine(input='@'+input_name, output=output_name, combine='average', reject='crreject', process='no', scale='mode', ccdtype='', lsigma='3.', hsigma='3.' )
iraf.flatcombine(input='@'+input_name, output=output_name, combine='median', reject='minmax', process='no', scale='mode', ccdtype='')
print(output_name+' is created. Normalizing...')
data, newhdr = fits.getdata(output_name, header=True)
x = np.mean(data)
nimage = data/x
newflat_name = curdate+'_n'+str(infilter[i])+'flat.fits'
fits.writeto(newflat_name, nimage, header=newhdr, overwrite=True)
#os.system('/usr/bin/cp '+newflat_name+' ../')
os.system('/usr/bin/cp '+newflat_name+' /data1/KHAO/MDFTS/red/masterflat_'+infilter[i]+'/')
print('Normalised master flats are created.')
iraf.imstat(images='*n?flat.fits')
def main(raw_dir):
if not os.path.exists("{}/step1.tmp".format(os.getcwd())):
print "Setting up directory"
list_dict = setup_dir(raw_dir)
print "OT'ing"
iraf.ccdproc(
"@all_raw.lst",
output="@all_ot.lst",
overscan=True,
trim=True,
zerocor=False,
darkcor=False,
flatcor=False,
illumco=False,
biassec="image",
trimsec="image",
zero="",
interact=True,
order=100,
)
cull_bias(list_dict["zeros"], "zeros.lst")
os.system("touch {}/step1.tmp".format(os.getcwd()))
return
print "Making master bias"
iraf.zerocombine("@zeros.lst", output="Zero.fits", combine="average", reject="crreject")
print "Subtracting bias"
iraf.ccdproc("@all_otz.lst", overscan=False, trim=False, zerocor=True, zero="Zero.fits")
detect_flats(os.getcwd())
flats_to_make = glob("dflat*.lst")
for flat in flats_to_make:
name = "dFlat_{}.fits".format(flat.split(".lst")[0].split("_")[1])
print "Making flat {}".format(name)
iraf.flatcombine("@{}".format(flat), output=name, combine="average", reject="crreject")
try:
print "Making master dark"
iraf.darkcombine("@darks.lst", output="Dark.fits", combine="average", reject="crreject")
except Exception as e:
print "\t{}".format(e)
print "\tNO DARK FOUND"
print "Making master comp"
iraf.imcombine("@comps.lst", "Comp.fits", combine="average", reject="crreject")
return
def makeflatPyraf(filelist, bias, output, noise, gain):
"""
Creates a master flat from given list of flat frames.
"""
from pyraf import iraf
from pyraf.iraf import imred, ccdred
combine = "median"
reject = "avsigclip"
iraf.flatcombine(input=filelist,
output=output,
combine=combine,
reject=reject,
rdnoise=noise,
gain=gain)
def make_flat(images, outflat, gain=1.0, rdnoise=0.0, xwindow=50,
ywindow=50, hmin=0, hmax=65535, lowclip=0.7, highclip=1.3):
'''Construct flat field from individual frames'''
flatimages=','.join(images)
iraf.flatcombine(flatimages, output='flat1', combine='median',
reject='avsigclip', ccdtype='', process=no, subsets=no,
delete=no, clobber=no, scale='median', lsigma=3.0,
hsigma=3.0, gain=gain, rdnoise=rdnoise)
iraf.fmedian('flat1', 'flat2', xwindow, ywindow, hmin=hmin, hmax=hmax)
iraf.imarith('flat1', '/', 'flat2', outflat)
iraf.imreplace(outflat, 1.0, lower=INDEF, upper=lowclip)
iraf.imreplace(outflat, 1.0, lower=highclip, upper=INDEF)
return
def optdomecomb(date, fwheel=['bias','B','V','R','I']):
'''
#combine biases and optical domes
#Requires: the uncombined fits images
# if you are combining a dome, you must have a bias from the same night as the dome to preform appropriate bias subtraction
#Input: the date the domes were observed YYMMDD, and fwheel, a list that contains the filters of the domes to be combined
#Outupt: combined dome fits frame for each color where uncombined frames are in the directory
'''
#convert date to string incase it was entered as an int of float
date=str(date)
if len(glob.glob('*bias*')) < 1:
print "no biases found, exiting"
return
else:
for color in fwheel:
if color=='bias':
biaslist=glob.glob('*bias.[0-9]*')
if len(biaslist) > 10:
print "only "+str(len(biaslist))+" biases found. you need at least 10"
else:
with open("bias.list",'w') as BILIS:
for i in biaslist:
BILIS.write(i+'\n')
iraf.zerocombine("@bias.list",output="ccd"+str(date)+".bias.fits",combine="average",reject="minmax",scale="none",ccdtype="",process="no",delete="no",clobber="no",nlow=1,nhigh=1,nkeep=1)
print "created ccd"+str(date)+".bias.fits"
os.system('rm bias.list')
elif color in ['B','V','R','I']:
domelist=glob.glob('*dome'+color+'.[0-9]*')
if len(domelist) < 1:
print 'no '+color+' domes found'
elif len(domelist) > 10:
print 'only '+str(len(domelist))+' domes found. you need at least 10'
else:
with open('flat'+color+'.list', 'w') as flist:
for i in domelist:
flist.write(i+'\n')
iraf.ccdproc("@flat"+color+".list", output="z@flat"+color+".list",ccdtype=" ",noproc="no", fixpix="no",overscan="yes", trim="no", zerocor="yes",darkcor="no",flatcor="no", illumcor="no", fringec="no", readcor="no", scancor="no", readaxis="line", biassec="[3:14,1:1024]", zero="ccd"+str(date)+".bias.fits", interactive="no", functio="spline3", order=11)
iraf.flatcombine("z@flat"+color+".list", output="ccd"+str(date)+".dome"+color+".fits",combine="average", reject="crreject", ccdtype="", process="no", subsets="no", delete="no", clobber="no", scale="mode", rdnoise=6.5, gain=2.3)
os.system('rm z*dome'+color+'*fits')
print "created ccd"+str(date)+".dome"+color+".fits"
os.system('rm flat'+color+'.list')
else:
print "your input for the filter was not recognized. Please use either 'bias', 'B', 'V', 'R', or 'I' and try again"
return
def combine_flats(flatdir):
"""
Co-add flat-field images. Check no variation night to night and then add all together.
Need to first run set_instrument
Once combined should use imarith to divide each by the average and check see only
noise and no residuals
First should lcpixmap flat fields
"""
print 'In directory ' + flatdir
print 'Combining flats...'
if os.path.exists( os.path.join(flatdir,'Flat.fits') ):
os.remove( os.path.join(flatdir,'Flat.fits') )
print 'Removing file ' + os.path.join(flatdir,'Flat.fits')
names = [name + '[1]' for name in os.listdir(flatdir) if (name.endswith('.fit')) & (name.startswith('cr')) ]
# # Give write permission to files
# for name in names:
# os.chmod(name.replace('[1]',''),0644)
#
with open( os.path.join(flatdir,'input.list'), 'w') as f:
for name in names:
f.write( os.path.join(flatdir,name + '\n') )
iraf.flatcombine.setParam('input', '@' + os.path.join(flatdir, 'input.list') )
iraf.flatcombine.setParam('rdnoise','readnois')
iraf.flatcombine.setParam('combine','average')
iraf.flatcombine.setParam('reject','crreject')
iraf.flatcombine.setParam('ccdtype','flat')
iraf.flatcombine.setParam('scale','mode')
iraf.flatcombine.setParam('lsigma',3.0)
iraf.flatcombine.setParam('hsigma',3.0)
iraf.flatcombine.setParam('output', os.path.join(flatdir,'Flat.fits') )
iraf.flatcombine()
return None
def makeFlat(fileList='flats.lst',
inputType=INPUT_LISTFNAME,
output='Flat',
combine='average',
reject='avsigclip',
ccdtype='flat',
process=iraf.yes,
subsets=iraf.yes,
scale='mode'):
'''create master Flats from fileList, if subsets is iraf.yes, then
create a different flat for each subset indicated in the subset header'''
iraf.flatcombine(_genIRAFString(fileList, inputType),
output=output,
combine=combine,
process=process,
ccdtype=ccdtype,
reject=reject,
scale=scale,
subsets=subsets)
return
def flatcom(inim_list,
outim_list,
combine='median',
reject='sigclip',
scale='mode'):
import os, sys
from pyraf import iraf
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.ccdred.setinst(instrume='camera',
directo='/iraf/iraf/noao/imred/ccdred/ccddb/',
query='q',
review='no')
iraf.flatcombine(input=inim_list,
output=outim_list,
combine=combine,
reject=reject,
process='no',
scale=scale,
ccdtype='')
print('Output masterflat is created.')
def combine_flat(filename):
outname = filename.replace('.lst','.fits')
print 'run function flatcombine...'
print 'make file', outname
iraf.flatcombine(input = 'tbo//@' + filename
, output = outname, combine = 'average', reject = 'avsigclip'
, ccdtype = '', process = False, subsets = True
, delete = False, clobber = False, scale = 'mode'
, statsec = '', nlow = 1, nhigh = 1, nkeep = 1
, mclip = True, lsigma = 3.0, hsigma = 3.0
, rdnoise = 9.4, gain = 0.35, snoise = 0.0
, pclip = -0.5, blank = 1.0)
iraf.noao()
iraf.twodspec()
iraf.longslit()
print 'run function response...'
print 'make file', 're' + outname
iraf.response(calibration = outname
, normalization = outname, response = 're' + outname
, interactive = True, threshold = 'INDEF', sample = '*'
, naverage = 1, function = 'spline3', order = 7
, low_reject = 0.0, high_reject = 0.0, niterate = 1
, grow = 0.0, graphics = 'stdgraph', cursor = '')
print 'run function illumination...'
print 'make file', 'il' + outname
iraf.illumination(images = 're' + outname
, illuminations = 'il' + outname, interactive = False
, bins = '', nbins = 5, sample = '*', naverage = 1
, function = 'spline3', order = 1, low_reject = 0.0
, high_reject = 0.0, niterate = 1, grow = 0.0
, interpolator = 'poly3', graphics = 'stdgraph', cursor = '')
print 'run function imarith...'
print 'make file', 'per' + outname
iraf.imarith(operand1 = 're' + outname
, op = '/', operand2 = 'il' + outname, result = 'per' + outname
, title = '', divzero = 0.0, hparams = '', pixtype = ''
, calctype = '', verbose = True, noact = False)
return outname, 're' + outname, 'il' + outname, 'per' + outname
def combine_flat(lstfile):
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.flatcombine(input = 'tbo//@' + lstfile
, output = 'Halogen', combine = 'average', reject = 'crreject'
, ccdtype = '', process = False, subsets = False
, delete = False, clobber = False, scale = 'mode'
, statsec = '', nlow = 1, nhigh = 1, nkeep = 1
, mclip = True, lsigma = 3.0, hsigma = 3.0
, rdnoise = 'rdnoise', gain = 'gain', snoise = 0.0
, pclip = -0.5, blank = 1.0)
script_path = os.path.split(os.path.realpath(__file__))[0]
iraf.twodspec()
iraf.longslit(dispaxis = 2, nsum = 1, observatory = 'observatory'
, extinction = script_path + os.sep + 'LJextinct.dat'
, caldir = script_path + os.sep + 'standarddir' + os.sep, interp = 'poly5')
iraf.response(calibration = 'Halogen'
, normalization = 'Halogen', response = 'Resp'
, interactive = True, threshold = 'INDEF', sample = '*'
, naverage = 1, function = 'spline3', order = 25
, low_reject = 10.0, high_reject = 10.0, niterate = 1
, grow = 0.0, graphics = 'stdgraph', cursor = '')
def FLAT(masterbias,flat): # FALTA NORMALIZAR EL MASTERFLAT
complete_reduction(flat,masterbias,'')
path_flat = os.path.dirname(flat)
flat = flat.replace(path_flat + '/','')
iraf.chdir(path_flat)
os.chdir(path_flat)
iraf.flatcombine.input = '@' + flat
if os.path.isfile('MasterFlat.fits'):
subprocess.call(['rm', 'MasterFlat.fits'])
iraf.flatcombine.output = 'MasterFlat.fits'
iraf.flatcombine.process = 'no'
iraf.flatcombine.subsets = 'no'
iraf.flatcombine.rdnoise = 'HIERARCH ESO DET OUT1 RON'
iraf.flatcombine.gain = 'HIERARCH ESO DET OUT1 GAIN'
iraf.flatcombine.ccdtype = ''
iraf.flatcombine.statsec = '[800:900,5:105]'
iraf.flatcombine.reject = 'minmax'
#iraf.lpar(iraf.flatcombine)
iraf.flatcombine(mode='h')
def skyflat(date, low=15000, high=23000, numimages=5):
'''
make a combined b skyflat.
Requires: a bias image in same directory to do the bias subtraction
skyflats must be offset and have appropriate count number
input: the date the skyflats were observed YYMMDD
output: flat.B, a text file that lists the names of the skyflat fits files
ccdYYMMDD.skyflatB.fits, the combined skyflat
'''
#check if biases are in this directory
if len(glob.glob('*.bias.*')) < 1:
print "no combined bias found, exiting"
return
#get image name and mean pixel value for all skyflat images
stats=iraf.imstat('*sky*',format=False,fields='image,mean',Stdout=1)
pairs=[i.split() for i in stats]
#write the names of the skyflats w right ammount of counts to file
#keep track of how many good ones there are
goodCount=0
with open("flat.B",'w') as FB:
for i in pairs:
if float(i[1]) > low and float(i[1]) < high:
FB.write(i[0]+'\n')
goodCount+=1
if goodCount < numimages:
print "only "+str(goodCount)+" skyflats have counts between "+str(low)+" and "+str(high)
print "no combined skyflat made"
return
else:
iraf.ccdproc(images="@flat.B",output=" ",fixpix="no",overscan="yes",trim="no",zerocor="yes",darkcor="no",flatcor="no",illumcor="no",fringecor="no",readcor="no",scancor="no",readaxis="line",biassec="[3:14,1:1024]",zero="*.bias.fits",interactive="no",functio="spline3",order=11)
iraf.flatcombine("@flat.B",output="FLAT",combine="median",reject="minmax",process="no",scale="mode",ccdtype="")
os.system("mv FLAT.fits ccd"+str(date)+".skyflatB.fits")
print ("made combined skyflat ccd"+str(date)+".skyflatB.fits")
return
def the_flatcombine(self,
in_file_list,
out_file,
dark=None,
zero=None,
ccdtype="",
method="Median",
rejection="minmax",
subset="yes",
overscan="no",
trim="no"):
self.etc.log(
"Flatcombine started for {} files using combine({}), rejection({}) and subset=({}) and zero({}) and dark({})"
.format(len(in_file_list), method, rejection, subset, zero, dark))
try:
if self.fop.is_file("{}*".format(out_file)):
self.fop.rm("{}*".format(out_file))
files = []
for file in in_file_list:
if self.fit.is_fit(file):
files.append(file)
if not len(files) == 0:
flats = ",".join(files)
# Load packages
# Unlearn settings
iraf.imred.unlearn()
iraf.ccdred.unlearn()
iraf.ccdred.ccdproc.unlearn()
iraf.ccdred.combine.unlearn()
iraf.ccdred.flatcombine.unlearn()
ccdred.instrument = "ccddb$kpno/camera.dat"
iraf.imred()
iraf.ccdred()
iraf.flatcombine(input=flats,
output=out_file,
combine=method,
reject=rejection,
ccdtype=ccdtype,
subset=subset,
process="no",
Stdout="/dev/null")
if dark is not None or zero is not None:
if zero is not None and self.fop.is_file(zero):
zerocor = 'yes'
else:
zerocor = 'no'
if dark is not None and self.fop.is_file(dark):
darkcor = 'yes'
else:
darkcor = 'no'
iraf.ccdproc(images=flats,
ccdtype='',
fixpix='no',
oversca=overscan,
trim=trim,
zerocor=zerocor,
darkcor=darkcor,
flatcor='no',
zero=zero,
dark=dark,
Stdout="/dev/null")
self.etc.log("flatcombine is done!")
except Exception as e:
self.etc.log(e)
def deimos_docalib(flats, arc):
'''Calculate y- and x- distortions, create flat-field, calculate wavelength
solution, for a given grating and central wavelength.'''
# Pre-process flats
for image in flats:
deimos_preproc(image)
# Create median combination of flats
iraf.flatcombine(",".join(["rtd%s_B.fits" % x[6:10] for x in flats]),
output=BFLAT, combine="median", reject="avsigclip")
iraf.flatcombine(",".join(["rtd%s_R.fits" % x[6:10] for x in flats]),
output=RFLAT, combine="median", reject="avsigclip")
# Pre-process arc
graname = get_head(arc, "GRATENAM", extn=0)
deimos_preproc(arc)
barc = "rtd%s_B.fits" % arc[6:10]; rarc = "rtd%s_R.fits" % arc[6:10]
# Generate flats with sharpened edges
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (BFLAT, BSHARP))
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (RFLAT, RSHARP))
# Calculate y-distortion from sharpened frame
os.system('getrect -ydist %s -nx 6 -dy 11 -x 1 -y 1 -peakwin 3 -dump' % BSHARP)
os.system('getrect -ydist %s -nx 12 -dy 11 -x 1 -y 1 -peakwin 3 -dump' % RSHARP)
# Copy y rectification to flats
os.system('copyrect -ydist %s %s' % (BSHARP, BFLAT))
os.system('copyrect -ydist %s %s' % (RSHARP, RFLAT))
# Find slits
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % BFLAT)
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % RFLAT)
# Correct Slit locations
update_head(BFLAT, ["NSLITS", "CSECT1A", "CSECT1B", "CSECT2A", "CSECT2B",
"CSECT3A", "CSECT3B", "CSECT4A", "CSECT4B", "CSECT5A",
"CSECT5B"], DGRATINGS[graname]["bslits"])
update_head(RFLAT, ["NSLITS", "CSECT1A", "CSECT1B", "CSECT2A", "CSECT2B",
"CSECT3A", "CSECT3B", "CSECT4A", "CSECT4B", "CSECT5A",
"CSECT5B"], DGRATINGS[graname]["rslits"])
# Create flat fields
os.system("makeflat -ydist %s" % BFLAT)
os.system("makeflat -ydist %s" % RFLAT)
# Apply distortion, slits and flat-fields to arcs
os.system("copyrect -ydist %s %s" % (BSHARP, barc))
os.system("copyrect -ydist %s %s" % (RSHARP, rarc))
os.system("copyslit %s %s -fft -ydist" % (BFLAT, barc))
os.system("copyslit %s %s -fft -ydist" % (RFLAT, rarc))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (BFLAT.split(".")[0], barc))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (RFLAT.split(".")[0], rarc))
os.system("flat2d %sflt.fits %s" % (BFLAT.split(".")[0], barc))
os.system("flat2d %sflt.fits %s" % (RFLAT.split(".")[0], rarc))
# Create new files for x-distortion
os.system("flat2d %sblz.fits %sf.fits" % (BFLAT.split(".")[0], barc.split(".")[0]))
os.system("flat2d %sblz.fits %sf.fits" % (RFLAT.split(".")[0], rarc.split(".")[0]))
os.system("efits %sblz.fits %sff.fits 'i2*greater(i1,0.1)' xdist_B.fits" %
(BFLAT.split(".")[0], barc.split(".")[0]))
os.system("efits %sblz.fits %sff.fits 'i2*greater(i1,0.1)' xdist_R.fits" %
(RFLAT.split(".")[0], rarc.split(".")[0]))
# Calculate x distortion
os.system("getrect -xdist -ydist xdist_B.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6")
os.system("getrect -xdist -ydist xdist_R.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6")
# Apply x distortion and calculate wavelength solution
os.system("copyrect -xdist xdist_B.fits %sf.fits" % barc.split(".")[0])
os.system("copyrect -xdist xdist_R.fits %sf.fits" % rarc.split(".")[0])
grating = get_head(arc, "GRATENAM", extn=0)
d0 = DGRATINGS[grating]["dispersion"]
cwlen = get_head(arc, DGRATINGS[grating]["tiltkey"], extn=0)
blen = get_head(barc, "NAXIS1"); rlen = get_head(rarc, "NAXIS1")
os.system("waverect -xdist -ydist %sf.fits -o 3 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5" % (barc.split(".")[0], cwlen - blen * d0, cwlen, d0, DARCLIST))
os.system("waverect -xdist -ydist %sf.fits -o 3 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5" % (rarc.split(".")[0], cwlen, cwlen + rlen * d0, d0, DARCLIST))
return
def main(argv):
import getopt, os
import param as p
try:
# accept options -h, -m, -t, -b, -d, -f, -r
opts, argv = getopt.getopt(argv, 'hm:t:b:d:f:r', ['help', 'master=', 'telescope=', 'bias=', 'dark=', 'flat=', 'reduce'])
except getopt.GetoptError:
usage()
sys.exit(2)
for opt, arg in opts:
if opt in ('-h', '--help'):
usage()
sys.exit(2)
# need to select the telescope, see param.py
if opt in ('-r', '--reduce') or opt in ('-m', '--master'):
for opt2, arg2 in opts:
if opt2 in ('-t', '--telescope'):
telescope = p.telescopes[arg2]
try: telescope
except:
usage()
sys.exit(2)
# option -r, reduce frames
if opt in ('-r', '--reduce'):
from pyraf import iraf
import pyfits
# inizialise variables
p = {}
framelist = []
filters = []
if arg:
# one can specify a list of frames, divided by a space
# @todo not tested!
framelist.append(arg.split(' '))
else:
# get list of frames, using function list_frames
frameType = 'Light Frame'
frames = list_frames('.')[0]
for date in frames[frameType]: # select just frameType
for temperature in frames[frameType][date]:
for bin in frames[frameType][date][temperature]:
framesProc = frames[frameType][date][temperature][bin]
for frameProc in framesProc:
# exclude master frames (shouldn't be necessary!)
if not os.path.basename(frameProc['path'])[0:6] == 'Master':
# add frame path to framelist
if not frameProc['filter'] in filters: filters.append(frameProc['filter'])
f = {}
f['path'] = os.path.abspath(frameProc['path'])
f['filter'] = frameProc['filter']
framelist.append(f)
# set bias, dark and flats.
# if nothing is specified, use default Master*.fits
for opt1, arg1 in opts:
if opt1 in ('-b', '--bias'):
if arg1: p['bias'] = arg1
if opt1 in ('-d', '--dark'):
if arg1: p['dark'] = arg1
if opt1 in ('-f', '--flat'):
if arg1: p['flat'] = arg1
if not 'bias' in p: p['bias'] = 'MasterBias.fits'
if not 'dark' in p: p['dark'] = 'MasterDark.fits'
# set flat fields, divided by filter
flatfiles = []
if not 'flats' in p:
import glob
for file in glob.glob('./MasterFlat*'):
f = {}
f['path'] = os.path.abspath(file)
header=pyfits.getheader(file)
f['filter'] = header['FILTER']
flatfiles.append(f)
# check that files exist
try:
with open(p['dark']): pass
except IOError:
print '%s not found' % p['dark']
try:
with open(p['bias']): pass
except IOError:
print '%s not found' % p['bias']
# create copies of original files, with prefix CORR_
import shutil
for f in framelist:
shutil.copy(f['path'], os.path.splitext(f['path'])[0]+'_reduced.fits')
# reduce images by filter
for filter in filters:
# create text list of files for iraf
tmplistcorr = open('tmplistcorr', 'w')
for frame in framelist:
if frame['filter'] == filter:
tmplistcorr.write("%s_reduced.fits\n" % (os.path.splitext(frame['path'])[0]))
tmplistcorr.close()
# select flat field
for flat in flatfiles:
if flat['filter'] == filter: p['flat'] = flat['path']
# reduce frames
print 'Calibrate images with %s filter' % filter
iraf.ccdproc(images='@tmplistcorr',output='', ccdtype='', fixpix='no', \
overscan='no', trim='No', zerocor='yes', zero=p['bias'], \
darkcor='Yes', dark=p['dark'], flatcor='yes', flat=p['flat'], readaxi='column', \
biassec='', trimsec='')
# option -m type: create master frames
elif opt in ('-m', '--master'):
from pyraf import iraf
import pyfits
p = {}
framelist = []
# select frame type
if arg == 'bias' or arg == 'dark' or arg == 'flat':
if arg == 'bias': frameType = 'Bias Frame' # we need 'Bias Frame', not just 'bias'!
elif arg == 'dark': frameType = 'Dark Frame' # same
elif arg == 'flat':
frameType = 'Flat Field' # same
frameflat = [] # frameflat is used to divide files by filter
filters = []
# use function list_frames from reduceall, to list all the frames
# divided by frametype, date, temperature, bin
# and add them to a single list
frames = list_frames('.')[0]
for date in frames[frameType]: # select just frameType
for temperature in frames[frameType][date]:
for bin in frames[frameType][date][temperature]:
framesProc = frames[frameType][date][temperature][bin]
for frameProc in framesProc:
# exclude master frames
if not os.path.basename(frameProc['path'])[0:6] == 'Master':
# add frame path to framelist
framelist.append(os.path.abspath(frameProc['path']))
# for flat fields, create a second list (frameflat)
# that specifies the filter name
if arg == 'flat':
if not frameProc['filter'] in filters: filters.append(frameProc['filter'])
flat = {}
flat['path'] = os.path.abspath(frameProc['path'])
flat['filter'] = frameProc['filter']
frameflat.append(flat)
# set bias frame and/or dark frame.
# if nothing is specified, use default Master*.fits
# check that files actually exist
if arg == 'dark' or arg == 'flat':
for opt1, arg1 in opts:
if opt1 in ('-b', '--bias'):
if arg1: p['bias'] = arg1
if opt1 in ('-d', '--dark') and arg == 'flat':
if arg1: p['dark'] = arg1
if not 'bias' in p: p['bias'] = 'MasterBias.fits'
if not 'dark' in p and arg == 'flat': p['dark'] = 'MasterDark.fits'
if 'dark' in p:
try:
with open(p['dark']): pass
except IOError:
print '%s not found' % p['dark']
if 'bias' in p:
try:
with open(p['bias']): pass
except IOError:
print '%s not found' % p['bias']
# create copies of original files, with prefix CORR_
import shutil
for f in framelist:
shutil.copy(f, os.path.join(os.path.dirname(f), 'CORR_'+os.path.basename(f)))
# create text lists of original and copied files
tmplistcorr = open('tmplistcorr', 'w')
tmplist = open('tmplist', 'w')
for f in framelist:
tmplistcorr.write("%s/CORR_%s\n" % (os.path.dirname(f), os.path.basename(f)))
tmplist.write("%s\n" % f)
tmplistcorr.close()
tmplist.close()
# create MasterBias
if arg == 'bias':
print 'Trim bias frames'
iraf.ccdproc (images='@tmplistcorr',output ='', ccdtype = '', fixpix='no', \
oversca = 'no', trim = 'yes', zerocor = 'no', zero = '', \
flatcor = 'no', darkcor = 'no', readaxi = 'column', \
biassec = '', trimsec=telescope.trimsection)
print 'Creating MasterBias.fits'
iraf.zerocombine (PYinput='@tmplistcorr', output='MasterBias.fits', ccdtype='', \
combine='median', reject='none', rdnoise=telescope.rdnoise, gain=telescope.egain)
# create MasterDark
if arg == 'dark':
print 'Bias subtract dark frames'
iraf.ccdproc(images='@tmplistcorr',output='', ccdtype='', fixpix='no', \
overscan='no', trim='yes', zerocor='yes', zero=p['bias'], \
darkcor='No', flatcor='no', readaxi='column', \
biassec='', trimsec=telescope.trimsection)
print 'Creating MasterDark.fits'
iraf.darkcombine (PYinput='@tmplistcorr', output='MasterDark.fits', ccdtype='', \
combine='median', reject='none', process='no', delete='no', \
clobber='no', scale='exposure', statsec='', nlow='0', nhigh='1', \
nkeep='1', mclip='yes', lsigma='5.', hsigma='4.',rdnoise=telescope.rdnoise,\
gain=telescope.egain, snoise='0.', pclip='-0.5', blank='0.')
# create FlatFields, divided by filter
if arg == 'flat':
print 'Bias and dark subtract flat fields'
iraf.ccdproc(images='@tmplistcorr',output='', ccdtype='', fixpix='no', \
overscan='no', trim='yes', zerocor='yes', zero=p['bias'], \
darkcor='Yes', dark=p['dark'], flatcor='no', readaxi='column', \
biassec='', trimsec=telescope.trimsection)
# divide images by filter and flat combine
for filter in filters:
print 'Combine flat fields - Filter %s' % filter
# create text list for iraf
flatlist = open('flatlist', 'w')
for frame in frameflat:
if frame['filter'] == filter:
flatlist.write("%s\n" % frame['path'])
flatlist.close()
iraf.flatcombine(PYinput = '@flatlist', output='MasterFlat-%s.fits' % filter, ccdtype='', \
combine='median', reject='none', process='no', subsets='no', \
delete='no', clobber='no', scale='mode', statsec=' ', nlow='1', \
nhigh='1', nkeep='1', mclip='yes', lsigma='5.', hsigma='5.', \
rdnoise=telescope.rdnoise, gain=telescope.egain, snoise='0.', pclip='-0.5',blank='1.')
os.remove('flatlist')
# remove working files & temp lists
for f in framelist:
os.remove(os.path.join(os.path.dirname(f), 'CORR_'+os.path.basename(f)))
os.remove('tmplist')
os.remove('tmplistcorr')
def process():
ovrsc = '[1:4,*]'
trim = '[200:2046,50:200]'
imglist = glob.glob('FORS2*.fits')
# objdict = {}
# objdict.clear()
for img in imglist:
hdu = pyfits.getheader(img)
if 'OBJECT' in hdu:
obj = hdu['OBJECT']
if obj not in objdict:
objdict[obj] = [img]
else:
objdict[obj].append(img)
if 'STD' not in objdict:
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')
stars = {'specphot-LTT7379':'l7379','specphot-LDS749B':'lds749b','specphot-EG274':'eg274','specphot-G138-31':'g13831','specphot-C-32d9927':'cd32','specphot-LTT9491':'l9491','specphot-LTT7987':'l7987'}
i = 0
for key in objdict.keys():
if 'STD' in key:
for img in objdict[key]:
i = i + 1
numstars = len(objdict['STD'])
hds = pyfits.getheader(img)
_starname = stars[hds['HIERARCH ESO OBS NAME']]
if _starname in ['lds749b','g13831']:
print 'Bad standard, copying from 2016-05-12'
if not os.path.isdir('badstd'):
os.mkdir('badstd')
os.system('mv '+img+' ./badstd')
if i >= numstars:
objdict.pop('STD')
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')
if os.path.isfile('biaslist'):
os.remove('biaslist')
if os.path.isfile('masterbias.fits'):
os.remove('masterbias.fits')
f = open('biaslist','w')
for img in objdict['BIAS']:
f.write(img+'\n')
f.close()
imglist = '@biaslist'
name = 'masterbias.fits'
hdb = pyfits.getheader(objdict['BIAS'][0])
_gain = hdb['HIERARCH ESO DET OUT1 GAIN']
_ron = hdb['HIERARCH ESO DET OUT1 RON']
iraf.zerocombine(imglist,output=name,combine='average',reject='minmax',ccdtype='none',process='no',gain=_gain,rdnoise=_ron,Stdout=1)
if os.path.isfile('flatlist'):
os.remove('flatlist')
if os.path.isfile('sciflatlist'):
os.remove('sciflatlist')
if os.path.isfile('stdflatlist'):
os.remove('stdflatlist')
if os.path.isfile('masterflat.fits'):
os.remove('masterflat.fits')
if os.path.isfile('scimasterflat.fits'):
os.remove('scimasterflat.fits')
if os.path.isfile('stdmasterflat.fits'):
os.remove('stdmasterflat.fits')
f = open('sciflatlist','w')
for img in objdict['FLAT,LAMP']:
hdu = pyfits.getheader(img)
if hdu['HIERARCH ESO DPR TECH'] == 'SPECTRUM':
f.write(img+'\n')
f.close()
j = 0
f = open('stdflatlist','w')
for img in objdict['FLAT,LAMP']:
hdu = pyfits.getheader(img)
if hdu['HIERARCH ESO DPR TECH'] == 'MOS':
f.write(img+'\n')
j = j + 1
f.close()
imglist = '@sciflatlist'
name = 'scimasterflat.fits'
hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
_gain = hdf['HIERARCH ESO DET OUT1 GAIN']
_ron = hdf['HIERARCH ESO DET OUT1 RON']
iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)
if j == 0:
imglist = '@sciflatlist'
elif j >= 1:
imglist = '@stdflatlist'
name = 'stdmasterflat.fits'
hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
_gain = hdf['HIERARCH ESO DET OUT1 GAIN']
_ron = hdf['HIERARCH ESO DET OUT1 RON']
iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)
if os.path.isfile('tmasterbias.fits'):
os.remove('tmasterbias.fits')
fits = 'masterbias.fits'
name = 'tmasterbias.fits'
zcor = 'no'
fcor = 'no'
_zero = ''
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
if os.path.isfile('tmasterflat.fits'):
os.remove('tmasterflat.fits')
if os.path.isfile('tscimasterflat.fits'):
os.remove('tscimasterflat.fits')
if os.path.isfile('tstdmasterflat.fits'):
os.remove('tstdmasterflat.fits')
fits = 'scimasterflat.fits'
name = 'tscimasterflat.fits'
zcor = 'yes'
fcor = 'no'
_zero = 'tmasterbias.fits'
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
fits = 'stdmasterflat.fits'
name = 'tstdmasterflat.fits'
zcor = 'yes'
fcor = 'no'
_zero = 'tmasterbias.fits'
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
if os.path.isfile('ntmasterflat.fits'):
os.remove('ntmasterflat.fits')
if os.path.isfile('ntscimasterflat.fits'):
os.remove('ntscimasterflat.fits')
if os.path.isfile('ntstdmasterflat.fits'):
os.remove('ntstdmasterflat.fits')
cal = 'tscimasterflat.fits'
resp = 'ntscimasterflat.fits'
_order = 100
iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')
cal = 'tstdmasterflat.fits'
resp = 'ntstdmasterflat.fits'
_order = 100
iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')
##########
ovrsc = '[1:4,*]'
trim = '[200:2046,50:200]'
fimg = 'ntscimasterflat'
_area = '[1:150,*]'
_value = 1.0
iraf.imreplace(images=fimg+_area,value=_value)
fimg = 'ntstdmasterflat'
_area = '[1:150,*]'
_value = 1.0
iraf.imreplace(images=fimg+_area,value=_value)
alist = glob.glob('*AGN*.fits')
for f in alist:
os.remove(f)
slist = glob.glob('STD*.fits')
for f in slist:
os.remove(f)
llist = glob.glob('LAMP*.fits')
for f in llist:
os.remove(f)
zcor = 'yes'
fcor = 'yes'
_zero = 'tmasterbias.fits'
sciflat = 'ntscimasterflat.fits'
stdflat = 'ntstdmasterflat.fits'
for key in objdict.keys():
if 'AGN' in key:
for img in objdict[key]:
hds = pyfits.getheader(img)
if 'EXPTIME' in hds:
if hds['EXPTIME'] >= 50.0:
num = img.rsplit('.',2)[1]
name = key+'_'+num
'''
iraf.ccdproc(img,output='trim_'+name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor='no',darkcor='no',flatcor='no',illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero='',flat='',Stdout=1)
_gain = hds['HIERARCH ESO DET OUT1 GAIN']
_ron = hds['HIERARCH ESO DET OUT1 RON']
cosmics.lacos('trim_'+name+'.fits', output='c_'+name+'.fits', gain=_gain, readn=_ron, xorder=9, yorder=9, sigclip=4.5, sigfrac=0.5, objlim=1, verbose=True, interactive=False)
iraf.ccdproc('c_'+name,output=name,ccdtype='',noproc='no',fixpix='no',overscan='no',trim='no',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec='',trimsec='',zero=_zero,flat=sciflat,Stdout=1)
'''
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=sciflat,Stdout=1)
else:
pass
elif 'STD' in key:
for img in objdict[key]:
num = img.rsplit('.',2)[1]
name = key+'_'+num
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)
elif 'WAVE' in key:
for img in objdict[key]:
num = img.rsplit('.',2)[1]
name = 'LAMP_'+num
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)
def skyflat(date, low=15000, high=23000, numimages=5):
'''
make a combined b skyflat.
Requires: a bias image in same directory to do the bias subtraction
skyflats must be offset and have appropriate count number
input: the date the skyflats were observed YYMMDD
output: flat.B, a text file that lists the names of the skyflat fits files
ccdYYMMDD.skyflatB.fits, the combined skyflat
'''
#check if biases are in this directory
if len(glob.glob('*.bias.*')) < 1:
print "no combined bias found, exiting"
return
#get image name and mean pixel value for all skyflat images
stats = iraf.imstat('*sky*', format=False, fields='image,mean', Stdout=1)
pairs = [i.split() for i in stats]
#write the names of the skyflats w right ammount of counts to file
#keep track of how many good ones there are
goodCount = 0
with open("flat.B", 'w') as FB:
for i in pairs:
if float(i[1]) > low and float(i[1]) < high:
FB.write(i[0] + '\n')
goodCount += 1
if goodCount < numimages:
print "only " + str(
goodCount) + " skyflats have counts between " + str(
low) + " and " + str(high)
print "no combined skyflat made"
return
else:
iraf.ccdproc(images="@flat.B",
output=" ",
fixpix="no",
overscan="yes",
trim="no",
zerocor="yes",
darkcor="no",
flatcor="no",
illumcor="no",
fringecor="no",
readcor="no",
scancor="no",
readaxis="line",
biassec="[3:14,1:1024]",
zero="*.bias.fits",
interactive="no",
functio="spline3",
order=11)
iraf.flatcombine("@flat.B",
output="FLAT",
combine="median",
reject="minmax",
process="no",
scale="mode",
ccdtype="")
os.system("mv FLAT.fits ccd" + str(date) + ".skyflatB.fits")
print("made combined skyflat ccd" + str(date) + ".skyflatB.fits")
return
def run_flatcom(input):
iraf.noao.imred(_doprint=0)
iraf.noao.imred.ccdred(_doprint=0)
iraf.flatcombine(input[0],output=input[1],combine="average",scale="mean",reject="none",nlow="0",nhigh="0",nkeep="1",ccdtype="",process="no",subsets="no",delete="no",clobber="no",statsec="[100:1000,100:1000]",mclip="no",lsigma="3",hsigma="3",rdnoise="6",gain="7",snoise="0",pclip="-0.5",blank="1.0")
def lris_do_bcalib(flats, arcs):
'''Calculate y- and x- distortions, create flat-field, calculate wavelength
solution, for a given grism.'''
# Pre-process flats
for image in flats:
os.system("lris_bpreproc.py %s" % image)
# Create median combination of flats
iraf.flatcombine(",".join(["rtb%s.fits" % x[8:12] for x in flats]),
output=BFLAT, combine="median", reject="avsigclip",
ccdtype="", process=no, subsets=no, delete=no, clobber=no,
scale="none", statsec="", lsigma=3.0, hsigma=3.0, rdnoise=0.,
gain=1.)
# Pre-process arc
grism = get_head("../rawdata/%s" % arcs[0], "GRISNAME", extn=0)
for image in arcs:
os.system("lris_bpreproc.py %s" % image)
#iraf.imarith("rtb%s.fits" % arcs[0][8:12], "+", "rtb%s.fits" % arcs[1][8:12], BARC)
os.system("cp rtb%s.fits %s" % (arcs[0][8:12], BARC))
# Generate flats with sharpened edges
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (BFLAT, BSHARP))
# Calculate y-distortion from sharpened frame
os.system('getrect -ydist %s -nx 12 -dy 11 -x 1 -y 1 -peakwin 3 -dump' % BSHARP)
# Copy y rectification to flats
os.system('copyrect -ydist %s %s' % (BSHARP, BFLAT))
# Find slits
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % BFLAT)
# Correct Slit locations
grism = get_head(BFLAT, "GRISNAME")
update_head(BFLAT, ["NSLITS", "CSECT1A", "CSECT1B", "CSECT2A", "CSECT2B"],
LGRISMS[grism]["slits"])
# Create flat fields
os.system("makeflat -ydist %s" % BFLAT)
# Bluest part of the flat has too few counts to be useful.
x = pyfits.open("%sflt.fits" % BFLAT.split(".")[0])
x[0].data[LGRISMS[grism]["slits"][1]:LGRISMS[grism]["slits"][2],0:1150] = 1
x[0].data[LGRISMS[grism]["slits"][3]:LGRISMS[grism]["slits"][4],0:1150] = 1
x.writeto("%sflt.fits" % BFLAT.split(".")[0], clobber=True)
# Apply distortion, slits and flat-fields to arcs
os.system("copyrect -ydist %s %s" % (BSHARP, BARC))
#os.system("copyslit %s %s -fft -ydist" % (BFLAT, BARC))
os.system("copyslit %s %s -ydist" % (BFLAT, BARC))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (BFLAT.split(".")[0], BARC))
os.system("flat2d %sflt.fits %s" % (BFLAT.split(".")[0], BARC))
# Create new files for x-distortion
#os.system("flat2d %sblz.fits %sf.fits" % (BFLAT.split(".")[0],
# BARC.split(".")[0]))
os.system("efits %sblz.fits %sf.fits 'i2*greater(i1,0.01)' xdist_B.fits" %
(BFLAT.split(".")[0], BARC.split(".")[0]))
# Calculate x distortion
os.system("getrect -xdist -ydist xdist_B.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6")
# Apply x distortion and calculate wavelength solution
os.system("copyrect -xdist xdist_B.fits %sf.fits" % BARC.split(".")[0])
d0 = LGRISMS[grism]["dispersion"]
[w1, w2] = LGRISMS[grism]["wavelengths"]
os.system("waverect -xdist -ydist %sf.fits -o 5 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5 -bth 5.0" % (BARC.split(".")[0], w1, w2, d0, LBARCLIST))
return
print "> creating master dark image"
iraf.darkcombine("Dark*.fit",
output="master_dark.fits",
combine="average",
reject="none",
ccdtype="dark",
scale="none",
process="no")
# creating master flat
unlearn(iraf.flatcombine)
print "> creating master flat image"
iraf.flatcombine("Flat*.fit",
output="master_flat_temp.fits",
combine="median",
reject="crreject",
ccdtype="flat",
process="no",
delete="no",
scale="median")
# removing 0s and negative values with standard ones (-9.0)
iraf.imcalc("master_flat_temp.fits",
output="master_flat.fits",
equals="if (im1.le.0.0) then -9.0 else im1",
verbose="no")
# creating the necessary lists for dark and flat processing
rd = open('rawdata.irafl', 'w')
cd = open('cordata.irafl', 'w')
for im in data:
# print im
def efoscreduction(imglist, _interactive, _doflat, _dobias, listflat, listbias, _dobadpixel, badpixelmask,
fringingmask, _archive, typefile, filenameobjects, _system, _cosmic, _verbose=False, method='iraf'):
# print "LOGX:: Entering `efoscreduction` method/function in %(__file__)s"
# % globals()
import ntt
from ntt.efoscphotredudef import searchbias
from ntt.util import delete, readhdr, readkey3, display_image, searchflat, rangedata, correctcard
from numpy import argmin, min, abs, sqrt
import string
import os
import re
import math
import sys
from pyraf import iraf
# ## Call and set parameters for useful iraf tasks
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.proto(_doprint=0)
toforget = ['ccdproc', 'zerocombine',
'flatcombine', 'imreplace', 'proto.fixpix']
for t in toforget:
iraf.unlearn(t)
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.overscan = 'no'
iraf.ccdproc.ccdtype = ''
iraf.ccdproc.biassec = ''
iraf.ccdred.instrument = "/dev/null"
if _verbose:
iraf.ccdred.verbose = 'yes'
else:
iraf.ccdred.verbose = 'no'
import datetime
import time
# starttime=time.time()
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
outputfile = []
reduceddata = rangedata(imglist)
img = re.sub('\n', '', imglist[0])
hdr = readhdr(img)
_gain = readkey3(hdr, 'gain')
_rdnoise = readkey3(hdr, 'ron')
_instrume = readkey3(hdr, 'instrume')
_trimsec = '[3:1010,1:1015]'
biaslist = {}
flatlist1 = {}
flatlist2 = {}
objectlist = {}
filterlist1 = []
filterlist2 = []
for img in imglist:
_type = ''
img = re.sub('\n', '', img)
hdr = readhdr(img)
_naxis1 = readkey3(hdr, 'NAXIS1')
_naxis2 = readkey3(hdr, 'NAXIS2')
if _naxis1 != 1030 or _naxis2 != 1030:
ntt.util.writeinthelog(
'image ' + str(img) + ' different dimension =\n', './logNTT.txt')
_type = 'not good'
if not _type and readkey3(hdr, 'speed') != 'fastL':
_type = 'not good'
if not _type and readkey3(hdr, 'instrume') != 'efosc':
_type = 'not good'
_imagetype = readkey3(hdr, 'tech')
if not _type and _imagetype == 'SPECTRUM':
_type = 'spectroscopic data'
if not _type:
_exptime = readkey3(hdr, 'exptime')
_date = readkey3(hdr, 'date-night')
_filter = readkey3(hdr, 'filter')
if float(_exptime) == 0.0:
if _date not in biaslist:
biaslist[_date] = []
biaslist[_date].append(img)
_type = 'bias'
if not _type:
_object = readkey3(hdr, 'object')
if _filter.lower() in ['g782', 'r784', 'z623', 'u640', 'b639', 'v641', 'r642',
'i705'] and _imagetype == 'IMAGE':
if 'sky,flat' in _object.lower():
_type = 'flat'
elif 'dome' in _object.lower() or 'flat' in _object.lower():
_type = 'flat dome'
if _type == 'flat':
if _filter not in filterlist1:
filterlist1.append(_filter)
flatlist1[_filter] = []
flatlist1[_filter].append(img)
if _type == 'flat dome':
if _filter not in filterlist2:
filterlist2.append(_filter)
flatlist2[_filter] = []
flatlist2[_filter].append(img)
if not _type:
_catg = readkey3(hdr, 'catg')
if 'science' in _catg.lower() or 'acquisition' in _catg.lower():
_type = 'object'
if _filter not in objectlist:
objectlist[_filter] = []
objectlist[_filter].append(img)
if 'acquisition' in _catg.lower():
try:
correctcard(img)
_ra1, _dec1, _name = ntt.util.correctobject(
img, 'standard_efosc_mab.txt')
_ra1, _dec1, _name = ntt.util.correctobject(
img, filenameobjects)
except:
pass
elif 'focus' in _object.lower():
_type = 'not good'
if not _type:
print '\n### warning: object not recognized '
_object = readkey3(hdr, 'object')
print img, _object, _imagetype
answ = raw_input(
'what is it: bias [1], flat [3], object[4], test [5] ? [5] ')
if not answ:
answ = '5'
if answ == '1':
if _date not in biaslist:
biaslist[_date] = ()
biaslist[_date].append(img)
elif answ == '4':
if _filter not in objectlist:
objectlist[_filter] = []
objectlist[_filter].append(img)
elif answ == '3':
tt = raw_input('dome or sky [d/[s]] ? ')
if tt == 's':
_type = 'flat'
_filter = readkey3(hdr, 'filter')
if _filter not in filterlist1:
filterlist1.append(_filter)
flatlist1[_filter] = []
flatlist1[_filter].append(img)
elif tt == 'd':
_type = 'flat dome'
_filter = readkey3(hdr, 'filter')
if _filter not in filterlist2:
filterlist2.append(_filter)
flatlist2[_filter] = []
flatlist2[_filter].append(img)
elif answ == '5':
_type = 'not good'
filterlist = list(set(filterlist1 + filterlist2))
if _verbose:
print filterlist1
print filterlist2
print flatlist1
print flatlist2
flatlist = {}
for _filt in filterlist:
if _filt not in flatlist1.keys():
if _filt in flatlist2.keys():
if len(flatlist2[_filt]) >= 3:
flatlist[_filt] = flatlist2[_filt]
elif len(flatlist1[_filt]) < 3:
if _filt in flatlist2.keys():
if len(flatlist2[_filt]) >= 3:
flatlist[_filt] = flatlist2[_filt]
elif _filt in flatlist1.keys():
if len(flatlist1[_filt]) >= 3:
flatlist[_filt] = flatlist1[_filt]
listaout = []
if _verbose:
print '\n### flat ', str(flatlist), '\n'
print '\n### bias ', str(biaslist), '\n'
print '\n### object ', str(objectlist), '\n'
###### masterbias #################
if _dobias:
if not _archive:
if listbias:
masterbiaslist = listbias
else:
masterbiaslist = []
if biaslist:
for _date in biaslist:
print '\n do bias ' + str(_date) + '\n'
biaslist[_date] = rejectbias(
biaslist[_date], False, 10)
if len(biaslist[_date]) >= 3:
masterbiasfile = 'bias_' + \
str(_date) + '_' + str(MJDtoday) + '.fits'
delete(masterbiasfile)
f = open('biaslist', 'w')
h = open('obiaslist', 'w')
for img in biaslist[_date]:
f.write(img + '\n')
h.write('o' + img + '\n')
delete('o' + img)
f.close()
h.close()
try:
print 'processing bias .....'
iraf.ccdproc('@biaslist', output='@obiaslist', overscan="no", trim="yes", zerocor='no',
fixpix='no', ccdtype='', flatcor='no', darkcor='no', biassec='',
trimsec=str(_trimsec), readaxi='column', Stdout=1)
iraf.zerocombine('@obiaslist', output=masterbiasfile, combine='median',
reject='ccdclip', ccdtype='', process='no',
rdnoise=_rdnoise, gain=_gain, Stdout=1)
correctcard(masterbiasfile)
num = 0
for img in biaslist[_date]:
num = num + 1
ntt.util.updateheader(masterbiasfile, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader(masterbiasfile, 0, {
'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
delete('o' + img)
ntt.util.updateheader(masterbiasfile, 0,
{'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})
ntt.util.updateheader(masterbiasfile, 0,
{'SINGLEXP': [False, 'TRUE if resulting from single exposure']})
ntt.util.updateheader(masterbiasfile, 0, {
'FILETYPE': [11201, 'bias']})
masterbiaslist.append(masterbiasfile)
if masterbiasfile not in outputfile:
outputfile.append(masterbiasfile)
except:
ntt.util.writeinthelog(
'Warning ' +
str(biaslist[_date]) +
' problem with this list of bias \n',
'./logNTT.txt')
if masterbiasfile and _interactive:
aa, bb, cc = display_image(
masterbiasfile, 1, '', '', False)
answ = raw_input(
'is the masterbias ok [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['n', 'no']:
sys.exit(
'remove bad bias from input list and restart')
else:
masterbiaslist = []
########## masterflat #########################
if _doflat:
if not _archive:
if listflat:
masterflatlist = listflat
else:
masterflatlist = []
if flatlist:
for _filter in flatlist:
print '\n do flat ' + str(_filter) + '\n'
flatlist[_filter] = rejectflat(
flatlist[_filter], False)
if len(flatlist[_filter]) >= 3:
_date = readkey3(
readhdr(flatlist[_filter][0]), 'date-night')
masterflat = 'flat_' + \
str(_date) + '_' + str(_filter) + \
'_' + str(MJDtoday) + '.fits'
listaflat = 'flatlist_' + \
str(_date) + '_' + str(_filter)
_bias = ''
if masterbiaslist:
_bias = searchbias(flatlist[_filter][
0], masterbiaslist)[0]
if not _bias:
_bias = searchbias(flatlist[_filter][0], '')[0]
if _bias:
if _bias[0] == '/':
os.system('cp ' + _bias + ' .')
_bias = string.split(_bias, '/')[-1]
_zerocor = 'yes'
else:
_zerocor = 'yes'
else:
_zerocor = 'no'
answ0 = 'n'
while answ0 != 'y':
f = open(listaflat, 'w')
h = open('o' + listaflat, 'w')
for img in flatlist[_filter]:
f.write(img + '\n')
h.write('o' + img + '\n')
delete('o' + img)
f.close()
h.close()
try:
print 'processing flat .....'
iraf.ccdproc('@' + listaflat, output='@o' + listaflat, overscan='no', trim='yes',
darkcor='no', fixpix='no',
zerocor=_zerocor, flatcor='no', trimsec=str(_trimsec), biassec='',
zero=_bias, readaxi='column', ccdtype='', Stdout=1)
delete(masterflat)
iraf.flatcombine('@o' + listaflat, output=masterflat, combine='average',
reject='avsigclip', ccdtype='', process='no',
rdnoise=_rdnoise, gain=_gain, statsec='[100:800,100:800]',
lsigma=3, hsigma=2, Stdout=1)
masterflatlist.append(masterflat)
correctcard(masterflat)
num = 0
for img in flatlist[_filter]:
num = num + 1
ntt.util.updateheader(masterflat, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader(masterflat, 0, {
'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
delete('o' + img)
ntt.util.updateheader(
masterflat, 0, {'ZEROCOR': [_bias, '']})
ntt.util.updateheader(masterflat, 0, {
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})
ntt.util.updateheader(masterflat, 0, {
'SINGLEXP': [False, 'TRUE if resulting from single exposure']})
ntt.util.updateheader(
masterflat, 0, {'FILETYPE': [11202, 'flat field']})
if masterflat not in outputfile:
outputfile.append(masterflat)
except:
ntt.util.writeinthelog(
'Warning ' +
str(flatlist[
_filter]) + ' problem with this list of flat \n',
'./logNTT.txt')
aa, bb, cc = display_image(
masterflat, 1, '', '', False)
if masterflat and _interactive:
answ = raw_input(
'is the masterflat ok [[y]/n] ?')
if not answ:
answ = 'y'
if answ.lower() in ['n', 'no']:
answ1 = raw_input(
'try again [[y]/n] ?')
if not answ1:
answ1 = 'y'
if answ1.lower() in ['y', 'yes']:
flatlist[_filter] = ntt.efoscphotredudef.rejectflat(
flatlist[_filter], True)
else:
sys.exit(
'error: problem with flat .... exit')
else:
answ0 = 'y'
else:
answ0 = 'y'
else:
masterflatlist = []
##########################################################################
if len(masterbiaslist) == 0:
masterbiaslist = ''
if len(masterflatlist) == 0:
masterflatlist = ''
######################################
if _verbose:
print ''
print '#############################'
print masterflatlist
print masterbiaslist
print '#############################'
print ''
if masterflatlist:
listaout = listaout + masterflatlist
if masterbiaslist:
listaout = listaout + masterbiaslist
if typefile == 'calib':
objectlist = {}
for _filter in objectlist:
for img in objectlist[_filter]:
hdr = readhdr(img)
print '\n#####################################################################\n'
_object = readkey3(hdr, 'object')
_object = re.sub(' ', '', _object)
_object = re.sub('/', '_', _object)
_object = re.sub('\n', '', _object)
_exptime = readkey3(hdr, 'exptime')
_date = readkey3(hdr, 'date-night')
nameout = ntt.util.name_duplicate(img, str(_object) + '_' + str(_date) + '_' + str(_filter) + '_' + str(
MJDtoday), '')
_bias = ''
if _dobias:
if masterbiaslist:
_bias = searchbias(img, masterbiaslist)[0]
if not _bias:
_bias = searchbias(img, '')[0]
_flat = ''
if _doflat:
if masterflatlist:
_flat = searchflat(img, masterflatlist)[0]
if not _flat:
_flat = searchflat(img, '')[0]
if _bias: # bias ###
if _bias[0] == '/':
os.system('cp ' + _bias + ' .')
_bias = string.split(_bias, '/')[-1]
_zerocor = 'yes'
else:
_zerocor = 'no'
else:
_zerocor = 'no'
if _flat: # flat ###
if _flat[0] == '/':
os.system('cp ' + _flat + ' .')
_flat = string.split(_flat, '/')[-1]
_flatcor = 'yes'
else:
_flatcor = 'no'
sss = str(_object) + '_' + str(_date) + '_' + str(_filter)
print '### input', img, sss
print '### bias ', _zerocor, _bias
print '### flat ', _flatcor, _flat
print '### name ', nameout
delete(nameout)
try:
iraf.ccdproc(img, output=nameout, overscan="no", trim="yes", zerocor=_zerocor, flatcor='no',
darkcor='no', trimsec=str(_trimsec), zero=_bias, biassec='', readaxi='column', Stdout=1)
try:
iraf.ccdproc(nameout, output='', overscan="no", trim="no", zerocor='no', flatcor=_flatcor,
darkcor='no', flat=_flat, readaxi='column', ccdtype='', Stdout=1)
except:
iraf.imrepla(images=_flat, value=0.01,
lower='INDEF', upper=0.01, radius=0)
iraf.ccdproc(nameout, output='', overscan="no", trim="no", zerocor='no', flatcor=_flatcor,
darkcor='no', flat=_flat, readaxi='column', ccdtype='', Stdout=1)
correctcard(nameout)
ntt.util.updateheader(nameout, 0, {'FILTER': [readkey3(readhdr(nameout), 'filter'), 'Filter name'],
'SINGLEXP': [True, 'TRUE if resulting from single exposure'],
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
'FLATCOR': [_flat, ''],
'ZEROCOR': [_bias, ''], 'FILETYPE': [12204, 'pre-reduced image'],
'PROV1': [readkey3(readhdr(nameout), 'ARCFILE'), 'Originating file'],
'NCOMBINE': [1, 'Number of raw science data'],
'TRACE1': [readkey3(readhdr(nameout), 'ARCFILE'),
'Originating file']})
ntt.util.airmass(nameout) # phase 3 definitions
ntt.util.writeinthelog('\n', './logNTT.txt')
ntt.util.writeinthelog(
'image= ' + str(img) + ' output= ' + str(nameout) + '\n', './logNTT.txt')
ntt.util.writeinthelog(
'bias= ' + str(_bias) + ', flat= ' + str(_flat) + '\n', './logNTT.txt')
ntt.util.writeinthelog('\n', './logNTT.txt')
if nameout not in outputfile:
outputfile.append(nameout)
except:
ntt.util.writeinthelog(
'image ' + str(img) + ' probably corrupted\n', './logNTT.txt')
if _dobadpixel:
if not badpixelmask:
badpixelmask = 'bad_pixel_mask.fits'
delete(badpixelmask)
os.system('cp ' + ntt.__path__[0] + '/archive/' + str(
_instrume) + '/badpixels/badpixel.fits ' + badpixelmask)
iraf.proto.fixpix(images=nameout, masks=badpixelmask,
linterp='INDEF', cinterp='INDEF', verbose='no')
ntt.util.updateheader(
nameout, 0, {'FIXPIX': [badpixelmask, '']})
ntt.util.writeinthelog('image ' + str(nameout) + ' bad pixel corrected with ' + badpixelmask + '\n',
'./logNTT.txt')
print '\n### bad pixel mask correction ..... done'
else:
ntt.util.writeinthelog(
'image ' + str(nameout) + ' bad pixel NOT corrected\n', './logNTT.txt')
if _cosmic:
try:
print '\n### cosmic ..... '
ntt.cosmics.lacos_im(nameout, _output='', gain=_gain, readn=_rdnoise, xorder=9, yorder=9,
sigclip=4.5, sigfrac=0.5, objlim=1, skyval=0, niter=0, verbose=True,
interactive=False)
ntt.util.updateheader(nameout, 0, {
'LACOSMIC': [True, 'TRUE if Laplacian cosmic ray rejection has been applied to the image']})
print '\n### cosmic ..... removed '
except Exception, e:
print e
else:
ntt.util.updateheader(nameout, 0, {
'LACOSMIC': [False, 'TRUE if Laplacian cosmic ray rejection has been applied to the image']})
try:
##########################
sexvec = ntt.efoscastrodef.sextractor(nameout)
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, 40, 40, 100, 'rxyscale', 100, 30, sexvec, True, 10, method)
if rmsx3 <= 2 and rmsy3 <= 2:
break
if rmsx3 > 2 and rmsy3 > 2:
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, 20, int(20), int(50), 'rxyscale', 100, 30, sexvec, True, 5, method)
if rmsx3 <= 2 and rmsy3 <= 2:
break
if rmsx3 > 2 and rmsy3 > 2:
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, int(10), int(10),
int(25), 'rxyscale', 100, 30, sexvec, True, int(3), method)
##########################
astrostring = str(rmsx3) + ' ' + str(rmsy3) + ' ' + str(num3)
ntt.util.updateheader(
nameout, 0, {'ASTROMET': [astrostring, 'rmsx rmsy nstars']})
print '\n### check astrometry: fine \n### rmsx rmsy nstars: ' + astrostring
except Exception, e:
print e
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = '', '', '', '', '', '', '', '', ''
print '\n### problem with astrometry, do you have network ? '
if fwhmgess and fwhmgess < 99:
ntt.util.updateheader(nameout, 0, {'PSF_FWHM': [fwhmgess, 'Spatial resolution (arcsec)'],
'ELLIPTIC': [ellgess, 'Average ellipticity of point sources'],
'CRDER1': [(1 / sqrt(2.)) * float(rmsx3) * (1. / 3600.),
'Random error (degree)'],
'CRDER2': [(1 / sqrt(2.)) * float(rmsy3) * (1. / 3600.),
'Random error (degree)'],
'CUNIT1': ['deg', 'unit of the coord. trans.'],
'CUNIT2': ['deg', 'unit of the coord. trans.'],
'CSYER1': [rasys3, 'Systematic error (RA_m - Ra_ref)'],
'CSYER2': [decsys3, 'Systematic error (DEC_m - DEC_ref)']})
else:
ntt.util.updateheader(nameout, 0, {'PSF_FWHM': [9999., 'FHWM (arcsec) - computed with sectractor'],
'ELLIPTIC': [9999., 'ellipticity of point sources (1-b/a)'],
'CRDER1': [9999., 'Random error in axis 1'],
'CRDER2': [9999., 'Random error in axis 2'],
'CUNIT1': ['deg', 'unit of the coord. trans.'],
'CUNIT2': ['deg', 'unit of the coord. trans.'],
'CSYER1': [9999., 'Systematic error (RA_m - Ra_ref)'],
'CSYER2': [9999., 'Systematic error (DEC_m - DEC_ref)']})
try:
result = ntt.efoscastrodef.zeropoint(
nameout, _system, method, False, False)
except:
result = ''
if result:
if os.path.isfile(re.sub('.fits', '.ph', nameout)):
if re.sub('.fits', '.ph', nameout) not in outputfile:
outputfile.append(
re.sub('.fits', '.ph', nameout))
print '\n### zeropoint ..... done'
for ll in result:
valore = '%3.3s %6.6s %6.6s' % (
str(ll), str(result[ll][1]), str(result[ll][0]))
print '### ', valore
ntt.util.updateheader(
nameout, 0, {'zp' + ll: [str(valore), '']})
if magsat3:
if readkey3(readhdr(nameout), 'FLUXCAL') == 'ABSOLUTE':
try:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3) + float(readkey3(readhdr(nameout)), 'PHOTZP'),
'Saturation limit for point sources (AB mags)']})
except:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3), 'Saturation limit for point sources (AB mags)']})
else:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3), 'Saturation limit for point sources (AB mags)']})
else:
ntt.util.updateheader(nameout, 0, {'ABMAGSAT': [
9999., 'Saturation limit for point sources (AB mags)']})
maglim = ntt.util.limmag(nameout)
if maglim:
ntt.util.updateheader(nameout, 0,
{'ABMAGLIM': [maglim, '5-sigma limiting AB magnitude for point sources']})
else:
ntt.util.updateheader(nameout, 0,
{'ABMAGLIM': [9999., '5-sigma limiting AB magnitude for point sources']})
if readkey3(readhdr(nameout), 'filter') in ['i705']:
try:
nameout, maskname = ntt.efoscphotredudef.fringing2(
nameout, fringingmask, _interactive, False)
if nameout not in outputfile:
outputfile.append(nameout)
if maskname not in outputfile:
outputfile.append(maskname)
except:
ntt.util.writeinthelog(
'image ' + str(nameout) + ' probably corrupted\n', './logNTT.txt')
print '\n### problem with fringing correction'
os.system('mv ' + 'ARC_blue.fits' + ' ' + 'pre_reduced' + '/')
os.system('mv ' + 'ARC_red.fits' + ' ' + 'pre_reduced' + '/')
if mkflat != 'n':
inter = 'yes'
iraf.specred.dispaxi = 1
if len(list_flat_b) == 1:
Flat_blue = list_flat_b[0]
else:
flat_str = ''
for flat in list_flat_b:
flat_str = flat_str + 't' + flat + ','
iraf.flatcombine(flat_str,
output='tFlat_blue',
ccdtype='',
rdnoise=3.7,
subsets='yes',
process='no')
Flat_blue = 'Flat_blue.fits'
iraf.specred.response('t' + Flat_blue,
normaliz='t' + Flat_blue,
response='RESP_blue',
interac=inter,
thresho='INDEF',
sample='*',
naverage=2,
function='legendre',
low_rej=3,
high_rej=3,
order=60,
def createAverageFlat(self):
iraf.flatcombine(input=self.inputFolder,process="No",output=self.avaragedFlat)
def run_flatcom(input):
iraf.noao.imred(_doprint=0)
iraf.noao.imred.ccdred(_doprint=0)
iraf.flatcombine(input[0],output=input[1],combine="median",scale="mode",reject="avsigclip",nlow="0",nhigh="0",nkeep="3",ccdtype="",process="no",subsets="no",delete="no",clobber="no",statsec="[300:800,300:800]",mclip="yes",lsigma="2.7",hsigma="2.7",rdnoise="6.0",gain="7.0")
infile=open('junkfile','r')
set_filter=set(filter)
set_ftype=set(ftype)
array_ftype=np.array(ftype)
array_filter=np.array(filter)
for f in set_ftype:
print "flat type=",f
for element in set_filter:
ftype_filter = str(f)+str(element)
print 'filter type = ',element
indices=np.where((array_ftype == f)&(array_filter == element))
print indices, len(indices[0])
if len(indices[0]) > 0:
outfile = open(ftype_filter,'w')
for i in indices[0]:
outfile.write(fnames[i]+'\n')
outfile.close()
os.remove('junkfile')
flats = glob.glob('*flat*')
for f in flats:
iraf.flatcombine(input='@'+f, output='c'+f, combine=median)
mean = iraf.imstat(input=f, fields='mean', lower='INDEF', format=0, Stdout=1)
iraf.imarith(operand1='@c'+f, op='/', operand2=mean, result='n'+f
for ifile in flist:
ifile = ifile.strip()
params = '../../dcr ' + ifile + ' ' + ifile + ' crmask >> reductionlog.txt'
call(params, shell=True)
flist.close()
# COMBINE 'RED' AND 'BLUE' FLATS SEPARATELY
# Separately combine 'red' and 'blue' flat field frames to produce mean flats
print 'Combining red and blue flats into separate fiducial flats...'
iraf.flatcombine('@flat_blue',
output='flat_blue_mean.fits',
combine='median',
ccdtype='',
reject='avsigclip',
process=no,
delete=no,
scale='none',
mclip=yes,
lsigma=3.0,
hsigma=3.0)
iraf.flatcombine('@flat_red',
output='flat_red_mean.fits',
combine='median',
ccdtype='',
reject='avsigclip',
process=no,
delete=no,
scale='none',
mclip=yes,
lsigma=3.0,
def lris_do_rcalib(flats, arcs):
'''Calculate y- and x- distortions, create flat-field, calculate wavelength
solution, for a given grating and central wavelength.'''
# Pre-process flats
for image in flats:
os.system("lris_rpreproc.py %s" % image)
# Create median combination of flats
iraf.flatcombine(",".join(["rtr%s.fits" % x[8:12] for x in flats]),
output=RFLAT, combine="median", reject="avsigclip",
ccdtype="", process=no, subsets=no, delete=no, clobber=no,
scale="none", statsec="", lsigma=3.0, hsigma=3.0, rdnoise=0.,
gain=1.)
# Pre-process arcs
for arc in arcs:
os.system("lris_rpreproc.py %s" % arc)
iraf.imcopy("rtr%s.fits" % arc[8:12], "%s%s.fits" % (RARC, arc[8:12]))
# Generate flats with sharpened edges
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (RFLAT, RSHARP))
# Calculate y-distortion from sharpened frame
os.system('getrect -ydist %s -nx 12 -dy 11 -x 1 -y 1 -peakwin 3 -dump' % RSHARP)
# Copy y rectification to flats
os.system('copyrect -ydist %s %s' % (RSHARP, RFLAT))
# Find slits
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % RFLAT)
# Correct Slit locations
[grating, binning] = get_head(RFLAT, ["GRANAME", "CCDSUM"])
update_head(RFLAT, "NSLITS", LGRATINGS[grating]["slits"][0])
update_head(RFLAT, "CSECT1A",
LGRATINGS[grating]["slits"][1] / int(binning.split()[0]))
update_head(RFLAT, "CSECT1B",
LGRATINGS[grating]["slits"][2] / int(binning.split()[0]))
update_head(RFLAT, "CSECT2A",
LGRATINGS[grating]["slits"][3] / int(binning.split()[0]))
update_head(RFLAT, "CSECT2B",
LGRATINGS[grating]["slits"][4] / int(binning.split()[0]))
# Create flat fields
os.system("makeflat -ydist %s" % RFLAT)
# Correct for bad pixels
x = pyfits.open("%sflt.fits" % RFLAT.split(".")[0])
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],0:21] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],0:20] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],1845:1930] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],1845:1930] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],3275:3355] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],3275:3355] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],3375:3475] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],3375:3475] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],4061:] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],4061] = 1
x.writeto("%sflt.fits" % RFLAT.split(".")[0], clobber=True)
for arc in arcs:
# Apply distortion, slits and flat-fields to arcs
os.system("copyrect -ydist %s %s%s.fits" % (RSHARP, RARC, arc[8:12]))
#os.system("copyslit %s %s -fft -ydist" % (RFLAT, RARC))
os.system("copyslit %s %s%s.fits -ydist" % (RFLAT, RARC, arc[8:12]))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (RFLAT.split(".")[0], RARC))
os.system("flat2d %sflt.fits %s%s.fits" % (RFLAT.split(".")[0], RARC,
arc[8:12]))
# Create new files for x-distortion
#os.system("flat2d %sblz.fits %sf.fits" % (RFLAT.split(".")[0],
# RARC.split(".")[0]))
os.system("efits %sblz.fits %s%sf.fits 'i2*greater(i1,0.0)' xdist_R%s.fits" %
(RFLAT.split(".")[0], RARC, arc[8:12], arc[8:12]))
# Calculate x distortion
os.system("getrect -xdist -ydist xdist_R%s.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6" % arc[8:12])
# Apply x distortion and calculate wavelength solution
os.system("copyrect -xdist xdist_R%s.fits %s%sf.fits" % (arc[8:12], RARC,
arc[8:12]))
[grating, cwlen] = get_head("../rawdata/%s" % arc, ["GRANAME", "WAVELEN"],
extn=0)
d0 = LGRATINGS[grating]["dispersion"]
rlen = get_head("%s%sf.fits" % (RARC, arc[8:12]), "NAXIS1")
os.system("waverect -xdist -ydist %s%sf.fits -o 4 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5 -bth 20.0" % (RARC, arc[8:12], cwlen - rlen * d0 / 2.0, cwlen + rlen * d0 / 2.0, d0, LRARCLIST))
return
def preprocess(path): #, mode='EABA'):
""" Pipeline that applies the basic CCD reduction tasks.
Asks for the path were all files are present, and
performs Bias combination, Dark combination, Flat
combination and the inherent substractions
Works in TORITOS mode, and EABA mode, the latter using UBVRI
filters.
Pipeline que aplica una reducción básica a imágenes de tipo CCD
Realiza combinación de Bias, de Darks, y Flats.
Luego realiza las restas inherentes
Funciona en dos modos, TORITOS y EABA, el último usando filtros UBVRI
"""
bornpath = os.path.abspath('.')
workdir = os.path.abspath(path)
biasdir = os.path.join(workdir, "bias")
flatdir = os.path.join(workdir, "flat")
darkdir = os.path.join(workdir, "dark")
scidir = os.path.join(workdir, "science")
for x in [biasdir, flatdir, darkdir, scidir]:
print "creating {} directory".format(str(x))
mkdir(x)
cd(workdir)
#-------------------------------------------------------------------------#
fitslist = []
for root, dirs, files in os.walk(path, topdown=True):
for name in fnmatch.filter(files, '*.fit*'):
fitslist.append(os.path.join(root, name))
#fitslist = glob.glob('*.fits') + glob.glob('*.fit')
#load_ccdred()
#-------------------------------------------------------------------------#
imagetypes = {}
for img in fitslist:
imghead = fits.getheader(img)
imgty = imghead['IMAGETYP']
imagetypes[str(img)] = imgty
#-------------------------------------------------------------------------#
# now we get the image types in the dictionary imagetypes
# now we make different lists for different image types
#-------------------------------------------------------------------------#
biaslist = []
sciencelist = []
flatlist = []
darklist = []
#-------------------------------------------------------------------------#
for k, v in imagetypes.iteritems():
print k, v
if v.upper() == 'LIGHT' or v.upper() == 'OBJECT' or v.upper(
) == 'LIGHT FRAME':
# print str(k)+' is a '+str(v)+' file'
sciencelist.append(str(k))
elif v.upper() == 'BIAS' or v.upper() == 'ZERO' or v.upper(
) == 'BIAS FRAME':
# print str(k)+' is a '+str(v)+' file'
biaslist.append(str(k))
elif v.upper() == 'FLAT' or v.upper() == 'FLAT FRAME' or v.upper(
) == 'FLAT FIELD':
# print str(k)+' is a '+str(v)+' file'
flatlist.append(str(k))
elif v.upper() == 'DARK' or v.upper() == 'DARK FRAME':
# print str(k)+' is a '+str(v)+' file'
darklist.append(str(k))
#-------------------------------------------------------------------------#
insp = raw_input("Inspeccionar imagenes de correccion con ds9? (si/NO)\n")
if insp.upper() in ('S', 'Y', 'SI', 'YES'):
print('\n')
print(u"""Comienza la inspeccion visual de bias
ante cualquier anomalia en alguna imagen se la vetará
Starting visual inspection of Bias frames. If there exists any
anomalies image will be discarded""")
for idx, img in enumerate(list(biaslist)):
I = raw_input("inspeccionar {} mediante ds9? (si/NO)\n".format(
str(img)))
if I.upper() in ('S', 'Y', 'SI', 'YES'):
print(""" inspeccionando {} mediante ds9""".format(str(img)))
print("\n")
command = shlex.split('ds9 {}'.format(str(img)))
subprocess.call(command)
V = raw_input(" Vetar la imagen? (si/NO)")
print("\n")
if V.upper() in ('S', 'Y', 'SI', 'YES'):
S = raw_input('Es una imagen de ciencia? (LIGHT) (SI/no)')
print("\n")
if S.upper() in ('S', 'Y', 'SI', 'YES'):
hdu = fits.open(img, mode='update')
hdr = hdu[0].header
hdr.set('IMAGETYP', 'LIGHT')
hdu.close()
sciencelist.append(img)
elif S.upper() in ('N', 'NO'):
os.rename(img, img + '.vet')
biaslist.remove(img)
#-------------------------------------------------------------------------#
print('\n')
print(u"""Comienza la inspeccion visual de flats
ante cualquier anomalia en alguna imagen se la vetará \n""")
for idx, img in enumerate(list(flatlist)):
I = raw_input("inspeccionar {} mediante ds9? (si/NO)\n".format(
str(img)))
if I.upper() in ('S', 'Y', 'SI', 'YES'):
print(""" inspeccionando {} mediante ds9""".format(str(img)))
print("\n")
command = shlex.split('ds9 {}'.format(str(img)))
subprocess.call(command)
V = raw_input(" Vetar la imagen? (si/NO) ")
print("\n")
if V.upper() in ('S', 'Y', 'SI', 'YES'):
S = raw_input(
'Es una imagen de ciencia? (LIGHT) (SI/no) ')
print("\n")
if S.upper() in ('S', 'Y', 'SI', 'YES'):
hdu = fits.open(img, mode='update')
hdr = hdu[0].header
hdr.set('IMAGETYP', 'LIGHT')
hdu.close()
sciencelist.append(img)
elif S.upper() in ('N', 'NO'):
os.rename(img, img + '.vet')
flatlist.remove(img)
#-------------------------------------------------------------------------#
print("\n")
print(u"""Comienza la inspeccion visual de darks
ante cualquier anomalia en alguna imagen se la vetará \n""")
for idx, img in enumerate(list(darklist)):
I = raw_input("inspeccionar {} mediante ds9? (si/NO)\n".format(
str(img)))
if I.upper() in ('S', 'Y', 'SI', 'YES'):
print(""" inspeccionando {} mediante ds9""".format(str(img)))
print("\n")
command = shlex.split('ds9 {}'.format(str(img)))
subprocess.call(command)
V = raw_input(" Vetar la imagen? (si/NO)")
print("\n")
if V.upper() in ('S', 'Y', 'SI', 'YES'):
S = raw_input('Es una imagen de ciencia? (LIGHT) (SI/no)')
print("\n")
if S.upper() in ('S', 'Y', 'SI', 'YES'):
hdu = fits.open(img, mode='update')
hdr = hdu[0].header
hdr.set('IMAGETYP', 'LIGHT')
hdu.close()
sciencelist.append(img)
elif S.upper() in ('N', 'NO'):
os.rename(img, img + '.vet')
darklist.remove(img)
#-------------------------------------------------------------------------#
#posee listas de todos los files.
#comienzo por los bias:
#primero creo una lista (file) para darle a zerocombine
write_tolist(biaslist, 'lbias', biasdir) #, workdir)
#-------------------------------------------------------------------------#
iraf.ccdhedit('@lbias', parameter='imagetype', value='zero')
iraf.zerocombine.unlearn()
iraf.zerocombine('@lbias', output='Zero.fits')
#baseflat = [os.path.basename(x) for x in flatlist]
#basesci = [os.path.basename(x) for x in sciencelist]
#basedark = [os.path.basename(x) for x in darklist]
#-------------------------------------------------------------------------#
#ahora corrijo las imagenes de flat, dark y objetos por bias.
load_ccdproc()
iraf.ccdproc.zero = 'Zero.fits'
for names in flatlist:
iraf.ccdproc(names,
output=os.path.join(flatdir,
'fz' + os.path.basename(names)))
for names in darklist:
iraf.ccdproc(names,
output=os.path.join(darkdir,
'dz' + os.path.basename(names)))
#-------------------------------------------------------------------------#
#recreate lists of new corrected objects
flatlist = []
darklist = []
for root, dirs, files in os.walk(path, topdown=True):
for name in fnmatch.filter(files, 'fz*'):
flatlist.append(os.path.join(flatdir, name))
for root, dirs, files in os.walk(path, topdown=True):
for name in fnmatch.filter(files, 'dz*'):
darklist.append(os.path.join(darkdir, name))
#-------------------------------------------------------------------------#
#combino darks tal como hice con los bias
#-------------------------------------------------------------------------#
write_tolist(darklist, 'ldark')
#write_tolist(sciencelist, 'lsci')
iraf.darkcombine.unlearn()
iraf.ccdhedit('@ldark', parameter='imagetype', value='dark')
iraf.darkcombine('@ldark', output='Dark.fits')
#-------------------------------------------------------------------------#
# discrimino por filtro los datos. es importante esta parte!
#.------------------------------------------------------------------------#
#SE USARON FILTROS?
filteruse = True
for v in flatlist:
try:
hdr = fits.getheader(v)
FILTER = hdr['FILTER']
print 'FILTROS HALLADOS'
except KeyError:
filteruse = False
print 'NO SE USARON FILTROS'
#---------------------------IF FILTERS USED ------------------------------#
if filteruse:
FD = {'U': [], 'B': [], 'V': [], 'R': [], 'I': []}
for idx, img in enumerate(list(flatlist)):
hdr = fits.getheader(img)
FR = hdr['FILTER']
#creo un diccionario de filtros
FD[str(FR)].append(img)
print(FD)
#-------------------------------------------------------------------------#
#combino los flats
#-------------------------------------------------------------------------#
for k, v in FD.iteritems():
if not v == []:
print('writing to list for {} filter'.format(k))
print(v)
lname = str(k) + 'flat'
#print(lname)
write_tolist(v, lname)
iraf.flatcombine.unlearn()
iraf.flatcombine.combine = 'median'
iraf.ccdhedit('@' + lname, parameter='imagetype', value='flat')
iraf.flatcombine('@' + lname, output='Flat' + k + '.fits')
#-------------------------------------------------------------------------#
SD = {'U': [], 'B': [], 'V': [], 'R': [], 'I': []}
for idx, img in enumerate(list(sciencelist)):
hdr = fits.getheader(img)
FR = hdr['FILTER']
#creo un diccionario de filtros
SD[str(FR)].append(img)
print(SD)
#-------------------------------------------------------------------------#
for k, v in SD.iteritems():
iraf.ccdproc.flatcor = 'yes'
iraf.ccdproc.darkcor = 'yes'
iraf.ccdproc.dark = 'Dark.fits'
for img in v:
if os.path.isfile('Flat' + k + '.fits'):
iraf.ccdproc.flat = 'Flat' + k + '.fits'
iraf.ccdproc(img, output='reduced_' + img)
#----------------------IF NO FILTERS--------------------------------------#
else:
lname = 'flist'
write_tolist(flatlist, lname)
iraf.flatcombine.unlearn()
iraf.flatcombine.combine = 'median'
iraf.ccdhedit('@' + lname, parameter='imagetype', value='flat')
iraf.flatcombine('@' + lname, output='Flat.fits')
iraf.ccdproc.flatcor = 'yes'
iraf.ccdproc.darkcor = 'yes'
iraf.ccdproc.dark = 'Dark.fits'
iraf.ccdproc.flat = 'Flat.fits'
iraf.ccdproc.ccdtype = ''
for sciname in sciencelist:
print 'ccdproc of ', sciname, '\n', os.path.join(
scidir, 'reduced_' + os.path.basename(sciname))
iraf.ccdproc(sciname, output=os.path.join(scidir,\
'reduced_'+os.path.basename(sciname)))
#[os.remove(x) for x in fitslist]
aux = glob.glob('sz*.fits') + glob.glob('dz*.fits') + glob.glob('fz*.fits')
[os.remove(x) for x in aux]
aux = glob.glob('reduced*.fits')
[os.rename(x, x[10:-4].upper() + x[-4:].lower()) for x in aux]
#-------------------------------------------------------------------------#
print(u""" Imágenes reducidas exitosamente APARENTEMENTE,
chequea todo obsesivamente, porque ni el desarrollador de
esto confia en que ande bien \n
""")
#-------------------------------------------------------------------------#
cd(bornpath)
#print sys.argv[:]
return ()
def flatcom(camera='STX16803', flattype='sky', dark=False):
"""
1. Description
: This function makes master-normalised images of KCT using Pyraf. Put flat images to 20XX-XX-XX/skyflat/ directory. Run this code on 20XX-XX-XX directory, then pyraf chdir task enter skyflat, directory and makes process. If dark=True, dark subtraction will perform on flat images. Use this keyword when you think dark subtraction is needed for flat images. If not, only bias subtraction will be perfromed. And then flatcombine and normalizing will be performed. Output image is nflatX.YYY.fits (X is filter and YYY is sky or dome. This function will classify each exposure of frames!) and they will be copied on upper directory. Due to iraf.chdir task, you should reset python when this code is finished in order to make confusion of current directory between iraf and python!
2. Usage
: Start on 20XX-XX-XX directory. Make skyflat or domeflat directory which contains each flat frame. Naming of each flat image should be *flat*.fit. And domeflat naming is Domeflat*.fit. Then just use flatcom().
>>> flatcom('sky') --> Use skyflat
>>> flatcom('dome') --> Use domeflat
*Default configuration is skyflat.
3. History
2018.03 Created by G. Lim.
2018.12.20 Edited by G. Lim. Define SAO_flatcom function.
2018.12.28 Edited by G. Lim. Add bias or dark keyword. Join function is used when performing imarith, combine tasks.
2019.02.07 Assign archive of masterflat in each date by G. Lim
2020.03.01 Remove process keyword from STX16803.
2020.03.06 Modified for KCT STX16803 process
"""
import glob
import os, sys
import itertools
import numpy as np
from pyraf import iraf
from astropy.io import fits
from astropy.io import ascii
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.ccdred.setinst(instrume='camera',
directo='/iraf/iraf/noao/imred/ccdred/ccddb/',
query='q',
review='no')
curdir = os.getcwd()
'''
yy = curdir.split('/')[-1].split('-')[0]
mm = curdir.split('/')[-1].split('-')[1]
dd = curdir.split('/')[-1].split('-')[2]
curdate = yy+mm+dd
'''
curdate = curdir.split('/')[-1]
savedir = '/data1/KCT/'
if flattype == 'dome':
iraf.chdir('./domeflat')
elif flattype == 'sky':
iraf.chdir('./skyflat')
flat = glob.glob('*flat*.fit')
flat.sort()
input_name = 'flat.list'
k = 0
f = open(input_name, 'w+')
for k in range(len(flat)):
f.write(flat[k] + '\n')
f.close()
print('zero subtraction with ' + flattype + 'flat images...')
iraf.imarith(operand1='@' + input_name,
op='-',
operand2='../*_zero.fits',
result='z@' + input_name)
if dark == True:
zflat = glob.glob('z*flat*.fit')
i = 0
allexptime = []
for i in range(len(zflat)):
hdr = fits.getheader(zflat[i])
allexptime.append(hdr['exptime'])
expset = set(allexptime)
exptime = list(sorted(expset))
i = 0
for i in range(len(exptime)):
print('Find images with exptime of ' + str(int(exptime[i])))
imlist = []
j = 0
for j in range(len(zflat)):
hdr = fits.getheader(zflat[j])
if hdr['exptime'] == exptime[i]:
imlist.append(zflat[j])
else:
pass
print(imlist)
imlist.sort()
input_name = 'zflat.list'
k = 0
f = open(input_name, 'w+')
for k in range(len(imlist)):
f.write(imlist[k] + '\n')
f.close()
iraf.imarith(operand1='@' + input_name,
op='-',
operand2='../*_dark' + str(int(exptime[i])) + '.fits',
result='d@' + input_name)
#iraf.imarith(operand1=darkjoin, op='-', operand2='../dark'+str(int(exptime[i]))+'.fits', result=outdarkjoin)
print('Dark subtracted flat images are created.')
# Flat combine
if dark == True:
calflat = glob.glob('dz*flat*.fit')
elif dark == False:
calflat = glob.glob('z*flat*.fit')
calflat.sort()
allfilter = []
i = 0
for i in range(len(calflat)):
hdr = fits.getheader(calflat[i])
allfilter.append(hdr['filter'])
filterset = set(allfilter)
infilter = list(sorted(filterset))
i = 0
for i in range(len(infilter)):
print('Find images with filter of ' + str(infilter[i]))
calimlist = []
for j in range(len(calflat)):
hdr = fits.getheader(calflat[j])
if hdr['filter'] == infilter[i]:
calimlist.append(calflat[j])
else:
pass
print(calimlist)
calimlist.sort()
input_name = str(infilter[i]) + 'flat.list'
k = 0
f = open(input_name, 'w+')
for k in range(len(calimlist)):
f.write(calimlist[k] + '\n')
f.close()
output_name = input_name[:-5] + '.fits'
iraf.flatcombine(input='@' + input_name,
output=output_name,
combine='average',
reject='crreject',
process='no',
scale='mode',
ccdtype='',
lsigma='3.',
hsigma='3.')
print(output_name + ' is created. Normalizing...')
data, newhdr = fits.getdata(output_name, header=True)
x = np.mean(data)
nimage = data / x
newflat_name = curdate + '_n' + str(
infilter[i]) + 'flat.' + flattype + '.fits'
fits.writeto(newflat_name, nimage, header=newhdr, overwrite=True)
os.system('/usr/bin/cp ' + newflat_name + ' ../')
os.system('mkdir ' + savedir + 'masterflat_' + infilter[i] + '/')
os.system('/usr/bin/cp ' + newflat_name + ' ' + savedir +
'masterflat_' + infilter[i] + '/')
print('Normalised master flats are created.')
iraf.imstat(images='*n?flat.' + flattype + '.fits')
os.system('/usr/bin/rm *.list ?flat.fits')
iraf.chdir('../')
iraf.dir('./')
def flatcombine(self):
'''Combine all of the flats into one flat.'''
iraf.flatcombine()
def optdomecomb(date, fwheel=['bias', 'B', 'V', 'R', 'I']):
'''
#combine biases and optical domes
#Requires: the uncombined fits images
# if you are combining a dome, you must have a bias from the same night as the dome to preform appropriate bias subtraction
#Input: the date the domes were observed YYMMDD, and fwheel, a list that contains the filters of the domes to be combined
#Outupt: combined dome fits frame for each color where uncombined frames are in the directory
'''
#convert date to string incase it was entered as an int of float
date = str(date)
if len(glob.glob('*bias*')) < 1:
print "no biases found, exiting"
return
else:
for color in fwheel:
if color == 'bias':
biaslist = glob.glob('*bias.[0-9]*')
if len(biaslist) > 10:
print "only " + str(
len(biaslist)) + " biases found. you need at least 10"
else:
with open("bias.list", 'w') as BILIS:
for i in biaslist:
BILIS.write(i + '\n')
iraf.zerocombine("@bias.list",
output="ccd" + str(date) + ".bias.fits",
combine="average",
reject="minmax",
scale="none",
ccdtype="",
process="no",
delete="no",
clobber="no",
nlow=1,
nhigh=1,
nkeep=1)
print "created ccd" + str(date) + ".bias.fits"
os.system('rm bias.list')
elif color in ['B', 'V', 'R', 'I']:
domelist = glob.glob('*dome' + color + '.[0-9]*')
if len(domelist) < 1:
print 'no ' + color + ' domes found'
elif len(domelist) > 10:
print 'only ' + str(
len(domelist)) + ' domes found. you need at least 10'
else:
with open('flat' + color + '.list', 'w') as flist:
for i in domelist:
flist.write(i + '\n')
iraf.ccdproc("@flat" + color + ".list",
output="z@flat" + color + ".list",
ccdtype=" ",
noproc="no",
fixpix="no",
overscan="yes",
trim="no",
zerocor="yes",
darkcor="no",
flatcor="no",
illumcor="no",
fringec="no",
readcor="no",
scancor="no",
readaxis="line",
biassec="[3:14,1:1024]",
zero="ccd" + str(date) + ".bias.fits",
interactive="no",
functio="spline3",
order=11)
iraf.flatcombine("z@flat" + color + ".list",
output="ccd" + str(date) + ".dome" +
color + ".fits",
combine="average",
reject="crreject",
ccdtype="",
process="no",
subsets="no",
delete="no",
clobber="no",
scale="mode",
rdnoise=6.5,
gain=2.3)
os.system('rm z*dome' + color + '*fits')
print "created ccd" + str(date) + ".dome" + color + ".fits"
os.system('rm flat' + color + '.list')
else:
print "your input for the filter was not recognized. Please use either 'bias', 'B', 'V', 'R', or 'I' and try again"
return
def lris_do_rcalib(flats, arcs):
'''Calculate y- and x- distortions, create flat-field, calculate wavelength
solution, for a given grating and central wavelength.'''
# Pre-process flats
for image in flats:
os.system("lris_rpreproc.py %s" % image)
# Create median combination of flats
iraf.flatcombine(",".join(["rtr%s.fits" % x[8:12] for x in flats]),
output=RFLAT,
combine="median",
reject="avsigclip",
ccdtype="",
process=no,
subsets=no,
delete=no,
clobber=no,
scale="none",
statsec="",
lsigma=3.0,
hsigma=3.0,
rdnoise=0.,
gain=1.)
# Pre-process arcs
for arc in arcs:
os.system("lris_rpreproc.py %s" % arc)
iraf.imcopy("rtr%s.fits" % arc[8:12], "%s%s.fits" % (RARC, arc[8:12]))
# Generate flats with sharpened edges
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (RFLAT, RSHARP))
# Calculate y-distortion from sharpened frame
os.system('getrect -ydist %s -nx 12 -dy 11 -x 1 -y 1 -peakwin 3 -dump' %
RSHARP)
# Copy y rectification to flats
os.system('copyrect -ydist %s %s' % (RSHARP, RFLAT))
# Find slits
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % RFLAT)
# Correct Slit locations
[grating, binning] = get_head(RFLAT, ["GRANAME", "CCDSUM"])
update_head(RFLAT, "NSLITS", LGRATINGS[grating]["slits"][0])
update_head(RFLAT, "CSECT1A",
LGRATINGS[grating]["slits"][1] / int(binning.split()[0]))
update_head(RFLAT, "CSECT1B",
LGRATINGS[grating]["slits"][2] / int(binning.split()[0]))
update_head(RFLAT, "CSECT2A",
LGRATINGS[grating]["slits"][3] / int(binning.split()[0]))
update_head(RFLAT, "CSECT2B",
LGRATINGS[grating]["slits"][4] / int(binning.split()[0]))
# Create flat fields
os.system("makeflat -ydist %s" % RFLAT)
# Correct for bad pixels
x = pyfits.open("%sflt.fits" % RFLAT.split(".")[0])
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],
0:21] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],
0:20] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],
1845:1930] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],
1845:1930] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],
3275:3355] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],
3275:3355] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],
3375:3475] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],
3375:3475] = 1
x[0].data[LGRATINGS[grating]["slits"][1]:LGRATINGS[grating]["slits"][2],
4061:] = 1
x[0].data[LGRATINGS[grating]["slits"][3]:LGRATINGS[grating]["slits"][4],
4061] = 1
x.writeto("%sflt.fits" % RFLAT.split(".")[0], clobber=True)
for arc in arcs:
# Apply distortion, slits and flat-fields to arcs
os.system("copyrect -ydist %s %s%s.fits" % (RSHARP, RARC, arc[8:12]))
#os.system("copyslit %s %s -fft -ydist" % (RFLAT, RARC))
os.system("copyslit %s %s%s.fits -ydist" % (RFLAT, RARC, arc[8:12]))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (RFLAT.split(".")[0], RARC))
os.system("flat2d %sflt.fits %s%s.fits" %
(RFLAT.split(".")[0], RARC, arc[8:12]))
# Create new files for x-distortion
#os.system("flat2d %sblz.fits %sf.fits" % (RFLAT.split(".")[0],
# RARC.split(".")[0]))
os.system(
"efits %sblz.fits %s%sf.fits 'i2*greater(i1,0.0)' xdist_R%s.fits" %
(RFLAT.split(".")[0], RARC, arc[8:12], arc[8:12]))
# Calculate x distortion
os.system(
"getrect -xdist -ydist xdist_R%s.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6"
% arc[8:12])
# Apply x distortion and calculate wavelength solution
os.system("copyrect -xdist xdist_R%s.fits %s%sf.fits" %
(arc[8:12], RARC, arc[8:12]))
[grating, cwlen] = get_head("../rawdata/%s" % arc,
["GRANAME", "WAVELEN"],
extn=0)
d0 = LGRATINGS[grating]["dispersion"]
rlen = get_head("%s%sf.fits" % (RARC, arc[8:12]), "NAXIS1")
os.system(
"waverect -xdist -ydist %s%sf.fits -o 4 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5 -bth 20.0"
% (RARC, arc[8:12], cwlen - rlen * d0 / 2.0,
cwlen + rlen * d0 / 2.0, d0, LRARCLIST))
return
iraf.zerocombine(input=bias,
reject='avsigclip',
ccdtype='',
rdnoise='rdnoise',
gain='gain')
iraf.imstat('bias*')
iraf.imstat('Zero')
iraf.imexamine('Zero')
iraf.flatcombine.unlearn()
iraf.flatcombine(input=flatcup,
output='Flat',
ccdtype='',
process=False,
subsets=False,
rdnoise='rdnoise',
gain='gain')
iraf.flatcombine(input=flati,
output='iFlat',
ccdtype='',
process=False,
subsets=False,
rdnoise='rdnoise',
gain='gain')
iraf.imstat(flatcup)
iraf.imstat('Flat')
def lris_do_bcalib(flats, arcs):
'''Calculate y- and x- distortions, create flat-field, calculate wavelength
solution, for a given grism.'''
# Pre-process flats
for image in flats:
os.system("lris_bpreproc.py %s" % image)
# Create median combination of flats
iraf.flatcombine(",".join(["rtb%s.fits" % x[8:12] for x in flats]),
output=BFLAT,
combine="median",
reject="avsigclip",
ccdtype="",
process=no,
subsets=no,
delete=no,
clobber=no,
scale="none",
statsec="",
lsigma=3.0,
hsigma=3.0,
rdnoise=0.,
gain=1.)
# Pre-process arc
grism = get_head("../rawdata/%s" % arcs[0], "GRISNAME", extn=0)
for image in arcs:
os.system("lris_bpreproc.py %s" % image)
#iraf.imarith("rtb%s.fits" % arcs[0][8:12], "+", "rtb%s.fits" % arcs[1][8:12], BARC)
os.system("cp rtb%s.fits %s" % (arcs[0][8:12], BARC))
# Generate flats with sharpened edges
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (BFLAT, BSHARP))
# Calculate y-distortion from sharpened frame
os.system('getrect -ydist %s -nx 12 -dy 11 -x 1 -y 1 -peakwin 3 -dump' %
BSHARP)
# Copy y rectification to flats
os.system('copyrect -ydist %s %s' % (BSHARP, BFLAT))
# Find slits
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % BFLAT)
# Correct Slit locations
grism = get_head(BFLAT, "GRISNAME")
update_head(BFLAT, ["NSLITS", "CSECT1A", "CSECT1B", "CSECT2A", "CSECT2B"],
LGRISMS[grism]["slits"])
# Create flat fields
os.system("makeflat -ydist %s" % BFLAT)
# Bluest part of the flat has too few counts to be useful.
x = pyfits.open("%sflt.fits" % BFLAT.split(".")[0])
x[0].data[LGRISMS[grism]["slits"][1]:LGRISMS[grism]["slits"][2],
0:1150] = 1
x[0].data[LGRISMS[grism]["slits"][3]:LGRISMS[grism]["slits"][4],
0:1150] = 1
x.writeto("%sflt.fits" % BFLAT.split(".")[0], clobber=True)
# Apply distortion, slits and flat-fields to arcs
os.system("copyrect -ydist %s %s" % (BSHARP, BARC))
#os.system("copyslit %s %s -fft -ydist" % (BFLAT, BARC))
os.system("copyslit %s %s -ydist" % (BFLAT, BARC))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (BFLAT.split(".")[0], BARC))
os.system("flat2d %sflt.fits %s" % (BFLAT.split(".")[0], BARC))
# Create new files for x-distortion
#os.system("flat2d %sblz.fits %sf.fits" % (BFLAT.split(".")[0],
# BARC.split(".")[0]))
os.system("efits %sblz.fits %sf.fits 'i2*greater(i1,0.01)' xdist_B.fits" %
(BFLAT.split(".")[0], BARC.split(".")[0]))
# Calculate x distortion
os.system(
"getrect -xdist -ydist xdist_B.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6"
)
# Apply x distortion and calculate wavelength solution
os.system("copyrect -xdist xdist_B.fits %sf.fits" % BARC.split(".")[0])
d0 = LGRISMS[grism]["dispersion"]
[w1, w2] = LGRISMS[grism]["wavelengths"]
os.system(
"waverect -xdist -ydist %sf.fits -o 5 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5 -bth 5.0"
% (BARC.split(".")[0], w1, w2, d0, LBARCLIST))
return
def main():
description = "> Performs pre-reduction steps"
usage = "%prog \t [option] \n Recommended syntax: %prog -i -c"
parser = OptionParser(usage=usage, description=description, version="0.1")
option, args = parser.parse_args()
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
mkarc = raw_input("Make arc? ([y]/n): ")
mkflat = raw_input("Make flat? ([y]/n): ")
if len(args) > 1:
files = []
sys.argv.append('--help')
option, args = parser.parse_args()
sys.exit()
elif len(args) == 1:
files = util.readlist(args[0])
sys.exit()
else:
listfile = glob.glob('*.fits')
files_science = []
files_arc = []
files_dflat = []
#print 'checking your files ...'
for img in listfile:
_type = ''
hdr0 = util.readhdr(img)
_type = util.readkey3(hdr0, 'object')
if 'flat' in _type.lower():
files_dflat.append(img)
elif 'arc' not in _type.lower() and 'arc' not in img.lower():
files_science.append(img)
if mkarc != 'n':
mkarc_b = raw_input(
"List blue arc files to combine (.fits will be added): "
).split()
mkarc_r = raw_input(
"List red arc files to combine (.fits will be added): ").split(
)
for arc in mkarc_b:
files_arc.append(arc + '.fits')
for arc in mkarc_r:
files_arc.append(arc + '.fits')
if mkarc != 'n':
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
if instruments.blue_or_red(arcs)[0] == 'blue':
list_arc_b.append(arcs)
elif instruments.blue_or_red(arcs)[0] == 'red':
list_arc_r.append(arcs)
else:
sys.exit()
if mkflat != 'n':
list_flat_b = []
list_flat_r = []
for dflats in files_dflat:
if instruments.blue_or_red(dflats)[0] == 'blue':
list_flat_b.append(dflats)
elif instruments.blue_or_red(dflats)[0] == 'red':
list_flat_r.append(dflats)
else:
sys.exit()
# make pre_reduced if it doesn't exist
if not os.path.isdir('pre_reduced/'):
os.mkdir('pre_reduced/')
# log the existing processed files (need to verify this works if pre_reduced is empty...)
pfiles = []
new_files = []
for root, dirnames, filenames in os.walk('pre_reduced'):
for file in filenames:
if file.startswith('to'):
pfiles.append(file)
print(pfiles)
# loop over each image in pre_reduced
for img in listfile:
hdr = util.readhdr(img)
targ = util.readkey3(hdr, 'object')
# if file is not not a processed file, run the overscan+trim code
if 'to' + img not in pfiles:
# if the file is a science file, grab the name for later
if 'arc' not in targ.lower() and 'flat' not in targ.lower():
new_files.append(img)
print('Adding data for: ' + targ)
inst = instruments.blue_or_red(img)[1]
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_biassec0 = inst.get('biassec')
_trimsec0 = inst.get('trimsec')
######################################################################
#
# JB: this chunk of code needs attention
# It seems incredibly hacky for anything but Kast...
#
# overscan
if not img.startswith('o') and inst.get('observatory') == 'lick':
if os.path.isfile('pre_reduced/o' + img):
os.remove('pre_reduced/o' + img)
util.kastbias(img, 'pre_reduced/o' + img)
elif not img.startswith('o') and inst.get('observatory') != 'lick':
if os.path.isfile('pre_reduced/o' + img):
os.remove('pre_reduced/o' + img)
os.system('cp ' + img + ' ' + 'pre_reduced/' + img)
# trim
if not img.startswith('t') and inst.get('observatory') == 'lick':
if os.path.isfile('pre_reduced/to' + img):
os.remove('pre_reduced/to' + img)
iraf.ccdproc('pre_reduced/o' + img,
output='pre_reduced/to' + img,
overscan='no',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='line',
trimsec=str(_trimsec0),
Stdout=1)
elif not img.startswith('t') and inst.get('observatory') != 'lick':
if os.path.isfile('pre_reduced/to' + img):
os.remove('pre_reduced/to' + img)
iraf.ccdproc('pre_reduced/' + img,
output='pre_reduced/to' + img,
overscan='yes',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='line',
trimsec=str(_trimsec0),
biassec=str(_biassec0),
Stdout=1)
# combine the arcs
if mkarc != 'n':
# blue arcs
if len(list_arc_b) > 0:
if len(list_arc_b) == 1:
arc_blue = list_arc_b[0]
os.system('cp ' + 'pre_reduced/to' + arc_blue + ' ' +
'pre_reduced/ARC_blue.fits')
else:
arc_str = ''
for arc in list_arc_b:
arc_str = arc_str + 'pre_reduced/to' + arc + ','
if os.path.isfile('pre_reduced/ARC_blue.fits'):
os.remove('pre_reduced/ARC_blue.fits')
iraf.imcombine(arc_str, output='pre_reduced/ARC_blue.fits')
# red arcs
if len(list_arc_r) > 0:
if len(list_arc_r) == 1:
arc_red = list_arc_r[0]
os.system('cp ' + 'pre_reduced/to' + arc_red + ' ' +
'pre_reduced/ARC_red.fits')
else:
arc_str = ''
for arc in list_arc_r:
arc_str = arc_str + 'pre_reduced/to' + arc + ','
if os.path.isfile('pre_reduced/ARC_red.fits'):
os.remove('pre_reduced/ARC_red.fits')
iraf.imcombine(arc_str, output='pre_reduced/ARC_red.fits')
# combine the flats
if mkflat != 'n':
inter = 'yes'
# blue flats
if len(list_flat_b) > 0:
br, inst = instruments.blue_or_red(list_flat_b[0])
iraf.specred.dispaxi = inst.get('dispaxis')
if len(list_flat_b) == 1:
# Flat_blue = 'pre_reduced/to'+ list_flat_b[0]
Flat_blue = list_flat_b[0]
else:
flat_str = ''
for flat in list_flat_b:
flat_str = flat_str + 'pre_reduced/to' + flat + ','
#subsets = 'no'
if os.path.isfile('pre_reduced/toFlat_blue'):
os.remove('pre_reduced/toFlat_blue')
iraf.flatcombine(flat_str,
output='pre_reduced/toFlat_blue',
ccdtype='',
rdnoise=3.7,
subsets='no',
process='no')
Flat_blue = 'Flat_blue.fits'
#What is the output here? Check for overwrite
iraf.specred.response('pre_reduced/to' + Flat_blue,
normaliz='pre_reduced/to' + Flat_blue,
response='pre_reduced/RESP_blue',
interac=inter,
thresho='INDEF',
sample='*',
naverage=2,
function='legendre',
low_rej=3,
high_rej=3,
order=60,
niterat=20,
grow=0,
graphic='stdgraph')
# red flats
if len(list_flat_r) > 0:
br, inst = instruments.blue_or_red(list_flat_r[0])
iraf.specred.dispaxi = inst.get('dispaxis')
if len(list_flat_r) == 1:
# Flat_red = 'pre_reduced/to' + list_flat_r[0]
Flat_red = list_flat_r[0]
else:
flat_str = ''
for flat in list_flat_r:
flat_str = flat_str + 'pre_reduced/to' + flat + ','
if os.path.isfile('pre_reduced/toFlat_red'):
os.remove('pre_reduced/toFlat_red')
iraf.flatcombine(flat_str,
output='pre_reduced/toFlat_red',
ccdtype='',
rdnoise=3.8,
subsets='yes',
process='no')
Flat_red = 'Flat_red.fits'
#What is the output here? Check for overwrite
iraf.specred.response('pre_reduced/to' + Flat_red,
normaliz='pre_reduced/to' + Flat_red,
response='pre_reduced/RESP_red',
interac=inter,
thresho='INDEF',
sample='*',
naverage=2,
function='legendre',
low_rej=3,
high_rej=3,
order=80,
niterat=20,
grow=0,
graphic='stdgraph')
# science files should have 't' in front now
# this just gets the base name, to prefix assumed below
if new_files is not None:
files_science = new_files
# get all the science objects for the night
science_targets = []
for obj in files_science:
hdr = util.readhdr(obj)
_type = util.readkey3(hdr, 'object')
science_targets.append(_type)
# make a dir for each sci object
science_targets = set(science_targets)
for targ in science_targets:
if not os.path.isdir('pre_reduced/' + targ + '/'):
os.mkdir('pre_reduced/' + targ + '/')
# copy the files into the obj dir
for obj in files_science:
hdr = util.readhdr(obj)
targ = util.readkey3(hdr, 'object')
if not obj.startswith('to'):
os.system('cp ' + 'pre_reduced/to' + obj + ' ' + 'pre_reduced/' +
targ + '/')
else:
os.system('cp ' + 'pre_reduced/' + obj + ' ' + 'pre_reduced/' +
targ + '/')
rawfiles = glob.glob('*.fits')
ofiles = glob.glob('pre_reduced/o' + '*.fits')
tfiles = glob.glob('pre_reduced/to' + '*.fits')
# delete raw files from the pre_reduced dir
# there shouldn't be any there though?
# maybe if the overscan isn't implemented for that detector
for img in rawfiles:
util.delete('pre_reduced/' + img)
# delete the ofiles from pre_reduced dir
for img in ofiles:
util.delete(img)
def deimos_docalib(flats, arc):
'''Calculate y- and x- distortions, create flat-field, calculate wavelength
solution, for a given grating and central wavelength.'''
# Pre-process flats
for image in flats:
deimos_preproc(image)
# Create median combination of flats
iraf.flatcombine(",".join(["rtd%s_B.fits" % x[6:10] for x in flats]),
output=BFLAT,
combine="median",
reject="avsigclip")
iraf.flatcombine(",".join(["rtd%s_R.fits" % x[6:10] for x in flats]),
output=RFLAT,
combine="median",
reject="avsigclip")
# Pre-process arc
graname = get_head(arc, "GRATENAM", extn=0)
deimos_preproc(arc)
barc = "rtd%s_B.fits" % arc[6:10]
rarc = "rtd%s_R.fits" % arc[6:10]
# Generate flats with sharpened edges
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (BFLAT, BSHARP))
os.system('efits %s "VTKLaplace(i1,numret=1)" %s' % (RFLAT, RSHARP))
# Calculate y-distortion from sharpened frame
os.system('getrect -ydist %s -nx 6 -dy 11 -x 1 -y 1 -peakwin 3 -dump' %
BSHARP)
os.system('getrect -ydist %s -nx 12 -dy 11 -x 1 -y 1 -peakwin 3 -dump' %
RSHARP)
# Copy y rectification to flats
os.system('copyrect -ydist %s %s' % (BSHARP, BFLAT))
os.system('copyrect -ydist %s %s' % (RSHARP, RFLAT))
# Find slits
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % BFLAT)
os.system('findslits -ydist %s -th 0.2 -fwhm 2' % RFLAT)
# Correct Slit locations
update_head(BFLAT, [
"NSLITS", "CSECT1A", "CSECT1B", "CSECT2A", "CSECT2B", "CSECT3A",
"CSECT3B", "CSECT4A", "CSECT4B", "CSECT5A", "CSECT5B"
], DGRATINGS[graname]["bslits"])
update_head(RFLAT, [
"NSLITS", "CSECT1A", "CSECT1B", "CSECT2A", "CSECT2B", "CSECT3A",
"CSECT3B", "CSECT4A", "CSECT4B", "CSECT5A", "CSECT5B"
], DGRATINGS[graname]["rslits"])
# Create flat fields
os.system("makeflat -ydist %s" % BFLAT)
os.system("makeflat -ydist %s" % RFLAT)
# Apply distortion, slits and flat-fields to arcs
os.system("copyrect -ydist %s %s" % (BSHARP, barc))
os.system("copyrect -ydist %s %s" % (RSHARP, rarc))
os.system("copyslit %s %s -fft -ydist" % (BFLAT, barc))
os.system("copyslit %s %s -fft -ydist" % (RFLAT, rarc))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (BFLAT.split(".")[0], barc))
#os.system("flat1d %sslt.fits %s -fft -ydist" % (RFLAT.split(".")[0], rarc))
os.system("flat2d %sflt.fits %s" % (BFLAT.split(".")[0], barc))
os.system("flat2d %sflt.fits %s" % (RFLAT.split(".")[0], rarc))
# Create new files for x-distortion
os.system("flat2d %sblz.fits %sf.fits" %
(BFLAT.split(".")[0], barc.split(".")[0]))
os.system("flat2d %sblz.fits %sf.fits" %
(RFLAT.split(".")[0], rarc.split(".")[0]))
os.system("efits %sblz.fits %sff.fits 'i2*greater(i1,0.1)' xdist_B.fits" %
(BFLAT.split(".")[0], barc.split(".")[0]))
os.system("efits %sblz.fits %sff.fits 'i2*greater(i1,0.1)' xdist_R.fits" %
(RFLAT.split(".")[0], rarc.split(".")[0]))
# Calculate x distortion
os.system(
"getrect -xdist -ydist xdist_B.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6"
)
os.system(
"getrect -xdist -ydist xdist_R.fits -nx 16 -dy 3 -b 3 -x 2 -y 2 -dump -normwt 1e6"
)
# Apply x distortion and calculate wavelength solution
os.system("copyrect -xdist xdist_B.fits %sf.fits" % barc.split(".")[0])
os.system("copyrect -xdist xdist_R.fits %sf.fits" % rarc.split(".")[0])
grating = get_head(arc, "GRATENAM", extn=0)
d0 = DGRATINGS[grating]["dispersion"]
cwlen = get_head(arc, DGRATINGS[grating]["tiltkey"], extn=0)
blen = get_head(barc, "NAXIS1")
rlen = get_head(rarc, "NAXIS1")
os.system(
"waverect -xdist -ydist %sf.fits -o 3 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5"
% (barc.split(".")[0], cwlen - blen * d0, cwlen, d0, DARCLIST))
os.system(
"waverect -xdist -ydist %sf.fits -o 3 -w1 %.2f -w2 %.2f -d0 %.2f -ll %s -arcamp -fwhm 8.5"
% (rarc.split(".")[0], cwlen, cwlen + rlen * d0, d0, DARCLIST))
return
def flatcombine(self,
files,
output,
dark=None,
zero=None,
ccdtype="",
method="Median",
rejection="minmax",
subset="no",
overwrite=True):
"""IRAF flatcombine"""
self.logger.info("Flatcombine Started")
try:
flats = ",".join(files)
iraf.imred.unlearn()
iraf.ccdred.unlearn()
iraf.ccdred.ccdproc.unlearn()
iraf.ccdred.combine.unlearn()
iraf.ccdred.flatcombine.unlearn()
ccdred.instrument = self.instrument_path
out_file = "{}/myraf_flats.flist".format(self.fop.tmp_dir)
with open(out_file, "w") as f2w:
for i in files:
f2w.write("{}\n".format(i))
flats = "@{}".format(out_file)
if self.fop.is_file(output) and overwrite:
self.logger.warning("Over Writing file({})".format(output))
self.fop.rm(output)
iraf.flatcombine(input=flats,
output=output,
combine=method,
reject=rejection,
ccdtype=ccdtype,
subset=subset,
process="no",
Stdout="/dev/null")
if dark is not None:
darkcor = 'yes'
else:
darkcor = 'no'
if zero is not None:
zerocor = 'yes'
else:
zerocor = 'no'
iraf.ccdproc(images=flats,
ccdtype='',
fixpix='no',
oversca="no",
trim="no",
zerocor=zerocor,
darkcor=darkcor,
flatcor='no',
zero=zero,
dark=dark,
Stdout="/dev/null")
return True
except Exception as e:
self.logger.error(e)
return False
def run_flatcombine(images, ccdtype="none"):
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.flatcombine(images, ccdtype=ccdtype, nlow=0, nhigh=0, nkeep=0)
