def zeroCorrection():
print "ZeroCorrection start"
#put all others params to no , they may be set by previous actions
zerofilename = os.path.join(OUTPUTDIR, "bias", "Zero.fits")
if os.path.isfile(zerofilename):
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
iraf.ccdproc.setParam('trim', 'no')
iraf.ccdproc.setParam('overscan', 'no')
#trim and overscan flat and object files
iraf.ccdproc.setParam('zerocor', 'yes')
iraf.ccdproc.setParam('zero', zerofilename)
iraf.ccdproc.setParam('trimsec', '')
iraf.ccdproc.setParam('biassec', '')
#online
iraf.ccdproc.setParam('output', '')
for imgtype in ["flat", "object"]:
for f in FILTERS:
iraf.ccdproc.setParam("images", os.path.join(OUTPUTDIR, imgtype, f) + "/*.fits")
iraf.ccdproc()
else:
print "Zero file %s not present" % zerofilename
print "ZeroCorrection end"
def subtract_bias(self):
"""Subtracting Bias Frame"""
iraf.unlearn(iraf.ccdproc)
iraf.ccdproc(self.data.iraf.modatfile(),
ccdtype="", fixpix="no", overscan="no", trim ="no", zerocor="yes", darkcor="no", flatcor ="no",
zero=self.bias.iin("Bias"))
self.data.idone()
def processImages(fileList,
inputType=INPUT_SINGLEFRAME,
output="",
outputType=INPUT_SINGLEFRAME,
ccdtype="object",
zerocor=iraf.yes,
darkcor=iraf.yes,
flatcor=iraf.yes,
zero="Zero.fits",
dark="Dark.fits",
flat="Flat*.fits"):
'''wrapper for IRAF's ccdproc command'''
iraf.ccdproc(_genIRAFString(fileList, inputType),
output=_genIRAFString(output, outputType),
ccdtype=ccdtype,
zerocor=zerocor,
darkcor=darkcor,
flatcor=flatcor,
zero=zero,
dark=dark,
flat=flat,
fixpix=iraf.no,
overscan=iraf.no,
trim=iraf.no)
return
def zeroCorrection():
print "ZeroCorrection start"
#put all others params to no , they may be set by previous actions
zerofilename = os.path.join(OUTPUTDIR, "bias", "Zero.fits")
if os.path.isfile(zerofilename):
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
iraf.ccdproc.setParam('trim', 'no')
iraf.ccdproc.setParam('overscan', 'no')
#trim and overscan flat and object files
iraf.ccdproc.setParam('zerocor', 'yes')
iraf.ccdproc.setParam('zero', zerofilename)
iraf.ccdproc.setParam('trimsec', '')
iraf.ccdproc.setParam('biassec', '')
#online
iraf.ccdproc.setParam('output', '')
for imgtype in ["flat", "object"]:
for f in FILTERS:
iraf.ccdproc.setParam(
"images",
os.path.join(OUTPUTDIR, imgtype, f) + "/*.fits")
iraf.ccdproc()
else:
print "Zero file %s not present" % zerofilename
print "ZeroCorrection end"
def trimAndOverscan():
print "TrimAndOverscan start"
#put all others params to no , they may be set by previous actions
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
#trim and overscan flat and object files
iraf.ccdproc.setParam('trim', 'yes')
iraf.ccdproc.setParam('trimsec', DATA_SEC)
iraf.ccdproc.setParam('overscan', 'yes')
iraf.ccdproc.setParam('biassec', BIAS_SEC)
#online
iraf.ccdproc.setParam('output', '')
for imgtype in ["flat", "object"]:
for f in FILTERS:
iraf.ccdproc.setParam(
"images",
os.path.join(OUTPUTDIR, imgtype, f) + "/*.fits")
iraf.ccdproc()
#only trim for bias files
if (os.listdir(os.path.join(OUTPUTDIR, "bias"))):
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('biassec', '')
iraf.hselect.setParam("images",
os.path.join(OUTPUTDIR, "bias") + "/*.fits")
iraf.ccdproc()
else:
print "No Bias Files present"
print "TrimAndOverscan end"
def trimAndOverscan():
print "TrimAndOverscan start"
#put all others params to no , they may be set by previous actions
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
#trim and overscan flat and object files
iraf.ccdproc.setParam('trim', 'yes')
iraf.ccdproc.setParam('trimsec', DATA_SEC)
iraf.ccdproc.setParam('overscan', 'yes')
iraf.ccdproc.setParam('biassec', BIAS_SEC)
#online
iraf.ccdproc.setParam('output', '')
for imgtype in ["flat", "object"]:
for f in FILTERS:
iraf.ccdproc.setParam("images", os.path.join(OUTPUTDIR, imgtype, f) + "/*.fits")
iraf.ccdproc()
#only trim for bias files
if(os.listdir(os.path.join(OUTPUTDIR, "bias"))):
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('biassec', '')
iraf.hselect.setParam("images", os.path.join(OUTPUTDIR, "bias") + "/*.fits")
iraf.ccdproc()
else:
print "No Bias Files present"
print "TrimAndOverscan end"
def flatCorrection():
print "FlatCorrection start"
#put all others params to no , they may be set by previous actions
iraf.ccdproc.setParam('flatcor', 'yes')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
iraf.ccdproc.setParam('trim', 'no')
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('trimsec', '')
iraf.ccdproc.setParam('biassec', '')
#online
iraf.ccdproc.setParam('output', '')
for f in FILTERS:
flatfilename = os.path.join(OUTPUTDIR, "flat", f, "Flat.fits")
if os.path.isfile(flatfilename):
iraf.ccdproc.setParam('flat', flatfilename)
iraf.ccdproc.setParam(
"images",
os.path.join(OUTPUTDIR, "object", f) + "/*.fits")
iraf.ccdproc()
else:
print "Flat file %s not present" % flatfilename
print "FlatCorrection end"
def coroverbiastrim(lstfile):
iraf.noao()
iraf.imred()
iraf.ccdred()
x1,x2,y1,y2 = get_trim_sec()
iraf.ccdproc(images = '@' + lstfile + '//[1]'
, output = '%bo%bo%@' + lstfile
, ccdtype = '', max_cache = 0, noproc = False
, fixpix = False, overscan = True, trim = False
, zerocor = True, darkcor = False, flatcor = False
, illumcor = False, fringecor = False, readcor = False
, scancor = False, readaxis = 'line', fixfile = ''
, biassec = '[5:45,%s:%s]'%(y1,y2), trimsec = '[%s:%s,%s:%s]'%(x1,x2,y1,y2)
, zero = 'Zero', dark = '', flat = '', illum = '', fringe = ''
, minreplace = 1.0, scantype = 'shortscan', nscan = 1
, interactive = False, function = 'chebyshev', order = 1
, sample = '*', naverage = 1, niterate = 1
, low_reject = 3.0, high_reject = 3.0, grow = 1.0)
iraf.ccdproc(images = '%bo%bo%@' + lstfile
, output = '%tbo%tbo%@' + lstfile
, ccdtype = '', max_cache = 0, noproc = False
, fixpix = False, overscan = False, trim = True
, zerocor = False, darkcor = False, flatcor = False
, illumcor = False, fringecor = False, readcor = False
, scancor = False, readaxis = 'line', fixfile = ''
, biassec = '[5:45,%s:%s]'%(y1,y2), trimsec = '[%s:%s,%s:%s]'%(x1,x2,y1,y2)
, zero = 'Zero', dark = '', flat = '', illum = '', fringe = ''
, minreplace = 1.0, scantype = 'shortscan', nscan = 1
, interactive = False, function = 'chebyshev', order = 1
, sample = '*', naverage = 1, niterate = 1
, low_reject = 3.0, high_reject = 3.0, grow = 1.0)
iraf.flpr()
def deimos_preproc(image):
'''Take a MEF DEIMOS image, extract the relevant extensions, trim the
LVM slit masks appropriately, and rename.'''
# Needing grating
graname = get_head(image, "GRATENAM", extn=0)
# "Blue" chip
iraf.imcopy("%s[%i]" % (image, BEXT), "d%s_B.fits" % image[6:10])
iraf.ccdproc("d%s_B.fits" % image[6:10], overscan=yes, trim=yes,
fixpix=yes, biassec=BBIASSEC, trimsec=BTRIMSEC,
fixfile="%s_%i.fits" % (MASK, BEXT))
iraf.imcopy("d%s_B.fits%s" % (image[6:10], DGRATINGS[graname]["blvmreg"]),
"td%s_B.fits" % image[6:10])
iraf.imtranspose("td%s_B.fits" % image[6:10], "rtd%s_B.fits" % image[6:10])
# "Red" chip
iraf.imcopy("%s[%i]" % (image, REXT), "d%s_R.fits" % image[6:10])
iraf.ccdproc("d%s_R.fits" % image[6:10], overscan=yes, trim=yes,
fixpix=yes, biassec=RBIASSEC, trimsec=RTRIMSEC,
fixfile="%s_%i.fits" % (MASK, BEXT))
iraf.imcopy("d%s_R.fits%s" % (image[6:10], DGRATINGS[graname]["rlvmreg"]),
"td%s_R.fits" % image[6:10])
iraf.imtranspose("td%s_R.fits" % image[6:10], "rtd%s_R.fits" % image[6:10])
iraf.rotate("rtd%s_R.fits" % image[6:10], "rtd%s_R.fits" % image[6:10], 180.0)
return
def subtract_dark(self):
"""Subtracting Dark Frame"""
iraf.unlearn(iraf.ccdproc)
iraf.ccdproc(self.data.iraf.modatfile(),
ccdtype="", fixpix="no", overscan="no", trim ="no", zerocor="no", darkcor="yes", flatcor ="no",
dark=self.dark.iin("Dark"))
self.data.idone()
def divide_flat(self):
"""Dividing by Flat Frame"""
iraf.unlearn(iraf.ccdproc)
iraf.ccdproc(self.data.iraf.inatfile(),
output=self.data.iraf.outatfile(append="-Flat"),
flat=self.flat.iin("Flat"),
ccdtype="", fixpix="no", overscan="no", trim ="no", zerocor="no", flatcor="yes", darkcor ="no")
self.data.iraf.done()
def darkcombine(self,
files,
output,
zero=None,
method="median",
rejection="minmax",
ccdtype="",
scale="exposure",
overwrite=True):
"""IRAF darkcombine"""
self.logger.info("Darkcombine Started")
try:
iraf.ccdred.unlearn()
iraf.ccdred.ccdproc.unlearn()
iraf.ccdred.combine.unlearn()
iraf.ccdred.darkcombine.unlearn()
iraf.imred.unlearn()
ccdred.instrument = self.instrument_path
darks = ",".join(files)
out_file = "{}/myraf_darks.flist".format(self.fop.tmp_dir)
with open(out_file, "w") as f2w:
for i in files:
f2w.write("{}\n".format(i))
darks = "@{}".format(out_file)
if zero is not None:
iraf.ccdproc(images=darks,
ccdtype='',
fixpix='no',
oversca="no",
trim="no",
zerocor='yes',
darkcor='no',
flatcor='no',
zero=zero,
Stdout="/dev/null")
if self.fop.is_file(output) and overwrite:
self.logger.warning("Over Writing file({})".format(output))
self.fop.rm(output)
iraf.darkcombine(input=darks,
output=output,
combine=method,
reject=rejection,
ccdtype=ccdtype,
scale=scale,
process="no",
Stdout="/dev/null")
return True
except Exception as e:
self.logger.error(e)
return False
def red_preproc(image):
'''Remove overscan and trim image from blue detector.'''
gratname = get_head(image, "GRATING_N", extn=0)
iraf.ccdproc(image, overscan=yes, trim=yes, biassec=BBIASSEC,
trimsec=KGRATINGS[gratname]["sreg"])
return
def red_science(image, flats, spath, object=None, arc='Arc-Red.fits',
smooth='BD284211.smooth.fits', telluric='telluric.fits',
sens='BD284211.sens', biassec=REDBIAS, trimsec=REDTRIM,
outflat='Flat-Red.fits', gain=REDGAIN, rdnoise=REDRDNOISE):
'''Full reduction of KAST science spectra on red CCD'''
# Bias subtract everything first
redbias(image, biassec=biassec, trimsec=trimsec)
redbias(flats, biassec=biassec, trimsec=trimsec)
# Create and apply flat-field
make_flat(flats, outflat, gain=gain, rdnoise=rdnoise)
iraf.ccdproc(image[0], ccdtype='', noproc=no, fixpix=no, overscan=no,
trim=no, zerocor=no, darkcor=no, flatcor=yes, illumcor=no,
fringecor=no, readcor=no, scancor=no, flat=outflat)
# Cosmic ray rejection
#iraf.lacos_spec(image[0], 'c%s' % image[0], 'cm%s' % image[0],
# gain=gain, readn=rdnoise, xorder=9, yorder=3,
# sigclip=4.5, sigfrac=0.5, objlim=1.0, niter=3)
# Extract spectrum
if object==None:
object=get_head(image, 'OBJECT')
iraf.apall('%s' % image[0], output=object, references='', interactive=yes,
find=yes, recenter=yes, resize=yes, edit=yes, trace=yes,
fittrace=yes, extract=yes, extras=yes, review=no,
background='fit', weights='variance', pfit='fit1d',
readnoise=rdnoise, gain=gain)
# Apply wavelength solution to standard
shutil.copy('%s/%s' % (spath, arc), '.')
shutil.copy('%s/database/id%s' % (spath, arc.rstrip('.fits')), 'database')
iraf.refspec(object, references=arc, sort="", group="", override=yes,
confirm=no, assign=yes)
iraf.dispcor(object, '%s.w' % object, confirm=no, listonly=no)
# Smooth
shutil.copy('%s/%s' % (spath, smooth), '.')
iraf.sarith('%s.w' % object, '/', smooth, '%s.s' % object)
# Create and apply telluric correction
shutil.copy('%s/%s' % (spath, telluric), '.')
iraf.telluric('%s.s' % object, '%s.t' % object, telluric, xcorr=yes,
tweakrms=yes, interactive=yes, sample='6850:6950,7575:7700')
# Flux calibration
shutil.copy('%s/%s.0001.fits' % (spath, sens), '.')
iraf.calibrate('%s.t' % object, '%s.f' % object, extinct=yes, flux=yes,
extinction='onedstds$kpnoextinct.dat',
observatory='Lick', sensitivity=sens, airmass='',
exptime='')
return
def ccdproc(self,
file,
output=None,
zero=None,
dark=None,
flat=None,
subset="no"):
"""
Does IRAF calibration
IRAF ccdproc
"""
self.logger.info("Ccdproc Started")
try:
iraf.imred.unlearn()
iraf.ccdred.unlearn()
iraf.ccdred.ccdproc.unlearn()
ccdred.instrument = self.instrument_path
yes_no = {True: "no", False: "yes"}
if output is None:
output = ""
zeroCor = yes_no[zero is None]
darkCor = yes_no[dark is None]
flatCor = yes_no[flat is None]
if flat is not None:
if subset == "no":
self.fts.update_header(flat, "subset", "")
iraf.ccdred.flatcombine.subset = subset
if subset == "no":
self.fts.update_header(file, "subset", "")
iraf.ccdproc(images=file,
output=output,
ccdtype='',
fixpix='no',
oversca="no",
trim="no",
zerocor=zeroCor,
darkcor=darkCor,
flatcor=flatCor,
zero=zero,
dark=dark,
flat=flat,
Stdout="/dev/null")
return True
except Exception as e:
self.logger.error(e)
return False
def reduceLight(inFn, outFn, mBiasFn, mDarkFn, mFlatFn):
logger = logging.getLogger(__name__)
if os.path.isfile(outFn + ".fits"):
return
logger.debug(inFn + " -> " + outFn)
flatcor = "no"
zerocor = "no"
darkcor = "no"
if os.path.isfile(mFlatFn + ".fits"):
flatcor = "yes"
if os.path.isfile(mBiasFn + ".fits"):
zerocor = "yes"
if os.path.isfile(mDarkFn + ".fits"):
darkcor = "yes"
iraf.ccdproc(
inFn,
output=outFn,
ccdtype=" ",
max_cache=0,
noproc="no",
fixpix="no",
overscan="no",
trim="no",
zerocor=zerocor,
darkcor=darkcor,
flatcor=flatcor,
illumcor="no",
fringecor="no",
readcor="no",
scancor="no",
readaxis="line",
fixfile="",
biassec="",
trimsec="",
zero=mBiasFn,
dark=mDarkFn,
flat=mFlatFn,
illum="",
fringe="",
minreplace=1.0,
scantype="shortscan",
nscan=1,
interactive="no",
function="legendre",
order=1,
sample="*",
naverage=1,
niterate=1,
low_reject=3.0,
high_reject=3.0,
grow=0.0,
)
def redbias(images, biassec=REDBIAS, trimsec=REDTRIM):
'''Subtract overscan and trim red frames'''
for image in images:
iraf.ccdproc(image, ccdtype='', noproc=no, fixpix=no, overscan=yes,
trim=yes, zerocor=no, darkcor=no, flatcor=no, illumcor=no,
fringecor=no, readcor=no, scancor=no, biassec=biassec,
trimsec=trimsec)
update_head(image, 'DISPAXIS', 1)
return
def blue_science(image, spath, object=None, flat='Flat-Blue.fits',
arc='Arc-Blue.fits', smooth='BD284211.smooth.fits',
sens='BD284211.sens', biassec1=BLUEBIAS1, trimsec1=BLUETRIM1,
biassec2=BLUEBIAS2, trimsec2=BLUETRIM2, gain=BLUEGAIN,
rdnoise=BLUERDNOISE):
'''Full reduction of KAST science spectra on blue CCD'''
# Bias subtract everything first
bluebias(image, biassec1=biassec1, trimsec1=trimsec1, biassec2=biassec2,
trimsec2=trimsec2)
# Apply flat-field
shutil.copy('%s/%s' % (spath, flat), '.')
iraf.ccdproc('j%s' % image[0], ccdtype='', noproc=no, fixpix=no,
overscan=no, trim=no, zerocor=no, darkcor=no, flatcor=yes,
illumcor=no, fringecor=no, readcor=no, scancor=no,
flat=flat)
# Cosmic ray rejection
#iraf.lacos_spec('j%s' % image[0], 'cj%s' % image[0], 'cmj%s' % image[0],
# gain=gain, readn=rdnoise, xorder=9, yorder=3,
# sigclip=4.5, sigfrac=0.5, objlim=1.0, niter=3)
# Extract spectrum
if object==None:
object=get_head(image, 'OBJECT')
iraf.apall('j%s' % image[0], output=object, references='',
interactive=yes, find=yes, recenter=yes, resize=yes, edit=yes,
trace=yes, fittrace=yes, extract=yes, extras=yes, review=no,
background='fit', weights='variance', pfit='fit1d',
readnoise=rdnoise, gain=gain)
# Apply wavelength solution to standard
shutil.copy('%s/%s' % (spath, arc), '.')
shutil.copy('%s/database/id%s' % (spath, arc.rstrip('.fits')), 'database')
iraf.refspec(object, references=arc, sort="", group="", override=yes,
confirm=no, assign=yes)
iraf.dispcor(object, '%s.w' % object, confirm=no, listonly=no)
# Smooth
shutil.copy('%s/%s' % (spath, smooth), '.')
iraf.sarith('%s.w' % object, '/', smooth, '%s.s' % object)
# Flux calibration
shutil.copy('%s/%s.0001.fits' % (spath, sens), '.')
iraf.calibrate('%s.s' % object, '%s.f' % object, extinct=yes, flux=yes,
extinction='onedstds$kpnoextinct.dat',
observatory='Lick', sensitivity=sens, airmass='',
exptime='')
return
def calibrateImages(imageList, masterBias, masterDark, masterFlat, outputFile):
calibrateString = getIrafString(imageList)
iraf.ccdproc(images=calibrateString,
zerocor="yes",
darkcor="yes",
flatcor="yes",
zero=masterBias,
dark=masterDark,
flat=masterFlat,
output=outputFile,
fixpix="No",
ccdtype="none",
overscan="no")
def flatten(flatFile):
"""grab any sky-subtracted images, and flatten them using flatFile"""
# first we have to clean up the flat, and get rid of any negative numbers or zeros
# just set them equal to 1 (??? why do we do this ???)
iraf.imreplace(flatFile, 1.0, imaginary=0.0, lower="INDEF", upper=1.0, radius=0.0)
# get a list of the sky subtracted images
skyims = glob.glob("scratch/s-binir*fits")
flatIms = ["s-f-" + i[10:] for i in skyims]
# organize input and output in iraf-friendly ways
inputFiles = joinStrList(skyims)
outputFiles = joinStrList(flatIms, scratch=True)
# now flatfield the sky subtracted images
iraf.ccdproc(
images=inputFiles,
output=outputFiles,
ccdtype="",
max_cache=0,
noproc="no",
fixpix="no",
overscan="no",
trim="no",
zerocor="no",
darkcor="no",
flatcor="yes",
illumcor="no",
fringecor="no",
readcor="no",
scancor="no",
readaxis="line",
fixfile="",
biassec="",
trimsec="",
zero="",
dark="",
flat=flatFile,
illum="",
fringe="",
minreplace=1.0,
scantype="shortscan",
nscan=1,
interactive="no",
function="legendre",
order=1,
sample="*",
naverage=1,
niterate=1,
low_reject=3.0,
high_reject=3.0,
grow=0.0,
)
return
def main(raw_dir):
if not os.path.exists("{}/step1.tmp".format(os.getcwd())):
print "Setting up directory"
list_dict = setup_dir(raw_dir)
print "OT'ing"
iraf.ccdproc(
"@all_raw.lst",
output="@all_ot.lst",
overscan=True,
trim=True,
zerocor=False,
darkcor=False,
flatcor=False,
illumco=False,
biassec="image",
trimsec="image",
zero="",
interact=True,
order=100,
)
cull_bias(list_dict["zeros"], "zeros.lst")
os.system("touch {}/step1.tmp".format(os.getcwd()))
return
print "Making master bias"
iraf.zerocombine("@zeros.lst", output="Zero.fits", combine="average", reject="crreject")
print "Subtracting bias"
iraf.ccdproc("@all_otz.lst", overscan=False, trim=False, zerocor=True, zero="Zero.fits")
detect_flats(os.getcwd())
flats_to_make = glob("dflat*.lst")
for flat in flats_to_make:
name = "dFlat_{}.fits".format(flat.split(".lst")[0].split("_")[1])
print "Making flat {}".format(name)
iraf.flatcombine("@{}".format(flat), output=name, combine="average", reject="crreject")
try:
print "Making master dark"
iraf.darkcombine("@darks.lst", output="Dark.fits", combine="average", reject="crreject")
except Exception as e:
print "\t{}".format(e)
print "\tNO DARK FOUND"
print "Making master comp"
iraf.imcombine("@comps.lst", "Comp.fits", combine="average", reject="crreject")
return
def trimPicture(image, output, trimsec):
iraf.ccdproc.setParam('images', image)
iraf.ccdproc.setParam('output', output)
iraf.ccdproc.setParam('ccdtype', '')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('trim', 'yes')
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('illumco', 'no')
iraf.ccdproc.setParam('readcor', 'no')
iraf.ccdproc.setParam('scancor', 'no')
iraf.ccdproc.setParam('trimsec', trimsec)
iraf.ccdproc()
def darkCorrect(image, output, dark):
iraf.ccdproc.setParam('images', image)
iraf.ccdproc.setParam('output', output)
iraf.ccdproc.setParam('ccdtype', '')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('trim', 'no')
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('darkcor', 'yes')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('illumco', 'no')
iraf.ccdproc.setParam('readcor', 'no')
iraf.ccdproc.setParam('scancor', 'no')
iraf.ccdproc.setParam('dark', dark)
iraf.ccdproc()
def trimPicture(image, output, trimsec):
iraf.ccdproc.setParam('images', image)
iraf.ccdproc.setParam('output', output)
iraf.ccdproc.setParam('ccdtype', '')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('trim', 'yes')
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('illumco', 'no')
iraf.ccdproc.setParam('readcor', 'no')
iraf.ccdproc.setParam('scancor', 'no')
iraf.ccdproc.setParam('trimsec', trimsec)
iraf.ccdproc()
def darkCorrect(image, output, dark):
iraf.ccdproc.setParam('images', image)
iraf.ccdproc.setParam('output', output)
iraf.ccdproc.setParam('ccdtype', '')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('trim', 'no')
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('darkcor', 'yes')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('illumco', 'no')
iraf.ccdproc.setParam('readcor', 'no')
iraf.ccdproc.setParam('scancor', 'no')
iraf.ccdproc.setParam('dark', dark)
iraf.ccdproc()
def correct_overscan(filename): iraf.ccdproc(images = '@' + filename + '//[1]' , output = '%o%o%@' + filename , ccdtype = '', max_cache = 0, noproc = False , fixpix = False, overscan = True, trim = False , zerocor = False, darkcor = False, flatcor = False , illumcor = False, fringecor = False, readcor = False , scancor = False, readaxis = 'line', fixfile = '' , biassec = '[5:45,1:4612]', trimsec = '', zero = '' , dark = '', flat = '', illum = '', fringe = '' , minreplace = 1.0, scantype = 'shortscan', nscan = 1 , interactive = False, function = 'legendre', order = 1 , sample = '*', naverage = 1, niterate = 1 , low_reject = 3.0, high_reject = 3.0, grow = 0.0) iraf.flpr()
def correct_bias(filename1, filename2):
iraf.ccdproc(images = 'o//@' + filename1
, output = 'bo//@' + filename1
, ccdtype = '', max_cache = 0, noproc = False
, fixpix = False, overscan = False, trim = False
, zerocor = True, darkcor = False, flatcor = False
, illumcor = False, fringecor = False, readcor = False
, scancor = False, readaxis = 'line', fixfile = ''
, biassec = '', trimsec = '', zero = filename2
, dark = '', flat = '', illum = '', fringe = ''
, minreplace = 1.0, scantype = 'shortscan', nscan = 1
, interactive = False, function = 'chebyshev', order = 1
, sample = '*', naverage = 1, niterate = 1
, low_reject = 3.0, high_reject = 3.0, grow = 0.0)
iraf.flpr()
def bluebias(images,
biassec1=BLUEBIAS1,
trimsec1=BLUETRIM1,
biassec2=BLUEBIAS2,
trimsec2=BLUETRIM2):
'''Subtract overscan and trim blue frames'''
for image in images:
root, ext = image.split('.')
iraf.ccdproc(image,
output='%s_1' % root,
ccdtype='',
noproc=no,
fixpix=no,
overscan=yes,
trim=yes,
zerocor=no,
darkcor=no,
flatcor=no,
illumcor=no,
fringecor=no,
readcor=no,
scancor=no,
biassec=biassec1,
trimsec=trimsec1)
iraf.ccdproc(image,
output='%s_2' % root,
ccdtype='',
noproc=no,
fixpix=no,
overscan=yes,
trim=yes,
zerocor=no,
darkcor=no,
flatcor=no,
illumcor=no,
fringecor=no,
readcor=no,
scancor=no,
biassec=biassec2,
trimsec=trimsec2)
iraf.imjoin('%s_1,%s_2' % (root, root), 'j%s' % image, 1)
update_head('j%s' % image, 'DISPAXIS', 1)
update_head('j%s' % image, ['CCDSEC', 'DATASEC'],
['[1:2048,1:270]', '[1:2048,1:270]'])
return
def basic_2d_proc(rawFile,imgType=None,CLOBBER=False):
# set up file names based on our convention
oScanFile = 'pre_reduced/o{}'.format(rawFile)
toScanFile = 'pre_reduced/to{}'.format(rawFile)
toScanFlat = 'pre_reduced/to{}'.format(rawFile.replace('.fits','_norm.fits'))
# run the basic 2D stuff on this image if necessary
if (not os.path.isfile(oScanFile)) or CLOBBER:
# get the instrument configuration
inst = instruments.blue_or_red(rawFile)[1]
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_biassec0 = inst.get('biassec')
_trimsec0 = inst.get('trimsec')
_flatsec0 = inst.get('flatsec')
# remove destination files
for file in [oScanFile,toScanFile]:
if os.path.isfile(file) and CLOBBER:
os.remove(file)
# check the ultimate desination file, since intermediates get deleted
if not os.path.isfile(toScanFile):
if inst.get('instrument')=='kast':
# do Lick specific bias operations
util.kastbias(rawFile,oScanFile)
elif inst.get('instrument')=='binospec':
util.binospecbias(rawFile,oScanFile,inst)
# do general (IRAF is in such a sorry state I'm not even sure if this works)
else:
iraf.ccdproc(rawFile, output=oScanFile,
overscan='yes', trim='yes',
zerocor="no", flatcor="no",readaxi='line',
trimsec=str(_trimsec0), biassec=str(_biassec0),
Stdout=1)
# trim (same trimming operation for all telescopes)
iraf.ccdproc(oScanFile, output=toScanFile,
overscan='no', trim='yes', zerocor="no", flatcor="no",
readaxi='line',trimsec=str(_trimsec0), Stdout=1)
#create trimmed flats for norm region
if imgType == 'CAL_FLAT' and inst.get('instrument')=='kast':
iraf.ccdproc(oScanFile, output=toScanFlat,
overscan='no', trim='yes', zerocor="no", flatcor="no",
readaxi='line',trimsec=str(_flatsec0), Stdout=1)
if imgType == 'CAL_FLAT' and inst.get('instrument')=='binospec':
iraf.ccdproc(oScanFile, output=toScanFlat,
overscan='no', trim='yes', zerocor="no", flatcor="no",
readaxi='line',trimsec=str(_flatsec0), Stdout=1)
os.remove(oScanFile)
return 0
def correct_flat(filename, flatname):
# flatname = 'per' + filename.replace('_no_flat.lst', '_flat.fits')
iraf.ccdproc(images = 'tbo//@' + filename
, output = 'ftbo//@' + filename
, ccdtype = '', max_cache = 0, noproc = False
, fixpix = False, overscan = False, trim = False
, zerocor = False, darkcor = False, flatcor = True
, illumcor = False, fringecor = False, readcor = False
, scancor = False, readaxis = 'line', fixfile = ''
, biassec = '', trimsec = '', zero = ''
, dark = '', flat = flatname, illum = '', fringe = ''
, minreplace = 1.0, scantype = 'shortscan', nscan = 1
, interactive = False, function = 'legendre', order = 1
, sample = '*', naverage = 1, niterate = 1
, low_reject = 3.0, high_reject = 3.0, grow = 0.0)
iraf.flpr()
def trimImages():
print "TrimAndOverscan start"
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
iraf.ccdproc.setParam('trim', 'yes')
iraf.ccdproc.setParam('trimsec', DATA_SEC)
iraf.ccdproc.setParam('overscan', 'yes')
iraf.ccdproc.setParam('biassec', BIAS_SEC)
iraf.ccdproc.setParam("images", "imagenesfits/*.fits")
#online
iraf.ccdproc.setParam('output', '')
iraf.ccdproc()
print "TrimAndOverscan end"
def trimImages():
print "TrimAndOverscan start"
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('flatcor', 'no')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
iraf.ccdproc.setParam('trim', 'yes')
iraf.ccdproc.setParam('trimsec', DATA_SEC)
iraf.ccdproc.setParam('overscan', 'yes')
iraf.ccdproc.setParam('biassec', BIAS_SEC)
iraf.ccdproc.setParam("images", "imagenesfits/*.fits")
#online
iraf.ccdproc.setParam('output', '')
iraf.ccdproc()
print "TrimAndOverscan end"
def optreduce(fwheel):
'''
reduces optical andicam data
required: combined optical biases and flats, unreduced data need to be in working directory
also in.{B,V,R,I} and out.{B,V,R,I}, which are text files that list data taken w respective filters
input: fwheel is a python list that holds the names of the filters for which you want to reduce
output: rccd versions of ccd*.fits images which are bias and flat corrected are output in working directory
make sure we have a bias so we can bias subtract the data
'''
if len(glob.glob('*.bias*')) < 1:
print "no combined bias found, exiting. Please place a combined bias in this directory and try agian"
return
else:
for color in fwheel:
#check that all necessary files exist for reduction, in.color, out.color
if len(glob.glob('in.'+color)) < 1:
print "in."+color+" not found. "+color+" data will not be reduced. Please create file and try again"
#elif len(glob.glob('out.'+color)) < 1:
# print "out."+color+" not found. "+color+" data will not be reduced. Please create file and try again"
else:
#B data uses the skyflat
if color=='B':
if len(glob.glob('*.skyflatB*')) < 1:
print "no combined B skyflat found. B data will not be reduced. Please create combined B skyflat and try again"
else:
with open("in.B") as f:
num_images=sum(1 for line in f)
if num_images > 1:
print str(num_images)+" B images found. Reducing ..."
iraf.ccdproc(images="@in.B",output="[email protected]",overscan="yes",trim="yes",zerocor="yes",darkcor="no",flatcor="yes",readaxis="line",biassec="[2:16,3:1026]",trimsec="[17:1040,3:1026]",zero="*.bias.fits",flat="*.skyflatB.fits",interactive="no",function="spline3",order="11")
else:
print "No B images found"
#all other data uses domes
elif color in ['V','R','I']:
if len(glob.glob('*.dome'+color+'.fits')) < 1:
print "no combined "+color+" dome found. "+color+" data will not be reduced. Please create combined "+color+" dome and try again"
else:
with open("in."+color) as f:
num_images=sum(1 for line in f)
if num_images > 1:
print str(num_images)+" "+color+" images found. Reducing ..."
iraf.ccdproc(images="@in."+color,output="r@in."+color,overscan="yes",trim="yes",zerocor="yes",darkcor="no",flatcor="yes",readaxis="line",biassec="[2:16,3:1026]",trimsec="[17:1040,3:1026]",zero="*.bias.fits",flat="*.dome"+color+".fits",interactive="no",function="spline3",order="11")
else:
print "No "+color+" images found."
else:
print color+" is not recognized as a filter. Please use 'B', 'V', 'R', or I"
return
def coroverscan(lstfn):
"""
call iraf command ccdproc, overscan correct.
lstfn : lst file name
type : string
output file : oYF*.fits
"""
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.ccdproc(images='@' + lstfn + '//[1]',
output='%o%o%@' + lstfn,
ccdtype='',
max_cache=0,
noproc=False,
fixpix=False,
overscan=True,
trim=False,
zerocor=False,
darkcor=False,
flatcor=False,
illumcor=False,
fringecor=False,
readcor=False,
scancor=False,
readaxis='line',
fixfile='',
biassec='[5:45,1:4612]',
trimsec='',
zero='',
dark='',
flat='',
illum='',
fringe='',
minreplace=1.0,
scantype='shortscan',
nscan=1,
interactive=False,
function='chebyshev',
order=1,
sample='*',
naverage=1,
niterate=1,
low_reject=3.0,
high_reject=3.0,
grow=1.0)
iraf.flpr()
def trim(tblst, trimsec):
iraf.ccdproc(images='b//@' + tblst,
output='tb//@' + tblst,
ccdtype='',
noproc=False,
overscan=False,
trim=True,
zerocor=False,
flatcor=False,
readaxis='line',
biassec='',
trimsec=trimsec,
interactive=False,
function='legendre',
order=1)
iraf.flpr()
print '<<<<<trim section successfully>>>>>'
def optdomecomb(date, fwheel=['bias','B','V','R','I']):
'''
#combine biases and optical domes
#Requires: the uncombined fits images
# if you are combining a dome, you must have a bias from the same night as the dome to preform appropriate bias subtraction
#Input: the date the domes were observed YYMMDD, and fwheel, a list that contains the filters of the domes to be combined
#Outupt: combined dome fits frame for each color where uncombined frames are in the directory
'''
#convert date to string incase it was entered as an int of float
date=str(date)
if len(glob.glob('*bias*')) < 1:
print "no biases found, exiting"
return
else:
for color in fwheel:
if color=='bias':
biaslist=glob.glob('*bias.[0-9]*')
if len(biaslist) > 10:
print "only "+str(len(biaslist))+" biases found. you need at least 10"
else:
with open("bias.list",'w') as BILIS:
for i in biaslist:
BILIS.write(i+'\n')
iraf.zerocombine("@bias.list",output="ccd"+str(date)+".bias.fits",combine="average",reject="minmax",scale="none",ccdtype="",process="no",delete="no",clobber="no",nlow=1,nhigh=1,nkeep=1)
print "created ccd"+str(date)+".bias.fits"
os.system('rm bias.list')
elif color in ['B','V','R','I']:
domelist=glob.glob('*dome'+color+'.[0-9]*')
if len(domelist) < 1:
print 'no '+color+' domes found'
elif len(domelist) > 10:
print 'only '+str(len(domelist))+' domes found. you need at least 10'
else:
with open('flat'+color+'.list', 'w') as flist:
for i in domelist:
flist.write(i+'\n')
iraf.ccdproc("@flat"+color+".list", output="z@flat"+color+".list",ccdtype=" ",noproc="no", fixpix="no",overscan="yes", trim="no", zerocor="yes",darkcor="no",flatcor="no", illumcor="no", fringec="no", readcor="no", scancor="no", readaxis="line", biassec="[3:14,1:1024]", zero="ccd"+str(date)+".bias.fits", interactive="no", functio="spline3", order=11)
iraf.flatcombine("z@flat"+color+".list", output="ccd"+str(date)+".dome"+color+".fits",combine="average", reject="crreject", ccdtype="", process="no", subsets="no", delete="no", clobber="no", scale="mode", rdnoise=6.5, gain=2.3)
os.system('rm z*dome'+color+'*fits')
print "created ccd"+str(date)+".dome"+color+".fits"
os.system('rm flat'+color+'.list')
else:
print "your input for the filter was not recognized. Please use either 'bias', 'B', 'V', 'R', or 'I' and try again"
return
def corhalogen(lstfile):
namelst = glob.glob('ftbo*.fits')
for name in namelst:
print 'remove ', name
os.remove(name)
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.ccdproc(images='tbo@' + lstfile,
output='%ftbo%ftbo%@' + lstfile,
ccdtype='',
max_cache=0,
noproc=False,
fixpix=False,
overscan=False,
trim=False,
zerocor=False,
darkcor=False,
flatcor=True,
illumcor=False,
fringecor=False,
readcor=False,
scancor=False,
readaxis='line',
fixfile='',
biassec='',
trimsec='',
zero='Zero',
dark='',
flat='Resp',
illum='',
fringe='',
minreplace=1.0,
scantype='shortscan',
nscan=1,
interactive=False,
function='chebyshev',
order=1,
sample='*',
naverage=1,
niterate=1,
low_reject=3.0,
high_reject=3.0,
grow=1.0)
iraf.flpr()
def corhalogen(lstfile):
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.ccdproc(images = 'tbo@' + lstfile
, output = '%ftbo%ftbo%@' + lstfile
, ccdtype = '', max_cache = 0, noproc = False
, fixpix = False, overscan = False, trim = False
, zerocor = False, darkcor = False, flatcor = True
, illumcor = False, fringecor = False, readcor = False
, scancor = False, readaxis = 'line', fixfile = ''
, biassec = '', trimsec = ''
, zero = 'Zero', dark = '', flat = 'Resp', illum = '', fringe = ''
, minreplace = 1.0, scantype = 'shortscan', nscan = 1
, interactive = False, function = 'chebyshev', order = 1
, sample = '*', naverage = 1, niterate = 1
, low_reject = 3.0, high_reject = 3.0, grow = 1.0)
iraf.flpr()
def reducePyraf(filelist, outputlist, bias, flat, trim, trimsec):
"""
Reduces the given science frame(s). Uses pyraf.
"""
from pyraf import iraf
from pyraf.iraf import imred, ccdred
ccdtype = " "
iraf.ccdproc(images=filelist,
output=outputlist,
ccdtype=ccdtype,
trim=trim,
zerocor="yes",
darkcor="no",
flatcor="yes",
trimsec=trimsec,
zero=bias,
flat=flat)
def bias_correct(file_list,xmin='18',xmax='4111',ymin='350',ymax='1570',bindefault=1):
irf_prm.set_ccdproc(iraf.ccdproc)
biassecs = {1:'[4114:4142,1:1896]',2:'[4114:4142,1:1896]',3:'[4114:4142,1:1896]',4:'[4114:4142,1:1896]'} #NEED UPDATED
#iraf.ccdproc.biassec = biassecs[binsize]
iraf.ccdproc.trimsec = '['+xmin+':'+xmax+','+ymin+':'+ymax+']'
for ff in file_list:
comm = scnd.get_comment(ff,'PARAM18')
if comm == '1 / Serial Binning,Pixels':
binsize = iraf.hselect(ff,'PARAM18','yes')
else:
param_name = scnd.find_param_with_comment(ff,'1 / Serial Binning,Pixels')
if param_name == 'NullReturn':
binsize=bindefault
else:
binsize = iraf.hselect(ff,param_name,'yes')
iraf.ccdproc.biassec = biassecs[binsize]
output = 'b'+ff
iraf.ccdproc(images=ff,output=output,ccdtype = "")
return
def correct_trim(filename, trimsec):
# print 'Please check the fits image and determine the size to trim...'
# os.popen('gedit ' + filename +' &')
# iraf.imexamine(input = 'bo//@' + filename, frame = 1)
# xy = raw_input("Please input x1, x2, y1, y2 to trim: ")
# xy = xy.split()
# trimsec = '[' + xy[0] + ':' + xy[1] + ',' + xy[2] + ':' + xy[3] +']'
iraf.ccdproc(images = 'bo//@' + filename
, output = 'tbo//@' + filename
, ccdtype = '', max_cache = 0, noproc = False
, fixpix = False, overscan = False, trim = True
, zerocor = False, darkcor = False, flatcor = False
, illumcor = False, fringecor = False, readcor = False
, scancor = False, readaxis = 'line', fixfile = ''
, biassec = '', trimsec = trimsec, zero = ''
, dark = '', flat = '', illum = '', fringe = ''
, minreplace = 1.0, scantype = 'shortscan', nscan = 1
, interactive = False, function = 'legendre', order = 1
, sample = '*', naverage = 1, niterate = 1
, low_reject = 3.0, high_reject = 3.0, grow = 0.0)
iraf.flpr()
def combineflats(inflats, outflat, outflatdc, darkflat, flat_sigmas=None):
ir.imcombine("@" + inflats,
output=outflat,
combine="average",
reject="crreject",
scale="median",
weight="median",
bpmasks="") # sigmas=flat_sigmas
ns.write_exptime(outflat, itime=itime)
print(outflat)
ir.ccdproc(outflat,
output=outflatdc,
ccdtype="",
fixpix="no",
overscan="no",
trim="no",
zerocor="no",
darkcor="yes",
flatcor="no",
dark=darkflat)
def cor_flat(corftblst):
print 1
iraf.ccdproc(images='tb//@' + corftblst,
output='ftb//@' + corftblst,
ccdtype='',
max_cache=0,
noproc=False,
fixpix=False,
overscan=False,
trim=False,
zerocor=False,
darkcor=False,
flatcor=True,
illumcor=False,
fringecor=False,
readcor=False,
scancor=False,
readaxis='line',
fixfile='',
biassec='',
trimsec='',
zero='',
dark='',
flat='flat',
illum='',
fringe='',
minreplace=1.0,
scantype='shortscan',
nscan=1,
interactive=False,
function='chebyshev',
order=1,
sample='*',
naverage=1,
niterate=1,
low_reject=3.0,
high_reject=3.0,
grow=1.0)
iraf.flpr()
print '<<<<<correct flat successfully>>>>>'
def flat_correction(targetdir,flatdir):
"""
Flat field correction
"""
print 'Target directory is ' + targetdir
print 'Flat directory is ' + flatdir
print 'Applying flat field correction...'
innames, outnames = [], []
for n in os.listdir(targetdir):
if (n.endswith('.fit')) & (n.startswith('cr')):
outnames.append( os.path.join(targetdir,'f' + n) )
innames.append( os.path.join( targetdir, n + '[1]') )
if os.path.exists( os.path.join( targetdir, 'f' + n) ):
print 'Removing file ' + os.path.join( targetdir, 'f' + n)
os.remove( os.path.join( targetdir, 'f' + n) )
with open( os.path.join(targetdir,'input.list'), 'w') as f:
for name in innames:
f.write( name + '\n' )
with open( os.path.join(targetdir,'output.list'), 'w') as f:
for name in outnames:
f.write( name + '\n' )
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.ccdproc.setParam('images', '@' + os.path.join(targetdir, 'input.list') )
iraf.ccdproc.setParam('flatcor','yes')
iraf.ccdproc.setParam('flat', os.path.join(flatdir,'nFlat.fits') )
iraf.ccdproc.setParam('output', '@' + os.path.join(targetdir, 'output.list'))
iraf.ccdproc()
return None
def redbias(images, biassec=REDBIAS, trimsec=REDTRIM):
'''Subtract overscan and trim red frames'''
for image in images:
iraf.ccdproc(image,
ccdtype='',
noproc=no,
fixpix=no,
overscan=yes,
trim=yes,
zerocor=no,
darkcor=no,
flatcor=no,
illumcor=no,
fringecor=no,
readcor=no,
scancor=no,
biassec=biassec,
trimsec=trimsec)
update_head(image, 'DISPAXIS', 1)
return
def clean_folder_write_header(WORK_DIR):
print '\n + Cleaning folder, truncating edges, fixing bad pixels\n'
for obj in observations[WORK_DIR]['objects']:
try: os.remove(obj+'.fits')
except: pass
try: os.remove(obj+'.ec.fits')
except: pass
iraf.imcopy(input=observations[WORK_DIR]['ORIG_DIR']+obj, output=obj)
for cal in observations[WORK_DIR]['calibs']:
try: os.remove(cal+'.fits')
except: pass
iraf.imcopy(input=observations[WORK_DIR]['ORIG_DIR']+cal, output=cal)
with open('badpix.txt', 'w+') as file:
file.write('704 704 1262 2048\n703 705 1262 1277')
iraf.hedit(images=','.join(observations[WORK_DIR]['objects']+observations[WORK_DIR]['calibs']), fields='CCDSEC', value='[51:2098,1:2048]', add='yes', verify='no')
iraf.hedit(images=','.join(observations[WORK_DIR]['objects']+observations[WORK_DIR]['calibs']), fields='DATASEC', value='[51:2098,1:2048]', add='yes', verify='no')
# FIX badpix
iraf.ccdproc(images=','.join(observations[WORK_DIR]['objects']+observations[WORK_DIR]['calibs']), ccdtype='', fixpix='yes', oversca='no', trim='yes', zerocor='no', darkcor='no', flatcor='no', fixfile='badpix.txt')
def flatCorrection():
print "FlatCorrection start"
#put all others params to no , they may be set by previous actions
iraf.ccdproc.setParam('flatcor', 'yes')
iraf.ccdproc.setParam('fixpix', 'no')
iraf.ccdproc.setParam('darkcor', 'no')
iraf.ccdproc.setParam('illumcor', 'no')
iraf.ccdproc.setParam('trim', 'no')
iraf.ccdproc.setParam('overscan', 'no')
iraf.ccdproc.setParam('zerocor', 'no')
iraf.ccdproc.setParam('trimsec', '')
iraf.ccdproc.setParam('biassec', '')
#online
iraf.ccdproc.setParam('output', '')
for f in FILTERS:
flatfilename = os.path.join(OUTPUTDIR, "flat", f, "Flat.fits")
if os.path.isfile(flatfilename):
iraf.ccdproc.setParam('flat', flatfilename)
iraf.ccdproc.setParam("images", os.path.join(OUTPUTDIR, "object", f) + "/*.fits")
iraf.ccdproc()
else:
print "Flat file %s not present" % flatfilename
print "FlatCorrection end"
def deimos_preproc(image):
'''Take a MEF DEIMOS image, extract the relevant extensions, trim the
LVM slit masks appropriately, and rename.'''
# Needing grating
graname = get_head(image, "GRATENAM", extn=0)
# "Blue" chip
iraf.imcopy("%s[%i]" % (image, BEXT), "d%s_B.fits" % image[6:10])
iraf.ccdproc("d%s_B.fits" % image[6:10],
overscan=yes,
trim=yes,
fixpix=yes,
biassec=BBIASSEC,
trimsec=BTRIMSEC,
fixfile="%s_%i.fits" % (MASK, BEXT))
iraf.imcopy("d%s_B.fits%s" % (image[6:10], DGRATINGS[graname]["blvmreg"]),
"td%s_B.fits" % image[6:10])
iraf.imtranspose("td%s_B.fits" % image[6:10], "rtd%s_B.fits" % image[6:10])
# "Red" chip
iraf.imcopy("%s[%i]" % (image, REXT), "d%s_R.fits" % image[6:10])
iraf.ccdproc("d%s_R.fits" % image[6:10],
overscan=yes,
trim=yes,
fixpix=yes,
biassec=RBIASSEC,
trimsec=RTRIMSEC,
fixfile="%s_%i.fits" % (MASK, BEXT))
iraf.imcopy("d%s_R.fits%s" % (image[6:10], DGRATINGS[graname]["rlvmreg"]),
"td%s_R.fits" % image[6:10])
iraf.imtranspose("td%s_R.fits" % image[6:10], "rtd%s_R.fits" % image[6:10])
iraf.rotate("rtd%s_R.fits" % image[6:10], "rtd%s_R.fits" % image[6:10],
180.0)
return
def bluebias(images, biassec1=BLUEBIAS1, trimsec1=BLUETRIM1,
biassec2=BLUEBIAS2, trimsec2=BLUETRIM2):
'''Subtract overscan and trim blue frames'''
for image in images:
root,ext=image.split('.')
iraf.ccdproc(image, output='%s_1' % root, ccdtype='', noproc=no,
fixpix=no, overscan=yes, trim=yes, zerocor=no,
darkcor=no, flatcor=no, illumcor=no, fringecor=no,
readcor=no, scancor=no, biassec=biassec1,
trimsec=trimsec1)
iraf.ccdproc(image, output='%s_2' % root, ccdtype='', noproc=no,
fixpix=no, overscan=yes, trim=yes, zerocor=no,
darkcor=no, flatcor=no, illumcor=no, fringecor=no,
readcor=no, scancor=no, biassec=biassec2,
trimsec=trimsec2)
iraf.imjoin('%s_1,%s_2' % (root, root), 'j%s' % image, 1)
update_head('j%s' % image, 'DISPAXIS', 1)
update_head('j%s' % image, ['CCDSEC', 'DATASEC'], ['[1:2048,1:270]',
'[1:2048,1:270]'])
return
def skyflat(date, low=15000, high=23000, numimages=5):
'''
make a combined b skyflat.
Requires: a bias image in same directory to do the bias subtraction
skyflats must be offset and have appropriate count number
input: the date the skyflats were observed YYMMDD
output: flat.B, a text file that lists the names of the skyflat fits files
ccdYYMMDD.skyflatB.fits, the combined skyflat
'''
#check if biases are in this directory
if len(glob.glob('*.bias.*')) < 1:
print "no combined bias found, exiting"
return
#get image name and mean pixel value for all skyflat images
stats=iraf.imstat('*sky*',format=False,fields='image,mean',Stdout=1)
pairs=[i.split() for i in stats]
#write the names of the skyflats w right ammount of counts to file
#keep track of how many good ones there are
goodCount=0
with open("flat.B",'w') as FB:
for i in pairs:
if float(i[1]) > low and float(i[1]) < high:
FB.write(i[0]+'\n')
goodCount+=1
if goodCount < numimages:
print "only "+str(goodCount)+" skyflats have counts between "+str(low)+" and "+str(high)
print "no combined skyflat made"
return
else:
iraf.ccdproc(images="@flat.B",output=" ",fixpix="no",overscan="yes",trim="no",zerocor="yes",darkcor="no",flatcor="no",illumcor="no",fringecor="no",readcor="no",scancor="no",readaxis="line",biassec="[3:14,1:1024]",zero="*.bias.fits",interactive="no",functio="spline3",order=11)
iraf.flatcombine("@flat.B",output="FLAT",combine="median",reject="minmax",process="no",scale="mode",ccdtype="")
os.system("mv FLAT.fits ccd"+str(date)+".skyflatB.fits")
print ("made combined skyflat ccd"+str(date)+".skyflatB.fits")
return
def reduce(imglist, files_arc, files_flat, _cosmic, _interactive_extraction,
_arc):
import string
import os
import re
import sys
import pdb
os.environ["PYRAF_BETA_STATUS"] = "1"
try:
from astropy.io import fits as pyfits
except:
import pyfits
import numpy as np
import util
import instruments
import combine_sides as cs
import cosmics
from pyraf import iraf
dv = util.dvex()
scal = np.pi / 180.
if not _interactive_extraction:
_interactive = False
else:
_interactive = True
if not _arc:
_arc_identify = False
else:
_arc_identify = True
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.disp(inlist='1', reference='1')
toforget = [
'ccdproc', 'imcopy', 'specred.apall', 'longslit.identify',
'longslit.reidentify', 'specred.standard', 'longslit.fitcoords',
'onedspec.wspectext'
]
for t in toforget:
iraf.unlearn(t)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
hdr = util.readhdr(arcs)
br, inst = instruments.blue_or_red(arcs)
if br == 'blue':
list_arc_b.append(arcs)
elif br == 'red':
list_arc_r.append(arcs)
else:
errStr = '{} '.format(str(util.readkey3(hdr, 'VERSION')))
errStr += 'not in database'
print(errStr)
sys.exit()
asci_files = []
newlist = [[], []]
print('\n### images to reduce :', imglist)
#raise TypeError
for img in imglist:
if 'b' in img:
newlist[0].append(img)
elif 'r' in img:
newlist[1].append(img)
if len(newlist[1]) < 1:
newlist = newlist[:-1]
elif len(newlist[0]) < 1:
newlist = newlist[1:]
else:
sides = raw_input("Reduce which side? ([both]/b/r): ")
if sides == 'b':
newlist = newlist[:-1]
elif sides == 'r':
newlist = newlist[1:]
for imgs in newlist:
hdr = util.readhdr(imgs[0])
br, inst = instruments.blue_or_red(imgs[0])
if br == 'blue':
flat_file = '../RESP_blue'
elif br == 'red':
flat_file = '../RESP_red'
else:
errStr = 'Not in intrument list'
print(errStr)
sys.exit()
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_gain = inst.get('gain')
_ron = inst.get('read_noise')
iraf.specred.apall.readnoi = _ron
iraf.specred.apall.gain = _gain
_object0 = util.readkey3(hdr, 'OBJECT')
_date0 = util.readkey3(hdr, 'DATE-OBS')
_object0 = re.sub(' ', '', _object0)
_object0 = re.sub('/', '_', _object0)
nameout0 = str(_object0) + '_' + inst.get('name') + '_' + str(_date0)
nameout0 = util.name_duplicate(imgs[0], nameout0, '')
timg = nameout0
print('\n### now processing :', timg, ' for -> ', inst.get('name'))
if len(imgs) > 1:
img_str = ''
for i in imgs:
img_str = img_str + i + ','
iraf.imcombine(img_str, output=timg)
else:
img = imgs[0]
if os.path.isfile(timg):
os.system('rm -rf ' + timg)
iraf.imcopy(img, output=timg)
# should just do this by hand
iraf.ccdproc(timg,
output='',
overscan='no',
trim='no',
zerocor="no",
flatcor="yes",
readaxi='line',
flat=flat_file,
Stdout=1)
img = timg
#raw_input("Press Enter to continue...")
if _cosmic:
print('\n### starting cosmic removal')
array, header = cosmics.fromfits(img)
c = cosmics.cosmicsimage(array,
gain=inst.get('gain'),
readnoise=inst.get('read_noise'),
sigclip=5,
sigfrac=0.5,
objlim=2.0)
c.run(maxiter=5)
cosmics.tofits('cosmic_' + img, c.cleanarray, header)
img = 'cosmic_' + img
print('\n### cosmic removal finished')
else:
print(
'\n### No cosmic removal, saving normalized image for inspection???'
)
if inst.get('arm') == 'blue' and len(list_arc_b) > 0:
arcfile = list_arc_b[0]
elif inst.get('arm') == 'red' and len(list_arc_r) > 0:
arcfile = list_arc_r[0]
else:
arcfile = None
if arcfile is not None and not arcfile.endswith(".fits"):
arcfile = arcfile + '.fits'
if not os.path.isdir('database/'):
os.mkdir('database/')
if _arc_identify:
os.system('cp ' + arcfile + ' .')
arcfile = string.split(arcfile, '/')[-1]
arc_ex = re.sub('.fits', '.ms.fits', arcfile)
arcref = inst.get('archive_arc_extracted')
arcref_img = string.split(arcref, '/')[-1]
arcref_img = arcref_img.replace('.ms.fits', '')
arcrefid = inst.get('archive_arc_extracted_id')
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
os.system('cp ' + arcrefid + ' ./database')
aperture = inst.get('archive_arc_aperture')
os.system('cp ' + aperture + ' ./database')
print('\n### arcfile : ', arcfile)
print('\n### arcfile extraction : ', arc_ex)
print('\n### arc reference : ', arcref)
# read for some meta data to get the row right
tmpHDU = pyfits.open(arcfile)
header = tmpHDU[0].header
try:
spatialBin = int(header['binning'].split(',')[0])
except KeyError:
spatialBin = 1
apLine = 700 // spatialBin
iraf.specred.apall(arcfile,
output=arc_ex,
ref=arcref_img,
line=apLine,
nsum=10,
interactive='no',
extract='yes',
find='yes',
nfind=1,
format='multispec',
trace='no',
back='no',
recen='no')
iraf.longslit.reidentify(referenc=arcref,
images=arc_ex,
interac='NO',
section=inst.get('section'),
coordli=inst.get('line_list'),
shift='INDEF',
search='INDEF',
mode='h',
verbose='YES',
step=0,
nsum=5,
nlost=2,
cradius=10,
refit='yes',
overrid='yes',
newaps='no')
print('\n### extraction using apall')
result = []
hdr_image = util.readhdr(img)
_type = util.readkey3(hdr_image, 'object')
if (_type.startswith("arc") or _type.startswith("dflat")
or _type.startswith("Dflat") or _type.startswith("Dbias")
or _type.startswith("Bias")):
print('\n### warning problem \n exit ')
sys.exit()
else:
imgex = util.extractspectrum(img, dv, inst, _interactive, 'obj')
print('\n### applying wavelength solution')
print(arc_ex)
iraf.disp(inlist=imgex, reference=arc_ex)
result = result + [imgex] + [timg]
# asci_files.append(imgasci)
if not os.path.isdir(_object0 + '_ex/'):
os.mkdir(_object0 + '_ex/')
if not _arc_identify:
util.delete(arcref)
else:
util.delete(arcfile)
util.delete(arc_ex)
util.delete(img)
util.delete(imgex)
util.delete(arcref)
util.delete('logfile')
#if _cosmic:
#util.delete(img[7:])
#util.delete("cosmic_*")
os.system('mv ' + 'd' + imgex + ' ' + _object0 + '_ex/')
use_sens = raw_input('Use archival flux calibration? [y]/n ')
if use_sens != 'no':
sensfile = inst.get('archive_sens')
os.system('cp ' + sensfile + ' ' + _object0 + '_ex/')
bstarfile = inst.get('archive_bstar')
os.system('cp ' + bstarfile + ' ' + _object0 + '_ex/')
return result
def process():
ovrsc = '[1:4,*]'
trim = '[200:2046,50:200]'
imglist = glob.glob('FORS2*.fits')
# objdict = {}
# objdict.clear()
for img in imglist:
hdu = pyfits.getheader(img)
if 'OBJECT' in hdu:
obj = hdu['OBJECT']
if obj not in objdict:
objdict[obj] = [img]
else:
objdict[obj].append(img)
if 'STD' not in objdict:
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')
stars = {'specphot-LTT7379':'l7379','specphot-LDS749B':'lds749b','specphot-EG274':'eg274','specphot-G138-31':'g13831','specphot-C-32d9927':'cd32','specphot-LTT9491':'l9491','specphot-LTT7987':'l7987'}
i = 0
for key in objdict.keys():
if 'STD' in key:
for img in objdict[key]:
i = i + 1
numstars = len(objdict['STD'])
hds = pyfits.getheader(img)
_starname = stars[hds['HIERARCH ESO OBS NAME']]
if _starname in ['lds749b','g13831']:
print 'Bad standard, copying from 2016-05-12'
if not os.path.isdir('badstd'):
os.mkdir('badstd')
os.system('mv '+img+' ./badstd')
if i >= numstars:
objdict.pop('STD')
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')
if os.path.isfile('biaslist'):
os.remove('biaslist')
if os.path.isfile('masterbias.fits'):
os.remove('masterbias.fits')
f = open('biaslist','w')
for img in objdict['BIAS']:
f.write(img+'\n')
f.close()
imglist = '@biaslist'
name = 'masterbias.fits'
hdb = pyfits.getheader(objdict['BIAS'][0])
_gain = hdb['HIERARCH ESO DET OUT1 GAIN']
_ron = hdb['HIERARCH ESO DET OUT1 RON']
iraf.zerocombine(imglist,output=name,combine='average',reject='minmax',ccdtype='none',process='no',gain=_gain,rdnoise=_ron,Stdout=1)
if os.path.isfile('flatlist'):
os.remove('flatlist')
if os.path.isfile('sciflatlist'):
os.remove('sciflatlist')
if os.path.isfile('stdflatlist'):
os.remove('stdflatlist')
if os.path.isfile('masterflat.fits'):
os.remove('masterflat.fits')
if os.path.isfile('scimasterflat.fits'):
os.remove('scimasterflat.fits')
if os.path.isfile('stdmasterflat.fits'):
os.remove('stdmasterflat.fits')
f = open('sciflatlist','w')
for img in objdict['FLAT,LAMP']:
hdu = pyfits.getheader(img)
if hdu['HIERARCH ESO DPR TECH'] == 'SPECTRUM':
f.write(img+'\n')
f.close()
j = 0
f = open('stdflatlist','w')
for img in objdict['FLAT,LAMP']:
hdu = pyfits.getheader(img)
if hdu['HIERARCH ESO DPR TECH'] == 'MOS':
f.write(img+'\n')
j = j + 1
f.close()
imglist = '@sciflatlist'
name = 'scimasterflat.fits'
hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
_gain = hdf['HIERARCH ESO DET OUT1 GAIN']
_ron = hdf['HIERARCH ESO DET OUT1 RON']
iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)
if j == 0:
imglist = '@sciflatlist'
elif j >= 1:
imglist = '@stdflatlist'
name = 'stdmasterflat.fits'
hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
_gain = hdf['HIERARCH ESO DET OUT1 GAIN']
_ron = hdf['HIERARCH ESO DET OUT1 RON']
iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)
if os.path.isfile('tmasterbias.fits'):
os.remove('tmasterbias.fits')
fits = 'masterbias.fits'
name = 'tmasterbias.fits'
zcor = 'no'
fcor = 'no'
_zero = ''
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
if os.path.isfile('tmasterflat.fits'):
os.remove('tmasterflat.fits')
if os.path.isfile('tscimasterflat.fits'):
os.remove('tscimasterflat.fits')
if os.path.isfile('tstdmasterflat.fits'):
os.remove('tstdmasterflat.fits')
fits = 'scimasterflat.fits'
name = 'tscimasterflat.fits'
zcor = 'yes'
fcor = 'no'
_zero = 'tmasterbias.fits'
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
fits = 'stdmasterflat.fits'
name = 'tstdmasterflat.fits'
zcor = 'yes'
fcor = 'no'
_zero = 'tmasterbias.fits'
_flat = ''
iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
if os.path.isfile('ntmasterflat.fits'):
os.remove('ntmasterflat.fits')
if os.path.isfile('ntscimasterflat.fits'):
os.remove('ntscimasterflat.fits')
if os.path.isfile('ntstdmasterflat.fits'):
os.remove('ntstdmasterflat.fits')
cal = 'tscimasterflat.fits'
resp = 'ntscimasterflat.fits'
_order = 100
iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')
cal = 'tstdmasterflat.fits'
resp = 'ntstdmasterflat.fits'
_order = 100
iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')
##########
ovrsc = '[1:4,*]'
trim = '[200:2046,50:200]'
fimg = 'ntscimasterflat'
_area = '[1:150,*]'
_value = 1.0
iraf.imreplace(images=fimg+_area,value=_value)
fimg = 'ntstdmasterflat'
_area = '[1:150,*]'
_value = 1.0
iraf.imreplace(images=fimg+_area,value=_value)
alist = glob.glob('*AGN*.fits')
for f in alist:
os.remove(f)
slist = glob.glob('STD*.fits')
for f in slist:
os.remove(f)
llist = glob.glob('LAMP*.fits')
for f in llist:
os.remove(f)
zcor = 'yes'
fcor = 'yes'
_zero = 'tmasterbias.fits'
sciflat = 'ntscimasterflat.fits'
stdflat = 'ntstdmasterflat.fits'
for key in objdict.keys():
if 'AGN' in key:
for img in objdict[key]:
hds = pyfits.getheader(img)
if 'EXPTIME' in hds:
if hds['EXPTIME'] >= 50.0:
num = img.rsplit('.',2)[1]
name = key+'_'+num
'''
iraf.ccdproc(img,output='trim_'+name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor='no',darkcor='no',flatcor='no',illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero='',flat='',Stdout=1)
_gain = hds['HIERARCH ESO DET OUT1 GAIN']
_ron = hds['HIERARCH ESO DET OUT1 RON']
cosmics.lacos('trim_'+name+'.fits', output='c_'+name+'.fits', gain=_gain, readn=_ron, xorder=9, yorder=9, sigclip=4.5, sigfrac=0.5, objlim=1, verbose=True, interactive=False)
iraf.ccdproc('c_'+name,output=name,ccdtype='',noproc='no',fixpix='no',overscan='no',trim='no',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec='',trimsec='',zero=_zero,flat=sciflat,Stdout=1)
'''
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=sciflat,Stdout=1)
else:
pass
elif 'STD' in key:
for img in objdict[key]:
num = img.rsplit('.',2)[1]
name = key+'_'+num
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)
elif 'WAVE' in key:
for img in objdict[key]:
num = img.rsplit('.',2)[1]
name = 'LAMP_'+num
iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)
# 2008-06-04 08:42 IJC: Scale and combine the flats appropriately (as lamp is warming up, flux changes)
ir.imcombine("@" + procflat,
output=_sflat,
combine="average",
reject="crreject",
scale="median",
weight="median",
bpmasks="") # sigmas=_sflats
ns.write_exptime(_sflat, itime=itime)
print _sflat, _sdark
ir.ccdproc(_sflat,
output=_sflatdc,
ccdtype="",
fixpix="no",
overscan="no",
trim="no",
zerocor="no",
darkcor="yes",
flatcor="no",
dark=_sdark)
if verbose: print "Done making flat frame!"
ir.imdelete(_sflatdcn)
ir.imdelete(_sflatdcn + 'big')
# ------------------------------------
# Do some FITS-file gymnastics to allow all 6 orders to be traced
# ------------------------------------
flatdat = pyfits.getdata(_sflatdc + postfn)
flathdr = pyfits.getheader(_sflatdc + postfn)
def efoscfastredu(imglist, _listsens, _listarc, _ext_trace, _dispersionline,
_cosmic, _interactive):
# print "LOGX:: Entering `efoscfastredu` method/function in %(__file__)s"
# % globals()
import string
import os
import re
import sys
os.environ["PYRAF_BETA_STATUS"] = "1"
try:
from astropy.io import fits as pyfits
except:
import pyfits
from ntt.util import readhdr, readkey3
import ntt
import numpy as np
dv = ntt.dvex()
scal = np.pi / 180.
if not _interactive:
_interactive = False
_inter = 'NO'
else:
_inter = 'YES'
from pyraf import iraf
iraf.noao(_doprint=0, Stdout=0)
iraf.imred(_doprint=0, Stdout=0)
iraf.ccdred(_doprint=0, Stdout=0)
iraf.twodspec(_doprint=0, Stdout=0)
iraf.longslit(_doprint=0, Stdout=0)
iraf.onedspec(_doprint=0, Stdout=0)
iraf.specred(_doprint=0, Stdout=0)
toforget = [
'ccdproc', 'imcopy', 'specred.apall', 'longslit.identify',
'longslit.reidentify', 'specred.standard', 'longslit.fitcoords',
'onedspec.wspectext'
]
for t in toforget:
iraf.unlearn(t)
iraf.ccdred.verbose = 'no' # not print steps
iraf.specred.verbose = 'no' # not print steps
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
_gain = ntt.util.readkey3(ntt.util.readhdr(imglist[0]), 'gain')
_ron = ntt.util.readkey3(ntt.util.readhdr(imglist[0]), 'ron')
iraf.specred.apall.readnoi = _ron
iraf.specred.apall.gain = _gain
iraf.specred.dispaxi = 2
iraf.longslit.dispaxi = 2
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
iraf.set(direc=ntt.__path__[0] + '/')
for img in imglist:
hdr = ntt.util.readhdr(img)
_tech = ntt.util.readkey3(hdr, 'tech')
if _tech != 'SPECTRUM':
sys.exit('error: ' + str(img) + ' is not a spectrum ')
print '\n#### image name = ' + img + '\n'
_grism0 = readkey3(hdr, 'grism')
_filter0 = readkey3(hdr, 'filter')
_slit0 = readkey3(hdr, 'slit')
_object0 = readkey3(hdr, 'object')
_date0 = readkey3(hdr, 'date-night')
setup = (_grism0, _filter0, _slit0)
_biassec0 = '[3:1010,1026:1029]'
if _grism0 == 'Gr16':
_trimsec0 = '[100:950,1:950]'
elif _grism0 == 'Gr13':
if _filter0 == 'Free':
_trimsec0 = '[100:950,1:1015]'
elif _filter0 == 'GG495':
_trimsec0 = '[100:950,208:1015]'
elif _filter0 == 'OG530':
_trimsec0 = '[100:950,300:1015]'
elif _grism0 == 'Gr11':
_trimsec0 = '[100:950,5:1015]'
else:
_trimsec0 = '[100:950,5:1015]'
_object0 = re.sub(' ', '', _object0)
_object0 = re.sub('/', '_', _object0)
nameout0 = 't' + str(_object0) + '_' + str(_date0)
for _set in setup:
nameout0 = nameout0 + '_' + _set
nameout0 = ntt.util.name_duplicate(img, nameout0, '')
timg = nameout0
if os.path.isfile(timg):
os.system('rm -rf ' + timg)
iraf.imcopy(img, output=timg)
iraf.ccdproc(timg,
output='',
overscan='no',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='column',
trimsec=str(_trimsec0),
biassec=_biassec0,
Stdout=1)
img = timg
if _listarc:
arcfile = ntt.util.searcharc(img, _listarc)[0]
else:
arcfile = ''
if not arcfile:
arcfile = ntt.util.searcharc(img, '')[0]
else:
iraf.ccdproc(arcfile,
output='t' + arcfile,
overscan='no',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='column',
trimsec=str(_trimsec0),
biassec=str(_biassec0),
Stdout=1)
arcfile = 't' + arcfile
if _cosmic:
# print cosmic rays rejection
ntt.cosmics.lacos(img,
output='',
gain=_gain,
readn=_ron,
xorder=9,
yorder=9,
sigclip=4.5,
sigfrac=0.5,
objlim=1,
verbose=True,
interactive=False)
print '\n### cosmic rays rejections ........ done '
if not arcfile:
print '\n### warning no arcfile \n exit '
else:
arcref = ntt.util.searcharc(img, '')[0]
if arcfile[0] == '/':
os.system('cp ' + arcfile + ' ' +
string.split(arcfile, '/')[-1])
arcfile = string.split(arcfile, '/')[-1]
arcref = string.split(arcref, '/')[-1]
if arcref:
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
if not os.path.isdir('database/'):
os.mkdir('database/')
if os.path.isfile(
ntt.util.searcharc(img, '')[1] + '/database/id' +
re.sub('.fits', '', arcref)):
os.system('cp ' + ntt.util.searcharc(img, '')[1] +
'/database/id' + re.sub('.fits', '', arcref) +
' database/')
iraf.longslit.reidentify(
referenc=arcref,
images=arcfile,
interac=_inter,
section='column 10',
coordli='direc$standard/ident/Lines_HgCdHeNeAr600.dat',
overrid='yes',
step=0,
newaps='no',
nsum=5,
nlost=2,
mode='h',
verbose='no')
else:
iraf.longslit.identify(
images=arcfile,
section='column 10',
coordli='direc$standard/ident/Lines_HgCdHeNeAr600.dat',
nsum=10,
fwidth=7,
order=3,
mode='h')
iraf.longslit.reident(
referenc=arcfile,
images=arcfile,
interac='NO',
section='column 10',
coordli='direc$standard/ident/Lines_HgCdHeNeAr600.dat',
overrid='yes',
step=10,
newaps='yes',
nsum=5,
nlost=2,
mode='h',
verbose='no')
qqq = iraf.longslit.fitcoords(images=re.sub('.fits', '', arcfile),
fitname=re.sub('.fits', '', arcfile),
interac='no',
combine='yes',
databas='database',
function='legendre',
yorder=4,
logfile='logfile',
plotfil='',
mode='h')
iraf.specred.transform(input=img,
output=img,
minput='',
fitnames=re.sub('.fits', '', arcfile),
databas='database',
x1='INDEF',
x2='INDEF',
y1='INDEF',
y2='INDEF',
flux='yes',
mode='h',
logfile='logfile')
# ###################### check wavelength calibration ############
_skyfile = ntt.__path__[0] + '/standard/ident/sky_' + setup[
0] + '_' + setup[1] + '.fits'
shift = ntt.efoscspec2Ddef.skyfrom2d(img, _skyfile)
print '\n### check in wavelengh performed ...... spectrum shifted of ' + str(
shift) + ' Angstrom \n'
zro = pyfits.open(img)[0].header.get('CRVAL2')
ntt.util.updateheader(img, 0, {'CRVAL2': [zro + int(shift), '']})
std, rastd, decstd, magstd = ntt.util.readstandard(
'standard_efosc_mab.txt')
hdrt = readhdr(img)
_ra = readkey3(hdrt, 'RA')
_dec = readkey3(hdrt, 'DEC')
_object = readkey3(hdrt, 'object')
dd = np.arccos(
np.sin(_dec * scal) * np.sin(decstd * scal) +
np.cos(_dec * scal) * np.cos(decstd * scal) * np.cos(
(_ra - rastd) * scal)) * ((180 / np.pi) * 3600)
if min(dd) < 100:
_type = 'stdsens'
ntt.util.updateheader(img, 0,
{'stdname': [std[np.argmin(dd)], '']})
ntt.util.updateheader(
img, 0, {'magstd': [float(magstd[np.argmin(dd)]), '']})
else:
_type = 'obj'
print '\n### EXTRACTION USING IRAF TASK APALL \n'
result = []
if _type == 'obj':
imgex = ntt.util.extractspectrum(img, dv, _ext_trace,
_dispersionline, _interactive,
_type)
ntt.util.updateheader(
imgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum ']})
ntt.util.updateheader(
imgex, 0, {
'PRODCATG': [
'SCIENCE.' +
readkey3(readhdr(imgex), 'tech').upper(),
'Data product category'
]
})
ntt.util.updateheader(imgex, 0, {'TRACE1': [img, '']})
result.append(imgex)
if _listsens:
sensfile = ntt.util.searchsens(img, _listsens)[0]
else:
sensfile = ''
if not sensfile:
sensfile = ntt.util.searchsens(img, '')[0]
if sensfile:
imgf = re.sub('.fits', '_f.fits', img)
_extinctdir = 'direc$standard/extinction/'
_extinction = 'extinction_lasilla.dat'
_observatory = 'lasilla'
_exptime = readkey3(hdrt, 'exptime')
_airmass = readkey3(hdrt, 'airmass')
ntt.util.delete(imgf)
iraf.specred.calibrate(input=imgex,
output=imgf,
sensiti=sensfile,
extinct='yes',
flux='yes',
ignorea='yes',
extinction=_extinctdir +
_extinction,
observatory=_observatory,
airmass=_airmass,
exptime=_exptime,
fnu='no')
hedvec = {
'SENSFUN': [
string.split(sensfile, '/')[-1],
'sensitivity function'
],
'FILETYPE':
[22208, '1D wavelength and flux calibrated spectrum '],
'SNR':
[ntt.util.StoN2(imgf, False), 'Average S/N ratio'],
'BUNIT':
['erg/cm2/s/Angstrom', 'Flux Calibration Units'],
'TRACE1': [imgex, '']
}
ntt.util.updateheader(imgf, 0, hedvec)
imgout = imgf
imgd = ntt.efoscspec1Ddef.fluxcalib2d(img, sensfile)
ntt.util.updateheader(
imgd, 0, {
'FILETYPE': [
22209,
'2D wavelength and flux calibrated spectrum '
]
})
ntt.util.updateheader(imgd, 0, {'TRACE1': [img, '']})
imgasci = re.sub('.fits', '.asci', imgout)
ntt.util.delete(imgasci)
iraf.onedspec.wspectext(imgout + '[*,1,1]',
imgasci,
header='no')
result = result + [imgout, imgd, imgasci]
else:
imgex = ntt.util.extractspectrum(img, dv, _ext_trace,
_dispersionline, _interactive,
'std')
imgout = ntt.efoscspec1Ddef.sensfunction(
imgex, 'spline3', 6, _inter)
result = result + [imgout]
for img in result:
if img[-5:] == '.fits':
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.airmass(img) # phase 3 definitions
ntt.util.updateheader(
img, 0, {'quality': ['Rapid', 'Final or Rapid reduction']})
return result
def reduce(imglist,files_arc, _cosmic, _interactive_extraction,_arc):
import string
import os
import re
import sys
os.environ["PYRAF_BETA_STATUS"] = "1"
try: from astropy.io import fits as pyfits
except: import pyfits
import numpy as np
import util
import instruments
import combine_sides as cs
import cosmics
from pyraf import iraf
dv = util.dvex()
scal = np.pi / 180.
if not _interactive_extraction:
_interactive = False
else:
_interactive = True
if not _arc:
_arc_identify = False
else:
_arc_identify = True
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.disp(inlist='1', reference='1')
toforget = ['ccdproc', 'imcopy', 'specred.apall', 'longslit.identify', 'longslit.reidentify', 'specred.standard',
'longslit.fitcoords', 'onedspec.wspectext']
for t in toforget:
iraf.unlearn(t)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
hdr = util.readhdr(arcs)
if util.readkey3(hdr, 'VERSION') == 'kastb':
list_arc_b.append(arcs)
elif util.readkey3(hdr, 'VERSION') == 'kastr':
list_arc_r.append(arcs)
else:
print util.readkey3(hdr, 'VERSION') + 'not in database'
sys.exit()
asci_files = []
newlist = [[],[]]
print '\n### images to reduce :',imglist
#raise TypeError
for img in imglist:
if 'b' in img:
newlist[0].append(img)
elif 'r' in img:
newlist[1].append(img)
if len(newlist[1]) < 1:
newlist = newlist[:-1]
for imgs in newlist:
hdr = util.readhdr(imgs[0])
if util.readkey3(hdr, 'VERSION') == 'kastb':
inst = instruments.kast_blue
elif util.readkey3(hdr, 'VERSION') == 'kastr':
inst = instruments.kast_red
else:
print util.readkey3(hdr, 'VERSION') + 'not in database'
sys.exit()
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_gain = inst.get('gain')
_ron = inst.get('read_noise')
iraf.specred.apall.readnoi = _ron
iraf.specred.apall.gain = _gain
_object0 = util.readkey3(hdr, 'OBJECT')
_date0 = util.readkey3(hdr, 'DATE-OBS')
_biassec0 = inst.get('biassec')
_trimsec0 = inst.get('trimsec')
_object0 = re.sub(' ', '', _object0)
_object0 = re.sub('/', '_', _object0)
nameout0 = str(_object0) + '_' + inst.get('name') + '_' + str(_date0)
nameout0 = util.name_duplicate(imgs[0], nameout0, '')
timg = nameout0
print '\n### now processing :',timg,' for -> ',inst.get('name')
if len(imgs) > 1:
img_str = ''
for i in imgs:
img_str = img_str + i + ','
iraf.imcombine(img_str, output=timg)
else:
img = imgs[0]
if os.path.isfile(timg):
os.system('rm -rf ' + timg)
iraf.imcopy(img, output=timg)
zero_file = inst.get('archive_zero_file')
os.system('cp ' + zero_file + ' .')
zero_file = string.split(zero_file, '/')[-1]
flat_file = inst.get('archive_flat_file')
os.system('cp ' + flat_file + ' .')
flat_file = string.split(flat_file, '/')[-1]
iraf.ccdproc(timg, output='', overscan='yes', trim='yes', zerocor="no", flatcor="no", readaxi='line',
trimsec=str(_trimsec0),biassec=str(_biassec0), Stdout=1)
iraf.ccdproc(timg, output='', overscan='no', trim='no', zerocor="yes", flatcor="no", readaxi='line',
zero=zero_file,order=3, Stdout=1)
iraf.ccdproc(timg, output='', overscan='no', trim='no', zerocor="no", flatcor="yes", readaxi='line',
flat=flat_file, Stdout=1)
img = timg
#raw_input("Press Enter to continue...")
print '\n### starting cosmic removal'
if _cosmic:
array, header = cosmics.fromfits(img)
c = cosmics.cosmicsimage(array, gain=inst.get('gain'), readnoise=inst.get('read_noise'), sigclip = 4.5, sigfrac = 0.5, objlim = 1.0)
c.run(maxiter = 4)
cosmics.tofits('cosmic_' + img, c.cleanarray, header)
print '\n### cosmic removal finished'
img='cosmic_' + img
if inst.get('name') == 'kast_blue':
arcfile = list_arc_b[0]
elif inst.get('name') == 'kast_red':
arcfile = list_arc_r[0]
if not arcfile.endswith(".fits"):
arcfile=arcfile+'.fits'
if os.path.isfile(arcfile):
util.delete('t' + arcfile)
iraf.ccdproc(arcfile, output= 't' + arcfile, overscan='yes', trim='yes', zerocor="no", flatcor="no",
readaxi='line', trimsec=str(_trimsec0), biassec=str(_biassec0), Stdout=1)
arcfile = 't' + arcfile
else:
print '\n### warning no arcfile \n exit '
sys.exit()
if not os.path.isdir('database/'):
os.mkdir('database/')
if _arc_identify:
arc_ex=re.sub('.fits', '.ms.fits', arcfile)
print '\n### arcfile : ',arcfile
print '\n### arcfile extraction : ',arc_ex
iraf.specred.apall(arcfile, output='', line = 'INDEF', nsum=10, interactive='no', extract='yes',find='yes', nfind=1 ,format='multispec', trace='no',back='no',recen='no')
iraf.longslit.identify(images=arc_ex, section=inst.get('section'),coordli=inst.get('line_list'),function = 'spline3',order=3, mode='h')
else:
arcref = inst.get('archive_arc_extracted')
arcrefid = inst.get('archive_arc_extracted_id')
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
os.system('cp ' + arcrefid + ' ./database')
arc_ex=re.sub('.fits', '.ms.fits', arcfile)
print '\n### arcfile : ',arcfile
print '\n### arcfile extraction : ',arc_ex
print '\n### arc referenece : ',arcref
iraf.specred.apall(arcfile, output=arc_ex, line = 'INDEF', nsum=10, interactive='no', extract='yes',find='yes', nfind=1 ,format='multispec', trace='no',back='no',recen='no')
iraf.longslit.reidentify(referenc=arcref, images=arc_ex, interac='NO', section=inst.get('section'),
coordli=inst.get('line_list'), shift='INDEF', search='INDEF',
mode='h', verbose='YES', step=0,nsum=5, nlost=2, cradius=10, refit='yes',overrid='yes',newaps='no')
#print '\n### checking sky lines '
#_skyfile = inst.get('sky_file')
#shift = util.skyfrom2d(img, _skyfile,'True')
#print '\n### I found a shift of : ',shift
print '\n### extraction using apall'
result = []
hdr_image = util.readhdr(img)
_type=util.readkey3(hdr_image, 'object')
if _type.startswith("arc") or _type.startswith("dflat") or _type.startswith("Dflat") or _type.startswith("Dbias") or _type.startswith("Bias"):
print '\n### warning problem \n exit '
sys.exit()
else:
imgex = util.extractspectrum(
img, dv, inst, _interactive, 'obj')
print '\n### applying wavelength solution'
iraf.disp(inlist=imgex, reference=arc_ex)
sensfile = inst.get('archive_sens')
os.system('cp ' + sensfile + ' .')
sensfile = string.split(sensfile, '/')[-1]
if sensfile:
print '\n### sensitivity function : ',sensfile
imgf = re.sub('.fits', '_f.fits', img)
_extinction = inst.get('extinction_file')
_observatory = inst.get('observatory')
_exptime = util.readkey3(hdr, 'EXPTIME')
_airmass = util.readkey3(hdr, 'AIRMASS')
util.delete(imgf)
dimgex='d'+imgex
iraf.specred.calibrate(input=dimgex, output=imgf, sensiti=sensfile, extinct='yes',
extinction=_extinction,flux='yes', ignorea='yes', airmass=_airmass, exptime=_exptime,
fnu='no')
imgout = imgf
imgasci = re.sub('.fits', '.asci', imgout)
errasci = re.sub('.fits', '_err.asci', imgout)
util.delete(imgasci)
iraf.onedspec.wspectext(imgout + '[*,1,1]', imgasci, header='no')
iraf.onedspec.wspectext(imgout + '[*,1,4]', errasci, header='no')
spec = np.transpose(np.genfromtxt(imgasci))
err = np.transpose(np.genfromtxt(errasci))
util.delete(errasci)
final = np.transpose([spec[0], spec[1], err[1]])
np.savetxt(imgasci, final)
result = result + [imgout, imgasci]
result = result + [imgex] + [timg]
asci_files.append(imgasci)
if not os.path.isdir(_object0 + '/'):
os.mkdir(_object0 + '/')
for img in result:
os.system('mv ' + img + ' ' + _object0 + '/')
else:
for img in result:
os.system('mv ' + img + ' ' + _object0 + '/')
if not _arc_identify:
util.delete(arcref)
util.delete(sensfile)
util.delete(zero_file)
util.delete(flat_file)
util.delete(arc_ex)
util.delete(arcfile)
util.delete('logfile')
util.delete(dimgex)
util.delete('cosmic_*')
print '\n### now i will merge ...'
if len(asci_files) > 1:
final = cs.combine_blue_red(asci_files[0], asci_files[1], _object0)
print '\n### final result in folder ',_object0,' is ',_object0+'_merged.asci'
return result
def flat_correct(WORK_DIR): print '\n + Correcting images and arcs for flat\n' iraf.ccdproc(images=','.join(observations[WORK_DIR]['objects']+observations[WORK_DIR]['calibs']), ccdtype='', fixpix='yes', oversca='no', trim='yes', zerocor='no', darkcor='no', flatcor='yes', flat='masterflat_norm.fits', fixfile='badpix.txt')
def main():
description = "> Performs pre-reduction steps"
usage = "%prog \t [option] \n Recommended syntax: %prog -i -c"
parser = OptionParser(usage=usage, description=description, version="0.1")
option, args = parser.parse_args()
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
mkarc = raw_input("Make arc? ([y]/n): ")
mkflat = raw_input("Make flat? ([y]/n): ")
if len(args) > 1:
files = []
sys.argv.append('--help')
option, args = parser.parse_args()
sys.exit()
elif len(args) == 1:
files = util.readlist(args[0])
sys.exit()
else:
listfile = glob.glob('*.fits')
files_science = []
files_arc = []
files_dflat = []
#print 'checking your files ...'
for img in listfile:
_type = ''
hdr0 = util.readhdr(img)
_type = util.readkey3(hdr0, 'object')
if 'flat' in _type.lower():
files_dflat.append(img)
elif 'arc' not in _type.lower() and 'arc' not in img.lower():
files_science.append(img)
if mkarc != 'n':
mkarc_b = raw_input(
"List blue arc files to combine (.fits will be added): "
).split()
mkarc_r = raw_input(
"List red arc files to combine (.fits will be added): ").split(
)
for arc in mkarc_b:
files_arc.append(arc + '.fits')
for arc in mkarc_r:
files_arc.append(arc + '.fits')
if mkarc != 'n':
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
if instruments.blue_or_red(arcs)[0] == 'blue':
list_arc_b.append(arcs)
elif instruments.blue_or_red(arcs)[0] == 'red':
list_arc_r.append(arcs)
else:
sys.exit()
if mkflat != 'n':
list_flat_b = []
list_flat_r = []
for dflats in files_dflat:
if instruments.blue_or_red(dflats)[0] == 'blue':
list_flat_b.append(dflats)
elif instruments.blue_or_red(dflats)[0] == 'red':
list_flat_r.append(dflats)
else:
sys.exit()
# make pre_reduced if it doesn't exist
if not os.path.isdir('pre_reduced/'):
os.mkdir('pre_reduced/')
# log the existing processed files (need to verify this works if pre_reduced is empty...)
pfiles = []
new_files = []
for root, dirnames, filenames in os.walk('pre_reduced'):
for file in filenames:
if file.startswith('to'):
pfiles.append(file)
print(pfiles)
# loop over each image in pre_reduced
for img in listfile:
hdr = util.readhdr(img)
targ = util.readkey3(hdr, 'object')
# if file is not not a processed file, run the overscan+trim code
if 'to' + img not in pfiles:
# if the file is a science file, grab the name for later
if 'arc' not in targ.lower() and 'flat' not in targ.lower():
new_files.append(img)
print('Adding data for: ' + targ)
inst = instruments.blue_or_red(img)[1]
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_biassec0 = inst.get('biassec')
_trimsec0 = inst.get('trimsec')
######################################################################
#
# JB: this chunk of code needs attention
# It seems incredibly hacky for anything but Kast...
#
# overscan
if not img.startswith('o') and inst.get('observatory') == 'lick':
if os.path.isfile('pre_reduced/o' + img):
os.remove('pre_reduced/o' + img)
util.kastbias(img, 'pre_reduced/o' + img)
elif not img.startswith('o') and inst.get('observatory') != 'lick':
if os.path.isfile('pre_reduced/o' + img):
os.remove('pre_reduced/o' + img)
os.system('cp ' + img + ' ' + 'pre_reduced/' + img)
# trim
if not img.startswith('t') and inst.get('observatory') == 'lick':
if os.path.isfile('pre_reduced/to' + img):
os.remove('pre_reduced/to' + img)
iraf.ccdproc('pre_reduced/o' + img,
output='pre_reduced/to' + img,
overscan='no',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='line',
trimsec=str(_trimsec0),
Stdout=1)
elif not img.startswith('t') and inst.get('observatory') != 'lick':
if os.path.isfile('pre_reduced/to' + img):
os.remove('pre_reduced/to' + img)
iraf.ccdproc('pre_reduced/' + img,
output='pre_reduced/to' + img,
overscan='yes',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='line',
trimsec=str(_trimsec0),
biassec=str(_biassec0),
Stdout=1)
# combine the arcs
if mkarc != 'n':
# blue arcs
if len(list_arc_b) > 0:
if len(list_arc_b) == 1:
arc_blue = list_arc_b[0]
os.system('cp ' + 'pre_reduced/to' + arc_blue + ' ' +
'pre_reduced/ARC_blue.fits')
else:
arc_str = ''
for arc in list_arc_b:
arc_str = arc_str + 'pre_reduced/to' + arc + ','
if os.path.isfile('pre_reduced/ARC_blue.fits'):
os.remove('pre_reduced/ARC_blue.fits')
iraf.imcombine(arc_str, output='pre_reduced/ARC_blue.fits')
# red arcs
if len(list_arc_r) > 0:
if len(list_arc_r) == 1:
arc_red = list_arc_r[0]
os.system('cp ' + 'pre_reduced/to' + arc_red + ' ' +
'pre_reduced/ARC_red.fits')
else:
arc_str = ''
for arc in list_arc_r:
arc_str = arc_str + 'pre_reduced/to' + arc + ','
if os.path.isfile('pre_reduced/ARC_red.fits'):
os.remove('pre_reduced/ARC_red.fits')
iraf.imcombine(arc_str, output='pre_reduced/ARC_red.fits')
# combine the flats
if mkflat != 'n':
inter = 'yes'
# blue flats
if len(list_flat_b) > 0:
br, inst = instruments.blue_or_red(list_flat_b[0])
iraf.specred.dispaxi = inst.get('dispaxis')
if len(list_flat_b) == 1:
# Flat_blue = 'pre_reduced/to'+ list_flat_b[0]
Flat_blue = list_flat_b[0]
else:
flat_str = ''
for flat in list_flat_b:
flat_str = flat_str + 'pre_reduced/to' + flat + ','
#subsets = 'no'
if os.path.isfile('pre_reduced/toFlat_blue'):
os.remove('pre_reduced/toFlat_blue')
iraf.flatcombine(flat_str,
output='pre_reduced/toFlat_blue',
ccdtype='',
rdnoise=3.7,
subsets='no',
process='no')
Flat_blue = 'Flat_blue.fits'
#What is the output here? Check for overwrite
iraf.specred.response('pre_reduced/to' + Flat_blue,
normaliz='pre_reduced/to' + Flat_blue,
response='pre_reduced/RESP_blue',
interac=inter,
thresho='INDEF',
sample='*',
naverage=2,
function='legendre',
low_rej=3,
high_rej=3,
order=60,
niterat=20,
grow=0,
graphic='stdgraph')
# red flats
if len(list_flat_r) > 0:
br, inst = instruments.blue_or_red(list_flat_r[0])
iraf.specred.dispaxi = inst.get('dispaxis')
if len(list_flat_r) == 1:
# Flat_red = 'pre_reduced/to' + list_flat_r[0]
Flat_red = list_flat_r[0]
else:
flat_str = ''
for flat in list_flat_r:
flat_str = flat_str + 'pre_reduced/to' + flat + ','
if os.path.isfile('pre_reduced/toFlat_red'):
os.remove('pre_reduced/toFlat_red')
iraf.flatcombine(flat_str,
output='pre_reduced/toFlat_red',
ccdtype='',
rdnoise=3.8,
subsets='yes',
process='no')
Flat_red = 'Flat_red.fits'
#What is the output here? Check for overwrite
iraf.specred.response('pre_reduced/to' + Flat_red,
normaliz='pre_reduced/to' + Flat_red,
response='pre_reduced/RESP_red',
interac=inter,
thresho='INDEF',
sample='*',
naverage=2,
function='legendre',
low_rej=3,
high_rej=3,
order=80,
niterat=20,
grow=0,
graphic='stdgraph')
# science files should have 't' in front now
# this just gets the base name, to prefix assumed below
if new_files is not None:
files_science = new_files
# get all the science objects for the night
science_targets = []
for obj in files_science:
hdr = util.readhdr(obj)
_type = util.readkey3(hdr, 'object')
science_targets.append(_type)
# make a dir for each sci object
science_targets = set(science_targets)
for targ in science_targets:
if not os.path.isdir('pre_reduced/' + targ + '/'):
os.mkdir('pre_reduced/' + targ + '/')
# copy the files into the obj dir
for obj in files_science:
hdr = util.readhdr(obj)
targ = util.readkey3(hdr, 'object')
if not obj.startswith('to'):
os.system('cp ' + 'pre_reduced/to' + obj + ' ' + 'pre_reduced/' +
targ + '/')
else:
os.system('cp ' + 'pre_reduced/' + obj + ' ' + 'pre_reduced/' +
targ + '/')
rawfiles = glob.glob('*.fits')
ofiles = glob.glob('pre_reduced/o' + '*.fits')
tfiles = glob.glob('pre_reduced/to' + '*.fits')
# delete raw files from the pre_reduced dir
# there shouldn't be any there though?
# maybe if the overscan isn't implemented for that detector
for img in rawfiles:
util.delete('pre_reduced/' + img)
# delete the ofiles from pre_reduced dir
for img in ofiles:
util.delete(img)
def reduceDark(dListFn, dMasterFn, mBiasFn):
if os.path.isfile(dMasterFn + ".fits"):
return
zerocor = "no"
if os.path.isfile(mBiasFn + ".fits"):
zerocor = "yes"
print("File list: @" + dListFn)
print("dMasterFn: " + dMasterFn)
iraf.reset(use_new_imt="no")
iraf.ccdproc(
"@" + dListFn,
output="@" + dListFn + "//atlas_b",
ccdtype=" ",
max_cache=0,
noproc="no",
fixpix="no",
overscan="no",
trim="no",
zerocor=zerocor,
darkcor="no",
flatcor="no",
illumcor="no",
fringecor="no",
readcor="no",
scancor="no",
readaxis="line",
fixfile="",
biassec="",
trimsec="",
zero=mBiasFn,
dark=" ",
flat="",
illum="",
fringe="",
minreplace=1.0,
scantype="shortscan",
nscan=1,
interactive="no",
function="legendre",
order=1,
sample="*",
naverage=1,
niterate=1,
low_reject=3.0,
high_reject=3.0,
grow=0.0,
)
iraf.imcombine(
"@" + dListFn + "//atlas_b",
output=dMasterFn,
headers="",
bpmasks="",
rejmasks="",
nrejmasks="",
expmasks="",
sigmas="",
imcmb="$I",
logfile=dMasterFn + ".log",
combine="median",
reject="crreject",
project="no",
outtype="real",
outlimits="",
offsets="none",
masktype="none",
maskvalue="0",
blank=0.0,
scale="exposure",
zero="none",
weight="none",
statsec="",
expname="",
lthreshold="INDEF",
hthreshold="INDEF",
nlow=1,
nhigh=1,
nkeep=1,
mclip="yes",
lsigma=3.0,
hsigma=3.0,
rdnoise="0.",
gain="1.",
snoise="0.",
sigscale=0.1,
pclip=-0.5,
grow=0.0,
)
def run_ccdp(input):
iraf.noao.imred(_doprint=0)
iraf.noao.imred.ccdred(_doprint=0)
iraf.ccdproc(input[0],output=input[1],trim="no",trimsec="",zerocor="no",flatcor="no",flat="",ccdtype="",max_cache="128",noproc="no",fixpix="no",zero="",overscan="no",darkcor="yes",illumcor="no",fringecor="no",readcor="no",scancor="no",readaxis="line",fixfile="",biassec="",dark=input[2],illum="",fringe="",minreplace="1.0",scantype="shortscan",nscan="1",interactive="no",function="legendre",order="1",sample="*",naverage="1",niterate="1",low_reject="3",high_reject="3",grow="0.0")
