def makeArcDark(arcdarklist, arcdark, calflat, over, log):
""""Prepare with iraf.nfprepare and combine the daytime arc darks.
Processing with NFPREPARE will rename the data extension and add
variance and data quality extensions. By default (see NSHEADERS)
the extension names are SCI for science data, VAR for variance, and
DQ for data quality (0 = good). Generation of the data quality
plane (DQ) is important in order to fix hot and dark pixels on the
NIFS detector in subsequent steps in the data reduction process.
Various header keywords (used later) are also added in NFPREPARE.
NFPREPARE will also add an MDF file (extension MDF) describing the
NIFS image slicer pattern and how the IFU maps to the sky field.
"""
# Update arc dark frames with mdf offset value and generate variance and data quality extensions.
for image in arcdarklist:
image = str(image).strip()
if over:
iraf.delete("n"+image+".fits")
iraf.nfprepare(image, rawpath='.', shiftimage="s"+calflat, \
bpm=sflat_bpm,fl_vardq='yes',fl_corr='no',fl_nonl='no', logfile=log)
arcdarklist = checkLists(arcdarklist, '.', 'n', '.fits')
# Combine arc dark frames, "n"+image+".fits". Output combined file will have the name of the first arc dark file.
if over:
iraf.delete("gn"+arcdark+".fits")
if len(arcdarklist) > 1:
iraf.gemcombine(listit(arcdarklist,"n"),output="gn"+arcdark, fl_dqpr='yes',fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+arcdark+'.fits', 'gn'+arcdark+'.fits')
# Put the name of the combined and prepared arc dark frame "gn"+arcdark into a text
# file called arcdarkfile to be used by the pipeline later.
open("arcdarkfile", "w").write("gn"+arcdark)
def makeArcDark(arcdarklist, arcdark, calflat, over, log):
# combine the daytime arc darks
for image in arcdarklist:
image = str(image).strip()
if over:
iraf.delete("n" + image + ".fits")
iraf.nfprepare(image,
rawpath='./',
shiftimage="s" + calflat,
bpm="rgn" + calflat + "_sflat_bpm.pl",
fl_vardq='yes',
fl_corr='no',
fl_nonl='no',
logfile=log)
arcdarklist = checkLists(arcdarklist, '.', 'n', '.fits')
if over:
iraf.delete("gn" + arcdark + ".fits")
if len(arcdarklist) > 1:
iraf.gemcombine(listit(arcdarklist, "n"),
output="gn" + arcdark,
fl_dqpr='yes',
fl_vardq='yes',
masktype="none",
logfile=log)
else:
iraf.copy('n' + arcdark + '.fits', 'gn' + arcdark + '.fits')
open("arcdarkfile", "w").write("gn" + arcdark)
def ronchi(ronchilist, ronchiflat, calflat, over, flatdark, log):
"""Establish Spatial-distortion calibration with nfsdist.
NFSDIST uses the information in the "Ronchi" Calibration images
to calibrate the spatial dimension of the NIFS IFU field. The
Ronchi frame is a dispersed flat field image with a slit-mask
in the field so that the illumination on the IFU is in a
pattern of ~10 different slitlets that are stacked in the
y-dimension on the field. Proper alignment of the slits across
the image slicer pattern can be used for spatial rectification
of the on-sky science data. The spatial solution determined by
NFSDIST is linked to the science data in NFFITCOORDS.
"""
# Update ronchi flat frames with offset value and generate variance and data quality extensions.
# Output: "n"+image+".fits".
for image in ronchilist:
image = str(image).strip()
if over:
iraf.delete("n"+image+'.fits')
iraf.nfprepare(image,rawpath=".", shiftimage="s"+calflat, \
bpm="rgn"+calflat+"_sflat_bpm.pl", fl_vardq="yes",fl_corr="no",fl_nonl="no", \
logfile=log)
ronchilist = checkLists(ronchilist, '.', 'n', '.fits')
# Combine nfprepared ronchi flat images "n"+image+".fits". Output: combined file will
# have the name of the first ronchi flat file, "gn"+ronchiflat+".fits".
if over:
iraf.delete("gn"+ronchiflat+".fits")
if len(ronchilist) > 1:
iraf.gemcombine(listit(ronchilist,"n"),output="gn"+ronchiflat,fl_dqpr="yes", \
masktype="none",fl_vardq="yes",logfile=log)
else:
iraf.copy("n"+ronchiflat+".fits","gn"+ronchiflat+".fits")
# NSREDUCE combined ronchi frame, "gn"+ronchiflat+".fits", to extract the slices into unique MEF extensions and
# apply an approximate wavelength calibration to each slice. Output: "rgn"+ronchiflat+".fits".
if over:
iraf.delete("rgn"+ronchiflat+".fits")
iraf.nsreduce("gn"+ronchiflat, outpref="r", dark="rgn"+flatdark+"_dark", flatimage="rgn"+calflat+"_flat", fl_cut="yes", fl_nsappw="yes", fl_flat="yes", fl_sky="no", fl_dark="yes", fl_vardq="no", logfile=log)
if os.path.exists("ronchifile"):
if over:
iraf.delete("ronchifile")
iraf.delete("rgn"+ronchiflat)
else:
print "\nOutput file exists and -over not set - ",\
"not performing ronchi calibration with iraf.nfsdist.\n"
return
else:
# Determine the spatial distortion correction. Output: overwrites "rgn"+ronchiflat+".fits" and makes
# changes to files in /database directory.
iraf.nfsdist("rgn"+ronchiflat,fwidth=6.0, cradius=8.0, glshift=2.8, minsep=6.5, thresh=2000.0, nlost=3, fl_inter='no',logfile=log)
# Put the name of the spatially referenced ronchi flat "rgn"+ronchiflat into a
# text file called ronchifile to be used by the pipeline later. Also associated files
# are in the "database/" directory.
open("ronchifile", "w").write("rgn"+ronchiflat)
def copyImage(input, output, over):
# copy an image (used to add the correct prefix)
if os.path.exists(output):
if over:
iraf.delete(output)
else:
print "Output file exists and -over not set - skipping copy_ima"
return
iraf.copy('n' + input[0] + '.fits', output)
def copyImage(input, output, over):
"""Copy a frame (used to add the correct prefix when skipping steps)."""
if os.path.exists(output):
if over:
iraf.delete(output)
else:
logging.info(
"Output file exists and -over not set - skipping copy_ima")
return
iraf.copy('n' + input[0] + '.fits', output)
def reduceArc(arclist, arc, arcdarklist, arcdark, log, over):
"""Flat field and cut the arc data with iraf.nfprepare and
iraf.nsreduce.
Processing with NFPREPARE will rename the data extension and add
variance and data quality extensions. By default (see NSHEADERS)
the extension names are SCI for science data, VAR for variance, and
DQ for data quality (0 = good). Generation of the data quality
plane (DQ) is important in order to fix hot and dark pixels on the
NIFS detector in subsequent steps in the data reduction process.
Various header keywords (used later) are also added in NFPREPARE.
NFPREPARE will also add an MDF file (extension MDF) describing the
NIFS image slicer pattern and how the IFU maps to the sky field.
NSREDUCE is used for basic reduction of raw data - it provides a
single, unified interface to several tasks and also allows for
the subtraction of dark frames and dividing by the flat. For
NIFS reduction, NSREDUCE is used to call the NSCUT and NSAPPWAVE
routines. NSREDUCE resides in the GNIRS package.
"""
# Store the name of the shift image in "shiftima".
shiftima = open("shiftfile", "r").readlines()[0].strip()
# Store the name of the bad pixel mask in "sflat_bpm".
sflat_bpm = open("sflat_bpmfile", "r").readlines()[0].strip()
# Store the name of the final flat field frame in "flat".
flat = open("flatfile", "r").readlines()[0].strip()
# Update arc images with offset value and generate variance and data
# quality extensions. Results in "n"+image+".fits"
for image in arclist:
image = str(image).strip()
if os.path.exists("n"+image+".fits"):
if over:
iraf.delete("n"+image+".fits")
else:
print "\nOutput file exists and -over not set - skipping nfprepare of arcs."
continue
iraf.nfprepare(image, rawpath=".", shiftimage=shiftima,bpm=sflat_bpm,\
fl_vardq="yes",fl_corr='no',fl_nonl='no',logfile=log)
# Check that output files for all arc images exists from nfprepare; if output does not
# exist remove corresponding arc images from arclist.
arclist = checkLists(arclist, '.', 'n', '.fits')
# Update arc dark frames with mdf offset value and generate variance and data
# quality extensions. Results in "n"+image+".fits"
for image in arcdarklist:
image = str(image).strip()
if os.path.exists("n"+image+".fits"):
if over:
iraf.delete("n"+image+".fits")
else:
print "\nOutput file exists and -over not set - skipping nfprepare of arcdarks."
continue
iraf.nfprepare(image, rawpath=".", shiftimage=shiftima, bpm=sflat_bpm, \
fl_vardq='yes',fl_corr='no',fl_nonl='no',logfile=log)
# Check that output files for all arc images exists from nfprepare; if output does not
# exist remove corresponding arc images from arclist.
arcdarklist = checkLists(arcdarklist, '.', 'n', '.fits')
# Combine arc frames, "n"+image+".fits". Output combined file will have the name of the first arc file.
if os.path.exists("gn"+arc+".fits"):
if over:
iraf.delete("gn"+arc+".fits")
if len(arclist) > 1:
iraf.gemcombine(listit(arclist,"n"),output="gn"+arc, fl_dqpr='yes',fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+arc+'.fits', 'gn'+arc+'.fits')
else:
print "\nOutput file exists and -over not set - skipping gemcombine of arcs."
else:
if len(arclist) > 1:
iraf.gemcombine(listit(arclist,"n"),output="gn"+arc, fl_dqpr='yes',fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+arc+'.fits', 'gn'+arc+'.fits')
# Combine arc dark frames, "n"+image+".fits". Output combined file will have the name of the first arc dark file.
if os.path.exists("gn"+arcdark+".fits"):
if over:
iraf.delete("gn"+arcdark+".fits")
if len(arcdarklist) > 1:
iraf.gemcombine(listit(arcdarklist,"n"),output="gn"+arcdark, fl_dqpr='yes',fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+arcdark+'.fits', 'gn'+arcdark+'.fits')
else:
print "\nOutput file exists and -over not set - skipping gemcombine of arcdarks."
else:
if len(arcdarklist) > 1:
iraf.gemcombine(listit(arcdarklist,"n"),output="gn"+arcdark, fl_dqpr='yes',fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+arcdark+'.fits', 'gn'+arcdark+'.fits')
# Put the name of the combined and prepared arc dark frame "gn"+arcdark into a text
# file called arcdarkfile to be used by the pipeline later.
open("arcdarkfile", "w").write("gn"+arcdark)
# NSREDUCE on arc images "gn"+arc+".fits" to extract the slices and apply an approximate
# wavelength calibration. Results in "rgn"+image+".fits"
if os.path.exists("rgn"+arc+".fits"):
if over:
iraf.delete("rgn"+arc+".fits")
else:
print "Output file exists and -over not set - skipping apply_flat_arc."
return
fl_dark = "no"
if arcdark != "":
fl_dark = "yes"
hdulist = astropy.io.fits.open(arc+'.fits')
if 'K_Long' in hdulist[0].header['GRATING']:
iraf.nsreduce("gn"+arc, darki=arcdark, fl_cut="yes", fl_nsappw="yes", crval = 23000., fl_dark="yes", fl_sky="no", fl_flat="yes", flatimage=flat, fl_vardq="no",logfile=log)
else:
iraf.nsreduce("gn"+arc, darki="gn"+arcdark, flatimage=flat, \
fl_vardq="no", fl_cut="yes", fl_nsappw="yes", fl_sky="no", fl_dark="yes", fl_flat="yes", \
logfile=log)
# Put the name of the combined and prepared arc dark frame "gn"+arcdark into a text
# file called arcdarkfile to be used by the pipeline later.
open("arcdarkfile", "w").write("gn"+arcdark)
os.system(com)
else:
com = "echo '{0:d} {1:d} 1 {2:d}' >> ".format(
s[0][k] - s[1][k] / 2, s[0][k] + s[1][k] / 2,
d.shape[0]) + fmsk
os.system(com)
iraf.text2mask(fmsk,
fmsk.strip('.txt') + '.pl', d.shape[1], d.shape[0])
iraf.imdelete('xeqxbrg@' + ic.lst_sci)
iraf.imdelete('tmp@' + ic.lst_sci)
iraf.imdelete('tmpdq@' + ic.lst_sci)
for sci in iraf.type(ic.lst_sci, Stdout=1):
sci = sci.strip()
iraf.copy('eqxbrg' + sci + '.fits', 'tmp' + sci + '.fits')
for j in np.arange(ic.nslit):
iraf.proto.fixpix('tmp' + sci + '.fits[sci,{0:1d}]'.format(j + 1),
'mskbadcol_' + sci + '_{0:1d}.pl'.format(j + 1),
linterp='1,2,3,4')
iraf.copy('tmp' + sci + '.fits', 'xeqxbrg' + sci + '.fits')
for j in np.arange(ic.nslit):
iraf.imarith('mskbadcol_' + sci + '_{0:1d}.pl'.format(j + 1), '+',
'xeqxbrg' + sci + '.fits[dq,{0:1d}]'.format(j + 1),
'tmpdq' + sci + '.fits[dq,{0:1d}]'.format(j + 1))
iraf.imcopy('tmpdq' + sci + '.fits[dq,{0:1d}][*,*]'.format(j + 1),
'xeqxbrg' + sci + '.fits[dq,{0:1d}][*,*]'.format(j + 1))
iraf.proto.fixpix('tmp' + sci + '.fits[sci,{0:1d}]'.format(j + 1),
'mskbadcol_' + sci + '_{0:1d}.pl'.format(j + 1),
linterp='1,2,3,4')
os.chdir(ic.dir_iraf)
from pyraf import iraf
from pyraf.iraf import gemini, gmos
os.chdir(current_dir)
iraf.chdir(current_dir)
iraf.unlearn('gfreduce')
iraf.unlearn('gfextract')
# ---------- Verifying the MDF ---------- #
os.system('rm -rfv ' + ic.dir_db + ' *.fits tmp*')
# Copy the MDF
iraf.copy('gmos$data/' + ic.mdf, '.', verbose='no')
# Extract a flat
iraf.imdelete('g@' + ic.lst_flat)
iraf.imdelete('rg@' + ic.lst_flat)
flat0 = iraf.type(ic.lst_flat, Stdout=1)[0]
iraf.gfreduce(flat0,
rawpath=ic.rawdir,
fl_extract='no',
bias=ic.bias,
fl_over='yes',
fl_trim='yes',
mdffile=ic.mdf,
mdfdir='./',
slits='both',
def ronchi(ronchilist, ronchiflat, calflat, over, flatdark, log):
for image in ronchilist:
image = str(image).strip()
if over:
iraf.delete("n" + image + '.fits')
iraf.nfprepare(image,
rawpath="",
shiftimage="s" + calflat,
bpm="rgn" + calflat + "_sflat_bpm.pl",
fl_vardq="yes",
fl_corr="no",
fl_nonl="no",
logfile=log)
ronchilist = checkLists(ronchilist, '.', 'n', '.fits')
# Determine the number of input Ronchi calibration mask files so that
# the routine runs automatically for single or multiple files.
if over:
iraf.delete("gn" + ronchiflat + ".fits")
if len(ronchilist) > 1:
iraf.gemcombine(listit(ronchilist, "n"),
output="gn" + ronchiflat,
fl_dqpr="yes",
masktype="none",
fl_vardq="yes",
logfile=log)
else:
iraf.copy("n" + ronchiflat + ".fits", "gn" + ronchiflat + ".fits")
if over:
iraf.delete("rgn" + ronchiflat + ".fits")
iraf.nsreduce("gn" + ronchiflat,
outpref="r",
dark="rgn" + flatdark + "_dark",
flatimage="rgn" + calflat + "_flat",
fl_cut="yes",
fl_nsappw="yes",
fl_flat="yes",
fl_sky="no",
fl_dark="yes",
fl_vardq="no",
logfile=log)
if over:
iraf.delete("brgn" + ronchiflat + ".fits")
iraf.nfsdist("rgn" + ronchiflat,
fwidth=6.0,
cradius=8.0,
glshift=2.8,
minsep=6.5,
thresh=2000.0,
nlost=3,
fl_inter="no",
logfile=log)
# Put the name of the distortion file into a file of fixed name to be
# used by the pipeline
open("ronchifile", "w").write("rgn" + ronchiflat)
def reduceArc(arclist, arc, log, over):
# flat field and cut the arc data
shiftima = open("shiftfile", "r").readlines()[0].strip()
sflat_bpm = open("sflat_bpmfile", "r").readlines()[0].strip()
flat = open("flatfile", "r").readlines()[0].strip()
dark = open("arcdarkfile", "r").readlines()[0].strip()
for image in arclist:
image = str(image).strip()
if os.path.exists("n" + image + ".fits"):
if over:
iraf.delete("n" + image + ".fits")
else:
print "Output file exists and -over not set - skipping combine_ima"
return
iraf.nfprepare(image,
rawpath="",
shiftimage=shiftima,
fl_vardq="yes",
bpm=sflat_bpm,
logfile=log)
if os.path.exists("gn" + image + ".fits"):
if over:
iraf.delete("gn" + image + ".fits")
else:
print "Output file exists and -over not set - skipping apply_flat_arc"
return
arclist = checkLists(arclist, '.', 'n', '.fits')
if len(arclist) > 1:
iraf.gemcombine(listit(arclist, "n"),
output="gn" + arc,
fl_dqpr='yes',
fl_vardq='yes',
masktype="none",
logfile=log)
else:
iraf.copy("n" + arc + ".fits", "gn" + arc + ".fits")
if os.path.exists("rgn" + image + ".fits"):
if over:
iraf.delete("rgn" + image + ".fits")
else:
print "Output file exists and -over not set - skipping apply_flat_arc"
return
fl_dark = "no"
if dark != "":
fl_dark = "yes"
hdulist = pyfits.open(arc + '.fits')
if 'K_Long' in hdulist[0].header['GRATING']:
iraf.nsreduce("gn" + arc,
darki=dark,
fl_cut="yes",
fl_nsappw="yes",
crval=23000.,
fl_dark="yes",
fl_sky="no",
fl_flat="yes",
flatimage=flat,
fl_vardq="no",
logfile=log)
else:
iraf.nsreduce("gn" + arc,
darki=dark,
fl_cut="yes",
fl_nsappw="yes",
fl_dark="yes",
fl_sky="no",
fl_flat="yes",
flatimage=flat,
fl_vardq="no",
logfile=log)
def process(infile, coords, fwhmpsf, sigma, aperture, annulus, dannulus,
output, zmag, debug):
"""
Entry point function to process science image.
Parameters
----------
infile : string
Science image or list of science images
coords : string
Input text file with coordinates of stars
fwhmpsf : float
FWHM of the stelar psf in pixels
sigma : float
Sky background sigma
annulus : int
Inner sky annulus radius in pixels
dannulus : int
Radius of sky annulus in pixels
output : string
Output file name
zmag : string
Photometric zero point
Returns
-------
None
"""
print("PHOTOMETRY: CHIMERA Aperture Photometry Routine")
fwhmpsf = float(fwhmpsf)
sigma = float(sigma)
annulus = int(annulus)
dannulus = int(dannulus)
# Check if input is a string of FITS images or a text file with file names
if infile[0] == "@":
infile = infile[1:]
if not os.path.exists(infile):
raise IOError("PHOTOMETRY: Not able to locate file %s. Stopping." %
infile)
image_cubes = []
with open(infile, "r") as fd:
for line in fd.readlines():
if len(line) > 1:
image_cubes.append(line.replace("\n", ""))
else:
image_cubes = infile.split(",")
# Number of images
ncubes = len(image_cubes)
# Check if the input coordinates file exists. Take a tmp copy of the input
# file and use that. Delete at the end of processing.
if not os.path.exists(coords):
raise IOError("Input coordinate file %s does not exist. Stopping." %
coords)
else:
tmp_coords = coords + ".tmp"
iraf.copy(coords, tmp_coords)
# Fields to extract from phot file
fields = "XCEN,YCEN,CIER,MSKY,STDEV,NSKY,SIER,SUM,AREA,FLUX,MERR,PIER"
total_phot_data = []
img_sec = []
for i in range(ncubes):
sci_file = image_cubes[i]
if not os.path.exists(sci_file):
raise IOError("FITS image %s does not exist. Stopping." % sci_file)
fpath, fname = os.path.split(sci_file)
print("\n Processing science image %s" % fname)
# Instantiate an Aperphot object
ap = chimera.Aperphot(sci_file, coords)
# Set fwhmpsf, sigma, annulus. dannulus and zero point
ap.fwhmpsf = fwhmpsf
ap.sigma = sigma
ap.annulus = annulus
ap.dannulus = dannulus
if zmag != "":
ap.zmag = float(zmag)
# Read the input FITS image
if i == 0:
img, imghdr = chimera.fitsread(ap.sci_file, header=True)
else:
img = chimera.fitsread(ap.sci_file)
# Determine nominal aperture radius for photometry
if i == 0:
if aperture:
nom_aper = float(aperture)
else:
nom_aper = ap.daocog()
print(" Nominal aperture radius : %4.1f pixels" % nom_aper)
# Perform aperture photometry on all the frames
dtype = [
("DATETIME", "S25"),
("XCEN", "f4"),
("YCEN", "f4"),
("CIER", "i4"),
("MSKY", "f4"),
("STDEV", "f4"),
("NSKY", "i4"),
("SIER", "i4"),
("SUM", "f4"),
("AREA", "f4"),
("FLUX_ADU", "f4"),
("FLUX_ELEC", "f4"),
("FERR", "f4"),
("MAG", "f4"),
("MERR", "f4"),
("PIER", "i4"),
]
phot_data = np.zeros([ap.nframes], dtype=dtype)
for j in range(ap.nframes):
print(" Processing frame number : %d" % (j + 1))
outfile = sci_file.replace(".fits", "_" + str(j) + ".phot.1")
ap.daophot(j + 1, tmp_coords, outfile, nom_aper)
objcen = dump(outfile, "XCEN,YCEN")
with open(tmp_coords, "w") as fd:
fd.write(objcen + '\n')
aperphot_data = dump(outfile, fields).split()
phot_data[j]['DATETIME'] = ap.addtime(j * ap.kintime).isoformat()
phot_data[j]['XCEN'] = float(aperphot_data[0])
phot_data[j]['YCEN'] = float(aperphot_data[1])
phot_data[j]['CIER'] = int(aperphot_data[2])
phot_data[j]['MSKY'] = float(aperphot_data[3])
phot_data[j]['STDEV'] = float(aperphot_data[4])
phot_data[j]['NSKY'] = int(aperphot_data[5])
phot_data[j]['SIER'] = int(aperphot_data[6])
phot_data[j]['SUM'] = float(aperphot_data[7])
phot_data[j]['AREA'] = float(aperphot_data[8])
phot_data[j]['FLUX_ADU'] = float(aperphot_data[9])
phot_data[j]['FLUX_ELEC'] = float(aperphot_data[9]) * ap.epadu
phot_data[j]['MAG'] = ap.zmag - 2.5 * np.log10(
phot_data[j]['FLUX_ELEC'] / ap.exptime)
if aperphot_data[10] == 'INDEF':
phot_data[j]['MERR'] = -10
else:
phot_data[j]['MERR'] = float(aperphot_data[10])
phot_data[j]['PIER'] = int(aperphot_data[11])
# Calculate error in flux - using the formula
# err = sqrt(flux * gain + npix * (1 + (npix/nsky)) * (flux_sky * gain + R**2))
phot_data[j]['FERR'] = np.sqrt(
phot_data[j]['FLUX_ELEC'] + phot_data[j]['AREA'] *
(1 + phot_data[j]['AREA'] / phot_data[j]['NSKY']) *
(phot_data[j]['MSKY'] * ap.epadu + ap.readnoise**2))
# Save a 51x51 image section of the object
xmin, xmax = int(phot_data[j]['XCEN']) - 25, int(
phot_data[j]['XCEN']) + 25
ymin, ymax = int(phot_data[j]['YCEN']) - 25, int(
phot_data[j]['YCEN']) + 25
img_sec.append(img[j, ymin:ymax, xmin:xmax])
# Save photometry of all the image cubes in a single file
total_phot_data.append(phot_data)
# If debug mode -
# 1. save DAOPHOT phot files
# 2. save individual phot data as npy file
# 3. Plot light cuve for each data cube separatly
if debug:
# Save photometry data in numpy binary format
print(" Saving photometry data as numpy binary")
if output != "":
npy_outfile = output + ".npy"
else:
npy_outfile = coords + "_phot.npy"
if os.path.exists(npy_outfile):
os.remove(npy_outfile)
np.save(npy_outfile, phot_data)
# Plot first pass light curve
if plot_flag:
print(" Plotting normalized light curve")
if output != "":
plt_outfile = output + ".png"
else:
plt_outfile = coords + "_lc.png"
plotter(phot_data, ap.nframes, ap.kintime, plt_outfile)
else:
# Delete intermediate files is not debug mode
iraf.delete(os.path.join(fpath, '*.phot.1'))
# Convert the total_phot_data to array and reshape it
print(' Saving consolidated photometry data...')
total_phot_data_arr = np.concatenate(total_phot_data)
# Save the array as npy file
np.save(coords + "_total.phot.npy", total_phot_data_arr)
# Save the image section with object as FITS file
print(' Saving image section with object as FITS image...')
img_sec_arr = np.asarray(img_sec)
img_fname = coords + "_obj.fits"
if os.path.exists(img_fname):
os.remove(img_fname)
chimera.fitswrite(img_sec_arr, coords + "_obj.fits", header=imghdr)
# Delete temporary coordinate file
if os.path.exists(tmp_coords):
os.remove(tmp_coords)
return
caldir = dir_iraf + "calibrations/"
if (glob.glob(caldir) == []):
os.mkdir(caldir)
lst_bias = 'bias.lis'
procbias = 'Mbias.fits'
# ----- Importing IRAF from the root directory ----- #
current_dir = os.getcwd()
os.chdir(dir_iraf)
from pyraf import iraf
from pyraf.iraf import gemini, gmos
iraf.unlearn()
os.chdir(current_dir)
iraf.chdir(current_dir)
# ----- Making a processed bias ----- #
iraf.imdelete(procbias)
iraf.imdelete('g@' + lst_bias)
iraf.gbias('@' + lst_bias, procbias, rawpath=rawdir, fl_vardq='yes')
iraf.copy(procbias, caldir)
# ----- Inspecting the processed bias ----- #
os.system('ds9 &')
iraf.sleep(5.0)
iraf.gdisplay(procbias, 1, fl_paste='no')
# Printing the running time
print('--- %s seconds ---' % (time.time() - start_time))
def start(obsDirList, calDirList, start, stop, tel, telinter, over):
path = os.getcwd()
# set up log
FORMAT = '%(asctime)s %(message)s'
DATEFMT = datefmt()
logging.basicConfig(filename='main.log',
format=FORMAT,
datefmt=DATEFMT,
level=logging.DEBUG)
log = os.getcwd() + '/main.log'
# loops through all the observation directories to perform the science reduction on each one
for obsDir in obsDirList:
os.chdir(obsDir)
tempObs = obsDir.split(os.sep)
# finds the Calibrations directory that corresponds to the science observation date
for calDir in calDirList:
tempCal = calDir.split(os.sep)
if tempObs[-3] == tempCal[-2]:
calDir = calDir + '/'
break
elif tempObs[-4] == tempCal[-2]:
calDir = calDir + '/'
break
obsid = tempObs[-1]
# reset iraf tasks
iraf.unlearn(iraf.gemini, iraf.gemtools, iraf.gnirs, iraf.nifs,
iraf.imcopy)
iraf.set(stdimage='imt2048')
# change the iraf directory to the current directory
pwd = os.getcwd()
iraffunctions.chdir(pwd)
iraf.nsheaders("nifs", logfile=log)
# define all the necessary variables and lists for the calibration and science images
shift = calDir + str(open(calDir + "shiftfile",
"r").readlines()[0]).strip()
flat = calDir + str(open(calDir + "flatfile",
"r").readlines()[0]).strip()
ronchi = open(calDir + "ronchifile", "r").readlines()[0].strip()
iraf.copy(calDir + ronchi + ".fits", output="./")
sflat_bpm = calDir + str(
open(calDir + "sflat_bpmfile", "r").readlines()[0]).strip()
arcdark = calDir + str(
open(calDir + "arcdarkfile", "r").readlines()[0]).strip()
# copy wavelength calibrated arc to obsDir
arc = "wrgn" + str(open(calDir + "arclist",
"r").readlines()[0]).strip()
iraf.copy(calDir + arc + ".fits", output="./")
# determines whether the data is science or telluric
if tempObs[-2] == 'Tellurics':
kind = 'Telluric'
objlist = open('tellist', 'r').readlines()
objlist = [image.strip() for image in objlist]
try:
skylist = open("skylist", "r").readlines()
skylist = [image.strip() for image in skylist]
except:
print "\nNo sky images were found for standard star. Please make a skylist in the telluric directory\n"
raise SystemExit
sky = skylist[0]
else:
kind = 'Object'
objlist = open("objlist", "r").readlines()
objlist = [image.strip() for image in objlist]
skylist = open("skylist", "r").readlines()
skylist = [image.strip() for image in skylist]
sky = skylist[0]
# check to see if the number of sky images matches the number of science images and if not duplicates sky images and rewrites the sky file and skylist
if not len(skylist) == len(objlist):
skylist = makeSkyList(skylist, objlist, obsDir)
centers = writeCenters(objlist)
# Make sure the database files are in place. Current understanding is that
# these should be local to the reduction directory, so need to be copied from
# the calDir.
if os.path.isdir("./database"):
if over:
shutil.rmtree("./database")
os.mkdir("./database")
elif not os.path.isdir("./database"):
os.mkdir('./database/')
iraf.copy(input=calDir + 'database/*', output="./database/")
#=========================================================
# Start main processing steps. Do this within a while loop
# to allow the use of start and stop positions
#=========================================================
logging.info('############################')
logging.info(' ')
logging.info(' Reducing Observations ')
logging.info(' ')
logging.info('############################')
print '############################'
print ' '
print ' Reducing Observations '
print ' '
print '############################'
valindex = start
if valindex > stop or valindex < 1 or stop > 9:
print "problem with start/stop values"
while valindex <= stop:
####################
## Prepare raw data ->n
logging.info('Prepare raw data ->n')
if valindex == 1:
objlist = prepare(objlist, shift, sflat_bpm, log, over)
skylist = prepare(skylist, shift, sflat_bpm, log, over)
#####################
## Combine multiple frames ->gn
elif valindex == 2:
if kind == 'Object':
logging.info('Combine multiple frames ->gn')
if len(skylist) > 1:
combineImages(skylist, "gn" + sky, log, over)
else:
copyImage(skylist, 'gn' + sky + '.fits', over)
else:
pass
##################
## Sky Subtraction ->gn
elif valindex == 3:
skySubtractObj(objlist, skylist, log, over)
logging.info('Sky Subtraction ->gn')
#################
## Flat field ->rgn
elif valindex == 4:
applyFlat(objlist, flat, log, over, kind)
logging.info('Flat field ->rgn')
#################
## Correct bad pixels ->brgn
elif valindex == 5:
fixBad(objlist, log, over)
logging.info('Correct bad pixels ->brgn')
#################
## Derive 2D->3D transformation ->fbrgn
elif valindex == 6:
fitCoords(objlist, arc, ronchi, log, over, kind)
logging.info('Derive 2D->3D transformation ->fbrgn')
#################
## Apply transformation ->tfbrgn
elif valindex == 7:
transform(objlist, log, over)
logging.info('Apply transformation ->tfbrgn')
#################
## Derive or apply telluric correction ->atfbrgn
elif valindex == 8:
logging.info('Derive or apply telluric correction ->atfbrgn')
if kind == 'Telluric':
makeTelluric(objlist, log, over)
elif kind == 'Object' and tel and telinter == 'no':
makeCube('tfbrgn', objlist, False, obsDir, log, over)
applyTelluric(objlist, obsid, skylist, telinter, log, over)
elif kind == 'Object' and tel and telinter == 'yes':
applyTelluric(objlist, obsid, skylist, telinter, log, over)
#################
## Create a 3D cube -> catfbrgn
elif valindex == 9:
if kind == "Telluric":
print "No cube being made for tellurics"
elif telinter == 'yes' and kind == 'Object' and tel:
logging.info('Create a 3D cube -> catfbrgn')
makeCube('atfbrgn', objlist, tel, obsDir, log, over)
elif kind == 'Object' and not tel and telinter == 'yes':
logging.info('Create a 3D cube -> ctfbrgn')
makeCube('tfbrgn', objlist, tel, obsDir, log, over)
valindex += 1
os.chdir(path)
return
def makeFlat(flatlist, flatdarklist, calflat, flatdark, grating, over, log):
"""Make flat and bad pixel mask.
Use NFPREPARE on the lamps on/lamps off flats to update the
raw data headers and attach the mask definition file (MDF) as a
binary table on all files. Note that dark frames will not have an
MDF attached by default. Instead, the appropriate MDF is added in
NSREDUCE or NSFLAT to match the data being reduced.
Use NSREDUCE to cut the calibration (flat/arc) spectra to
the size specified by the MDF, placing different IFU slices in
separate image extensions.
Use NSFLAT to generate a normalized flat field (for each
IFU slice or cross-dispersed order) from lamp flats. A mask (BPM)
will also be generated by thresholding - this can be used to flag
bad pixels in other data.
Use NSSLITFUNCTION to produce the final flat.
NSSLITFUNCTION extends the flatfield produced by correcting the
NSFLAT normalization for inter-slice variations. The output
from this task is used as the flatfield image for further
reduction.
"""
# Update lamps on flat frames with mdf offset value and generate variance and data quality extensions.
for image in flatlist:
image = str(image).strip()
if os.path.exists('n'+image+'.fits'):
if over:
os.remove('n'+image+'.fits')
iraf.nfprepare(image+'.fits',rawpath='.',shiftim="s"+calflat, fl_vardq='yes',fl_corr='no',fl_nonl='no', logfile=log)
else:
print "Output exists and -over- not set - skipping nfprepare of lamps on flats"
else:
iraf.nfprepare(image+'.fits',rawpath='.',shiftim="s"+calflat, fl_vardq='yes',fl_corr='no',fl_nonl='no', logfile=log)
flatlist = checkLists(flatlist, '.', 'n', '.fits')
# Update lamps off flat images with offset value and generate variance and data quality extensions.
for image in flatdarklist:
image = str(image).strip()
if os.path.exists('n'+image+'.fits'):
if over:
iraf.delete('n'+image+'.fits')
iraf.nfprepare(image+'.fits',rawpath='.',shiftim="s"+calflat, fl_vardq='yes',fl_corr='no',fl_nonl='no', logfile=log)
else:
print "\nOutput exists and -over- not set - skipping nfprepare of lamps off flats."
else:
iraf.nfprepare(image+'.fits',rawpath='.',shiftim="s"+calflat, fl_vardq='yes',fl_corr='no',fl_nonl='no', logfile=log)
flatdarklist = checkLists(flatdarklist, '.', 'n', '.fits')
# Combine lamps on flat images, "n"+image+".fits". Output combined file will have name of the first flat file with "gn" prefix.
if os.path.exists('gn'+calflat+'.fits'):
if over:
iraf.delete("gn"+calflat+".fits")
if len(flatlist) > 1:
iraf.gemcombine(listit(flatlist,"n"),output="gn"+calflat,fl_dqpr='yes', fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+calflat+'.fits', 'gn'+calflat+'.fits')
else:
print "\nOutput exists and -over- not set - skipping gemcombine of lamps on flats."
else:
if len(flatlist) > 1:
iraf.gemcombine(listit(flatlist,"n"),output="gn"+calflat,fl_dqpr='yes', fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+calflat+'.fits', 'gn'+calflat+'.fits')
# Combine lamps off flat images, "n"+image+".fits". Output combined file will have name of the first darkflat file with "gn" prefix.
if os.path.exists('gn'+flatdark+'.fits'):
if over:
iraf.delete("gn"+flatdark+".fits")
if len(flatdarklist) > 1:
iraf.gemcombine(listit(flatdarklist,"n"),output="gn"+flatdark,fl_dqpr='yes', fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+flatdark+'.fits', 'gn'+flatdark+'.fits')
else:
print "\nOutput exists and -over- not set - skipping gemcombine of lamps on flats."
else:
if len(flatdarklist) > 1:
iraf.gemcombine(listit(flatdarklist,"n"),output="gn"+flatdark,fl_dqpr='yes', fl_vardq='yes',masktype="none",logfile=log)
else:
iraf.copy('n'+flatdark+'.fits', 'gn'+flatdark+'.fits')
# NSREDUCE on lamps on flat images, "gn"+calflat+".fits", to extract the slices and apply an approximate wavelength calibration.
if os.path.exists('rgn'+calflat+'.fits'):
if over:
iraf.delete("rgn"+calflat+".fits")
iraf.nsreduce ("gn"+calflat,fl_cut='yes',fl_nsappw='yes',fl_vardq='yes', fl_sky='no',fl_dark='no',fl_flat='no',logfile=log)
else:
print "\nOutput exists and -over- not set - skipping nsreduce of lamps on flats."
else:
iraf.nsreduce ("gn"+calflat,fl_cut='yes',fl_nsappw='yes',fl_vardq='yes', fl_sky='no',fl_dark='no',fl_flat='no',logfile=log)
# NSREDUCE on lamps off flat frames, "gn"+flatdark+".fits", to extract the slices and apply an approximate wavelength calibration.
if os.path.exists('rgn'+flatdark+'.fits'):
if over:
iraf.delete("rgn"+flatdark+".fits")
iraf.nsreduce ("gn"+flatdark,fl_cut='yes',fl_nsappw='yes',fl_vardq='yes', fl_sky='no',fl_dark='no',fl_flat='no',logfile=log)
else:
print "\nOutput exists and -over- not set - skipping nsreduce of lamps off flats."
else:
iraf.nsreduce ("gn"+flatdark,fl_cut='yes',fl_nsappw='yes',fl_vardq='yes', fl_sky='no',fl_dark='no',fl_flat='no',logfile=log)
# Create slice-by-slice flat field image and BPM image from the darkflats, using NSFLAT.
# Lower and upper limit of bad pixels are 0.15 and 1.55.
# Get the spectral band so we can fine tune thr_flo and thr_fup. This fine tunes the number of
# bad pixels caught to an approximately constant level for each band. A few bad pixels will
# not be caught, but it was found that catching them marked large portions of the top and bottom
# rows of pixels of each slice as bad pixels.
header = astropy.io.fits.open(calflat+'.fits')
grat = header[0].header['GRATING'][0:1]
if grat == 'Z' or grat == 'J':
flo = 0.07
fup = 1.55
inter = 'no'
elif grat == 'H' or grat == 'K':
flo = 0.05
fup = 1.55
inter = 'no'
else:
print "\n#####################################################################"
print "#####################################################################"
print ""
print " WARNING in baselineCalibration: nsflat detected a non-standard wavelength "
print " configuration. Running interactively. "
print ""
print "#####################################################################"
print "#####################################################################\n"
# Arbitrary; not tested with non-standard configurations.
flo = 0.15
fup = 1.55
inter = 'yes'
if os.path.exists("rgn"+calflat+"_sflat.fits"):
if over:
iraf.delete("rgn"+flatdark+"_dark.fits")
iraf.delete("rgn"+calflat+"_sflat.fits")
iraf.delete("rgn"+calflat+"_sflat_bpm.pl")
iraf.nsflat("rgn"+calflat,darks="rgn"+flatdark,flatfile="rgn"+calflat+"_sflat", \
darkfile="rgn"+flatdark+"_dark", fl_save_dark='yes',process="fit", \
thr_flo=flo,thr_fup=fup, fl_vardq='yes', fl_int=inter, logfile=log )
else:
print "\nOutput exists and -over- not set - skipping nsflat creation of preliminary flat"
else:
iraf.nsflat("rgn"+calflat,darks="rgn"+flatdark,flatfile="rgn"+calflat+"_sflat", \
darkfile="rgn"+flatdark+"_dark", fl_save_dark='yes',process="fit", \
thr_flo=flo,thr_fup=fup, fl_vardq='yes', fl_int=inter, logfile=log )
# Renormalize the slices to account for slice-to-slice variations using NSSLITFUNCTION - make the final flat field image.
if os.path.exists("rgn"+calflat+"_flat.fits"):
if over:
iraf.delete("rgn"+calflat+"_flat.fits")
iraf.nsslitfunction("rgn"+calflat,"rgn"+calflat+"_flat", \
flat="rgn"+calflat+"_sflat",dark="rgn"+flatdark+"_dark",combine="median", \
order=3,fl_vary='no',logfile=log)
else:
print "\nOutput exists and -over- not set - skipping nsslitfunction flat rectification"
else:
iraf.nsslitfunction("rgn"+calflat,"rgn"+calflat+"_flat", \
flat="rgn"+calflat+"_sflat",dark="rgn"+flatdark+"_dark",combine="median", \
order=3,fl_vary='no',logfile=log)
# Put the name of the final flat field and bad pixel mask (BPM) into text files of fixed name to be used by the pipeline later.
open("flatfile", "w").write("rgn"+calflat+"_flat") # Final flat field
open("sflatfile", "w").write("rgn"+calflat+"_sflat") # Flat field before renormalization (before nsslitfunction)
open("sflat_bpmfile", "w").write("rgn"+calflat+"_sflat_bpm.pl") # Bad Pixel Mask
# Copy to relevant science directory calibrations/ directories as well
copyCalibration("rgn"+calflat+"_flat.fits", 'finalFlat.fits', grating, over)
copyCalibration("rgn"+calflat+"_sflat.fits", 'preliminaryFlat.fits', grating, over)
copyCalibration("rgn"+calflat+"_sflat_bpm.pl", 'finalBadPixelMask.pl', grating, over)
import os, glob
from pyraf import iraf
# takes a number of list files, creates folders in the working directory, and copies the files in each list to the new folder
prefixDict = {'n1':'17','n2':'18','n3':'19','n4':'20','n5':'21'}
linePrefix1 = 'data/201205'
linePrefix3 = '/reduce/'
iraf.copy.setParam('verbose','yes')
for file_ in glob.glob('*.list'):
with open(file_) as f:
for line in f:
if line.startswith('#'):
print "creating folder", line[2:-1]
os.mkdir(line[2:-1])
folder = line[2:-1]
else :
cpfile = linePrefix1+prefixDict[line[0:2]]+linePrefix3+line[0:-1]+'.fits'
dest = folder+os.sep+line[0:-1]+'.fits'
iraf.copy(cpfile,dest)
def adjustDQ(self, inIm, ext, txtMask):
print "\nADJUSTING MASK FOR " + inIm + ext.upper()
# grab dimensions of mask from header of image
# TODO: will expression for num work in one-slit case?
hdu = fits.open(inIm)
num = 3 * (int(ext[-2]) - 1) + 2
xDim = hdu[num].header["NAXIS1"]
yDim = hdu[num].header["NAXIS2"]
hdu.close()
# display input image
print "Displaying 2D spectra"
iraf.display(inIm + ext)
if ext[-2] == "1":
iraf.imdelete("x" + inIm, verify="yes")
# allow user to adjust mask iteratively
while True:
# ask user to input coordinate ranges to mask
fixDQ = bool(input("\nDoes mask need improvement? (True/False): "))
if fixDQ:
os.system("rm -iv " + txtMask)
ranges = raw_input(
"Coord ranges to be masked (space-separated): ")
cmd = "echo '" + ranges + "' >> " + txtMask
os.system(cmd)
else:
break
# transform text mask to pixel list format
plMask = txtMask[:txtMask.find(".")] + ".pl"
iraf.delete(plMask)
iraf.text2mask(txtMask, plMask, xDim, yDim)
# interpolate over bad pixels and mark them in DQ plane
tmp = "tmp_" + inIm[:inIm.find(".")] + "_" + ext[-2] + ".fits"
tmpdq = "tmpdq_" + inIm[:inIm.find(".")] + "_" + ext[-2] + ".fits"
images = tmp + "," + tmpdq
iraf.imdelete(images)
if ext[-2] == "1": iraf.copy(inIm, "x" + inIm)
iraf.imcopy(inIm + ext, tmp)
iraf.fixpix(tmp, plMask, linterp="1,2,3,4")
iraf.imcopy(tmp + "[*,*]", "x" + inIm + ext + "[*,*]")
iraf.imarith(plMask, "+", "x" + inIm + "[DQ," + ext[-2:], tmpdq)
iraf.imcopy(tmpdq + "[*,*]", "x" + inIm + "[DQ," + ext[-2:] + \
"[*,*]")
# display result
iraf.display("x" + inIm + ext)
# delete temporary files
iraf.imdelete(images)
return
def makeWaveCal(arclist, arc, arcdarklist, arcdark, log, over, path):
"""Determine the wavelength solution of each slice of the observation and
set the arc coordinate file.
If the user wishes to change the coordinate file to a different
one, they need only to change the "clist" variable to their line list
in the coordli= parameter in the nswavelength call.
Uses NSWAVELENGTH to calibrate arc data (after cutting and
optionally applying a flatfield with NSREDUCE in a previous step).
DATA REDUCTION HINT -
For the nswavelength call, the different wavelength settings
use different vaues for some of the parameters. For optimal auto
results, use:
K-band: thresho=50.0, cradius=8.0 --> (gives rms of 0.1 to 0.3)
H-band: thresho=100.0, cradius=8.0 --> (gives rms of 0.05 to 0.15)
J-band: thresho=100.0 --> (gives rms of 0.03 to 0.09)
Z-band: Currently not working very well for non-interactive mode
Note that better RMS fits can be obtained by running the wavelength
calibration interactively and identifying all of the lines
manually. Tedious, but will give more accurate results than the
automatic mode (i.e., fl_inter-). Use fl_inter+ for manual mode.
"""
# Store the name of the shift image in "shiftima".
shiftima = open("shiftfile", "r").readlines()[0].strip()
# Store the name of the bad pixel mask in "sflat_bpm".
sflat_bpm = open("sflat_bpmfile", "r").readlines()[0].strip()
# Store the name of the final flat field frame in "flat".
flat = open("flatfile", "r").readlines()[0].strip()
# Update arc images with offset value and generate variance and data
# quality extensions. Results in "n"+image+".fits"
for image in arclist:
image = str(image).strip()
if os.path.exists("n" + image + ".fits"):
if over:
iraf.delete("n" + image + ".fits")
else:
print "\nOutput file exists and -over not set - skipping nfprepare of arcs."
continue
iraf.nfprepare(image, rawpath=".", shiftimage=shiftima,bpm=sflat_bpm,\
fl_vardq="yes",fl_corr='no',fl_nonl='no',logfile=log)
# Check that output files for all arc images exists from nfprepare; if output does not
# exist remove corresponding arc images from arclist.
arclist = checkLists(arclist, '.', 'n', '.fits')
# Update arc dark frames with mdf offset value and generate variance and data
# quality extensions. Results in "n"+image+".fits"
for image in arcdarklist:
image = str(image).strip()
if os.path.exists("n" + image + ".fits"):
if over:
iraf.delete("n" + image + ".fits")
else:
print "\nOutput file exists and -over not set - skipping nfprepare of arcdarks."
continue
iraf.nfprepare(image, rawpath=".", shiftimage=shiftima, bpm=sflat_bpm, \
fl_vardq='yes',fl_corr='no',fl_nonl='no',logfile=log)
# Check that output files for all arc images exists from nfprepare; if output does not
# exist remove corresponding arc images from arclist.
arcdarklist = checkLists(arcdarklist, '.', 'n', '.fits')
# Combine arc frames, "n"+image+".fits". Output combined file will have the name of the first arc file.
if os.path.exists("gn" + arc + ".fits"):
if over:
iraf.delete("gn" + arc + ".fits")
if len(arclist) > 1:
iraf.gemcombine(listit(arclist, "n"),
output="gn" + arc,
fl_dqpr='yes',
fl_vardq='yes',
masktype="none",
logfile=log)
else:
iraf.copy('n' + arc + '.fits', 'gn' + arc + '.fits')
else:
print "\nOutput file exists and -over not set - skipping gemcombine of arcs."
else:
if len(arclist) > 1:
iraf.gemcombine(listit(arclist, "n"),
output="gn" + arc,
fl_dqpr='yes',
fl_vardq='yes',
masktype="none",
logfile=log)
else:
iraf.copy('n' + arc + '.fits', 'gn' + arc + '.fits')
# Combine arc dark frames, "n"+image+".fits". Output combined file will have the name of the first arc dark file.
if os.path.exists("gn" + arcdark + ".fits"):
if over:
iraf.delete("gn" + arcdark + ".fits")
if len(arcdarklist) > 1:
iraf.gemcombine(listit(arcdarklist, "n"),
output="gn" + arcdark,
fl_dqpr='yes',
fl_vardq='yes',
masktype="none",
logfile=log)
else:
iraf.copy('n' + arcdark + '.fits', 'gn' + arcdark + '.fits')
else:
print "\nOutput file exists and -over not set - skipping gemcombine of arcdarks."
else:
if len(arcdarklist) > 1:
iraf.gemcombine(listit(arcdarklist, "n"),
output="gn" + arcdark,
fl_dqpr='yes',
fl_vardq='yes',
masktype="none",
logfile=log)
else:
iraf.copy('n' + arcdark + '.fits', 'gn' + arcdark + '.fits')
# Put the name of the combined and prepared arc dark frame "gn"+arcdark into a text
# file called arcdarkfile to be used by the pipeline later.
open("arcdarkfile", "w").write("gn" + arcdark)
# NSREDUCE on arc images "gn"+arc+".fits" to extract the slices and apply an approximate
# wavelength calibration. Results in "rgn"+image+".fits"
if os.path.exists("rgn" + arc + ".fits"):
if over:
iraf.delete("rgn" + arc + ".fits")
else:
print "Output file exists and -over not set - skipping apply_flat_arc."
return
fl_dark = "no"
if arcdark != "":
fl_dark = "yes"
hdulist = astropy.io.fits.open(arc + '.fits')
if 'K_Long' in hdulist[0].header['GRATING']:
iraf.nsreduce("gn" + arc,
darki=arcdark,
fl_cut="yes",
fl_nsappw="yes",
crval=23000.,
fl_dark="yes",
fl_sky="no",
fl_flat="yes",
flatimage=flat,
fl_vardq="no",
logfile=log)
else:
iraf.nsreduce("gn"+arc, darki="gn"+arcdark, flatimage=flat, \
fl_vardq="no", fl_cut="yes", fl_nsappw="yes", fl_sky="no", fl_dark="yes", fl_flat="yes", \
logfile=log)
# Put the name of the combined and prepared arc dark frame "gn"+arcdark into a text
# file called arcdarkfile to be used by the pipeline later.
open("arcdarkfile", "w").write("gn" + arcdark)
if os.path.exists("wrgn" + arc + ".fits"):
if over:
iraf.delete("wrgn" + arc + ".fits")
else:
print "\nOutput file exists and -over not set - ",\
"not determining wavelength solution and recreating the wavelength reference arc.\n"
return
# Determine the wavelength setting.
hdulist = astropy.io.fits.open("rgn" + arc + ".fits")
band = hdulist[0].header['GRATING'][0:1]
central_wavelength = float(hdulist[0].header['GRATWAVE'])
# Set interactive mode. Default False for standard configurations (and True for non-standard wavelength configurations ).
pauseFlag = False
if band == "K" and central_wavelength == 2.20:
clist = RUNTIME_DATA_PATH + "k_ar.dat"
my_thresh = 50.0
elif band == "J":
clist = RUNTIME_DATA_PATH + "j_ar.dat"
my_thresh = 100.0
elif band == "H":
clist = RUNTIME_DATA_PATH + "h_ar.dat"
my_thresh = 100.0
elif band == "Z":
clist = "nifs$data/ArXe_Z.dat"
my_thresh = 100.0
else:
# Print a warning that the pipeline is being run with non-standard grating.
print "\n#####################################################################"
print "#####################################################################"
print ""
print " WARNING in calibrate: found a non-standard (non Z, J, H or K) "
print " wavelength configuration."
print " NSWAVELENGTH will be run interactively."
print ""
print "#####################################################################"
print "#####################################################################\n"
clist = "gnirs$data/argon.dat"
my_thresh = 100.0
interactive = 'yes'
if not pauseFlag:
# Establish wavelength calibration for arclamp spectra. Output: A series of
# files in a "database/" directory containing the wavelength solutions of
# each slice and a reduced arc frame "wrgn"+ARC+".fits".
iraf.nswavelength("rgn"+arc, coordli=clist, nsum=10, thresho=my_thresh, \
trace='yes', fwidth=2.0, match=-6,cradius=8.0,fl_inter='no',nfound=10,nlost=10, \
logfile=log)
else:
a = raw_input("For now, interactive Z or non-standard wavelength calibrations are unsupported. " + \
"Bugs running IRAF tasks interactively from python mean iraf.nswavelength cannot be activated automatically. " + \
"Therefore please run iraf.nswavelength() interactively from Pyraf to do a wavelength calibration by hand.")
def start(kind, telluricDirectoryList="", scienceDirectoryList=""):
"""
start(kind): Do a full reduction of either Science or Telluric data.
nifsReduce- for the telluric and science data reduction.
Reduces NIFS telluric and science frames and attempts a flux calibration.
Parameters are loaded from runtimeData/config.cfg. This script will
automatically detect if it is being run on telluric data or science data.
There are 6 steps.
INPUT:
+ Raw files
- Science frames
- Sky frames
+ Calibration files
- MDF shift file
- Bad Pixel Mask (BPM)
- Flat field frame
- Reduced arc frame
- Reduced ronchi mask frame
- arc and ronchi database/ files
OUTPUT:
- If telluric reduction an efficiency spectrum used to telluric correct and absolute flux
calibrate science frames
- If science reduction a reduced science data cube.
Args:
kind (string): either 'Telluric' or 'Science'.
telluricDirectoryList (string): Used by low memory pipeline.
scienceDirectoryList (string): Used by low memory pipeline.
"""
# TODO(nat): Right now the pipeline will crash if you decide to skip, say, doing a bad
# pixel correction. This is because each step adds a prefix to the frame name, and most following
# steps depend on that prefix being there.
# One way to fix this is if a step is to be skipped, iraf.copy() is called instead to copy the frame and
# add the needed prefix. Messy but it might work for now.
###########################################################################
## ##
## BEGIN - GENERAL REDUCTION SETUP ##
## ##
###########################################################################
# Store current working directory for later use.
path = os.getcwd()
# Set up the logging file.
log = os.getcwd() + '/Nifty.log'
logging.info('\n#################################################')
logging.info('# #')
logging.info('# Start the NIFS Science and Telluric Reduction #')
logging.info('# #')
logging.info('#################################################\n')
# Set up/prepare IRAF.
iraf.gemini()
iraf.gemtools()
iraf.gnirs()
iraf.nifs()
# Reset to default parameters the used IRAF tasks.
iraf.unlearn(iraf.gemini, iraf.gemtools, iraf.gnirs, iraf.nifs,
iraf.imcopy)
# From http://bishop.astro.pomona.edu/Penprase/webdocuments/iraf/beg/beg-image.html:
# Before doing anything involving image display the environment variable
# stdimage must be set to the correct frame buffer size for the display
# servers (as described in the dev$graphcap file under the section "STDIMAGE
# devices") or to the correct image display device. The task GDEVICES is
# helpful for determining this information for the display servers.
iraf.set(stdimage='imt2048')
# Prepare the IRAF package for NIFS.
# NSHEADERS lists the header parameters used by the various tasks in the
# NIFS package (excluding headers values which have values fixed by IRAF or
# FITS conventions).
iraf.nsheaders("nifs", logfile=log)
# Set clobber to 'yes' for the script. This still does not make the gemini
# tasks overwrite files, so:
# YOU WILL LIKELY HAVE TO REMOVE FILES IF YOU RE_RUN THE SCRIPT.
user_clobber = iraf.envget("clobber")
iraf.reset(clobber='yes')
# This helps make sure all variables are initialized to prevent bugs.
scienceSkySubtraction = None
scienceOneDExtraction = None
extractionXC = None
extractionYC = None
extractionRadius = None
telluricSkySubtraction = None
# Load reduction parameters from runtimeData/config.cfg.
with open('./config.cfg') as config_file:
config = ConfigObj(config_file, unrepr=True)
# Read general pipeline config.
over = config['over']
manualMode = config['manualMode']
calDirList = config['calibrationDirectoryList']
scienceOneDExtraction = config['scienceOneDExtraction']
extractionXC = config['extractionXC']
extractionYC = config['extractionYC']
extractionRadius = config['extractionRadius']
if kind == 'Telluric':
# Telluric reduction specific config.
telluricReductionConfig = config['telluricReductionConfig']
if telluricDirectoryList:
observationDirectoryList = telluricDirectoryList
elif not telluricDirectoryList:
observationDirectoryList = config['telluricDirectoryList']
start = telluricReductionConfig['telStart']
stop = telluricReductionConfig['telStop']
telluricSkySubtraction = telluricReductionConfig[
'telluricSkySubtraction']
if kind == 'Science':
# Science reduction specific config.
scienceReductionConfig = config['scienceReductionConfig']
if scienceDirectoryList:
observationDirectoryList = scienceDirectoryList
elif not scienceDirectoryList:
observationDirectoryList = config['scienceDirectoryList']
start = scienceReductionConfig['sciStart']
stop = scienceReductionConfig['sciStop']
scienceSkySubtraction = scienceReductionConfig[
'scienceSkySubtraction']
###########################################################################
## ##
## COMPLETE - GENERAL REDUCTION SETUP ##
## ##
###########################################################################
# nifsReduce has two nested loops that reduced data.
# It loops through each science (or telluric) directory, and
# runs through a series of calibrations steps on the data in that directory.
# Loop through all the observation (telluric or science) directories to perform a reduction on each one.
for observationDirectory in observationDirectoryList:
###########################################################################
## ##
## BEGIN - OBSERVATION SPECIFIC SETUP ##
## ##
###########################################################################
# Print the current directory of data being reduced.
logging.info(
"\n#################################################################################"
)
logging.info(" ")
logging.info(" Currently working on reductions in")
logging.info(" in " + str(observationDirectory))
logging.info(" ")
logging.info(
"#################################################################################\n"
)
os.chdir(observationDirectory)
tempObs = observationDirectory.split(os.sep)
obsid = tempObs[-1]
# Change the iraf directory to the current directory.
pwd = os.getcwd()
iraffunctions.chdir(pwd)
# Copy relevant calibrations over to the science directory.
# Open and store the name of the MDF shift reference file from shiftfile into shift.
shift = 'calibrations/shiftFile'
# Open and store the name of the flat frame
flat = 'calibrations/finalFlat'
# Open and store the bad pixel mask
finalBadPixelMask = 'calibrations/finalBadPixelMask'
# Ronchi, arc and database must all be in local calibrations directory
# Open and store the name of the reduced spatial correction ronchi flat frame name from ronchifile in ronchi.
ronchi = 'finalRonchi'
# Open and store the name of the reduced wavelength calibration arc frame from arclist in arc.
arc = 'finalArc'
if os.path.exists(os.getcwd() + '/' + ronchi + ".fits"):
if over:
iraf.delete(os.getcwd() + '/calibrations/finalRonchi.fits')
# Copy the spatial calibration ronchi flat frame from Calibrations_grating to the observation directory.
shutil.copy(os.getcwd() + '/calibrations/finalRonchi.fits',
ronchi + '.fits')
else:
print "\nOutput exists and -over not set - skipping copy of reduced ronchi"
else:
shutil.copy(os.getcwd() + '/calibrations/finalRonchi.fits',
ronchi + '.fits')
if os.path.exists(os.getcwd() + '/' + arc + ".fits"):
if over:
iraf.delete(os.getcwd() + '/calibrations/finalArc.fits')
# Copy the spatial calibration arc flat frame from Calibrations_grating to the observation directory.
shutil.copy(os.getcwd() + '/calibrations/finalArc.fits',
arc + '.fits')
else:
print "\nOutput exists and -over not set - skipping copy of reduced arc"
else:
shutil.copy(os.getcwd() + '/calibrations/finalArc.fits',
arc + '.fits')
# Make sure the database files are in place. Current understanding is that
# these should be local to the reduction directory, so need to be copied from
# the calDir.
if os.path.isdir("./database"):
if over:
shutil.rmtree("./database")
os.mkdir("./database")
for item in glob.glob("calibrations/database/*"):
shutil.copy(item, "./database/")
else:
print "\nOutput exists and -over not set - skipping copy of database directory"
else:
os.mkdir('./database/')
for item in glob.glob("calibrations/database/*"):
shutil.copy(item, "./database/")
if telluricSkySubtraction or scienceSkySubtraction:
# Read the list of sky frames in the observation directory.
try:
skyFrameList = open("skyFrameList", "r").readlines()
skyFrameList = [frame.strip() for frame in skyFrameList]
except:
logging.info(
"\n#####################################################################"
)
logging.info(
"#####################################################################"
)
logging.info("")
logging.info(
" WARNING in reduce: No sky frames were found in a directory."
)
logging.info(" Please make a skyFrameList in: " +
str(os.getcwd()))
logging.info("")
logging.info(
"#####################################################################"
)
logging.info(
"#####################################################################\n"
)
raise SystemExit
sky = skyFrameList[0]
# If we are doing a telluric reduction, open the list of telluric frames in the observation directory.
# If we are doing a science reduction, open the list of science frames in the observation directory.
if kind == 'Telluric':
tellist = open('tellist', 'r').readlines()
tellist = [frame.strip() for frame in tellist]
elif kind == 'Science':
scienceFrameList = open("scienceFrameList", "r").readlines()
scienceFrameList = [frame.strip() for frame in scienceFrameList]
# For science frames, check to see if the number of sky frames matches the number of science frames.
# IF NOT duplicate the sky frames and rewrite the sky file and skyFrameList.
if scienceSkySubtraction:
if not len(skyFrameList) == len(scienceFrameList):
skyFrameList = makeSkyList(skyFrameList, scienceFrameList,
observationDirectory)
###########################################################################
## ##
## COMPLETE - OBSERVATION SPECIFIC SETUP ##
## BEGIN DATA REDUCTION FOR AN OBSERVATION ##
## ##
###########################################################################
# Check start and stop values for reduction steps. Ask user for a correction if
# input is not valid.
valindex = start
while valindex > stop or valindex < 1 or stop > 6:
logging.info(
"\n#####################################################################"
)
logging.info(
"#####################################################################"
)
logging.info("")
logging.info(
" WARNING in reduce: invalid start/stop values of observation"
)
logging.info(" reduction steps.")
logging.info("")
logging.info(
"#####################################################################"
)
logging.info(
"#####################################################################\n"
)
valindex = int(
raw_input(
"\nPlease enter a valid start value (1 to 7, default 1): ")
)
stop = int(
raw_input(
"\nPlease enter a valid stop value (1 to 7, default 7): "))
while valindex <= stop:
###########################################################################
## STEP 1: Prepare raw data; science, telluric and sky frames ->n ##
###########################################################################
if valindex == 1:
if manualMode:
a = raw_input(
"About to enter step 1: locate the spectrum.")
if kind == 'Telluric':
tellist = prepare(tellist, shift, finalBadPixelMask, log,
over)
elif kind == 'Science':
scienceFrameList = prepare(scienceFrameList, shift,
finalBadPixelMask, log, over)
if telluricSkySubtraction or scienceSkySubtraction:
skyFrameList = prepare(skyFrameList, shift,
finalBadPixelMask, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 1: Locate the Spectrum (and prepare raw data) ->n - COMPLETED "
)
logging.info("")
logging.info(
"##############################################################################\n"
)
###########################################################################
## STEP 2: Sky Subtraction ->sn ##
###########################################################################
elif valindex == 2:
if manualMode:
a = raw_input("About to enter step 2: sky subtraction.")
# Combine telluric sky frames.
if kind == 'Telluric':
if telluricSkySubtraction:
if len(skyFrameList) > 1:
combineImages(skyFrameList, "gn" + sky, log, over)
else:
copyImage(skyFrameList, 'gn' + sky + '.fits', over)
skySubtractTel(tellist, "gn" + sky, log, over)
else:
for image in tellist:
iraf.copy('n' + image + '.fits',
'sn' + image + '.fits')
if kind == 'Science':
if scienceSkySubtraction:
skySubtractObj(scienceFrameList, skyFrameList, log,
over)
else:
for image in scienceFrameList:
iraf.copy('n' + image + '.fits',
'sn' + image + '.fits')
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(" STEP 2: Sky Subtraction ->sn - COMPLETED ")
logging.info("")
logging.info(
"##############################################################################\n"
)
##############################################################################
## STEP 3: Flat field, slice, subtract dark and correct bad pixels ->brsn ##
##############################################################################
elif valindex == 3:
if manualMode:
a = raw_input(
"About to enter step 3: flat fielding and bad pixels correction."
)
if kind == 'Telluric':
applyFlat(tellist, flat, log, over, kind)
fixBad(tellist, log, over)
elif kind == 'Science':
applyFlat(scienceFrameList, flat, log, over, kind)
fixBad(scienceFrameList, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 3: Flat fielding and Bad Pixels Correction ->brsn - COMPLETED "
)
logging.info("")
logging.info(
"##############################################################################\n"
)
###########################################################################
## STEP 4: Derive and apply 2D to 3D transformation ->tfbrsn ##
###########################################################################
elif valindex == 4:
if manualMode:
a = raw_input(
"About to enter step 4: 2D to 3D transformation and Wavelength Calibration."
)
if kind == 'Telluric':
fitCoords(tellist, arc, ronchi, log, over, kind)
transform(tellist, log, over)
elif kind == 'Science':
fitCoords(scienceFrameList, arc, ronchi, log, over, kind)
transform(scienceFrameList, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 4: 2D to 3D transformation and Wavelength Calibration ->tfbrsn - COMPLETED "
)
logging.info("")
logging.info(
"##############################################################################\n"
)
############################################################################
## STEP 5 (tellurics): For telluric data derive a telluric ##
## correction ->gxtfbrsn ##
## STEP 5 (science): For science apply an efficiency correction and make ##
## a data cube (not necessarily in that order). ##
## (i) Python method applies correction to nftransformed cube. ##
## Good for faint objects. ->cptfbrsn ##
## (ii) iraf.telluric method applies correction to ##
## nftransformed result (not quite a data cube) then ##
## nftransforms cube. ->catfbrsn ##
## (iii) If no telluric correction/flux calibration to be ##
## applied make a plain data cube. ->ctfbrsn ##
############################################################################
elif valindex == 5:
if manualMode:
a = raw_input("About to enter step 5.")
# For telluric data:
# Make a combined extracted 1D standard star spectrum.
if kind == 'Telluric':
extractOneD(tellist, kind, log, over, extractionXC,
extractionYC, extractionRadius)
# TODO(nat): add this as a parameter; encapsulate this.
copyToScience = True
if copyToScience:
# Copy final extracted results to science directory.
try:
with open("scienceMatchedTellsList", "r") as f:
lines = f.readlines()
lines = [x.strip() for x in lines]
for i in range(len(lines)):
if "obs" in lines[i]:
k = 1
while i + k != len(
lines) and "obs" not in lines[i +
k]:
copyResultsToScience(
"gxtfbrsn" + tellist[0] + ".fits",
"0_tel" + lines[i + k] + ".fits",
over)
k += 1
except IOError:
logging.info(
"\nNo scienceMatchedTellsList found in " +
os.getcwd() +
" . Skipping copy of extracted spectra to science directory."
)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5a: Extract 1D Spectra and Make Combined 1D Standard Star Spectrum"
)
logging.info(" ->gxtfbrsn - COMPLETED")
logging.info("")
logging.info(
"##############################################################################\n"
)
#TODO(nat): add this as a parameter.
makeTelluricCube = True
if makeTelluricCube:
makeCube('tfbrsn', tellist, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5b: Make uncorrected standard star data cubes, ->ctfbrsn - COMPLETED"
)
logging.info("")
logging.info(
"##############################################################################\n"
)
# For Science data:
# Possibly extract 1D spectra, and make uncorrected cubes.
elif kind == 'Science':
if scienceOneDExtraction:
extractOneD(scienceFrameList, kind, log, over,
extractionXC, extractionYC,
extractionRadius)
copyExtracted(scienceFrameList, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5a: Make extracted 1D Science spectra, ->ctgbrsn - COMPLETED"
)
logging.info("")
logging.info(
"##############################################################################\n"
)
makeCube('tfbrsn', scienceFrameList, log, over)
# TODO(nat): encapsulate this inside a function.
if os.path.exists('products_uncorrected'):
if over:
shutil.rmtree('products_uncorrected')
os.mkdir('products_uncorrected')
else:
logging.info(
"\nOutput exists and -over not set - skipping creating of products_uncorrected directory"
)
else:
os.mkdir('products_uncorrected')
for item in scienceFrameList:
if os.path.exists('products_uncorrected/ctfbrsn' +
item + '.fits'):
if over:
os.remove('products_uncorrected/ctfbrsn' +
item + '.fits')
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_uncorrected/ctfbrsn' + item +
'.fits')
else:
logging.info(
"\nOutput exists and -over not set - skipping copy of uncorrected cube"
)
else:
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_uncorrected/ctfbrsn' + item +
'.fits')
if os.path.exists('products_telluric_corrected'):
if over:
shutil.rmtree('products_telluric_corrected')
os.mkdir('products_telluric_corrected')
else:
logging.info(
"\nOutput exists and -over not set - skipping creating of products_telluric_corrected directory"
)
else:
os.mkdir('products_telluric_corrected')
for item in scienceFrameList:
if os.path.exists(
'products_telluric_corrected/ctfbrsn' + item +
'.fits'):
if over:
os.remove(
'products_telluric_corrected/ctfbrsn' +
item + '.fits')
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_telluric_corrected/ctfbrsn' +
item + '.fits')
else:
logging.info(
"\nOutput exists and -over not set - skipping copy of uncorrected cube"
)
else:
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_telluric_corrected/ctfbrsn' + item +
'.fits')
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5b: Make uncorrected science data cubes, ->ctfbrsn - COMPLETED"
)
logging.info("")
logging.info(
"##############################################################################\n"
)
valindex += 1
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(" COMPLETE - Reductions completed for " +
str(observationDirectory))
logging.info("")
logging.info(
"##############################################################################\n"
)
# Return to directory script was begun from.
os.chdir(path)
import os, glob
from pyraf import iraf
# takes a number of list files, creates folders in the working directory, and copies the files in each list to the new folder
prefixDict = {'n1': '17', 'n2': '18', 'n3': '19', 'n4': '20', 'n5': '21'}
linePrefix1 = 'data/201205'
linePrefix3 = '/reduce/'
iraf.copy.setParam('verbose', 'yes')
for file_ in glob.glob('*.list'):
with open(file_) as f:
for line in f:
if line.startswith('#'):
print "creating folder", line[2:-1]
os.mkdir(line[2:-1])
folder = line[2:-1]
else:
cpfile = linePrefix1 + prefixDict[
line[0:2]] + linePrefix3 + line[0:-1] + '.fits'
dest = folder + os.sep + line[0:-1] + '.fits'
iraf.copy(cpfile, dest)
def extractOneD(inputList,
kind,
log,
over,
extractionXC=15.0,
extractionYC=33.0,
extractionRadius=2.5):
"""Extracts 1-D spectra with iraf.nfextract and combines them with iraf.gemcombine.
iraf.nfextract is currently only done interactively. Output: -->xtfbrsn and gxtfbrsn
NFEXTRACT - Extract NIFS spectra.
This could be used to extract a 1D spectra from IFU data and is
particularly useful for extracting the bright spectra of
telluric calibrator stars. Note that this routine only works
on data that has been run through NFTRANSFORM.
"""
for frame in inputList:
frame = str(frame).strip()
if os.path.exists("xtfbrsn" + frame + ".fits"):
if over:
iraf.delete("xtfbrsn" + frame + ".fits")
else:
logging.info(
"Output file exists and -over not set - skipping nfextract in extract1D"
)
continue
iraf.nfextract("tfbrsn" + frame,
outpref="x",
xc=extractionXC,
yc=extractionYC,
diameter=extractionRadius,
fl_int='no',
logfile=log)
inputList = checkLists(inputList, '.', 'xtfbrsn', '.fits')
# Combine all the 1D spectra to one final output file with the name of the first input file.
combined = str(inputList[0]).strip()
if len(inputList) > 1:
if os.path.exists("gxtfbrsn" + combined + ".fits"):
if over:
iraf.delete("gxtfbrsn" + combined + ".fits")
else:
logging.info(
"Output file exists and -over not set - skipping gemcombine in extract1D"
)
return
iraf.gemcombine(listit(inputList, "xtfbrsn"),
output="gxtfbrsn" + combined,
statsec="[*]",
combine="median",
masktype="none",
fl_vardq="yes",
logfile=log)
else:
if over:
iraf.delete("gxtfbrsn" + combined + ".fits")
iraf.copy(input="xtfbrsn" + combined + ".fits",
output="gxtfbrsn" + combined + ".fits")
if kind == 'Telluric':
# Put the name of the final combined file into a text file called
# telluricfile to be used by the pipeline later.
open("telluricfile", "w").write("gxtfbrsn" + combined)
elif kind == 'Science':
open("combinedOneD", "w").write("gxtfbrsn" + combined)
def process(infile, coords, fwhmpsf, sigma, aperture, annulus, dannulus, output, zmag, debug):
"""
Entry point function to process science image.
Parameters
----------
infile : string
Science image or list of science images
coords : string
Input text file with coordinates of stars
fwhmpsf : float
FWHM of the stelar psf in pixels
sigma : float
Sky background sigma
annulus : int
Inner sky annulus radius in pixels
dannulus : int
Radius of sky annulus in pixels
output : string
Output file name
zmag : string
Photometric zero point
Returns
-------
None
"""
print "PHOTOMETRY: CHIMERA Aperture Photometry Routine"
fwhmpsf = float(fwhmpsf)
sigma = float(sigma)
annulus = int(annulus)
dannulus = int(dannulus)
# Check if input is a string of FITS images or a text file with file names
if infile[0] == "@":
infile = infile[1:]
if not os.path.exists(infile):
raise IOError("PHOTOMETRY: Not able to locate file %s. Stopping." %infile)
image_cubes = []
with open(infile, "r") as fd:
for line in fd.readlines():
if len(line) > 1:
image_cubes.append(line.replace("\n", ""))
else:
image_cubes = infile.split(",")
# Number of images
ncubes = len(image_cubes)
# Check if the input coordinates file exists. Take a tmp copy of the input
# file and use that. Delete at the end of processing.
if not os.path.exists(coords):
raise IOError("Input coordinate file %s does not exist. Stopping." %coords)
else:
tmp_coords = coords + ".tmp"
iraf.copy(coords, tmp_coords)
# Fields to extract from phot file
fields = "XCEN,YCEN,CIER,MSKY,STDEV,NSKY,SIER,SUM,AREA,FLUX,MERR,PIER"
total_phot_data = []
img_sec = []
for i in range(ncubes):
sci_file = image_cubes[i]
if not os.path.exists(sci_file):
raise IOError("FITS image %s does not exist. Stopping." %sci_file)
fpath, fname = os.path.split(sci_file)
print "\n Processing science image %s" %fname
# Instantiate an Aperphot object
ap = chimera.Aperphot(sci_file, coords)
# Set fwhmpsf, sigma, annulus. dannulus and zero point
ap.fwhmpsf = fwhmpsf
ap.sigma = sigma
ap.annulus = annulus
ap.dannulus = dannulus
if zmag != "":
ap.zmag = float(zmag)
# Read the input FITS image
if i == 0:
img, imghdr = chimera.fitsread(ap.sci_file, header = True)
else:
img = chimera.fitsread(ap.sci_file)
# Determine nominal aperture radius for photometry
if i == 0:
if aperture:
nom_aper = float(aperture)
else:
nom_aper = ap.daocog()
print " Nominal aperture radius : %4.1f pixels" %nom_aper
# Perform aperture photometry on all the frames
dtype = [("DATETIME", "S25"),("XCEN", "f4"),("YCEN", "f4"),("CIER", "i4"),("MSKY", "f4"),("STDEV", "f4"),("NSKY", "i4"),("SIER", "i4"),("SUM", "f4"),("AREA", "f4"),("FLUX_ADU", "f4"),("FLUX_ELEC", "f4"),("FERR", "f4"),("MAG", "f4"),("MERR", "f4"),("PIER", "i4"),]
phot_data = np.zeros([ap.nframes], dtype = dtype)
for j in range(ap.nframes):
print " Processing frame number : %d" %(j+1)
outfile = sci_file.replace(".fits", "_" + str(j) + ".phot.1")
ap.daophot(j+1, tmp_coords, outfile, nom_aper)
objcen = dump(outfile, "XCEN,YCEN")
with open(tmp_coords, "w") as fd:
fd.write(objcen + '\n')
aperphot_data = dump(outfile, fields).split()
phot_data[j]['DATETIME'] = ap.addtime(j * ap.kintime).isoformat()
phot_data[j]['XCEN'] = float(aperphot_data[0])
phot_data[j]['YCEN'] = float(aperphot_data[1])
phot_data[j]['CIER'] = int(aperphot_data[2])
phot_data[j]['MSKY'] = float(aperphot_data[3])
phot_data[j]['STDEV'] = float(aperphot_data[4])
phot_data[j]['NSKY'] = int(aperphot_data[5])
phot_data[j]['SIER'] = int(aperphot_data[6])
phot_data[j]['SUM'] = float(aperphot_data[7])
phot_data[j]['AREA'] = float(aperphot_data[8])
phot_data[j]['FLUX_ADU'] = float(aperphot_data[9])
phot_data[j]['FLUX_ELEC'] = float(aperphot_data[9]) * ap.epadu
phot_data[j]['MAG'] = ap.zmag - 2.5 * np.log10(phot_data[j]['FLUX_ELEC']/ap.exptime)
if aperphot_data[10] == 'INDEF':
phot_data[j]['MERR'] = -10
else:
phot_data[j]['MERR'] = float(aperphot_data[10])
phot_data[j]['PIER'] = int(aperphot_data[11])
# Calculate error in flux - using the formula
# err = sqrt(flux * gain + npix * (1 + (npix/nsky)) * (flux_sky * gain + R**2))
phot_data[j]['FERR'] = np.sqrt(phot_data[j]['FLUX_ELEC'] + phot_data[j]['AREA'] * (1 + phot_data[j]['AREA']/phot_data[j]['NSKY']) * (phot_data[j]['MSKY'] * ap.epadu + ap.readnoise**2))
# Save a 51x51 image section of the object
xmin, xmax = int(phot_data[j]['XCEN']) - 25, int(phot_data[j]['XCEN']) + 25
ymin, ymax = int(phot_data[j]['YCEN']) - 25, int(phot_data[j]['YCEN']) + 25
img_sec.append(img[j, ymin:ymax, xmin:xmax])
# Save photometry of all the image cubes in a single file
total_phot_data.append(phot_data)
# If debug mode -
# 1. save DAOPHOT phot files
# 2. save individual phot data as npy file
# 3. Plot light cuve for each data cube separatly
if debug:
# Save photometry data in numpy binary format
print " Saving photometry data as numpy binary"
if output != "":
npy_outfile = output + ".npy"
else:
npy_outfile = coords + "_phot.npy"
if os.path.exists(npy_outfile):
os.remove(npy_outfile)
np.save(npy_outfile, phot_data)
# Plot first pass light curve
if plot_flag:
print " Plotting normalized light curve"
if output != "":
plt_outfile = output + ".png"
else:
plt_outfile = coords + "_lc.png"
plotter(phot_data, ap.nframes, ap.kintime, plt_outfile)
else:
# Delete intermediate files is not debug mode
iraf.delete(os.path.join(fpath, '*.phot.1'))
# Convert the total_phot_data to array and reshape it
print ' Saving consolidated photometry data...'
total_phot_data_arr = np.concatenate(total_phot_data)
# Save the array as npy file
np.save(coords + "_total.phot.npy", total_phot_data_arr)
# Save the image section with object as FITS file
print ' Saving image section with object as FITS image...'
img_sec_arr = np.asarray(img_sec)
img_fname = coords + "_obj.fits"
if os.path.exists(img_fname):
os.remove(img_fname)
chimera.fitswrite(img_sec_arr, coords + "_obj.fits", header = imghdr)
# Delete temporary coordinate file
if os.path.exists(tmp_coords):
os.remove(tmp_coords)
return
