def apply_flat(scifilename, maskname, band):
''' Divides the contents of scifilename by the flat field and
overwrites scifilename with the same file divided by the flat
Args:
scifilename: Path to science file name.
maskname: The mask name
band: The filter bands
Results:
Overwrites scifilename where the data contents of the file
are divided by the pixel flat
'''
flat = IO.load_flat(maskname, band, {})
flat_data = flat[1].filled(1.0)
header, data = IO.readfits(scifilename)
print("Applying flat to file {0}".format(scifilename))
IO.writefits(data / flat_data,
maskname,
scifilename, {},
header=header,
overwrite=True)
def imcombine(filelist, maskname, fname, options, sum_type):
""" combine the images in file list into fname.
Sum type:
rate -- filelist is in cnt/s
ivar-rate -- filelist is in s/cnt
snr-rate -- filelist is in SNR
"""
ARR = None
hdr = None
i = 1
itime = 0
for file in filelist:
this_hdr, img = IO.readfits(file)
cards = this_hdr.ascardlist()
thisitime = this_hdr["truitime"]
itime += thisitime
if ARR is None:
ARR = np.zeros(img.shape)
if sum_type == "rate":
ARR += img * thisitime
if sum_type == "ivar-rate":
ARR += thisitime / img
if sum_type == "snr-rate":
ARR += img * thisitime
if hdr is None:
hdr = this_hdr
hdr.update("fno%2.2i" % i, file, "--")
for card in cards:
key, value, comment = (card.key, card.value, card.comment)
if hdr.has_key(key) and hdr[key] != value:
key = key + ("_img%2.2i" % i)
if len(key) > 8:
key = "HIERARCH " + key
try:
hdr.update(key, value, comment)
except ValueError:
pass
hdr.update("itime", itime, "Itime for %i rectified images" % len(filelist))
if sum_type == "rate":
ARR /= itime
if sum_type == "ivar-rate":
ARR = itime / ARR
if sum_type == "snr-rate":
ARR /= itime
IO.writefits(ARR, maskname, fname, options, header=hdr, overwrite=True, lossy_compress=True)
def imcombine(filelist, maskname, fname, options, sum_type):
''' combine the images in file list into fname.
Sum type:
rate -- filelist is in cnt/s
ivar-rate -- filelist is in s/cnt
snr-rate -- filelist is in SNR
'''
ARR = None
hdr = None
i = 1
itime = 0
for file in filelist:
this_hdr, img = IO.readfits(file)
cards = this_hdr.ascardlist()
thisitime = this_hdr['truitime']
itime += thisitime
if ARR is None: ARR = np.zeros(img.shape)
if sum_type == 'rate': ARR += img * thisitime
if sum_type == 'ivar-rate': ARR += thisitime / img
if sum_type == 'snr-rate': ARR += img * thisitime
if hdr is None:
hdr = this_hdr
hdr["fno%2.2i" % i] = (file, "--")
for card in cards:
key, value, comment = (card.key, card.value, card.comment)
if hdr.has_key(key) and hdr[key] != value:
key = key + ("_img%2.2i" % i)
if len(key) > 8: key = 'HIERARCH ' + key
try:
hdr[key] = (value, comment)
except ValueError:
pass
hdr['itime'] = (itime, 'Itime for %i rectified images' % len(filelist))
if sum_type == 'rate': ARR /= itime
if sum_type == 'ivar-rate': ARR = itime / ARR
if sum_type == 'snr-rate': ARR /= itime
IO.writefits(ARR,
maskname,
fname,
options,
header=hdr,
overwrite=True,
lossy_compress=True)
def combine(flatlist, maskname, band, options):
'''
combine list of flats into a flat file'''
out = os.path.join("combflat_2d_%s.fits"
% (band))
if os.path.exists(out):
os.remove(out)
IO.imcombine(flatlist, out, options, reject="minmax", nlow=1, nhigh=1)
def combine(flatlist, maskname, band, options, lampsOff=False):
'''
combine list of flats into a flat file'''
if lampsOff:
out = os.path.join("combflat_lamps_off_2d_%s.fits" % (band))
else:
out = os.path.join("combflat_2d_%s.fits" % (band))
if os.path.exists(out):
os.remove(out)
IO.imcombine(flatlist, out, options, reject="minmax", nlow=1, nhigh=1)
def __init__(self, maskname, bandname, options, fig):
self.fig = fig
self.flat = IO.read_drpfits(maskname, "combflat_2d_%s.fits" % bandname,
options)
self.edges, meta = IO.load_edges(maskname, bandname, options)
self.edgeno=2
self.cid = self.fig.canvas.mpl_connect('key_press_event', self)
self.draw()
def rectify(dname, lamdat, A, B, maskname, band, wavoptions, longoptions):
header, data = IO.readfits(dname)
raw_img = data * Detector.gain / header['TRUITIME']
dlam = Wavelength.grating_results(band)
hpp = np.array(Filters.hpp[band])
ll_fid = np.arange(hpp[0], hpp[1], dlam)
rectified = np.zeros((2048, len(ll_fid)))
from scipy.interpolate import interp1d
for i in xrange(2048):
ll = lamdat[i, :]
ss = raw_img[i, :]
ok = np.isfinite(ll) & np.isfinite(ss) & (ll < hpp[1]) & (ll > hpp[0])
if len(np.where(ok)[0]) < 30:
continue
f = interp1d(ll[ok], ss[ok], bounds_error=False)
rectified[i, :] = f(ll_fid)
header.update("wat0_001", "system=world")
header.update("wat1_001", "wtype=linear")
header.update("wat2_001", "wtype=linear")
header.update("dispaxis", 1)
header.update("dclog1", "Transform")
header.update("dc-flag", 0)
header.update("ctype1", "AWAV")
header.update("cunit1", "Angstrom")
header.update("crval1", ll_fid[0])
header.update("crval2", 0)
header.update("crpix1", 1)
header.update("crpix2", 1)
header.update("cdelt1", 1)
header.update("cdelt2", 1)
header.update("cname1", "angstrom")
header.update("cname2", "pixel")
header.update("cd1_1", dlam)
header.update("cd1_2", 0)
header.update("cd2_1", 0)
header.update("cd2_2", 1)
header.update("object", "rectified [eps]")
IO.writefits(rectified,
maskname,
"rectified_%s" % (dname),
wavoptions,
header=header,
overwrite=True,
lossy_compress=True)
def __init__(self, maskname, bandname, options, fig):
self.fig = fig
self.flat = IO.read_drpfits(maskname, "combflat_2d_%s.fits" % bandname,
options)
self.edges, meta = IO.load_edges(maskname, bandname, options)
self.edgeno = 2
self.cid = self.fig.canvas.mpl_connect('key_press_event', self)
self.draw()
def rectify(dname, lamdat, A, B, maskname, band, wavoptions,
longoptions):
header, data = IO.readfits(dname)
raw_img = data * Detector.gain / header['TRUITIME']
dlam = Wavelength.grating_results(band)
hpp = np.array(Filters.hpp[band])
ll_fid = np.arange(hpp[0], hpp[1], dlam)
rectified = np.zeros((2048, len(ll_fid)))
from scipy.interpolate import interp1d
for i in xrange(2048):
ll = lamdat[i,:]
ss = raw_img[i,:]
ok = np.isfinite(ll) & np.isfinite(ss) & (ll < hpp[1]) & (ll >
hpp[0])
if len(np.where(ok)[0]) < 30:
continue
f = interp1d(ll[ok], ss[ok], bounds_error=False)
rectified[i,:] = f(ll_fid)
header.update("wat0_001", "system=world")
header.update("wat1_001", "wtype=linear")
header.update("wat2_001", "wtype=linear")
header.update("dispaxis", 1)
header.update("dclog1", "Transform")
header.update("dc-flag", 0)
header.update("ctype1", "AWAV")
header.update("cunit1", "Angstrom")
header.update("crval1", ll_fid[0])
header.update("crval2", 0)
header.update("crpix1", 1)
header.update("crpix2", 1)
header.update("cdelt1", 1)
header.update("cdelt2", 1)
header.update("cname1", "angstrom")
header.update("cname2", "pixel")
header.update("cd1_1", dlam)
header.update("cd1_2", 0)
header.update("cd2_1", 0)
header.update("cd2_2", 1)
header.update("object", "rectified [eps]")
IO.writefits(rectified, maskname, "rectified_%s" % (dname),
wavoptions, header=header, overwrite=True, lossy_compress=True)
def isEmpty(self,file):
if not os.path.exists(file):
return True
fname = IO.list_file_to_strings(file)
if len(fname):
return False
else:
return True
def combine(flatlist, maskname, band, options, lampsOff=False):
'''
combine list of flats into a flat file'''
if lampsOff:
out = os.path.join("combflat_lamps_off_2d_%s.fits"
% (band))
else:
out = os.path.join("combflat_2d_%s.fits"
% (band))
if os.path.exists(out):
os.remove(out)
if len(flatlist)>1:
IO.imcombine(flatlist, out, options, reject="minmax", nlow=1, nhigh=1)
else:
IO.imcombine(flatlist, out, options, reject="none", nlow=1, nhigh=1)
def isEmpty(self, file):
if not os.path.exists(file):
return True
fname = IO.list_file_to_strings(file)
if len(fname):
return False
else:
return True
def set_header(self, header, ssl=None, msl=None, asl=None, targs=None):
'''Passed "header" a FITS header dictionary and converts to a Barset'''
self.pos = np.array(IO.parse_header_for_bars(header))
self.set_pos_pix()
self.ssl = ssl
self.msl = msl
self.asl = asl
self.targs = targs
def is_alignment_slit(slit):
return (np.float(slit["Target_Priority"]) < 0)
# If len(ssl) == 0 then the header is for a long slit
if (header['MASKNAME'] == 'long2pos'):
info("long2pos mode in CSU slit determination")
self.long2pos_slit = True
if (len(ssl) == 0):
self.long_slit = True
start = np.int(msl[0]["Slit_Number"])
stop = np.int(msl[-1]["Slit_Number"])
for mech_slit in msl:
mech_slit["Target_in_Slit"] = "long"
self.ssl = np.array([("1", "??", "??", "??", "??", "??", "??", msl[0]['Slit_width'],
(stop-start+1)*7.6, "0", "long", "0")],
dtype= [ ('Slit_Number', '|S2'),
('Slit_RA_Hours', '|S2'), ('Slit_RA_Minutes', '|S2'), ('Slit_RA_Seconds', '|S5'),
('Slit_Dec_Degrees', '|S3'), ('Slit_Dec_Minutes', '|S2'), ('Slit_Dec_Seconds', '|S5'),
('Slit_width', '|S5'), ('Slit_length', '|S5'), ('Target_to_center_of_slit_distance', '|S5'),
('Target_Name', '|S80'), ('Target_Priority', '|S1')])
self.scislit_to_slit = [ np.arange(start,stop) ]
ssl = None
# Create a map between scislit number and mechanical slit
# recall that slits count from 1
if ssl is not None:
prev = self.msl[0]["Target_in_Slit"]
v = []
for science_slit in ssl:
targ = science_slit["Target_Name"]
v.append([int(x) for x in self.msl.field("Slit_Number")[np.where(self.msl.field("Target_in_Slit").rstrip() == targ)[0]]])
self.scislit_to_slit = v
if (len(self.scislit_to_slit) != len(ssl)) and not (self.long_slit
and len(self.scislit_to_slit) == 1):
error("SSL should match targets in slit")
raise Exception("SSL should match targets in slit")
def test_trace_edge(self):
(header, data1, targs, ssl, msl, asl) = \
IO.readfits_all("/users/npk/desktop/c9/m110326_3242.fits")
data = data1
ssl = ssl[ssl["Slit_Number"] != ' ']
numslits = np.round(np.array(ssl["Slit_length"],
dtype=np.float) / 7.02)
for i in range(len(ssl)):
print ssl[i]["Target_Name"], numslits[i]
def test_trace_edge(self):
(header, data1, targs, ssl, msl, asl) = \
IO.readfits_all("/users/npk/desktop/c9/m110326_3242.fits")
data = data1
ssl = ssl[ssl["Slit_Number"] != ' ']
numslits = np.round(
np.array(ssl["Slit_length"], dtype=np.float) / 7.02)
for i in range(len(ssl)):
print ssl[i]["Target_Name"], numslits[i]
def audit(filename):
header, data = IO.readfits(filename)
ll0 = header['crval1']
dlam = header['cd1_1']
ls = ll0 + dlam * np.arange(data.shape[1])
linelist = Wavelength.pick_linelist(header)
deltas = []
sigs = []
xpos = []
ys = []
for y in np.linspace(750, 1100, 30):
#for y in np.linspace(5, 640, 50):
sp = np.ma.array(data[y,:])
xs, sxs, sigmas = Wavelength.find_known_lines(linelist, ls, sp,
Options.wavelength)
xpos.append(xs)
ys.append([y] * len(xs))
deltas.append(xs - (linelist - ll0)/dlam)
sigs.append(sxs)
xpos, ys, deltas, sigs = map(np.array, [xpos, ys, deltas, sigs])
deltas[np.abs(deltas) > .75] = np.nan
sigs[np.abs(sigs) > .001] = np.nan
pl.clf()
size = 0.003/sigs
size[size > 30] = 30
size[size < 1] = 1
pl.scatter( xpos, ys, c=deltas, s=size)
pl.xlim([0, data.shape[1]])
pl.ylim([0, data.shape[0]])
pl.xlabel("Spectral pixel")
pl.ylabel("Spatial pixel")
pl.title("Night sky line deviation from solution [pixel]")
pl.colorbar()
pl.savefig("audit.pdf")
pdb.set_trace()
def printMaskAndBand(self):
offsetfile = self.offsetFiles[0]
fname = IO.list_file_to_strings(offsetfile)
if os.path.isabs(fname[0]): path = fname[0]
else: path = os.path.join(fname_to_path(fname[0]), fname[0])
hdulist = pf.open(path)
header = hdulist[0].header
self.maskName = header['maskname']
self.band = header['filter']
self.addLine("maskname = '"+str(self.maskName)+"'")
self.addLine("band = '"+str(self.band)+"'")
self.addLine("")
def rename_files(wavenames, maskname, band, wavops):
lamname = Wavelength.filelist_to_wavename(wavenames[0], band, maskname, wavops).rstrip(".fits")
suffix = lamname.lstrip("wave_stack_%s_" % band)
path = os.path.join(wavops["outdir"], maskname)
fnames = ["rectified_%s%s.fits", "rectified_ivar_%s%s.fits", "rectified_sn_%s%s.fits"]
for fname in fnames:
try:
a = get_path(os.path.join(path, fname % (band, "_" + suffix)))
b = os.path.join(path, fname % (band, "")) + gz(a)
os.rename(a, b)
except:
print "Ignoring renaming of: ", fname
pass
edges = IO.load_edges(maskname, band, wavops)
n_slits = len(edges[0])
for i in xrange(1, n_slits + 1):
S = "S%2.2i" % (i)
a = get_path(os.path.join(path, "eps_%s_%s_%s.fits" % (band, suffix, S)))
a_h = pf.open(a)[0].header
obj = a_h["object"]
b = os.path.join(path, "%s_%s_%s_eps.fits" % (maskname, band, obj)) + gz(a)
os.rename(a, b)
a = get_path(os.path.join(path, "ivar_%s_%s_%s.fits" % (band, suffix, S)))
a_h = pf.open(a)[0].header
obj = a_h["object"]
b = os.path.join(path, "%s_%s_%s_ivar.fits" % (maskname, band, obj)) + gz(a)
os.rename(a, b)
a = get_path(os.path.join(path, "eps_%s_%s_%s.fits" % (maskname, suffix, band)))
b = os.path.join(path, "%s_%s_eps.fits" % (maskname, band)) + gz(a)
os.rename(a, b)
a = get_path(os.path.join(path, "snrs_%s_%s_%s.fits" % (maskname, suffix, band)))
b = os.path.join(path, "%s_%s_snrs.fits" % (maskname, band)) + gz(a)
os.rename(a, b)
a = get_path(os.path.join(path, "ivars_%s_%s_%s.fits" % (maskname, suffix, band)))
b = os.path.join(path, "%s_%s_ivars.fits" % (maskname, band)) + gz(a)
os.rename(a, b)
def apply_flat(scifilename, maskname, band):
''' Divides the contents of scifilename by the flat field and
overwrites scifilename with the same file divided by the flat
Args:
scifilename: Path to science file name.
maskname: The mask name
band: The filter bands
Results:
Overwrites scifilename where the data contents of the file
are divided by the pixel flat
'''
flat = IO.load_flat(maskname, band, {})
flat_data = flat[1].filled(1.0)
header, data = IO.readfits(scifilename)
print("Applying flat to file {0}".format(scifilename))
IO.writefits(data/flat_data, maskname, scifilename, {}, header=header,
overwrite=True)
def printMaskAndBand(self):
offsetfile = self.offsetFiles[0]
fname = IO.list_file_to_strings(offsetfile)
if os.path.isabs(fname[0]): path = fname[0]
else: path = os.path.join(fname_to_path(fname[0]), fname[0])
hdulist = pf.open(path)
header = hdulist[0].header
self.maskName = header['maskname']
self.band = header['filter']
self.addLine("maskname = '" + str(self.maskName) + "'")
self.addLine("band = '" + str(self.band) + "'")
self.addLine("")
def audit(filename):
header, data = IO.readfits(filename)
ll0 = header['crval1']
dlam = header['cd1_1']
ls = ll0 + dlam * np.arange(data.shape[1])
linelist = Wavelength.pick_linelist(header)
deltas = []
sigs = []
xpos = []
ys = []
for y in np.linspace(750, 1100, 30):
#for y in np.linspace(5, 640, 50):
sp = np.ma.array(data[y, :])
xs, sxs, sigmas = Wavelength.find_known_lines(linelist, ls, sp,
Options.wavelength)
xpos.append(xs)
ys.append([y] * len(xs))
deltas.append(xs - (linelist - ll0) / dlam)
sigs.append(sxs)
xpos, ys, deltas, sigs = map(np.array, [xpos, ys, deltas, sigs])
deltas[np.abs(deltas) > .75] = np.nan
sigs[np.abs(sigs) > .001] = np.nan
pl.clf()
size = 0.003 / sigs
size[size > 30] = 30
size[size < 1] = 1
pl.scatter(xpos, ys, c=deltas, s=size)
pl.xlim([0, data.shape[1]])
pl.ylim([0, data.shape[0]])
pl.xlabel("Spectral pixel")
pl.ylabel("Spatial pixel")
pl.title("Night sky line deviation from solution [pixel]")
pl.colorbar()
pl.savefig("audit.pdf")
pdb.set_trace()
def imdiff(A, B, maskname, band, header, options):
s = "[0]"
targname = A[1]["targname"].rstrip(" ")
if targname == "":
objname = A[1]["object"].replace(" ", "_")
else:
objname = targname.replace(" ", "_")
operand1 = A[0] + '[0]'
operand2 = B[0] + '[0]'
imnumA = A[0].split('_')[-1].rstrip(".fits")
imnumB = B[0].split('_')[-1].rstrip(".fits")
dname = "{0}_{1}_{2}_{3}-{4}_{5}-{6}.fits".format(maskname, objname, band,
A[1]["frameid"],
B[1]["frameid"], imnumA,
imnumB)
try:
os.remove(dname)
except:
pass
print "Data Diff {0}-{1}".format(operand1, operand2)
IO.imarith(operand1, '-', operand2, dname)
''' Now handle variance '''
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
varname = "var_{0}_{1}_{2}_{3}+{4}_{5}+{6}.fits".format(
maskname, objname, band, A[1]["frameid"], B[1]["frameid"], imnumA,
imnumB)
print "Var Sum {0}+{1}".format(operand1, operand2)
IO.imarith(operand1, '+', operand2, "tmp_" + varname)
try:
os.remove(varname)
except:
pass
print "Var add RN {0}+{1}".format(operand1, RN_adu**2)
IO.imarith("tmp_" + varname, '+', RN_adu**2, varname)
try:
os.remove("tmp_" + varname)
except:
pass
return dname, varname
def imdiff(A, B, maskname, band, header, options):
s = "[0]"
targname = A[1]["targname"].rstrip(" ")
if targname == "":
objname = A[1]["object"].replace(" ", "_")
else:
objname = targname.replace(" ", "_")
operand1 = A[0] + '[0]'
operand2 = B[0] + '[0]'
imnumA = A[0].split('_')[-1].rstrip(".fits")
imnumB = B[0].split('_')[-1].rstrip(".fits")
dname = "{0}_{1}_{2}_{3}-{4}_{5}-{6}.fits".format(maskname, objname, band,
A[1]["frameid"], B[1]["frameid"], imnumA, imnumB)
try: os.remove(dname)
except:pass
print "Data Diff {0}-{1}".format(operand1,operand2)
IO.imarith(operand1, '-', operand2, dname)
''' Now handle variance '''
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
varname = "var_{0}_{1}_{2}_{3}+{4}_{5}+{6}.fits".format(maskname, objname, band,
A[1]["frameid"], B[1]["frameid"], imnumA, imnumB)
print "Var Sum {0}+{1}".format(operand1,operand2)
IO.imarith(operand1, '+', operand2, "tmp_" + varname)
try: os.remove(varname)
except: pass
print "Var add RN {0}+{1}".format(operand1,RN_adu**2)
IO.imarith("tmp_" + varname, '+', RN_adu**2, varname)
try: os.remove("tmp_" + varname)
except: pass
return dname, varname
import numpy as np
import time
import pylab as pl
import scipy as sp
from scipy import interpolate as II
import spline
import MOSFIRE.Options as options
from MOSFIRE import IO, Fit, Wavelength
path = "/scr2/mosfire/c9_npk/npk_calib4_q1700_pa_0/"
mdat = IO.readmosfits(path + "m110323_2737.fits")
fdat = IO.readfits(path + "pixelflat_2d_H.fits")
ldat = IO.readfits(path + "lambda_solution_m110323_2737.fits")
dat = mdat[1]/fdat[1]
lam = ldat[1]
yroi=slice(707,736)
slit = dat[yroi,:]
lslit = lam[yroi,:]
DRAW = False
if DRAW:
pl.figure(1)
pl.clf()
pl.subplot(2,2,1)
def handle_flats(flatlist, maskname, band, options, extension=None):
'''
handle_flats is the primary entry point to the Flats module.
handle_flats takes a list of individual exposure FITS files and creates:
1. A CRR, dark subtracted, pixel-response flat file.
2. A set of polynomials that mark the edges of a slit
Inputs:
flatlist: Either a string of an input file or a list of file names
maskname: The name of a mask
band: A string indicating the bandceil
Outputs:
file {maskname}/flat_2d_{band}.fits -- pixel response flat
file {maskname}/edges.np
'''
tick = time.time()
# Check
bpos = np.ones(92) * -1
flatlist = IO.list_file_to_strings(flatlist)
print flatlist
for fname in flatlist:
hdr, dat, bs = IO.readmosfits(fname, options, extension=extension)
try: bs0
except: bs0 = bs
if np.any(bs0.pos != bs.pos):
raise Exception("Barsets do not seem to match")
if hdr["filter"] != band:
print ("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
for i in xrange(len(bpos)):
b = hdr["B{0:02d}POS".format(i+1)]
if bpos[i] == -1:
bpos[i] = b
else:
if bpos[i] != b:
raise Exception("Bar positions are not all the same in "
"this set of flat files")
bs = bs0
# Imcombine
if True:
print "Attempting to combine: ", flatlist
combine(flatlist, maskname, band, options)
print "Combined '%s' to '%s'" % (flatlist, maskname)
path = "combflat_2d_%s.fits" % band
(header, data) = IO.readfits(path, use_bpm=True)
print "Flat written to %s" % path
# Edge Trace
results = find_and_fit_edges(data, header, bs, options)
results[-1]["maskname"] = maskname
results[-1]["band"] = band
np.save("slit-edges_{0}".format(band), results)
save_ds9_edges(results, options)
# Generate Flat
out = "pixelflat_2d_%s.fits" % (band)
make_pixel_flat(data, results, options, out, flatlist)
print "Pixel flat took {0:6.4} s".format(time.time()-tick)
def go(maskname,
band,
filenames,
wavefile,
wavoptions,
longoptions,
use_flat=False):
'''
The go command is the main entry point into this module.
Inputs:
maskname: String of the mask name
band: String of 'Y', 'J', 'H', or 'K'
filenames: List of filenames to reduce
wavefile: String of path to FITS file with the wavelength solution
wavoptions: The Wavelength Options dictionary
longoptions: Dictionary containing:
{'yrange': The pixel range to extract over
'row_position': The row to solve the initial wavelength solution on}
use_flat: Boolean False [default] means to use no flat field
Boolean True means to divide by the pixelflat
'''
wavename = Wavelength.filelist_to_wavename(filenames, band, maskname,
wavoptions).rstrip(".fits")
print "Wavefile: {0}".format(wavefile)
lamhdr, lamdat = IO.readfits(wavefile)
positions = []
objname = None
for listfile in filenames:
fnames = IO.list_file_to_strings(listfile)
if len(fnames) != 1:
raise Exception(
"I currently expect only one file per position. Remove multiple entries and try again"
)
header, data, bs = IO.readmosfits(fnames[0], wavoptions)
if objname is None:
objname = header["object"]
if objname != header["object"]:
print(
"Trying to combine longslit stack of object {0} "
"with object {1}".format(objname, header["object"]))
print("{0:18s} {1:30s} {2:2s} {3:4.1f}".format(file, header["object"],
header["frameid"],
header["yoffset"]))
positions.append([fnames[0], header, data, bs])
print("{0:2g} nod positions found. Producing stacked difference" \
" image.".format(len(positions)))
for i in xrange(len(positions) - 1):
A = positions[i]
B = positions[i + 1]
print("----------- -----".format(A, B))
dname, varname = imdiff(A, B, maskname, band, header, wavoptions)
if use_flat:
apply_flat(dname, maskname, band)
apply_flat(varname, maskname, band)
rectify(dname, lamdat, A, B, maskname, band, wavoptions, longoptions)
rectify(varname, lamdat, A, B, maskname, band, wavoptions, longoptions)
print dname
dname, vname = imdiff(B, A, maskname, band, header, wavoptions)
if use_flat:
apply_flat(dname, maskname, band)
apply_flat(vname, maskname, band)
rectify(dname, lamdat, B, A, maskname, band, wavoptions, longoptions)
rectify(vname, lamdat, B, A, maskname, band, wavoptions, longoptions)
if False:
fname = os.path.join(path, wavename + ".fits")
B = IO.readfits(fname)
B = [fname, B[0], B[1]]
for i in xrange(len(positions)):
A = positions[i]
imdiff(A, B, maskname, band, wavoptions)
rectify(path, dname, lamdat, A, B, maskname, band, wavoptions,
longoptions)
imdiff(B, A, maskname, band, wavoptions)
rectify(path, dname, lamdat, B, A, maskname, band, wavoptions,
longoptions)
def imcombine(files, maskname, options, flat, outname=None, shifts=None,
extension=None):
'''
From a list of files it imcombine returns the imcombine of several values.
The imcombine code also estimates the readnoise ad RN/sqrt(numreads) so
that the variance per frame is equal to (ADU + RN^2) where RN is computed
in ADUs.
Arguments:
files[]: list of full path to files to combine
maskname: Name of mask
options: Options dictionary
flat[2048x2048]: Flat field (values should all be ~ 1.0)
outname: If set, will write (see notes below for details)
eps_[outname].fits: electron/sec file
itimes_[outname].fits: integration time
var_[outname].fits: Variance files
shifts[len(files)]: If set, will "roll" each file by the
amount in the shifts vector in pixels. This argument
is used when telescope tracking is poor. If you need
to use this, please notify Keck staff about poor
telescope tracking.
Returns 6-element tuple:
header: The combined header
electrons [2048x2048]: e- (in e- units)
var [2048x2048]: electrons + RN**2 (in e-^2 units)
bs: The MOSFIRE.Barset instance
itimes [2048x2048]: itimes (in s units)
Nframe: The number of frames that contribute to the summed
arrays above. If Nframe > 5 I use the sigma-clipping
Cosmic Ray Rejection tool. If Nframe < 5 then I drop
the max/min elements.
Notes:
header -- fits header
ADUs -- The mean # of ADUs per frame
var -- the Variance [in adu] per frame.
bs -- Barset
itimes -- The _total_ integration time in second
Nframe -- The number of frames in a stack.
Thus the number of electron per second is derived as:
e-/sec = (ADUs * Gain / Flat) * (Nframe/itimes)
The total number of electrons is:
el = ADUs * Gain * Nframe
'''
ADUs = np.zeros((len(files), 2048, 2048))
itimes = np.zeros((len(files), 2048, 2048))
prevssl = None
prevmn = None
patternid = None
maskname = None
header = None
if shifts is None:
shifts = np.zeros(len(files))
warnings.filterwarnings('ignore')
for i in xrange(len(files)):
fname = files[i]
thishdr, data, bs = IO.readmosfits(fname, options, extension=extension)
itimes[i,:,:] = thishdr["truitime"]
base = os.path.basename(fname).rstrip(".fits")
fnum = int(base.split("_")[1])
if shifts[i] == 0:
ADUs[i,:,:] = data.filled(0.0) / flat
else:
ADUs[i,:,:] = np.roll(data.filled(0.0) / flat, np.int(shifts[i]), axis=0)
''' Construct Header'''
if header is None:
header = thishdr
header["imfno%3.3i" % (fnum)] = (fname, "img%3.3i file name" % fnum)
map(lambda x: rem_header_key(header, x), ["CTYPE1", "CTYPE2", "WCSDIM",
"CD1_1", "CD1_2", "CD2_1", "CD2_2", "LTM1_1", "LTM2_2", "WAT0_001",
"WAT1_001", "WAT2_001", "CRVAL1", "CRVAL2", "CRPIX1", "CRPIX2",
"RADECSYS"])
for card in header.cards:
if card == '': continue
key,val,comment = card
if key in thishdr:
if val != thishdr[key]:
newkey = key + ("_img%2.2i" % fnum)
try: header[newkey.rstrip()] = (thishdr[key], comment)
except: pass
''' Now handle error checking'''
if maskname is not None:
if thishdr["maskname"] != maskname:
raise Exception("File %s uses mask '%s' but the stack is of '%s'" %
(fname, thishdr["maskname"], maskname))
maskname = thishdr["maskname"]
if thishdr["aborted"]:
raise Exception("Img '%s' was aborted and should not be used" %
fname)
if prevssl is not None:
if len(prevssl) != len(bs.ssl):
# todo Improve these checks
raise Exception("The stack of input files seems to be of "
"different masks")
prevssl = bs.ssl
if patternid is not None:
if patternid != thishdr["frameid"]:
raise Exception("The stack should be of '%s' frames only, but "
"the current image is a '%s' frame." % (patternid,
thishdr["frameid"]))
patternid = thishdr["frameid"]
if maskname is not None:
if maskname != thishdr["maskname"]:
raise Exception("The stack should be of CSU mask '%s' frames "
"only but contains a frame of '%s'." % (maskname,
thishdr["maskname"]))
maskname = thishdr["maskname"]
if thishdr["BUNIT"] != "ADU per coadd":
raise Exception("The units of '%s' are not in ADU per coadd and "
"this violates an assumption of the DRP. Some new code "
"is needed in the DRP to handle the new units of "
"'%s'." % (fname, thishdr["BUNIT"]))
''' Error checking is complete'''
print "%s %s[%s]/%s: %5.1f s, Shift: %i px" % (fname, maskname, patternid,
header['filter'], np.mean(itimes[i]), shifts[i])
warnings.filterwarnings('always')
# the electrons and el_per_sec arrays are:
# [2048, 2048, len(files)] and contain values for
# each individual frame that is being combined.
# These need to be kept here for CRR reasons.
electrons = np.array(ADUs) * Detector.gain
el_per_sec = electrons / itimes
output = np.zeros((2048, 2048))
exptime = np.zeros((2048, 2048))
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
# Cosmic ray rejection code begins here. This code construction the
# electrons and itimes arrays.
if len(files) >= 9:
print "Sigclip CRR"
srt = np.argsort(electrons, axis=0, kind='quicksort')
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
el_per_sec = el_per_sec[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
# Construct the mean and standard deviation by dropping the top and bottom two
# electron fluxes. This is temporary.
mean = np.mean(el_per_sec[2:-2,:,:], axis = 0)
std = np.std(el_per_sec[2:-2,:,:], axis = 0)
drop = np.where( (el_per_sec > (mean+std*4)) | (el_per_sec < (mean-std*4)) )
print "dropping: ", len(drop[0])
electrons[drop] = 0.0
itimes[drop] = 0.0
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
elif len(files) > 5:
print "WARNING: Drop min/max CRR"
srt = np.argsort(el_per_sec,axis=0)
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
electrons = np.sum(electrons[1:-1,:,:], axis=0)
itimes = np.sum(itimes[1:-1,:,:], axis=0)
Nframe = len(files) - 2
else:
print "WARNING: CRR through median filtering"
for i in xrange(len(files)):
el = electrons[i,:,:]
it = itimes[i,:,:]
el_mf = scipy.signal.medfilt(el, 5)
bad = np.abs(el - el_mf) / np.abs(el) > 10.0
el[bad] = 0.0
it[bad] = 0.0
electrons[i,:,:] = el
itimes[i,:,:] = it
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
''' Now handle variance '''
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
var = (electrons + RN**2)
''' Now mask out bad pixels '''
electrons[data.mask] = np.nan
var[data.mask] = np.inf
if "RN" in header: raise Exception("RN Already populated in header")
header['RN'] = ("%1.3f" , "Read noise in e-")
header['NUMFRM'] = (Nframe, 'Typical number of frames in stack')
if outname is not None:
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(np.float32(electrons/itimes), maskname, "eps_%s" % (outname),
options, header=header, overwrite=True)
# Update itimes after division in order to not introduce nans
itimes[data.mask] = 0.0
header['BUNIT'] = 'ELECTRONS^2'
IO.writefits(var, maskname, "var_%s" % (outname),
options, header=header, overwrite=True, lossy_compress=True)
header['BUNIT'] = 'SECOND'
IO.writefits(np.float32(itimes), maskname, "itimes_%s" % (outname),
options, header=header, overwrite=True, lossy_compress=True)
return header, electrons, var, bs, itimes, Nframe
#For questions and comments, email [email protected], submit a ticket on the ticketing system, or contact Luca Rizzi @ WMKO # THIS DRIVER IS SETUP TO REDUCE A STAR CALLED HIP85871_7.25, change this string on the entried below maskname = 'long2pos_specphot (align)' band = 'H' #Set bypass to True to autofit wavelenth solution instead of manually fitting. bypassflag = False # these are the narrow slits obsfiles_posCnarrow = [ 'Offset_-21_HIP85871_7.25_PosC.txt', 'Offset_-7_HIP85871_7.25_PosC.txt' ] target_posCnarrow = "HIP85871_7.25_POSC_NARROW" IO.fix_long2pos_headers(obsfiles_posCnarrow) obsfiles_posAnarrow = [ 'Offset_7_HIP85871_7.25_PosA.txt', 'Offset_21_HIP85871_7.25_PosA.txt' ] target_posAnarrow = "HIP85871_7.25_POSA_NARROW" IO.fix_long2pos_headers(obsfiles_posAnarrow) # these are the wide slits, comment out if you are not using specphot obsfiles_posCwide = [ 'Offset_-14_HIP85871_7.25_PosC.txt', 'Offset_-7_HIP85871_7.25_PosC.txt' ] target_posCwide = "HIP85871_7.25_POSC_WIDE" IO.fix_long2pos_headers(obsfiles_posCwide) obsfiles_posAwide = [ 'Offset_14_HIP85871_7.25_PosA.txt', 'Offset_21_HIP85871_7.25_PosA.txt' ] target_posAwide = "HIP85871_7.25_POSA_WIDE"
fs = ['m120406_0291.fits']
maskname = 'NGC5053'
options = Options.wavelength
path = os.path.join(options["outdir"], maskname)
if not os.path.exists(path):
raise Exception("Output directory '%s' does not exist. This "
"directory should exist." % path)
if False:
for fname in fs:
fp = os.path.join(path, fname)
mfits = IO.readmosfits(fp)
header, data, bs = mfits
Wavelength.plot_mask_solution_ds9(fname, maskname, options)
Wavelength.fit_lambda(mfits, fname, maskname, options)
Wavelength.apply_lambda(mfits, fname, maskname, options)
Wavelength.plot_data_quality(maskname, fname, options)
Wavelength.plot_sky_spectra(maskname, fname, options)
Wavelength.plot_mask_fits(maskname, fname, options)
if True:
for fname in fs:
pass
def handle_background(filelist, wavename, maskname, band_name, options, shifts=None, plan=None, extension=None):
'''
Perform difference imaging and subtract residual background.
The plan looks something like: [['A', 'B']]
In this case, the number of output files is equal to the length of the list (1).
If you choose to use an ABA'B' pattern then the plan will be: [["A", "B"], ["A'", "B'"]]
the background subtraction code will make and handle two files, "A-B" and "A'-B'".
'''
global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band
band = band_name
flatname = "pixelflat_2d_%s.fits" % band_name
hdr, flat = IO.readfits("pixelflat_2d_%s.fits" % (band_name), options)
if np.abs(np.median(flat) - 1) > 0.1:
raise Exception("Flat seems poorly behaved.")
'''
This next section of the code figures out the observing plan
and then deals with the bookeeping of sending the plan
to the background subtracter.
'''
hdrs = []
epss = {}
vars = {}
bss = []
times = {}
Nframes = []
i = 0
header = pf.Header()
for i in xrange(len(filelist)):
fl = filelist[i]
files = IO.list_file_to_strings(fl)
print "Combining"
if shifts is None: shift = None
else: shift = shifts[i]
hdr, electron, var, bs, time, Nframe = imcombine(files, maskname,
options, flat, outname="%s.fits" % (fl),
shifts=shift, extension=extension)
hdrs.append(hdr)
header = merge_headers(header, hdr)
epss[hdr['FRAMEID']] = electron/time
vars[hdr['FRAMEID']] = var
times[hdr['FRAMEID']] = time
bss.append(bs)
Nframes.append(Nframe)
positions = {}
i = 0
for h in hdrs:
positions[h['FRAMEID']] = i
i += 1
posnames = set(positions.keys())
if plan is None:
plan = guess_plan_from_positions(posnames)
num_outputs = len(plan)
edges, meta = IO.load_edges(maskname, band, options)
lam = IO.readfits(wavename, options)
bs = bss[0]
for i in xrange(num_outputs):
posname0 = plan[i][0]
posname1 = plan[i][1]
print "Handling %s - %s" % (posname0, posname1)
data = epss[posname0] - epss[posname1]
Var = vars[posname0] + vars[posname1]
itime = np.mean([times[posname0], times[posname1]], axis=0)
p = Pool()
solutions = p.map(background_subtract_helper, xrange(len(bs.ssl)))
p.close()
write_outputs(solutions, itime, header, maskname, band, plan[i], options)
def go(fname):
global plot_arr
print fname
(header, data) = IO.readfits(fname)
bs = CSU.Barset()
bs.set_header(header)
bars = []
means = []
sds = []
deltas = []
deltas_mm = []
poss = []
poss_mm = []
poss_y = []
request = []
qs = []
bars = range(1, 93)
fit_fun = Fit.residual_single
for bar in bars:
pos = bs.get_bar_pix(bar)
if bar % 8 == 0:
print "%2.0i: (%7.2f, %7.2f)" % (bar, pos[0], pos[1])
if is_odd(bar):
if (bs.pos[bar] - bs.pos[bar-1]) < 2.7:
fit_fun = Fit.residual_pair
else:
fit_fun = Fit.residual_single
width = 19
[[xslice, yslice],extent,ystart] = make_slice(pos,0,width,30)
if not is_in_bounds(extent):
fits = [0,0,0,0,0]
[ff,ok] = [np.poly1d(0,0), []]
means.append(fits)
sds.append(fits)
drop_this = True
else:
drop_this = False
fits = []
ys = np.arange(-10,10, dtype=np.float32)
for i in ys:
tofit = data[ystart-i, xslice]
y = median_tails(tofit)
ps = Fit.do_fit(y, fit_fun)
fits.append(ps[0])
fits = np.array(fits)
fits[:,1] += 1
# fit to the ridgeline
[ff, ok] = fit_line_with_sigclip(ys, fits[:,1])
m = [np.mean(fits[:,i]) for i in range(5)]
s = [np.std(fits[:,i]) for i in range(5)]
means.append(m)
sds.append(s)
slit_center_offset = pos[1] - ystart
fc = ff(slit_center_offset)
slit_center_pos = np.float(extent[0] + fc )
if drop_this:
poss.append(NaN)
poss_y.append(NaN)
poss_mm.append(NaN)
else:
poss.append(slit_center_pos)
poss_y.append(ystart)
poss_mm.append(CSU.csu_pix_to_mm_poly(slit_center_pos, ystart)[0])
delta = np.float(slit_center_pos - pos[0])
if drop_this:
deltas.append(NaN)
deltas_mm.append(NaN)
else:
deltas.append(delta)
b = CSU.csu_pix_to_mm_poly(slit_center_pos + delta, ystart)[0]
deltas_mm.append(b - poss_mm[-1])
q = np.float(np.degrees(np.tan(ff(1)-ff(0))))
if drop_this: qs.append(NaN)
qs.append(q)
means = np.array(means)
f = lambda x: np.array(x).ravel()
sds = f(sds)
deltas = f(deltas)
poss = f(poss)
poss_y = f(poss_y)
poss_mm = f(poss_mm)
deltas_mm = f(deltas_mm)
qs = f(qs)
bars = f(bars)
fout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".sav"
print "saving"
tosav = {"bars": bars, "request": bs.pos, "deltas_mm": deltas_mm, "poss": poss, "poss_mm": poss_mm, "deltas": deltas, "means": means, "qs": qs}
scipy.io.savemat(fout, tosav)
save_region(bs, "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".reg")
print "saved"
regout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".meas.reg"
pairs = np.array([poss,poss_y]).transpose()
s = CSU.to_ds9_region(pairs, dash=0, color="blue", label=False)
try:
f = open(regout, "w")
f.writelines(s)
f.close()
except:
print "Couldn't write: %s" % regout
return [tosav, bs]
def handle_flats(flatlist,
maskname,
band,
options,
extension=None,
edgeThreshold=450,
lampOffList=None,
longslit=None):
'''
handle_flats is the primary entry point to the Flats module.
handle_flats takes a list of individual exposure FITS files and creates:
1. A CRR, dark subtracted, pixel-response flat file.
2. A set of polynomials that mark the edges of a slit
Inputs:
flatlist:
maskname: The name of a mask
band: A string indicating the bandceil
Outputs:
file {maskname}/flat_2d_{band}.fits -- pixel response flat
file {maskname}/edges.np
'''
tick = time.time()
# Check
bpos = np.ones(92) * -1
#Retrieve the list of files to use for flat creation.
flatlist = IO.list_file_to_strings(flatlist)
# Print the filenames to Standard-out
for flat in flatlist:
info(str(flat))
#Determine if flat files headers are in agreement
for fname in flatlist:
hdr, dat, bs = IO.readmosfits(fname, options, extension=extension)
try:
bs0
except:
bs0 = bs
if np.any(bs0.pos != bs.pos):
print "bs0: " + str(bs0.pos) + " bs: " + str(bs.pos)
error("Barset do not seem to match")
raise Exception("Barsets do not seem to match")
if hdr["filter"] != band:
error("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
raise Exception("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
for i in xrange(len(bpos)):
b = hdr["B{0:02d}POS".format(i + 1)]
if bpos[i] == -1:
bpos[i] = b
else:
if bpos[i] != b:
error("Bar positions are not all the same in "
"this set of flat files")
raise Exception("Bar positions are not all the same in "
"this set of flat files")
bs = bs0
# Imcombine the lamps ON flats
info("Attempting to combine previous files")
combine(flatlist, maskname, band, options)
# Imcombine the lamps OFF flats and subtract the off from the On sets
if lampOffList != None:
#Retrieve the list of files to use for flat creation.
lampOffList = IO.list_file_to_strings(lampOffList)
# Print the filenames to Standard-out
for flat in lampOffList:
info(str(flat))
print "Attempting to combine Lamps off data"
combine(lampOffList, maskname, band, options, lampsOff=True)
combine_off_on(maskname, band, options)
debug("Combined '%s' to '%s'" % (flatlist, maskname))
info("Comgined to '%s'" % (maskname))
path = "combflat_2d_%s.fits" % band
(header, data) = IO.readfits(path, use_bpm=True)
info("Flat written to %s" % path)
# Edge Trace
if bs.long_slit:
info("Long slit mode recognized")
info("Central row position: " + str(longslit["row_position"]))
info("Upper and lower limits: " + str(longslit["yrange"][0]) + " " +
str(longslit["yrange"][1]))
results = find_longslit_edges(data,
header,
bs,
options,
edgeThreshold=edgeThreshold,
longslit=longslit)
elif bs.long2pos_slit:
info("Long2pos mode recognized")
results = find_long2pos_edges(data,
header,
bs,
options,
edgeThreshold=edgeThreshold,
longslit=longslit)
else:
results = find_and_fit_edges(data,
header,
bs,
options,
edgeThreshold=edgeThreshold)
results[-1]["maskname"] = maskname
results[-1]["band"] = band
np.save("slit-edges_{0}".format(band), results)
save_ds9_edges(results, options)
# Generate Flat
out = "pixelflat_2d_%s.fits" % (band)
if lampOffList != None:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=True)
else:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=False)
info("Pixel flat took {0:6.4} s".format(time.time() - tick))
# Driver file automatically generated on Wed Jul 29 15:04:02 2015 # For questions and comments, email [email protected], submit a ticket on the ticketing system, or contact Luca Rizzi @ WMKO # THIS DRIVER IS SETUP TO REDUCE A STAR CALLED HIP85871_7.25, change this string on the entried below maskname = "long2pos_specphot (align)" band = "H" # Set bypass to True to autofit wavelenth solution instead of manually fitting. bypassflag = False # these are the narrow slits obsfiles_posCnarrow = ["Offset_-21_HIP85871_7.25_PosC.txt", "Offset_-7_HIP85871_7.25_PosC.txt"] target_posCnarrow = "HIP85871_7.25_POSC_NARROW" IO.fix_long2pos_headers(obsfiles_posCnarrow) obsfiles_posAnarrow = ["Offset_7_HIP85871_7.25_PosA.txt", "Offset_21_HIP85871_7.25_PosA.txt"] target_posAnarrow = "HIP85871_7.25_POSA_NARROW" IO.fix_long2pos_headers(obsfiles_posAnarrow) # these are the wide slits, comment out if you are not using specphot obsfiles_posCwide = ["Offset_-14_HIP85871_7.25_PosC.txt", "Offset_-7_HIP85871_7.25_PosC.txt"] target_posCwide = "HIP85871_7.25_POSC_WIDE" IO.fix_long2pos_headers(obsfiles_posCwide) obsfiles_posAwide = ["Offset_14_HIP85871_7.25_PosA.txt", "Offset_21_HIP85871_7.25_PosA.txt"] target_posAwide = "HIP85871_7.25_POSA_WIDE" IO.fix_long2pos_headers(obsfiles_posAwide) # Note: for long2pos, the row position is ignored, and the middle point of the slit is used longslit = {"yrange": [[1062, 1188], [887, 1010]], "row_position": 0, "mode": "long2pos"} Flats.handle_flats("Flat.txt", maskname, band, flatops, longslit=longslit)
def isEmpty(self,file):
fname = IO.list_file_to_strings(file)
if len(fname):
return False
else:
return True
def make():
"""Make the database"""
db = load_db()
c = db.cursor()
create(c)
dirs = os.walk(Options.indir)
Options.indir = Options.indir.rstrip("/")
for root, dirs, files in dirs:
if root == Options.indir: continue
ignore, path = root.split(Options.indir)
if len(path.split("/")) != 2: continue
try: date = int(path.split("/")[1][0:4])
except: continue
if (date < 2012) or (date > 2030): continue
for file in files:
if len(file) != 17: continue
p = os.path.join(root, file)
num = db.execute('select count(*) from files where path = "%s"' %
p).fetchall()
if num[0][0] > 0:
print("Skipping: " + p + " [already in db]")
continue
print(p)
hdr = IO.readheader(p)
try:
fdate = file.split("_")[0][1:]
number = file.split("_")[1][:-5]
except:
print("Skipping: " + p)
continue
insert_sql = "insert into files(path,fdate,number,"
vals = "?,?,?,"
values = [p, fdate, number]
for key in list(hdr.keys()):
if key == 'COMMENT': continue
value = hdr[key]
T = type(value)
key = key.replace("-","_")
insert_sql += key + ","
vals += "?,"
values.append(value)
if key in keys: continue
keys.append(key)
if T == int: typename = 'integer'
if T == float: typename = 'real'
else: typename = 'text'
append_column(c, key, typename)
insert_sql = insert_sql[:-1] + ") values (" + vals[:-1] + ")"
try:
c.execute(insert_sql, tuple(values))
except:
print("Query failed on:")
print(insert_sql)
traceback.print_exc()
#sys.exit()
db.commit()
def make():
"""Make the database"""
db = load_db()
c = db.cursor()
create(c)
dirs = os.walk(Options.indir)
Options.indir = Options.indir.rstrip("/")
for root, dirs, files in dirs:
if root == Options.indir: continue
ignore, path = root.split(Options.indir)
if len(path.split("/")) != 2: continue
try:
date = int(path.split("/")[1][0:4])
except:
continue
if (date < 2012) or (date > 2030): continue
for file in files:
if len(file) != 17: continue
p = os.path.join(root, file)
num = db.execute('select count(*) from files where path = "%s"' %
p).fetchall()
if num[0][0] > 0:
print("Skipping: " + p + " [already in db]")
continue
print(p)
hdr = IO.readheader(p)
try:
fdate = file.split("_")[0][1:]
number = file.split("_")[1][:-5]
except:
print("Skipping: " + p)
continue
insert_sql = "insert into files(path,fdate,number,"
vals = "?,?,?,"
values = [p, fdate, number]
for key in hdr.keys():
if key == 'COMMENT': continue
value = hdr[key]
T = type(value)
key = key.replace("-", "_")
insert_sql += key + ","
vals += "?,"
values.append(value)
if key in keys: continue
keys.append(key)
if T == int: typename = 'integer'
if T == float: typename = 'real'
else: typename = 'text'
append_column(c, key, typename)
insert_sql = insert_sql[:-1] + ") values (" + vals[:-1] + ")"
try:
c.execute(insert_sql, tuple(values))
except:
print "Query failed on:"
print insert_sql
traceback.print_exc()
#sys.exit()
db.commit()
x1 = pos[0] + w
if x1 > 2047: x1 = 2047
if x0 > x1: x0 = x1
xs = slice(x0, x1)
y0 = pos[1] - h
if y0 < 0: y0 = 0
y1 = pos[1] + h
if y1 > 2047: y1 = 2047
if y0 > y1: y0 = y1
ys = slice(y0, y1)
return [[xs, ys], [x0, x1, y0, y1]]
(header, data) = IO.readfits("/users/npk/desktop/c8/m101029_0233.ref.fits")
(header2, data2) = IO.readfits("/users/npk/desktop/c8/m101029_0425.ref.fits")
(header3, data3) = IO.readfits("/users/npk/desktop/c8/m101029_0427.ref.fits")
data = data3
deg = np.pi / 180.
np.set_printoptions(precision=3)
np.set_printoptions(suppress=True)
reload(CSU)
reload(IO)
reload(Fit)
reload(Detector)
pl.ion()
def handle_rectification(maskname, in_files, wavename, band_pass, barset_file, options,
commissioning_shift=3.0):
'''Handle slit rectification and coaddition.
Args:
maskname: The mask name string
in_files: List of stacked spectra in electron per second. Will look
like ['electrons_Offset_1.5.txt.fits', 'electrons_Offset_-1.5.txt.fits']
wavename: path (relative or full) to the wavelength stack file, string
band_pass: Band pass name, string
barset_file: Path to a mosfire fits file containing the full set of
FITS extensions for the barset. It can be any file in the list
of science files.
Returns:
None
Writes files:
[maskname]_[band]_[object name]_eps.fits --
The rectified, background subtracted, stacked eps spectrum
[maskname]_[band]_[object name]_sig.fits --
Rectified, background subtracted, stacked weight spectrum (STD/itime)
[maskname]_[band]_[object_name]_itime.fits
Rectified, CRR stacked integration time spectrum
[maskname]_[band]_[object_name]_snrs.fits
Rectified signal to noise spectrum
'''
global edges, dats, vars, itimes, shifts, lambdas, band, fidl, all_shifts
band = band_pass
dlambda = Wavelength.grating_results(band)
hpp = Filters.hpp[band]
fidl = np.arange(hpp[0], hpp[1], dlambda)
lambdas = IO.readfits(wavename, options)
if np.any(lambdas[1].data < 0) or np.any(lambdas[1].data > 29000):
print "***********WARNING ***********"
print "The file {0} may not be a wavelength file.".format(wavename)
print "Check before proceeding."
print "***********WARNING ***********"
edges, meta = IO.load_edges(maskname, band, options)
shifts = []
posnames = []
postoshift = {}
for file in in_files:
print ":: ", file
II = IO.read_drpfits(maskname, file, options)
off = np.array((II[0]["decoff"], II[0]["raoff"]),dtype=np.float64)
if "yoffset" in II[0]:
off = -II[0]["yoffset"]
else:
# Deal with data taken during commissioning
if II[0]["frameid"] == 'A': off = 0.0
else: off = commissioning_shift
try: off0
except: off0 = off
shift = off - off0
shifts.append(shift)
posnames.append(II[0]["frameid"])
postoshift[II[0]['frameid']] = shift
print "Position {0} shift: {1:2.2f} as".format(off, shift)
plans = Background.guess_plan_from_positions(set(posnames))
all_shifts = []
for plan in plans:
to_append = []
for pos in plan:
to_append.append(postoshift[pos])
all_shifts.append(to_append)
# Reverse the elements in all_shifts to deal with an inversion
all_shifts.reverse()
theBPM = IO.badpixelmask()
all_solutions = []
cntr = 0
for plan in plans:
p0 = plan[0].replace("'", "p")
p1 = plan[1].replace("'", "p")
suffix = "%s-%s" % (p0,p1)
print "Handling plan %s" % suffix
fname = "bsub_{0}_{1}_{2}.fits".format(maskname,band,suffix)
EPS = IO.read_drpfits(maskname, fname, options)
EPS[1] = np.ma.masked_array(EPS[1], theBPM, fill_value=0)
fname = "var_{0}_{1}_{2}.fits".format(maskname, band, suffix)
VAR = IO.read_drpfits(maskname, fname, options)
VAR[1] = np.ma.masked_array(VAR[1], theBPM, fill_value=np.inf)
fname = "itime_{0}_{1}_{2}.fits".format(maskname, band, suffix)
ITIME = IO.read_drpfits(maskname, fname, options)
ITIME[1] = np.ma.masked_array(ITIME[1], theBPM, fill_value=0)
dats = EPS
vars = VAR
itimes = ITIME
EPS[0]["ORIGFILE"] = fname
tock = time.time()
sols = range(len(edges)-1,-1,-1)
shifts = all_shifts[cntr]
cntr += 1
p = Pool()
solutions = p.map(handle_rectification_helper, sols)
#solutions = map(handle_rectification_helper, [15])
p.close()
all_solutions.append(solutions)
tick = time.time()
print "-----> Mask took %i. Writing to disk." % (tick-tock)
output = np.zeros((1, len(fidl)))
snrs = np.zeros((1, len(fidl)))
sdout= np.zeros((1, len(fidl)))
itout= np.zeros((1, len(fidl)))
# the barset [bs] is used for determining object position
x, x, bs = IO.readmosfits(barset_file, options)
for i_slit in xrange(len(solutions)):
solution = all_solutions[0][i_slit]
header = EPS[0].copy()
obj = header['OBJECT']
target_name = bs.ssl[-(i_slit+1)]['Target_Name']
header['OBJECT'] = target_name
pixel_dist = np.float(bs.ssl[-(i_slit+1)]['Target_to_center_of_slit_distance'])/0.18
pixel_dist -= solution['offset']
ll = solution["lambda"]
header["wat0_001"] = "system=world"
header["wat1_001"] = "wtype=linear"
header["wat2_001"] = "wtype=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["ctype1"] = "AWAV"
header["cunit1"] = "Angstrom"
header["crval1"] = ll[0]
header["crval2"] = -solution["eps_img"].shape[0]/2 - pixel_dist
header["crpix1"] = 1
header["crpix2"] = 1
header["cdelt1"] = 1
header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = ll[1]-ll[0]
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = 1
S = output.shape
img = solution["eps_img"]
std = solution["sd_img"]
tms = solution["itime_img"]
for i_solution in xrange(1,len(all_solutions)):
print "Combining solution %i" %i_solution
solution = all_solutions[i_solution][i_slit]
img += solution["eps_img"]
std += solution["sd_img"]
tms += solution["itime_img"]
output = np.append(output, img, 0)
output = np.append(output, np.nan*np.zeros((3,S[1])), 0)
snrs = np.append(snrs, img*tms/std, 0)
snrs = np.append(snrs, np.nan*np.zeros((3,S[1])), 0)
sdout = np.append(sdout, std, 0)
sdout = np.append(sdout, np.nan*np.zeros((3,S[1])), 0)
itout = np.append(itout, tms, 0)
itout = np.append(itout, np.nan*np.zeros((3,S[1])), 0)
header['bunit'] = ('electron/second', 'electron power')
IO.writefits(img, maskname,
"{0}_{1}_{2}_eps.fits".format(maskname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('electron/second', 'sigma/itime')
IO.writefits(std/tms, maskname,
"{0}_{1}_{2}_sig.fits".format(maskname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('second', 'exposure time')
IO.writefits(tms, maskname,
"{0}_{1}_{2}_itime.fits".format(maskname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('', 'SNR')
IO.writefits(img*tms/std, maskname,
"{0}_{1}_{2}_snrs.fits".format(maskname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header = EPS[0].copy()
header["wat0_001"] = "system=world"
header["wat1_001"] = "wtype=linear"
header["wat2_001"] = "wtype=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["ctype1"] = "AWAV"
header["cunit1"] = ("Angstrom", 'Start wavelength')
header["crval1"] = ll[0]
header["crval2"] = 1
header["crpix1"] = 1
header["crpix2"] = 1
header["cdelt1"] = 1
header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = (ll[1]-ll[0], 'Angstrom/pixel')
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = 1
header["bunit"] = "ELECTRONS/SECOND"
IO.writefits(output, maskname, "{0}_{1}_eps.fits".format(maskname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = ""
IO.writefits(snrs, maskname, "{0}_{1}_snrs.fits".format(maskname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = "ELECTRONS/SECOND"
IO.writefits(sdout/itout, maskname, "{0}_{1}_sig.fits".format(maskname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = "SECOND"
IO.writefits(itout, maskname, "{0}_{1}_itime.fits".format(maskname,
band), options, overwrite=True, header=header,
lossy_compress=False)
def rename_files(wavenames, maskname, band, wavops):
lamname = Wavelength.filelist_to_wavename(wavenames[0], band, maskname,
wavops).rstrip(".fits")
suffix = lamname.lstrip("wave_stack_%s_" % band)
path = os.path.join(wavops["outdir"], maskname)
fnames = ["rectified_%s%s.fits", "rectified_ivar_%s%s.fits",
"rectified_sn_%s%s.fits"]
for fname in fnames:
try:
a = get_path(os.path.join(path, fname % (band, "_" + suffix)))
b = os.path.join(path, fname % (band, "")) + gz(a)
os.rename(a, b)
except:
print "Ignoring renaming of: ", fname
pass
edges = IO.load_edges(maskname, band, wavops)
n_slits = len(edges[0])
for i in xrange(1, n_slits+1):
S = "S%2.2i" % (i)
a = get_path(os.path.join(path,
"eps_%s_%s_%s.fits" % (band, suffix, S)))
a_h = pf.open(a)[0].header
obj = a_h['object']
b = os.path.join(path, "%s_%s_%s_eps.fits" % (maskname, band, obj)) + \
gz(a)
os.rename(a,b)
a = get_path(os.path.join(path,
"ivar_%s_%s_%s.fits" % (band, suffix, S)))
a_h = pf.open(a)[0].header
obj = a_h['object']
b = os.path.join(path, "%s_%s_%s_ivar.fits" % (maskname, band, obj)) + \
gz(a)
os.rename(a,b)
a = get_path(os.path.join(path,
"eps_%s_%s_%s.fits" % (maskname, suffix, band)))
b = os.path.join(path,
"%s_%s_eps.fits" % (maskname, band)) + gz(a)
os.rename(a,b)
a = get_path(os.path.join(path,
"snrs_%s_%s_%s.fits" % (maskname, suffix, band)))
b = os.path.join(path,
"%s_%s_snrs.fits" % (maskname, band)) + gz(a)
os.rename(a, b)
a = get_path(os.path.join(path,
"ivars_%s_%s_%s.fits" % (maskname, suffix, band)))
b = os.path.join(path,
"%s_%s_ivars.fits" % (maskname, band)) + gz(a)
os.rename(a, b)
nums = [26, 27, 28, 29]
roll = 0
elif False: # test_marc H
name = "test_marc"
band = "H"
nums = [19, 20, 21, 22, 23, 24]
roll = 0
flatlist = []
if roll == 0:
for num in nums:
flatlist.append("/users/npk/desktop/5apr/m120406_%4.4i.fits" % num)
else:
for num in nums:
header, ff, bs, targs, ssl, msl, asl = IO.readmosfits(
"/users/npk/desktop/5apr/m120406_%4.4i.fits" % num)
hdu = pyfits.PrimaryHDU(np.roll(ff, -5, 0), header)
hdulist = pyfits.HDUList([hdu])
for tbl in [targs, ssl, msl, asl]:
hdu = pyfits.new_table(tbl)
hdulist.append(hdu)
fn = "/users/npk/desktop/5apr/roll_m120406_%4.4i.fits" % num
os.remove(fn)
hdulist.writeto(fn)
flatlist.append(fn)
print flatlist
def imcombine(files,
maskname,
options,
flat,
outname=None,
shifts=None,
extension=None):
'''
From a list of files it imcombine returns the imcombine of several values.
The imcombine code also estimates the readnoise ad RN/sqrt(numreads) so
that the variance per frame is equal to (ADU + RN^2) where RN is computed
in ADUs.
Arguments:
files[]: list of full path to files to combine
maskname: Name of mask
options: Options dictionary
flat[2048x2048]: Flat field (values should all be ~ 1.0)
outname: If set, will write (see notes below for details)
eps_[outname].fits: electron/sec file
itimes_[outname].fits: integration time
var_[outname].fits: Variance files
shifts[len(files)]: If set, will "roll" each file by the
amount in the shifts vector in pixels. This argument
is used when telescope tracking is poor. If you need
to use this, please notify Keck staff about poor
telescope tracking.
Returns 6-element tuple:
header: The combined header
electrons [2048x2048]: e- (in e- units)
var [2048x2048]: electrons + RN**2 (in e-^2 units)
bs: The MOSFIRE.Barset instance
itimes [2048x2048]: itimes (in s units)
Nframe: The number of frames that contribute to the summed
arrays above. If Nframe > 5 I use the sigma-clipping
Cosmic Ray Rejection tool. If Nframe < 5 then I drop
the max/min elements.
Notes:
header -- fits header
ADUs -- The mean # of ADUs per frame
var -- the Variance [in adu] per frame.
bs -- Barset
itimes -- The _total_ integration time in second
Nframe -- The number of frames in a stack.
Thus the number of electron per second is derived as:
e-/sec = (ADUs * Gain / Flat) * (Nframe/itimes)
The total number of electrons is:
el = ADUs * Gain * Nframe
'''
ADUs = np.zeros((len(files), 2048, 2048))
itimes = np.zeros((len(files), 2048, 2048))
prevssl = None
prevmn = None
patternid = None
maskname = None
header = None
if shifts is None:
shifts = np.zeros(len(files))
warnings.filterwarnings('ignore')
for i in xrange(len(files)):
fname = files[i]
thishdr, data, bs = IO.readmosfits(fname, options, extension=extension)
itimes[i, :, :] = thishdr["truitime"]
base = os.path.basename(fname).rstrip(".fits")
fnum = int(base.split("_")[1])
if shifts[i] == 0:
ADUs[i, :, :] = data.filled(0.0) / flat
else:
ADUs[i, :, :] = np.roll(data.filled(0.0) / flat,
np.int(shifts[i]),
axis=0)
''' Construct Header'''
if header is None:
header = thishdr
header["imfno%3.3i" % (fnum)] = (fname, "img%3.3i file name" % fnum)
map(lambda x: rem_header_key(header, x), [
"CTYPE1", "CTYPE2", "WCSDIM", "CD1_1", "CD1_2", "CD2_1", "CD2_2",
"LTM1_1", "LTM2_2", "WAT0_001", "WAT1_001", "WAT2_001", "CRVAL1",
"CRVAL2", "CRPIX1", "CRPIX2", "RADECSYS"
])
for card in header.cards:
if card == '': continue
key, val, comment = card
if key in thishdr:
if val != thishdr[key]:
newkey = key + ("_img%2.2i" % fnum)
try:
header[newkey.rstrip()] = (thishdr[key], comment)
except:
pass
''' Now handle error checking'''
if maskname is not None:
if thishdr["maskname"] != maskname:
raise Exception(
"File %s uses mask '%s' but the stack is of '%s'" %
(fname, thishdr["maskname"], maskname))
maskname = thishdr["maskname"]
if thishdr["aborted"]:
raise Exception("Img '%s' was aborted and should not be used" %
fname)
if prevssl is not None:
if len(prevssl) != len(bs.ssl):
# todo Improve these checks
raise Exception("The stack of input files seems to be of "
"different masks")
prevssl = bs.ssl
if patternid is not None:
if patternid != thishdr["frameid"]:
raise Exception("The stack should be of '%s' frames only, but "
"the current image is a '%s' frame." %
(patternid, thishdr["frameid"]))
patternid = thishdr["frameid"]
if maskname is not None:
if maskname != thishdr["maskname"]:
raise Exception("The stack should be of CSU mask '%s' frames "
"only but contains a frame of '%s'." %
(maskname, thishdr["maskname"]))
maskname = thishdr["maskname"]
if thishdr["BUNIT"] != "ADU per coadd":
raise Exception(
"The units of '%s' are not in ADU per coadd and "
"this violates an assumption of the DRP. Some new code "
"is needed in the DRP to handle the new units of "
"'%s'." % (fname, thishdr["BUNIT"]))
''' Error checking is complete'''
print "%s %s[%s]/%s: %5.1f s, Shift: %i px" % (
fname, maskname, patternid, header['filter'], np.mean(
itimes[i]), shifts[i])
warnings.filterwarnings('always')
# the electrons and el_per_sec arrays are:
# [2048, 2048, len(files)] and contain values for
# each individual frame that is being combined.
# These need to be kept here for CRR reasons.
electrons = np.array(ADUs) * Detector.gain
el_per_sec = electrons / itimes
output = np.zeros((2048, 2048))
exptime = np.zeros((2048, 2048))
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
# Cosmic ray rejection code begins here. This code construction the
# electrons and itimes arrays.
if len(files) >= 9:
print "Sigclip CRR"
srt = np.argsort(electrons, axis=0, kind='quicksort')
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
el_per_sec = el_per_sec[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
# Construct the mean and standard deviation by dropping the top and bottom two
# electron fluxes. This is temporary.
mean = np.mean(el_per_sec[2:-2, :, :], axis=0)
std = np.std(el_per_sec[2:-2, :, :], axis=0)
drop = np.where((el_per_sec > (mean + std * 4))
| (el_per_sec < (mean - std * 4)))
print "dropping: ", len(drop[0])
electrons[drop] = 0.0
itimes[drop] = 0.0
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
elif len(files) > 5:
print "WARNING: Drop min/max CRR"
srt = np.argsort(el_per_sec, axis=0)
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
electrons = np.sum(electrons[1:-1, :, :], axis=0)
itimes = np.sum(itimes[1:-1, :, :], axis=0)
Nframe = len(files) - 2
else:
print "WARNING: CRR through median filtering"
for i in xrange(len(files)):
el = electrons[i, :, :]
it = itimes[i, :, :]
el_mf = scipy.signal.medfilt(el, 5)
bad = np.abs(el - el_mf) / np.abs(el) > 10.0
el[bad] = 0.0
it[bad] = 0.0
electrons[i, :, :] = el
itimes[i, :, :] = it
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
''' Now handle variance '''
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
var = (electrons + RN**2)
''' Now mask out bad pixels '''
electrons[data.mask] = np.nan
var[data.mask] = np.inf
if "RN" in header: raise Exception("RN Already populated in header")
header['RN'] = ("%1.3f", "Read noise in e-")
header['NUMFRM'] = (Nframe, 'Typical number of frames in stack')
if outname is not None:
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(np.float32(electrons / itimes),
maskname,
"eps_%s" % (outname),
options,
header=header,
overwrite=True)
# Update itimes after division in order to not introduce nans
itimes[data.mask] = 0.0
header['BUNIT'] = 'ELECTRONS^2'
IO.writefits(var,
maskname,
"var_%s" % (outname),
options,
header=header,
overwrite=True,
lossy_compress=True)
header['BUNIT'] = 'SECOND'
IO.writefits(np.float32(itimes),
maskname,
"itimes_%s" % (outname),
options,
header=header,
overwrite=True,
lossy_compress=True)
return header, electrons, var, bs, itimes, Nframe
try:
print "%(datafile)12s %(object)40s %(truitime)6.1f s %(maskname)35s %(lamps)3s %(filter)4s %(mgtname)7s" % (header)
except:
try:
print "%(datafile)12s %(object)25s %(truitime)6.1f s %(lamps)3s %(filter)6s %(mgtname)7s" % (header)
except:
print "%s Skipped" % fname
continue
datafile = header['datafile'] + '.fits'
maskname = header['maskname']
target = header['targname']
filter = header['filter']
yr,mn,dy = IO.fname_to_date_tuple(datafile)
date = str(yr)+mn+str(dy)
object = header['object']
itime = header['truitime']
grating_turret = header['mgtname']
if object.find("MIRA") == -1: mira = False
else: mira = True
if maskname.find(" (align)") == -1:
align = False
else:
maskname = maskname.replace(" (align)", "")
align = True
def write_outputs(solutions, itime, header, maskname, band_name, plan,
options):
sky_sub_out = np.zeros((2048, 2048), dtype=np.float)
sky_model_out = np.zeros((2048, 2048), dtype=np.float)
p0 = plan[0].replace("'", "p")
p1 = plan[1].replace("'", "p")
suffix = "%s-%s" % (p0, p1)
xroi = slice(0, 2048)
for sol in solutions:
if not sol["ok"]:
continue
yroi = slice(sol["bottom"], sol["top"])
sky_sub_out[yroi, xroi] = sol["output"]
sky_model_out[yroi, xroi] = sol["model"]
header['BUNIT'] = 'SECOND'
IO.writefits(itime,
maskname,
"itime_%s_%s_%s.fits" % (maskname, band, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(data,
maskname,
"sub_%s_%s_%s.fits" % (maskname, band, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(sky_sub_out,
maskname,
"bsub_%s_%s_%s.fits" % (maskname, band, suffix),
options,
header=header,
overwrite=True)
header['BUNIT'] = 'ELECTRONS'
IO.writefits(Var,
maskname,
"var_%s_%s_%s.fits" % (maskname, band, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(sky_model_out,
maskname,
"bmod_%s_%s_%s.fits" % (maskname, band, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
'''Now create rectified solutions'''
dlam = Wavelength.grating_results(band)
hpp = np.array(Filters.hpp[band])
ll_fid = np.arange(hpp[0], hpp[1], dlam)
nspec = len(ll_fid)
rectified = np.zeros((2048, nspec), dtype=np.float32)
rectified_var = np.zeros((2048, nspec), dtype=np.float32)
rectified_itime = np.zeros((2048, nspec), dtype=np.float32)
from scipy.interpolate import interp1d
for i in xrange(2048):
ll = lam[1][i, :]
ss = sky_sub_out[i, :]
ok = np.isfinite(ll) & np.isfinite(ss) & (ll < hpp[1]) & (ll > hpp[0])
if len(np.where(ok)[0]) < 100:
continue
f = interp1d(ll[ok], ss[ok], bounds_error=False)
rectified[i, :] = f(ll_fid)
f = interp1d(ll, Var[i, :], bounds_error=False)
rectified_var[i, :] = f(ll_fid)
f = interp1d(ll, itime[i, :], bounds_error=False)
rectified_itime[i, :] = f(ll_fid)
header["wat0_001"] = "system=world"
header["wat1_001"] = "type=linear"
header["wat2_001"] = "type=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["type1"] = "AWAV"
header["cunit1"] = "Angstrom"
header["crval1"] = (ll_fid[0], "Starting wavelength Angstrom")
header["crval2"] = 0
header["crpix1"] = 1
header["crpix2"] = 1
header["cdelt1"] = 1
header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = (dlam, "Angstrom/pixel")
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = (1, "pixel/pixel")
IO.writefits(rectified_itime,
maskname,
"%s_rectified_itime_%s_%s.fits" %
(maskname, band_name, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
IO.writefits(rectified,
maskname,
"%s_rectified_%s_%s.fits" % (maskname, band_name, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
IO.writefits(rectified_var,
maskname,
"%s_rectified_var_%s_%s.fits" % (maskname, band_name, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
IO.writefits(rectified * rectified_itime / np.sqrt(rectified_var),
maskname,
"%s_rectified_sn_%s_%s.fits" % (maskname, band_name, suffix),
options,
header=header,
overwrite=True,
lossy_compress=True)
def write_outputs(solutions, itime, header, maskname, band_name, plan, options):
sky_sub_out = np.zeros((2048, 2048), dtype=np.float)
sky_model_out = np.zeros((2048, 2048), dtype=np.float)
p0 = plan[0].replace("'", "p")
p1 = plan[1].replace("'", "p")
suffix = "%s-%s" % (p0,p1)
xroi = slice(0,2048)
for sol in solutions:
if not sol["ok"]:
continue
yroi = slice(sol["bottom"], sol["top"])
sky_sub_out[yroi, xroi] = sol["output"]
sky_model_out[yroi, xroi] = sol["model"]
header['BUNIT'] = 'SECOND'
IO.writefits(itime, maskname, "itime_%s_%s_%s.fits" % (maskname, band,
suffix), options, header=header, overwrite=True, lossy_compress=True)
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(data, maskname, "sub_%s_%s_%s.fits" % (maskname, band,
suffix), options, header=header, overwrite=True, lossy_compress=True)
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(sky_sub_out, maskname, "bsub_%s_%s_%s.fits" % (maskname, band,
suffix), options, header=header, overwrite=True)
header['BUNIT'] = 'ELECTRONS'
IO.writefits(Var, maskname, "var_%s_%s_%s.fits" % (maskname, band,
suffix), options, header=header, overwrite=True, lossy_compress=True)
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(sky_model_out, maskname, "bmod_%s_%s_%s.fits" % (maskname,
band, suffix), options, header=header, overwrite=True,
lossy_compress=True)
'''Now create rectified solutions'''
dlam = Wavelength.grating_results(band)
hpp = np.array(Filters.hpp[band])
ll_fid = np.arange(hpp[0], hpp[1], dlam)
nspec = len(ll_fid)
rectified = np.zeros((2048, nspec), dtype=np.float32)
rectified_var = np.zeros((2048, nspec), dtype=np.float32)
rectified_itime = np.zeros((2048, nspec), dtype=np.float32)
from scipy.interpolate import interp1d
for i in xrange(2048):
ll = lam[1][i,:]
ss = sky_sub_out[i,:]
ok = np.isfinite(ll) & np.isfinite(ss) & (ll < hpp[1]) & (ll >
hpp[0])
if len(np.where(ok)[0]) < 100:
continue
f = interp1d(ll[ok], ss[ok], bounds_error=False)
rectified[i,:] = f(ll_fid)
f = interp1d(ll, Var[i,:], bounds_error=False)
rectified_var[i,:] = f(ll_fid)
f = interp1d(ll, itime[i,:], bounds_error=False)
rectified_itime[i,:] = f(ll_fid)
header["wat0_001"] = "system=world"
header["wat1_001"] = "type=linear"
header["wat2_001"] = "type=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["type1"] = "AWAV"
header["cunit1"] = "Angstrom"
header["crval1"] = (ll_fid[0], "Starting wavelength Angstrom")
header["crval2"] = 0
header["crpix1"] = 1
header["crpix2"] = 1
header["cdelt1"] = 1
header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = (dlam, "Angstrom/pixel")
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = (1, "pixel/pixel")
IO.writefits(rectified_itime, maskname,
"%s_rectified_itime_%s_%s.fits" % (maskname, band_name,
suffix), options, header=header, overwrite=True, lossy_compress=True)
IO.writefits(rectified, maskname, "%s_rectified_%s_%s.fits" % (maskname,
band_name, suffix), options, header=header, overwrite=True,
lossy_compress=True)
IO.writefits(rectified_var, maskname, "%s_rectified_var_%s_%s.fits" %
(maskname, band_name, suffix), options, header=header, overwrite=True,
lossy_compress=True)
IO.writefits(rectified*rectified_itime/np.sqrt(rectified_var), maskname,
"%s_rectified_sn_%s_%s.fits" % (maskname, band_name,
suffix), options, header=header, overwrite=True, lossy_compress=True)
npk April 14th 2011
'''
import MOSFIRE
import time
from MOSFIRE import Fit, IO
import numpy as np, pylab as pl
reload(Fit)
reload(IO)
if __name__ == "__main__":
(header, data1, targs, ssl, msl, asl) = IO.readfits_all("/users/npk/desktop/c9/m110326_3242.fits")
data = data1
ssl = ssl[ssl["Slit_Number"] != ' ']
numslits = np.round(np.array(ssl["Slit_length"], dtype=np.float) / 7.02)
for i in range(len(ssl)):
print ssl[i]["Target_Name"], numslits[i]
Outputs:
xposs []: Array of x positions along the slit edge [pix]
yposs []: The fitted y positions of the "top" edge of the slit [pix]
os.remove('filelist.txt')
fl = open('filelist.txt', 'w')
files = []
for i in range(1, len(sys.argv)):
files.extend(glob.iglob(os.path.abspath(sys.argv[i])))
files = [file for file in files if os.path.splitext(file)[1] != '.original']
masks = {}
info('Examining {} files'.format(len(files)))
for fname in files:
try:
header = IO.readheader(fname)
except IOError: #, err:
fl.write("Couldn't IO %s\n" % fname)
continue
except:
fl.write("%s is unreadable\n" % fname)
continue
lamps = ""
try:
if header["pwstata7"] == 1:
lamps += header["pwloca7"][0:2]
if header["pwstata8"] == 1:
lamps += header["pwloca8"][0:2]
except KeyError:
lamps = "???"
os.remove('filelist.txt')
fl = open('filelist.txt', 'w')
files = []
for i in range(1, len(sys.argv)):
files.extend(glob.iglob(sys.argv[i]))
masks = {}
info('Examining {} files'.format(len(files)))
for fname in files:
try:
header = IO.readheader(fname)
except IOError:#, err:
fl.write("Couldn't IO %s\n" % fname)
continue
except:
fl.write("%s is unreadable\n" % fname)
continue
lamps = ""
try:
if header["pwstata7"] == 1:
lamps += header["pwloca7"][0:2]
if header["pwstata8"] == 1:
lamps += header["pwloca8"][0:2]
except KeyError:
lamps = "???"
import pylab as pl
import scipy as sp
from MOSFIRE import IO, Fit, Bspline
reload(IO)
reload(Fit)
reload(Bspline)
np.set_printoptions(precision=3)
# use the following file and id'd spectrum to guess
(header, data, targs, ssl, msl, asl) = IO.readfits_all(
"/users/npk/desktop/c9/m110319_1949.fits")
band = 'H'
pl.ion()
DRAW = True
def fit_spec(data, pos, alpha, sinbeta, gamma, delta, band):
global DRAW
ar_h_lines = np.array([1.465435, 1.474317, 1.505062, 1.517684, 1.530607,
1.533334, 1.540685, 1.590403, 1.599386, 1.618444, 1.644107,
1.674465, 1.744967, 1.791961])
Ar_K_lines = np.array([1.982291, 1.997118, 2.032256, 2.057443,
2.062186, 2.099184, 2.133871, 2.15409, 2.204558, 2.208321,
2.313952, 2.385154])
def printWavelengthFit(self):
if self.type is 'longslit' or self.type is 'long2pos':
addLongSlit = ",longslit=longslit"
else:
addLongSlit = ""
if self.type is 'slitmask' or self.type is 'longslit':
self.useNeon = False
self.useArgon = False
# determine is Argon and Neon files contain data for K bands
if self.isEmpty('Ar.txt') is False and self.band is 'K':
self.useArgon = True
if self.isEmpty('Ne.txt') is False and self.band is 'K':
self.useNeon = True
self.addLine(
"Wavelength.imcombine(obsfiles, maskname, band, waveops)")
if self.useArgon:
self.addLine(
"Wavelength.imcombine('Ar.txt', maskname, band, waveops)")
if self.useNeon:
self.addLine(
"Wavelength.imcombine('Ne.txt', maskname, band, waveops)")
self.addLine(
"Wavelength.fit_lambda_interactively(maskname, band, obsfiles,waveops"
+ addLongSlit + ", noninteractive=noninteractiveflag)")
if self.useArgon:
self.addLine(
"Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ar.txt', argon=True)"
)
if self.useNeon:
self.addLine(
"Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ne.txt', neon=True)"
)
self.addLine(
"Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles,waveops"
+ addLongSlit + ")")
if self.useArgon and self.useNeon:
self.addLine(
"Wavelength.fit_lambda(maskname, band, 'Ne.txt', 'Ne.txt',waveops, wavenames2='Ar.txt'"
+ addLongSlit + ")")
if self.useArgon and not self.useNeon:
self.addLine(
"Wavelength.fit_lambda(maskname, band, 'Ar.txt', 'Ar.txt',waveops"
+ addLongSlit + ")")
if self.useNeon and not self.useArgon:
self.addLine(
"Wavelength.fit_lambda(maskname, band, 'Ne.txt', 'Ne.txt',waveops"
+ addLongSlit + ")")
if self.useNeon or self.useArgon:
self.addLine("LROI = [[21000,22800]]*1")
if self.useNeon:
self.addLine(
"LROIs = Wavelength.check_wavelength_roi(maskname, band, obsfiles, 'Ne.txt', LROI, waveops)"
)
if self.useArgon and not self.useNeon:
self.addLine(
"LROIs = Wavelength.check_wavelength_roi(maskname, band, obsfiles, 'Ar.txt', LROI, waveops)"
)
self.addLine(
"Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops"
+ addLongSlit + ")")
if self.useArgon and self.useNeon:
self.addLine(
"Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ne.txt', LROIs, waveops)"
)
if self.useArgon and not self.useNeon:
self.addLine(
"Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ar.txt', LROIs, waveops)"
)
if self.useNeon and not self.useArgon:
self.addLine(
"Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ne.txt', LROIs, waveops)"
)
# determine waveleng name
files = IO.list_file_to_strings(self.offsetFiles)
if self.useNeon:
neon_files = IO.list_file_to_strings('Ne.txt')
self.waveName = "merged_lambda_solution_" + str(
Wavelength.filelist_to_wavename(
files, self.band, self.maskName,
"")).rstrip(".fits") + "_and_" + str(
Wavelength.filelist_to_wavename(
neon_files, self.band, self.maskName, ""))
elif self.useArgon and not self.useNeon:
argon_files = IO.list_file_to_strings('Ar.txt')
self.waveName = "merged_lambda_solution_" + str(
Wavelength.filelist_to_wavename(
files, self.band, self.maskName,
"")).rstrip(".fits") + "_and_" + str(
Wavelength.filelist_to_wavename(
argon_files, self.band, self.maskName, ""))
else:
self.waveName = "lambda_solution_" + str(
Wavelength.filelist_to_wavename(files, self.band,
self.maskName, ""))
if self.type is 'long2pos' or self.type is 'long2pos_specphot':
calibWith = ""
if self.isEmpty('Ar.txt') is False:
self.addLine("argon = ['Ar.txt']")
calibWith = "argon"
waveFiles = IO.list_file_to_strings('Ar.txt')
if self.isEmpty('Ne.txt') is False:
self.addLine("neon = ['Ne.txt']")
calibWith = "neon"
waveFiles = IO.list_file_to_strings('Ne.txt')
if calibWith:
# we have either Argon, or Neon, or both, so we can use arcs for the reduction
self.addLine("Wavelength.imcombine(" + str(calibWith) +
", maskname, band, waveops)")
self.addLine(
"Wavelength.fit_lambda_interactively(maskname, band, " +
str(calibWith) + ",waveops,longslit=longslit, " +
str(calibWith) +
"=True, noninteractive=noninteractiveflag)")
self.addLine("Wavelength.fit_lambda(maskname, band, " +
str(calibWith) + "," + str(calibWith) +
",waveops,longslit=longslit)")
self.addLine(
"Wavelength.apply_lambda_simple(maskname, band, " +
str(calibWith) +
", waveops, longslit=longslit, smooth=True)")
self.waveName = "lambda_solution_" + str(
Wavelength.filelist_to_wavename(waveFiles, self.band,
self.maskName, ""))
else:
# we have no arcs. For the time being, we can try with sky lines but this only works with long2pos specphot
print "#####################################################################################################"
print "WARNING: There are no arc calibration files"
print " The pipeline will try to use sky lines but this only works if the observation is long enough"
print " and if you are only using long2pos. It will NOT work on long2pos_specphot"
print " Please contact the MosfireDRP team to obtain a standard wavelength solution"
print "#####################################################################################################"
self.addLine(
"obsfiles = obsfiles_posAnarrow + obsfiles_posCnarrow")
self.addLine(
"Wavelength.imcombine(obsfiles, maskname, band, waveops)")
self.addLine(
"Wavelength.fit_lambda_interactively(maskname, band, obsfiles ,waveops,longslit=longslit, noninteractive=noninteractiveflag)"
)
self.addLine(
"Wavelength.fit_lambda(maskname, band, obsfiles,obsfiles ,waveops,longslit=longslit)"
)
self.addLine(
"Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops, longslit=longslit, smooth=True)"
)
files = IO.list_file_to_strings(self.offsetFiles)
self.waveName = "lambda_solution_" + str(
Wavelength.filelist_to_wavename(files, self.band,
self.maskName, ""))
self.addLine("")
self.addLine("Wavelength_file = '" + str(self.waveName) + "'")
self.addLine("")
def handle_flats(flatlist, maskname, band, options, extension=None,edgeThreshold=450,lampOffList=None,longslit=None):
'''
handle_flats is the primary entry point to the Flats module.
handle_flats takes a list of individual exposure FITS files and creates:
1. A CRR, dark subtracted, pixel-response flat file.
2. A set of polynomials that mark the edges of a slit
Inputs:
flatlist:
maskname: The name of a mask
band: A string indicating the bandceil
Outputs:
file {maskname}/flat_2d_{band}.fits -- pixel response flat
file {maskname}/edges.np
'''
tick = time.time()
# Check
bpos = np.ones(92) * -1
#Retrieve the list of files to use for flat creation.
flatlist = IO.list_file_to_strings(flatlist)
# Print the filenames to Standard-out
for flat in flatlist:
info(str(flat))
#Determine if flat files headers are in agreement
for fname in flatlist:
hdr, dat, bs = IO.readmosfits(fname, options, extension=extension)
try: bs0
except: bs0 = bs
if np.any(bs0.pos != bs.pos):
print "bs0: "+str(bs0.pos)+" bs: "+str(bs.pos)
error("Barset do not seem to match")
raise Exception("Barsets do not seem to match")
if hdr["filter"] != band:
error ("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
raise Exception("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
for i in xrange(len(bpos)):
b = hdr["B{0:02d}POS".format(i+1)]
if bpos[i] == -1:
bpos[i] = b
else:
if bpos[i] != b:
error("Bar positions are not all the same in "
"this set of flat files")
raise Exception("Bar positions are not all the same in "
"this set of flat files")
bs = bs0
# Imcombine the lamps ON flats
info("Attempting to combine previous files")
combine(flatlist, maskname, band, options)
# Imcombine the lamps OFF flats and subtract the off from the On sets
if lampOffList != None:
#Retrieve the list of files to use for flat creation.
lampOffList = IO.list_file_to_strings(lampOffList)
# Print the filenames to Standard-out
for flat in lampOffList:
info(str(flat))
print "Attempting to combine Lamps off data"
combine(lampOffList, maskname, band, options, lampsOff=True)
combine_off_on( maskname, band, options)
debug("Combined '%s' to '%s'" % (flatlist, maskname))
info("Comgined to '%s'" % (maskname))
path = "combflat_2d_%s.fits" % band
(header, data) = IO.readfits(path, use_bpm=True)
info("Flat written to %s" % path)
# Edge Trace
if bs.long_slit:
info( "Long slit mode recognized")
info( "Central row position: "+str(longslit["row_position"]))
info( "Upper and lower limits: "+str(longslit["yrange"][0])+" "+str(longslit["yrange"][1]))
results = find_longslit_edges(data,header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit)
elif bs.long2pos_slit:
info( "Long2pos mode recognized")
results = find_long2pos_edges(data,header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit)
else:
results = find_and_fit_edges(data, header, bs, options,edgeThreshold=edgeThreshold)
results[-1]["maskname"] = maskname
results[-1]["band"] = band
np.save("slit-edges_{0}".format(band), results)
save_ds9_edges(results, options)
# Generate Flat
out = "pixelflat_2d_%s.fits" % (band)
if lampOffList != None:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=True)
else:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=False)
info( "Pixel flat took {0:6.4} s".format(time.time()-tick))
try:
print "%(datafile)12s %(object)40s %(truitime)6.1f s %(maskname)35s %(lamps)3s %(filter)4s %(mgtname)7s" % (
header)
except:
try:
print "%(datafile)12s %(object)25s %(truitime)6.1f s %(lamps)3s %(filter)6s %(mgtname)7s" % (
header)
except:
print "%s Skipped" % fname
continue
datafile = header['datafile'] + '.fits'
maskname = str(header['maskname'])
target = str(header['targname'])
filter = header['filter']
yr, mn, dy = IO.fname_to_date_tuple(datafile)
date = str(yr) + mn + str(dy)
object = header['object']
itime = header['truitime']
grating_turret = header['mgtname']
if object.find("MIRA") == -1:
mira = False
else:
mira = True
if header['MGTNAME'] is not 'mirror':
mira = False
if maskname.find(" (align)") == -1:
def handle_rectification(maskname, in_files, wavename, band_pass, files, options,
commissioning_shift=3.0, target='default'):
'''Handle slit rectification and coaddition.
Args:
maskname: The mask name string
in_files: List of stacked spectra in electron per second. Will look
like ['electrons_Offset_1.5.txt.fits', 'electrons_Offset_-1.5.txt.fits']
wavename: path (relative or full) to the wavelength stack file, string
band_pass: Band pass name, string
barset_file: Path to a mosfire fits file containing the full set of
FITS extensions for the barset. It can be any file in the list
of science files.
Returns:
None
Writes files:
[maskname]_[band]_[object name]_eps.fits --
The rectified, background subtracted, stacked eps spectrum
[maskname]_[band]_[object name]_sig.fits --
Rectified, background subtracted, stacked weight spectrum (STD/itime)
[maskname]_[band]_[object_name]_itime.fits
Rectified, CRR stacked integration time spectrum
[maskname]_[band]_[object_name]_snrs.fits
Rectified signal to noise spectrum
'''
global edges, dats, vars, itimes, shifts, lambdas, band, fidl, all_shifts
band = band_pass
dlambda = Wavelength.grating_results(band)
hpp = Filters.hpp[band]
fidl = np.arange(hpp[0], hpp[1], dlambda)
lambdas = IO.readfits(wavename, options)
if np.any(lambdas[1].data < 0) or np.any(lambdas[1].data > 29000):
info("***********WARNING ***********")
info("The file {0} may not be a wavelength file.".format(wavename))
info("Check before proceeding.")
info("***********WARNING ***********")
edges, meta = IO.load_edges(maskname, band, options)
shifts = []
posnames = []
postoshift = {}
for file in in_files:
info(":: "+str(file))
II = IO.read_drpfits(maskname, file, options)
off = np.array((II[0]["decoff"], II[0]["raoff"]),dtype=np.float64)
if "yoffset" in II[0]:
off = -II[0]["yoffset"]
else:
# Deal with data taken during commissioning
if II[0]["frameid"] == 'A': off = 0.0
else: off = commissioning_shift
try: off0
except: off0 = off
shift = off - off0
shifts.append(shift)
posnames.append(II[0]["frameid"])
postoshift[II[0]['frameid']] = shift
info("Position {0} shift: {1:2.2f} as".format(off, shift))
# this is to deal with cases in which we want to rectify one single file
if len(set(posnames)) is 1:
plans = [['A']]
else:
plans = Background.guess_plan_from_positions(set(posnames))
all_shifts = []
for plan in plans:
to_append = []
for pos in plan:
to_append.append(postoshift[pos])
all_shifts.append(to_append)
# Reverse the elements in all_shifts to deal with an inversion
all_shifts.reverse()
theBPM = IO.badpixelmask()
all_solutions = []
cntr = 0
if target is 'default':
outname = maskname
else:
outname = target
for plan in plans:
if len(plan) is 1:
p0 = 'A'
p1 = 'B'
else:
p0 = plan[0].replace("'", "p")
p1 = plan[1].replace("'", "p")
suffix = "%s-%s" % (p0,p1)
info("Handling plan %s" % suffix)
fname = "bsub_{0}_{1}_{2}.fits".format(outname,band,suffix)
EPS = IO.read_drpfits(maskname, fname, options)
EPS[1] = np.ma.masked_array(EPS[1], theBPM, fill_value=0)
fname = "var_{0}_{1}_{2}.fits".format(outname, band, suffix)
VAR = IO.read_drpfits(maskname, fname, options)
VAR[1] = np.ma.masked_array(VAR[1], theBPM, fill_value=np.inf)
fname = "itime_{0}_{1}_{2}.fits".format(outname, band, suffix)
ITIME = IO.read_drpfits(maskname, fname, options)
ITIME[1] = np.ma.masked_array(ITIME[1], theBPM, fill_value=0)
dats = EPS
vars = VAR
itimes = ITIME
EPS[0]["ORIGFILE"] = fname
tock = time.time()
sols = range(len(edges)-1,-1,-1)
shifts = all_shifts[cntr]
cntr += 1
p = Pool()
solutions = p.map(handle_rectification_helper, sols)
p.close()
all_solutions.append(solutions)
tick = time.time()
info("-----> Mask took %i. Writing to disk." % (tick-tock))
output = np.zeros((1, len(fidl)))
snrs = np.zeros((1, len(fidl)))
sdout= np.zeros((1, len(fidl)))
itout= np.zeros((1, len(fidl)))
# the barset [bs] is used for determining object position
files = IO.list_file_to_strings(files)
info("Using "+str(files[0])+" for slit configuration.")
x, x, bs = IO.readmosfits(files[0], options)
for i_slit in xrange(len(solutions)):
solution = all_solutions[0][i_slit]
header = EPS[0].copy()
obj = header['OBJECT']
target_name = bs.ssl[-(i_slit+1)]['Target_Name']
header['OBJECT'] = target_name
pixel_dist = np.float(bs.ssl[-(i_slit+1)]['Target_to_center_of_slit_distance'])/0.18
pixel_dist -= solution['offset']
ll = solution["lambda"]
header["wat0_001"] = "system=world"
header["wat1_001"] = "wtype=linear"
header["wat2_001"] = "wtype=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["ctype1"] = "AWAV"
header["cunit1"] = "Angstrom"
header["crval1"] = ll[0]
header["crval2"] = -solution["eps_img"].shape[0]/2 - pixel_dist
header["crpix1"] = 1
header["crpix2"] = 1
#remove redundant CDELTi due to wavelength issues with ds9
#see: https://github.com/Keck-DataReductionPipelines/MosfireDRP/issues/44
#header["cdelt1"] = 1
#header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = ll[1]-ll[0]
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = 1
try:
header["BARYCORR"]= (lambdas[0]['BARYCORR'],lambdas[0].comments['BARYCORR'])
except KeyError:
warning( "Barycentric corrections not applied to the wavelength solution")
pass
S = output.shape
img = solution["eps_img"]
std = solution["sd_img"]
tms = solution["itime_img"]
for i_solution in xrange(1,len(all_solutions)):
info("Combining solution %i" %i_solution)
solution = all_solutions[i_solution][i_slit]
img += solution["eps_img"]
std += solution["sd_img"]
tms += solution["itime_img"]
#print "adding in quadrature"
output = np.append(output, img, 0)
output = np.append(output, np.nan*np.zeros((3,S[1])), 0)
snrs = np.append(snrs, img*tms/std, 0)
snrs = np.append(snrs, np.nan*np.zeros((3,S[1])), 0)
sdout = np.append(sdout, std, 0)
sdout = np.append(sdout, np.nan*np.zeros((3,S[1])), 0)
itout = np.append(itout, tms, 0)
itout = np.append(itout, np.nan*np.zeros((3,S[1])), 0)
header['bunit'] = ('electron/second', 'electron power')
IO.writefits(img, maskname,
"{0}_{1}_{2}_eps.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('electron/second', 'sigma/itime')
IO.writefits(std/tms, maskname,
"{0}_{1}_{2}_sig.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('second', 'exposure time')
IO.writefits(tms, maskname,
"{0}_{1}_{2}_itime.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('', 'SNR')
IO.writefits(img*tms/std, maskname,
"{0}_{1}_{2}_snrs.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header = EPS[0].copy()
header["wat0_001"] = "system=world"
header["wat1_001"] = "wtype=linear"
header["wat2_001"] = "wtype=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["ctype1"] = "AWAV"
header["cunit1"] = ("Angstrom", 'Start wavelength')
header["crval1"] = ll[0]
header["crval2"] = 1
header["crpix1"] = 1
header["crpix2"] = 1
#remove redundant CDELTi due to wavelength issues with ds9
#see: https://github.com/Keck-DataReductionPipelines/MosfireDRP/issues/44
#header["cdelt1"] = 1
#header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = (ll[1]-ll[0], 'Angstrom/pixel')
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = 1
try:
header["BARYCORR"]= (lambdas[0]['BARYCORR'],lambdas[0].comments['BARYCORR'])
except KeyError:
warning( "Barycentric corrections not applied to the wavelength solution")
pass
header["bunit"] = "ELECTRONS/SECOND"
info("############ Final reduced file: {0}_{1}_eps.fits".format(outname,band))
IO.writefits(output, maskname, "{0}_{1}_eps.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = ""
IO.writefits(snrs, maskname, "{0}_{1}_snrs.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = "ELECTRONS/SECOND"
IO.writefits(sdout/itout, maskname, "{0}_{1}_sig.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = "SECOND"
IO.writefits(itout, maskname, "{0}_{1}_itime.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
def isEmpty(self, file):
fname = IO.list_file_to_strings(file)
if len(fname):
return False
else:
return True
def printWavelengthFit(self):
if self.type is 'longslit' or self.type is 'long2pos':
addLongSlit = ",longslit=longslit"
else:
addLongSlit = ""
if self.type is 'slitmask' or self.type is 'longslit':
self.useNeon = False
self.useArgon = False
# determine is Argon and Neon files contain data for K bands
if self.isEmpty('Ar.txt') is False and self.band is 'K':
self.useArgon = True
if self.isEmpty('Ne.txt') is False and self.band is 'K':
self.useNeon = True
self.addLine("Wavelength.imcombine(obsfiles, maskname, band, waveops)")
if self.useArgon:
self.addLine("Wavelength.imcombine('Ar.txt', maskname, band, waveops)")
if self.useNeon:
self.addLine("Wavelength.imcombine('Ne.txt', maskname, band, waveops)")
self.addLine("Wavelength.fit_lambda_interactively(maskname, band, obsfiles,waveops"+addLongSlit+", bypass=bypassflag)")
if self.useArgon:
self.addLine("Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ar.txt', argon=True)")
if self.useNeon:
self.addLine("Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ne.txt', neon=True)")
self.addLine("Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles,waveops"+addLongSlit+")")
if self.useArgon and self.useNeon:
self.addLine("Wavelength.fit_lambda(maskname, band, 'Ne.txt', 'Ne.txt',waveops, wavenames2='Ar.txt'"+addLongSlit+")")
if self.useArgon and not self.useNeon:
self.addLine("Wavelength.fit_lambda(maskname, band, 'Ar.txt', 'Ar.txt',waveops"+addLongSlit+")")
if self.useNeon and not self.useArgon:
self.addLine("Wavelength.fit_lambda(maskname, band, 'Ne.txt', 'Ne.txt',waveops"+addLongSlit+")")
if self.useNeon or self.useArgon:
self.addLine("LROI = [[21000,22800]]*1")
if self.useNeon:
self.addLine("LROIs = Wavelength.check_wavelength_roi(maskname, band, obsfiles, 'Ne.txt', LROI, waveops)")
if self.useArgon and not self.useNeon:
self.addLine("LROIs = Wavelength.check_wavelength_roi(maskname, band, obsfiles, 'Ar.txt', LROI, waveops)")
self.addLine("Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops"+addLongSlit+")")
if self.useArgon and self.useNeon:
self.addLine("Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ne.txt', LROIs, waveops)")
if self.useArgon and not self.useNeon:
self.addLine("Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ar.txt', LROIs, waveops)")
if self.useNeon and not self.useArgon:
self.addLine("Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ne.txt', LROIs, waveops)")
# determine waveleng name
files = IO.list_file_to_strings(self.offsetFiles)
if self.useNeon:
neon_files = IO.list_file_to_strings('Ne.txt')
self.waveName = "merged_lambda_solution_"+str(Wavelength.filelist_to_wavename(files, self.band, self.maskName,"")).rstrip(".fits")+"_and_"+str(Wavelength.filelist_to_wavename(neon_files, self.band, self.maskName,""))
elif self.useArgon and not self.useNeon:
argon_files = IO.list_file_to_strings('Ar.txt')
self.waveName = "merged_lambda_solution_"+str(Wavelength.filelist_to_wavename(files, self.band, self.maskName,"")).rstrip(".fits")+"_and_"+str(Wavelength.filelist_to_wavename(argon_files, self.band, self.maskName,""))
else:
self.waveName = "lambda_solution_"+str(Wavelength.filelist_to_wavename(files, self.band, self.maskName,""))
if self.type is 'long2pos' or self.type is 'long2pos_specphot':
calibWith = ""
if self.isEmpty('Ar.txt') is False:
self.addLine("argon = ['Ar.txt']")
calibWith = "argon"
waveFiles = IO.list_file_to_strings('Ar.txt')
if self.isEmpty('Ne.txt') is False:
self.addLine("neon = ['Ne.txt']")
calibWith = "neon"
waveFiles = IO.list_file_to_strings('Ne.txt')
if calibWith:
# we have either Argon, or Neon, or both, so we can use arcs for the reduction
self.addLine("Wavelength.imcombine("+str(calibWith)+", maskname, band, waveops)")
self.addLine("Wavelength.fit_lambda_interactively(maskname, band, "+str(calibWith)+",waveops,longslit=longslit, "+str(calibWith)+"=True, bypass=bypassflag)")
self.addLine("Wavelength.fit_lambda(maskname, band, "+str(calibWith)+","+str(calibWith)+",waveops,longslit=longslit)")
self.addLine("Wavelength.apply_lambda_simple(maskname, band, "+str(calibWith)+", waveops, longslit=longslit, smooth=True)")
self.waveName = "lambda_solution_"+str(Wavelength.filelist_to_wavename(waveFiles, self.band, self.maskName,""))
else:
# we have no arcs. For the time being, we can try with sky lines but this only works with long2pos specphot
print "#####################################################################################################"
print "WARNING: There are no arc calibration files"
print " The pipeline will try to use sky lines but this only works if the observation is long enough"
print " and if you are only using long2pos. It will NOT work on long2pos_specphot"
print " Please contact the MosfireDRP team to obtain a standard wavelength solution"
print "#####################################################################################################"
self.addLine("obsfiles = obsfiles_posAnarrow + obsfiles_posCnarrow")
self.addLine("Wavelength.imcombine(obsfiles, maskname, band, waveops)")
self.addLine("Wavelength.fit_lambda_interactively(maskname, band, obsfiles ,waveops,longslit=longslit, bypass=bypassflag)")
self.addLine("Wavelength.fit_lambda(maskname, band, obsfiles,obsfiles ,waveops,longslit=longslit)")
self.addLine("Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops, longslit=longslit, smooth=True)")
files = IO.list_file_to_strings(self.offsetFiles)
self.waveName = "lambda_solution_"+str(Wavelength.filelist_to_wavename(files, self.band, self.maskName,""))
self.addLine("")
self.addLine("Wavelength_file = '"+str(self.waveName)+"'")
self.addLine("Wavelength.bary_corr(obsfiles, Wavelength_file, maskname, band, waveops)")
self.addLine("")
def handle_background(filelist,
wavename,
maskname,
band_name,
options,
shifts=None,
plan=None,
extension=None):
'''
Perform difference imaging and subtract residual background.
The plan looks something like: [['A', 'B']]
In this case, the number of output files is equal to the length of the list (1).
If you choose to use an ABA'B' pattern then the plan will be: [["A", "B"], ["A'", "B'"]]
the background subtraction code will make and handle two files, "A-B" and "A'-B'".
'''
global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band
band = band_name
flatname = "pixelflat_2d_%s.fits" % band_name
hdr, flat = IO.readfits("pixelflat_2d_%s.fits" % (band_name), options)
if np.abs(np.median(flat) - 1) > 0.1:
raise Exception("Flat seems poorly behaved.")
'''
This next section of the code figures out the observing plan
and then deals with the bookeeping of sending the plan
to the background subtracter.
'''
hdrs = []
epss = {}
vars = {}
bss = []
times = {}
Nframes = []
i = 0
header = pf.Header()
for i in xrange(len(filelist)):
fl = filelist[i]
files = IO.list_file_to_strings(fl)
print "Combining"
if shifts is None: shift = None
else: shift = shifts[i]
hdr, electron, var, bs, time, Nframe = imcombine(files,
maskname,
options,
flat,
outname="%s.fits" %
(fl),
shifts=shift,
extension=extension)
hdrs.append(hdr)
header = merge_headers(header, hdr)
epss[hdr['FRAMEID']] = electron / time
vars[hdr['FRAMEID']] = var
times[hdr['FRAMEID']] = time
bss.append(bs)
Nframes.append(Nframe)
positions = {}
i = 0
for h in hdrs:
positions[h['FRAMEID']] = i
i += 1
posnames = set(positions.keys())
if plan is None:
plan = guess_plan_from_positions(posnames)
num_outputs = len(plan)
edges, meta = IO.load_edges(maskname, band, options)
lam = IO.readfits(wavename, options)
bs = bss[0]
for i in xrange(num_outputs):
posname0 = plan[i][0]
posname1 = plan[i][1]
print "Handling %s - %s" % (posname0, posname1)
data = epss[posname0] - epss[posname1]
Var = vars[posname0] + vars[posname1]
itime = np.mean([times[posname0], times[posname1]], axis=0)
p = Pool()
solutions = p.map(background_subtract_helper, xrange(len(bs.ssl)))
p.close()
write_outputs(solutions, itime, header, maskname, band, plan[i],
options)
